@teaflask/assistant 0.0.0 → 0.3.0

package/src/reader/render.ts

@@ -0,0 +1,150 @@
+// The snapshot's text form: an indented outline the model reads top to
+// bottom, one `- role "name" [ref=eN] [flags]` line per node, string
+// children as `- text: "…"` rows, and a two-line page header. Selects
+// render their options inline — the accessibility walk never descends
+// into them, but the choice list is exactly what a "what am I looking
+// at" answer needs.
+
+import { boundedTextOf } from "../transport/door-budget.js";
+import type { SnapshotNode } from "./snapshot.js";
+
+const MAX_TITLE_LENGTH = 300;
+const MAX_URL_LENGTH = 2_000;
+const MAX_OPTION_TEXT_LENGTH = 100;
+
+export function pageHeaderOf(title: string, url: string): string {
+  const boundedTitle = boundedTextOf(
+    title.replace(/\s+/g, " ").trim(),
+    MAX_TITLE_LENGTH,
+  );
+  const boundedUrl = boundedTextOf(url, MAX_URL_LENGTH);
+  return `# page: "${_escapedText(boundedTitle)}"\n# url: ${boundedUrl}`;
+}
+
+export function renderedTreeOf(root: SnapshotNode): string {
+  const lines: string[] = [];
+  _renderChildren(root, 0, lines);
+  return lines.join("\n");
+}
+
+function _renderChildren(
+  node: SnapshotNode,
+  depth: number,
+  lines: string[],
+): void {
+  for (const child of node.children) {
+    if (typeof child === "string") {
+      const text = child.trim();
+      if (text !== "") {
+        lines.push(`${_indentOf(depth)}- text: "${_escapedText(text)}"`);
+      }
+    } else {
+      _renderNode(child, depth, lines);
+    }
+  }
+}
+
+function _renderNode(node: SnapshotNode, depth: number, lines: string[]): void {
+  const indent = _indentOf(depth);
+  const label = _nodeLineOf(node);
+  const isSelect = node.element instanceof HTMLSelectElement;
+  const textChildren = node.children.filter(
+    (child): child is string => typeof child === "string",
+  );
+  const elementChildren = node.children.filter(
+    (child): child is SnapshotNode => typeof child !== "string",
+  );
+
+  if (textChildren.length === 1 && elementChildren.length === 0 && !isSelect) {
+    const text = textChildren[0].trim();
+    if (text === "") {
+      lines.push(`${indent}- ${label}`);
+    } else {
+      lines.push(`${indent}- ${label}: "${_escapedText(text)}"`);
+    }
+    return;
+  }
+
+  if (node.children.length === 0 && !isSelect) {
+    lines.push(`${indent}- ${label}`);
+    return;
+  }
+
+  lines.push(`${indent}- ${label}:`);
+  if (isSelect && node.element instanceof HTMLSelectElement) {
+    _renderSelectOptions(node.element, depth + 1, lines);
+    return;
+  }
+  _renderChildren(node, depth + 1, lines);
+}
+
+function _renderSelectOptions(
+  select: HTMLSelectElement,
+  depth: number,
+  lines: string[],
+): void {
+  const indent = _indentOf(depth);
+  for (const option of Array.from(select.options)) {
+    const text = option.text
+      .replace(/\s+/g, " ")
+      .trim()
+      .substring(0, MAX_OPTION_TEXT_LENGTH);
+    let line = `${indent}- option "${_escapedText(text)}"`;
+    if (option.selected) {
+      line += " (selected)";
+    }
+    if (option.value !== text) {
+      line += ` value="${_escapedText(option.value)}"`;
+    }
+    lines.push(line);
+  }
+}
+
+function _nodeLineOf(node: SnapshotNode): string {
+  const parts = [node.role];
+  if (node.name !== "") {
+    parts.push(`"${_escapedText(node.name)}"`);
+  }
+  if (node.refId !== null) {
+    parts.push(`[ref=${node.refId}]`);
+  }
+  if (node.level !== null && node.level > 0) {
+    parts.push(`[level=${String(node.level)}]`);
+  }
+  if (node.hidden) {
+    parts.push("[hidden]");
+  }
+  if (node.offscreen) {
+    parts.push("[offscreen]");
+  }
+  if (node.scrollable) {
+    parts.push("[scrollable]");
+  }
+  if (node.checked) {
+    parts.push("[checked]");
+  }
+  if (node.disabled) {
+    parts.push("[disabled]");
+  }
+  if (node.focused) {
+    parts.push("[focused]");
+  }
+  if (node.selected) {
+    parts.push("[selected]");
+  }
+  if (node.placeholder !== null && node.placeholder !== "") {
+    parts.push(`[placeholder="${_escapedText(node.placeholder)}"]`);
+  }
+  if (node.hints.length > 0) {
+    parts.push(`[${node.hints.join(" ")}]`);
+  }
+  return parts.join(" ");
+}
+
+function _escapedText(text: string): string {
+  return text.replace(/"/g, '\\"').replace(/\n/g, "\\n");
+}
+
+function _indentOf(depth: number): string {
+  return "  ".repeat(depth);
+}

package/src/reader/roles.ts

@@ -0,0 +1,277 @@
+// Role resolution for the page reader: which ARIA role a rendered
+// element effectively carries. Explicit role attributes win when they
+// name a real role; everything else falls back to the tag's implicit
+// role. role="presentation"/"none" deliberately fall back too — sites
+// slap presentation on styled tables and lists whose semantics the
+// reader still wants. Pure attribute logic, no layout reads.
+
+// Tag → implicit ARIA role, consulted when the element is not an input
+// and not contentEditable.
+const IMPLICIT_ROLE_BY_TAG: Record<string, string> = {
+  a: "link",
+  button: "button",
+  iframe: "iframe",
+  select: "combobox",
+  textarea: "textbox",
+  h1: "heading",
+  h2: "heading",
+  h3: "heading",
+  h4: "heading",
+  h5: "heading",
+  h6: "heading",
+  canvas: "canvas",
+  svg: "image",
+  img: "image",
+  nav: "navigation",
+  main: "main",
+  header: "banner",
+  footer: "contentinfo",
+  section: "region",
+  article: "article",
+  aside: "complementary",
+  form: "form",
+  fieldset: "group",
+  table: "table",
+  ul: "list",
+  ol: "list",
+  li: "listitem",
+  p: "paragraph",
+  label: "label",
+};
+
+// An explicit role="…" is honored only when it names a role in this
+// allowlist (first valid token wins).
+const VALID_ARIA_ROLES = new Set([
+  "alert",
+  "alertdialog",
+  "application",
+  "article",
+  "banner",
+  "blockquote",
+  "button",
+  "caption",
+  "cell",
+  "checkbox",
+  "code",
+  "columnheader",
+  "combobox",
+  "complementary",
+  "contentinfo",
+  "definition",
+  "deletion",
+  "dialog",
+  "directory",
+  "document",
+  "emphasis",
+  "feed",
+  "figure",
+  "form",
+  "generic",
+  "grid",
+  "gridcell",
+  "group",
+  "heading",
+  "img",
+  "insertion",
+  "link",
+  "list",
+  "listbox",
+  "listitem",
+  "log",
+  "main",
+  "mark",
+  "marquee",
+  "math",
+  "meter",
+  "menu",
+  "menubar",
+  "menuitem",
+  "menuitemcheckbox",
+  "menuitemradio",
+  "navigation",
+  "none",
+  "note",
+  "option",
+  "paragraph",
+  "presentation",
+  "progressbar",
+  "radio",
+  "radiogroup",
+  "region",
+  "row",
+  "rowgroup",
+  "rowheader",
+  "scrollbar",
+  "search",
+  "searchbox",
+  "separator",
+  "slider",
+  "spinbutton",
+  "status",
+  "strong",
+  "subscript",
+  "superscript",
+  "switch",
+  "tab",
+  "table",
+  "tablist",
+  "tabpanel",
+  "term",
+  "textbox",
+  "time",
+  "timer",
+  "toolbar",
+  "tooltip",
+  "tree",
+  "treegrid",
+  "treeitem",
+]);
+
+// Roles whose nodes carry the bracketed state that matches the set.
+const ARIA_LEVEL_ROLES = new Set(["heading", "listitem", "row", "treeitem"]);
+
+export const ARIA_SELECTED_ROLES = new Set([
+  "gridcell",
+  "option",
+  "row",
+  "tab",
+  "rowheader",
+  "columnheader",
+  "treeitem",
+]);
+
+export const ARIA_CHECKED_ROLES = new Set([
+  "checkbox",
+  "radio",
+  "menuitemcheckbox",
+  "menuitemradio",
+  "switch",
+]);
+
+export const ARIA_DISABLED_ROLES = new Set([
+  "application",
+  "button",
+  "composite",
+  "gridcell",
+  "group",
+  "input",
+  "link",
+  "menuitem",
+  "scrollbar",
+  "separator",
+  "tab",
+  "checkbox",
+  "columnheader",
+  "combobox",
+  "grid",
+  "listbox",
+  "menu",
+  "menubar",
+  "menuitemcheckbox",
+  "menuitemradio",
+  "option",
+  "radio",
+  "radiogroup",
+  "row",
+  "rowheader",
+  "searchbox",
+  "select",
+  "slider",
+  "spinbutton",
+  "switch",
+  "tablist",
+  "textbox",
+  "toolbar",
+  "tree",
+  "treegrid",
+  "treeitem",
+]);
+
+const NATIVE_HEADING_LEVELS: Record<string, number | undefined> = {
+  h1: 1,
+  h2: 2,
+  h3: 3,
+  h4: 4,
+  h5: 5,
+  h6: 6,
+};
+
+/** The element's effective role: explicit when valid, implicit otherwise. */
+export function roleOf(element: Element): string {
+  const explicit = explicitRoleOf(element);
+  if (explicit === null) {
+    return _implicitRoleOf(element);
+  }
+  if (explicit === "presentation" || explicit === "none") {
+    return _implicitRoleOf(element);
+  }
+  return explicit;
+}
+
+/** The first valid token of role="…", or null when none names a role. */
+export function explicitRoleOf(element: Element): string | null {
+  const tokens = (element.getAttribute("role") ?? "")
+    .split(" ")
+    .map((token) => token.trim().toLowerCase())
+    .filter((token) => token !== "");
+  return tokens.find((token) => VALID_ARIA_ROLES.has(token)) ?? null;
+}
+
+/** The heading/listitem/row/treeitem level, or null when the role has none. */
+export function ariaLevelOf(element: Element, role: string): number | null {
+  if (!ARIA_LEVEL_ROLES.has(role)) {
+    return null;
+  }
+  if (role === "heading") {
+    const native = NATIVE_HEADING_LEVELS[element.tagName.toLowerCase()];
+    if (native !== undefined) {
+      return native;
+    }
+  }
+  const declared = Number(element.getAttribute("aria-level"));
+  return Number.isInteger(declared) && declared >= 1 ? declared : null;
+}
+
+/** aria-* boolean tri-state: true / false / absent. */
+export function ariaBooleanOf(attribute: string | null): boolean | undefined {
+  if (attribute === null) {
+    return undefined;
+  }
+  return attribute.toLowerCase() === "true";
+}
+
+function _implicitRoleOf(element: Element): string {
+  const tag = element.tagName.toLowerCase();
+  if (isContentEditableElement(element)) {
+    return "textbox";
+  }
+  if (tag === "input") {
+    const type = element.getAttribute("type");
+    if (type === "submit" || type === "button" || type === "file") {
+      return "button";
+    }
+    if (type === "checkbox") {
+      return "checkbox";
+    }
+    if (type === "radio") {
+      return "radio";
+    }
+    return "textbox";
+  }
+  return IMPLICIT_ROLE_BY_TAG[tag] ?? "generic";
+}
+
+/** Editable regions read as textboxes. The attribute fallback covers
+ * engines that don't reflect the contentEditable property (jsdom). */
+export function isContentEditableElement(element: Element): boolean {
+  if (element instanceof HTMLElement) {
+    const reflected = element.contentEditable;
+    if (reflected === "true" || reflected === "plaintext-only") {
+      return true;
+    }
+  }
+  const attribute = element.getAttribute("contenteditable");
+  return (
+    attribute === "" || attribute === "true" || attribute === "plaintext-only"
+  );
+}

package/src/reader/snapshot-compaction.ts

@@ -0,0 +1,208 @@
+// The compaction passes over a walked tree: merge text runs, unwrap
+// anonymous wrappers, fold text leaves into their parent's name, and
+// refuse a tree that minted one ref twice.
+
+import type { SnapshotNode } from "./snapshot.js";
+import { _normalizedNodeName } from "./snapshot-node.js";
+
+/** Merge adjacent string children, fold bare paragraphs into the run,
+ * and drop a lone text child that just repeats the node's name. */
+export function _mergeStringChildren(node: SnapshotNode): void {
+  for (const child of node.children) {
+    if (typeof child !== "string") {
+      _mergeStringChildren(child);
+    }
+  }
+  const merged: (SnapshotNode | string)[] = [];
+  let buffer: string[] = [];
+  const flush = (): void => {
+    const text = _mergedTextOf(buffer);
+    if (text !== "") {
+      merged.push(text);
+    }
+    buffer = [];
+  };
+  for (const child of node.children) {
+    if (typeof child === "string") {
+      buffer.push(child);
+    } else if (_isBareTextParagraph(child)) {
+      buffer.push(
+        _mergedTextOf(child.children.filter((c) => typeof c === "string")),
+      );
+    } else {
+      flush();
+      merged.push(child);
+    }
+  }
+  flush();
+  node.children = merged;
+  if (
+    node.children.length === 1 &&
+    typeof node.children[0] === "string" &&
+    node.children[0] === node.name
+  ) {
+    node.children = [];
+  }
+}
+
+function _isBareTextParagraph(node: SnapshotNode): boolean {
+  return (
+    node.role === "paragraph" &&
+    node.refId === null &&
+    !node.hidden &&
+    !node.scrollable &&
+    node.children.every((child) => typeof child === "string")
+  );
+}
+
+/** Drop hidden ref-less subtrees and unwrap anonymous wrappers whose
+ * only content is a single ref-bearing child. */
+export function _unwrapBareGenerics(
+  node: SnapshotNode,
+  showHidden: boolean,
+): void {
+  const result: (SnapshotNode | string)[] = [];
+  for (const child of node.children) {
+    if (typeof child === "string") {
+      result.push(child);
+      continue;
+    }
+    _unwrapBareGenerics(child, showHidden);
+    // Hidden ref-less noise is pruned from a default read (only the
+    // branded checkbox/radio escape reaches here, and it carries a ref)
+    // — but a show_hidden read exists precisely to surface hidden text
+    // and structure, so there the drop would gut the knob.
+    if (
+      !showHidden &&
+      child.hidden &&
+      child.refId === null &&
+      !_hasRefInSubtree(child)
+    ) {
+      continue;
+    }
+    if (
+      child.role === "paragraph" &&
+      child.refId === null &&
+      child.children.length === 1 &&
+      typeof child.children[0] !== "string" &&
+      child.children[0].role === "paragraph"
+    ) {
+      result.push(child.children[0]);
+      continue;
+    }
+    if (
+      child.role === "generic" &&
+      !child.hidden &&
+      child.name === "" &&
+      child.children.length === 1 &&
+      typeof child.children[0] !== "string" &&
+      child.children[0].refId !== null
+    ) {
+      result.push(child.children[0]);
+      continue;
+    }
+    result.push(child);
+  }
+  node.children = result;
+}
+
+function _hasRefInSubtree(node: SnapshotNode): boolean {
+  if (node.refId !== null) {
+    return true;
+  }
+  return node.children.some(
+    (child) => typeof child !== "string" && _hasRefInSubtree(child),
+  );
+}
+
+const TEXT_LEAF_ROLES = new Set(["generic", "heading", "paragraph", "label"]);
+
+/** Fold a ref-bearing node's few text-only children into its own name. */
+export function _foldTextLeaves(node: SnapshotNode): void {
+  for (const child of node.children) {
+    if (typeof child !== "string") {
+      _foldTextLeaves(child);
+    }
+  }
+  if (node.refId === null) {
+    return;
+  }
+  if (node.children.length < 1 || node.children.length > 3) {
+    return;
+  }
+  const texts: string[] = [];
+  for (const child of node.children) {
+    if (typeof child === "string") {
+      texts.push(child);
+      continue;
+    }
+    const isFoldableLeaf =
+      !child.hidden &&
+      TEXT_LEAF_ROLES.has(child.role) &&
+      child.name !== "" &&
+      child.children.length === 0 &&
+      child.refId === null;
+    if (!isFoldableLeaf) {
+      return;
+    }
+    texts.push(child.name);
+  }
+  const mergedText = _normalizedNodeName(texts.join(" "));
+  if (node.name === "") {
+    node.name = mergedText;
+    node.children = [];
+  } else if (node.name.replace(/\s+/g, "") === mergedText.replace(/\s+/g, "")) {
+    node.children = [];
+  }
+}
+
+function _mergedTextOf(parts: string[]): string {
+  let merged = "";
+  for (const part of parts) {
+    if (part === "") {
+      continue;
+    }
+    if (merged === "") {
+      merged = part;
+      continue;
+    }
+    if (_wordsMeetAtTheSeam(merged, part)) {
+      merged += " ";
+    }
+    merged += part;
+  }
+  return merged.trim();
+}
+
+function _wordsMeetAtTheSeam(left: string, right: string): boolean {
+  // Raw boundary characters, not trimmed ends: a fragment that already
+  // carries its own boundary space ("Welcome " + "back") must not gain a
+  // second one — the doubled space would also defeat the repeats-the-name
+  // dedup above.
+  if (/\s$/u.test(left) || /^\s/u.test(right)) {
+    return false;
+  }
+  return /[\p{L}\p{N}]$/u.test(left) && /^[\p{L}\p{N}]/u.test(right);
+}
+
+export function _duplicateRefOf(node: SnapshotNode): string | null {
+  const seen = new Set<string>();
+  const walk = (current: SnapshotNode): string | null => {
+    if (current.refId !== null) {
+      if (seen.has(current.refId)) {
+        return current.refId;
+      }
+      seen.add(current.refId);
+    }
+    for (const child of current.children) {
+      if (typeof child !== "string") {
+        const duplicate = walk(child);
+        if (duplicate !== null) {
+          return duplicate;
+        }
+      }
+    }
+    return null;
+  };
+  return walk(node);
+}