@djangocfg/ui-tools 2.1.404 → 2.1.408

package/src/lib/page-snapshot/capture/scope.ts

@@ -0,0 +1,169 @@
+/**
+ * Scope detection — find the root element(s) the DOM walk starts from.
+ *
+ * Implements the 6-tier fallback ladder: try each tier
+ * top-to-bottom, first non-empty result wins. `[data-ai-context]` is an
+ * optional refinement (Tier 1), NOT a requirement — auto-detection
+ * works on un-annotated pages.
+ *
+ */
+
+import { isChrome } from './chrome-filter';
+import { elementArea, isVisible, linkDensity } from './dom-utils';
+
+/** Strategy for choosing the capture root. */
+export type ScopeStrategy = 'container' | 'viewport' | 'full';
+
+/** Options influencing scope resolution. */
+export interface ScopeOptions {
+  strategy: ScopeStrategy;
+  /** Optional explicit selector; overrides auto-detection entirely. */
+  targetSelector?: string;
+}
+
+/** Which tier produced the resolved scope (for telemetry / debugging). */
+export type ScopeTier =
+  | 'explicit-selector'
+  | 'tier0-dialog'
+  | 'tier1-annotation'
+  | 'tier2-main'
+  | 'tier3-heuristic'
+  | 'tier4-body'
+  | 'tier5-viewport';
+
+/** Resolved capture scope. */
+export interface ResolvedScope {
+  /** Root element(s) to walk. Usually one; dialogs/multi-panel give more. */
+  roots: HTMLElement[];
+  /** The tier that produced this result. */
+  tier: ScopeTier;
+}
+
+/** Block-level tags considered as heuristic content candidates (Tier 3). */
+const CANDIDATE_SELECTOR =
+  'div, section, article, form, table, ul, ol, [role="region"]';
+
+/** Tier 0 — a visible open modal / dialog overrides everything. */
+function tierDialog(): HTMLElement | null {
+  const selectors = [
+    '[role="dialog"][aria-modal="true"]',
+    '[role="alertdialog"][aria-modal="true"]',
+    'dialog[open]',
+  ];
+  for (const sel of selectors) {
+    const nodes = Array.from(document.querySelectorAll(sel));
+    const visible = nodes.filter((n) => isVisible(n)) as HTMLElement[];
+    if (visible.length > 0) {
+      // Topmost / largest open dialog.
+      return visible.sort((a, b) => elementArea(b) - elementArea(a))[0];
+    }
+  }
+  return null;
+}
+
+/** Tier 1 — explicit `[data-ai-context]` annotation (optional refinement). */
+function tierAnnotation(): HTMLElement[] {
+  const nodes = Array.from(
+    document.querySelectorAll('[data-ai-context]'),
+  ) as HTMLElement[];
+  return nodes.filter(
+    (n) => isVisible(n) && n.dataset.aiContext !== 'exclude',
+  );
+}
+
+/** Tier 2 — ARIA `<main>` / `role="main"`. */
+function tierMain(): HTMLElement | null {
+  const nodes = Array.from(
+    document.querySelectorAll('main, [role="main"]'),
+  ) as HTMLElement[];
+  const visible = nodes.filter((n) => isVisible(n));
+  if (visible.length === 0) return null;
+  if (visible.length === 1) return visible[0];
+  // Multiple — take the largest by rendered area.
+  return visible.sort((a, b) => elementArea(b) - elementArea(a))[0];
+}
+
+/** Tier 3 — heuristic largest non-chrome content block. */
+function tierHeuristic(): HTMLElement | null {
+  const docArea = Math.max(
+    1,
+    document.documentElement.clientWidth *
+      document.documentElement.clientHeight,
+  );
+  const candidates = Array.from(
+    document.body.querySelectorAll(CANDIDATE_SELECTOR),
+  ) as HTMLElement[];
+
+  let best: HTMLElement | null = null;
+  let bestScore = 0;
+
+  for (const el of candidates) {
+    if (!isVisible(el) || isChrome(el)) continue;
+    const normalizedArea = Math.min(1, elementArea(el) / docArea);
+    if (normalizedArea < 0.05) continue; // too small to be main content
+    const density = linkDensity(el);
+    // Favor large, low-link-density blocks (boilerplate is link-dense).
+    const score = normalizedArea * (1 - density);
+    if (score > bestScore) {
+      bestScore = score;
+      best = el;
+    }
+  }
+  return best;
+}
+
+/**
+ * Tier 5 — intersect a root with the viewport. Last resort for huge
+ * pages: returns the root itself (the walk's own viewport filtering
+ * trims off-screen nodes).
+ */
+function tierViewport(root: HTMLElement): HTMLElement {
+  return root;
+}
+
+/**
+ * Resolve the scope to walk.
+ *
+ * Honors an explicit `targetSelector` first; otherwise runs the ladder.
+ * The `strategy` option biases the ladder: `'viewport'` jumps to the
+ * viewport tier, `'full'` forces body-minus-chrome.
+ */
+export function resolveScope(options: ScopeOptions): ResolvedScope {
+  if (typeof document === 'undefined') {
+    throw new Error('resolveScope: no document (non-browser runtime).');
+  }
+
+  // Explicit selector — caller knows best.
+  if (options.targetSelector) {
+    const el = document.querySelector(options.targetSelector);
+    if (el instanceof HTMLElement) {
+      return { roots: [el], tier: 'explicit-selector' };
+    }
+  }
+
+  // Strategy shortcuts.
+  if (options.strategy === 'full') {
+    return { roots: [document.body], tier: 'tier4-body' };
+  }
+  if (options.strategy === 'viewport') {
+    return { roots: [tierViewport(document.body)], tier: 'tier5-viewport' };
+  }
+
+  // The ladder (strategy === 'container').
+  const dialog = tierDialog();
+  if (dialog) return { roots: [dialog], tier: 'tier0-dialog' };
+
+  const annotated = tierAnnotation();
+  if (annotated.length > 0) {
+    return { roots: annotated, tier: 'tier1-annotation' };
+  }
+
+  const main = tierMain();
+  if (main) return { roots: [main], tier: 'tier2-main' };
+
+  const heuristic = tierHeuristic();
+  if (heuristic) return { roots: [heuristic], tier: 'tier3-heuristic' };
+
+  // Tier 4 — body minus chrome (chrome is filtered during the walk).
+  return { roots: [document.body], tier: 'tier4-body' };
+}

package/src/lib/page-snapshot/capture/walk.ts

@@ -0,0 +1,250 @@
+/**
+ * DOM → CST walker.
+ *
+ * Recursively walks a scope root into CST nodes: visibility filtering,
+ * chrome exclusion, role classification, ref assignment, and a
+ * redaction hook for every captured value/text.
+ *
+ * Layout reads are confined to dom-utils helpers; the walk itself does
+ * no interleaved geometry queries beyond the up-front visibility check.
+ *
+ */
+
+import type {
+  CSTContainerNode,
+  CSTInteractiveNode,
+  CSTNode,
+  CSTRefId,
+  CSTTextNode,
+} from '../cst/types';
+import { accessibleName } from './accessible-name';
+import {
+  classify,
+  containerRole,
+  interactiveRole,
+} from './classify';
+import { isChrome } from './chrome-filter';
+import {
+  PLACEHOLDER_TAGS,
+  SKIP_TAGS,
+  isVisible,
+  tagName,
+} from './dom-utils';
+
+/** A value passed through redaction before entering the tree. */
+export type RedactValue = (value: string, context: RedactContext) => string;
+
+/** Context for a redaction decision. */
+export interface RedactContext {
+  /** Element the value came from. */
+  element: HTMLElement;
+  /** Where the value sits — an input value or static text. */
+  kind: 'value' | 'text';
+  /** Ref id assigned to the element, if any. */
+  ref?: CSTRefId;
+}
+
+/** No-op redaction — passes values through unchanged. */
+export const noopRedact: RedactValue = (value) => value;
+
+/** Result of walking the DOM under a scope root. */
+export interface WalkResult {
+  /** Captured child nodes (the root node is assembled by the engine). */
+  children: CSTNode[];
+  /** Map of assigned ref id → element — kept so directives can later
+   *  resolve a CST ref back to the live DOM node. */
+  refMap: Map<CSTRefId, HTMLElement>;
+  /** Number of elements visited. */
+  nodesVisited: number;
+}
+
+/** Options for a walk. */
+export interface WalkOptions {
+  /** Redaction hook; defaults to no-op. */
+  redactValue?: RedactValue;
+  /** Hard cap on elements visited (safety valve); default 50_000. */
+  maxNodes?: number;
+}
+
+/** Mutable state threaded through one walk. */
+interface WalkState {
+  redact: RedactValue;
+  refMap: Map<CSTRefId, HTMLElement>;
+  refCounter: number;
+  visited: number;
+  maxNodes: number;
+}
+
+/** Minimum trimmed length for a text node to be worth keeping. */
+const MIN_TEXT_LEN = 2;
+
+/** Read the current value of a form control as a string. */
+function controlValue(el: Element): string | undefined {
+  if (el instanceof HTMLInputElement) {
+    if (el.type === 'checkbox' || el.type === 'radio') return undefined;
+    return el.value || undefined;
+  }
+  if (el instanceof HTMLTextAreaElement) return el.value || undefined;
+  if (el instanceof HTMLSelectElement) {
+    return el.selectedOptions[0]?.textContent?.trim() || undefined;
+  }
+  if (el.getAttribute('contenteditable') === 'true') {
+    return el.textContent?.trim() || undefined;
+  }
+  return undefined;
+}
+
+/** Build an interactive CST node for an element. */
+function buildInteractive(el: HTMLElement, state: WalkState): CSTInteractiveNode {
+  const role = interactiveRole(el)!;
+  const ref = `@e${state.refCounter++}` as CSTRefId;
+  state.refMap.set(ref, el);
+
+  const rawValue = controlValue(el);
+  const value =
+    rawValue === undefined
+      ? undefined
+      : state.redact(rawValue, { element: el, kind: 'value', ref });
+
+  const node: CSTInteractiveNode = {
+    type: 'interactive',
+    role,
+    ref,
+    name: accessibleName(el),
+  };
+  if (value !== undefined) node.value = value;
+
+  const placeholder = el.getAttribute('placeholder');
+  if (placeholder) node.placeholder = placeholder;
+
+  if (el instanceof HTMLInputElement) {
+    if (el.type === 'checkbox' || el.type === 'radio') {
+      node.checked = el.checked;
+    }
+  }
+  const ariaChecked = el.getAttribute('aria-checked');
+  if (ariaChecked === 'true' || ariaChecked === 'false') {
+    node.checked = ariaChecked === 'true';
+  }
+  const ariaExpanded = el.getAttribute('aria-expanded');
+  if (ariaExpanded === 'true' || ariaExpanded === 'false') {
+    node.expanded = ariaExpanded === 'true';
+  }
+  if (
+    'disabled' in el &&
+    (el as HTMLInputElement).disabled === true
+  ) {
+    node.disabled = true;
+  }
+  if (el.getAttribute('aria-disabled') === 'true') node.disabled = true;
+  if (
+    el.getAttribute('required') !== null ||
+    el.getAttribute('aria-required') === 'true'
+  ) {
+    node.required = true;
+  }
+
+  return node;
+}
+
+/** Collect direct-child text not owned by a child element. */
+function ownText(el: Element, state: WalkState): string {
+  let text = '';
+  el.childNodes.forEach((n) => {
+    if (n.nodeType === 3 /* TEXT_NODE */) {
+      text += n.textContent ?? '';
+    }
+  });
+  const trimmed = text.replace(/\s+/g, ' ').trim();
+  if (trimmed.length < MIN_TEXT_LEN) return '';
+  return state.redact(trimmed, {
+    element: el as HTMLElement,
+    kind: 'text',
+  });
+}
+
+/** Walk one element into zero or more CST nodes. */
+function walkElement(el: Element, state: WalkState): CSTNode[] {
+  if (state.visited >= state.maxNodes) return [];
+  state.visited++;
+
+  const tag = tagName(el);
+  if (SKIP_TAGS.has(tag)) return [];
+  if (!(el instanceof HTMLElement)) return [];
+  if (isChrome(el)) return [];
+  if (!isVisible(el)) return [];
+
+  // data-ai-redact — drop the whole subtree, replace with a placeholder.
+  // Checked on the element itself; the walk descends top-down so an
+  // annotated ancestor is caught before any descendant is reached.
+  if (el.hasAttribute('data-ai-redact')) {
+    return [{ type: 'text', content: '[redacted]' } satisfies CSTTextNode];
+  }
+
+  // Canvas / SVG / iframe — placeholder, no descent (also redaction-safe).
+  if (PLACEHOLDER_TAGS.has(tag)) {
+    const label = accessibleName(el) || tag;
+    return [{ type: 'text', content: `[${label}]` } satisfies CSTTextNode];
+  }
+
+  const kind = classify(el);
+
+  if (kind === 'interactive') {
+    return [buildInteractive(el, state)];
+  }
+
+  // Container or text-host: gather children, plus own text.
+  const childNodes: CSTNode[] = [];
+  const text = ownText(el, state);
+  if (text) childNodes.push({ type: 'text', content: text });
+  for (const child of Array.from(el.children)) {
+    childNodes.push(...walkElement(child, state));
+  }
+
+  if (childNodes.length === 0) return [];
+
+  if (kind === 'container') {
+    const node: CSTContainerNode = {
+      type: 'container',
+      role: containerRole(el)!,
+      children: childNodes,
+    };
+    const name = accessibleName(el);
+    if (name && name.length <= 80) node.name = name;
+    return [node];
+  }
+
+  // text-host: a plain wrapper — flatten, don't add a tree level.
+  return childNodes;
+}
+
+/**
+ * Walk the DOM subtree under `root` into CST child nodes.
+ */
+export function walkDOM(
+  root: HTMLElement,
+  options: WalkOptions = {},
+): WalkResult {
+  const state: WalkState = {
+    redact: options.redactValue ?? noopRedact,
+    refMap: new Map(),
+    refCounter: 1,
+    visited: 0,
+    maxNodes: options.maxNodes ?? 50_000,
+  };
+
+  // Walk the root's children — the root element itself becomes the CST
+  // root node, assembled by the engine.
+  const children: CSTNode[] = [];
+  const text = ownText(root, state);
+  if (text) children.push({ type: 'text', content: text });
+  for (const child of Array.from(root.children)) {
+    children.push(...walkElement(child, state));
+  }
+
+  return {
+    children,
+    refMap: state.refMap,
+    nodesVisited: state.visited,
+  };
+}

package/src/lib/page-snapshot/cst/__tests__/serialize.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, it } from 'vitest';
+
+import { serializeCST } from '../serialize';
+import type { CSTRootNode } from '../types';
+
+const sampleRoot: CSTRootNode = {
+  type: 'root',
+  title: 'Billing Settings',
+  url: 'https://app.example.com/settings/billing',
+  children: [
+    {
+      type: 'container',
+      role: 'form',
+      name: 'Billing',
+      children: [
+        {
+          type: 'interactive',
+          role: 'textbox',
+          ref: '@e1',
+          name: 'Company',
+          value: 'Acme',
+        },
+        { type: 'text', content: 'Configure your subscription.' },
+      ],
+    },
+  ],
+};
+
+describe('serializeCST', () => {
+  it('produces a deterministic string for the same tree', () => {
+    expect(serializeCST(sampleRoot)).toBe(serializeCST(sampleRoot));
+  });
+
+  it('drops undefined optional fields', () => {
+    const json = serializeCST(sampleRoot);
+    // The textbox has no `placeholder` — it must not appear.
+    expect(json).not.toContain('placeholder');
+    expect(json).toContain('"ref":"@e1"');
+  });
+
+  it('emits keys in a fixed order regardless of input key order', () => {
+    const reordered: CSTRootNode = {
+      children: sampleRoot.children,
+      url: sampleRoot.url,
+      type: 'root',
+      title: sampleRoot.title,
+    };
+    expect(serializeCST(reordered)).toBe(serializeCST(sampleRoot));
+  });
+});

package/src/lib/page-snapshot/cst/directives.ts

@@ -0,0 +1,47 @@
+/**
+ * UI directives — how the AI points at the user's screen.
+ *
+ * The chat response can carry directives; for now the only one is
+ * `point` (highlight / focus an element). The union is a discriminated
+ * type so a future write-style directive (e.g. `fill`) can be added
+ * without breaking older clients — unknown `type`s are simply ignored.
+ */
+
+import type { CSTRefId } from './types';
+
+/**
+ * Point at an element: highlight and/or focus it. Read-only — neither
+ * changes the user's data.
+ */
+export interface PointDirective {
+  type: 'point';
+  /** CST ref of the target element. */
+  ref: CSTRefId;
+  /** Draw the highlight overlay (default: true). */
+  highlight?: boolean;
+  /** Call element.focus() once visible (default: false). */
+  focus?: boolean;
+  /** Optional short caption shown beside the highlight. */
+  label?: string;
+}
+
+/**
+ * The directive union. MVP ships `point` only.
+ *
+ * Level 3 (future, NOT built) would add:
+ *   | { type: 'fill'; ref: CSTRefId; value: string }
+ *
+ * Any consumer must `switch` on `type` and ignore unknown variants —
+ * never throw — so an older client meeting a newer directive is safe.
+ */
+export type UIDirective = PointDirective;
+
+/** Carried in the structured side-channel of the SSE response. */
+export interface ChatResponseMeta {
+  directives?: UIDirective[];
+}
+
+/** Type guard: is this a `point` directive. */
+export function isPointDirective(d: UIDirective): d is PointDirective {
+  return d.type === 'point';
+}

package/src/lib/page-snapshot/cst/payload.ts

@@ -0,0 +1,50 @@
+/**
+ * The request-side payload: a snapshot plus its route context and
+ * metadata, sent to the backend as a separate field of the chat request.
+ *
+ */
+
+import type { CSTRootNode } from './types';
+
+/** Current CST schema version. Client and server check this. */
+export const CST_SCHEMA_VERSION = '1.0.0';
+
+/** Snapshot representation tag. Only CST for now. */
+export type SnapshotRepresentation = 'CST';
+
+/** Metadata describing how a snapshot was produced. */
+export interface SnapshotMetadata {
+  /** Representation format. */
+  representation: SnapshotRepresentation;
+  /** Approximate token count of the serialized snapshot. */
+  tokenEstimate: number;
+  /** Epoch ms when the snapshot was captured. */
+  captureTimestamp: number;
+  /** CST schema version. */
+  schemaVersion: string;
+  /** Stable content hash — used for staleness detection. */
+  contentHash: string;
+  /** How many nodes/values were redacted during capture. */
+  redactedCount: number;
+  /** How many repetitive sibling chains were folded. */
+  foldedCount: number;
+}
+
+/**
+ * The full page-context payload. Travels as its own request field —
+ * never concatenated into the user's message.
+ */
+export interface PageContextPayload {
+  /** Full URL at capture time. */
+  url: string;
+  /** Route pathname. */
+  route: string;
+  /** Route params, if resolvable. */
+  params: Record<string, string>;
+  /** document.title. */
+  title: string;
+  /** The captured tree. */
+  snapshot: CSTRootNode;
+  /** Capture metadata. */
+  metadata: SnapshotMetadata;
+}

package/src/lib/page-snapshot/cst/serialize.ts

@@ -0,0 +1,84 @@
+/**
+ * CST serialization — tree → deterministic JSON string.
+ *
+ * Key order is fixed so the same tree always produces the same string,
+ * which makes the content hash (staleness detection) stable.
+ */
+
+import type {
+  CSTContainerNode,
+  CSTInteractiveNode,
+  CSTNode,
+  CSTRootNode,
+  CSTTextNode,
+} from './types';
+
+/** Build a plain object with keys in a fixed order (undefined dropped). */
+function ordered(entries: Array<[string, unknown]>): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  for (const [key, value] of entries) {
+    if (value !== undefined) out[key] = value;
+  }
+  return out;
+}
+
+/** Normalize one node into a deterministic plain object. */
+function normalizeNode(node: CSTNode): Record<string, unknown> {
+  switch (node.type) {
+    case 'root':
+      return normalizeRoot(node);
+    case 'interactive':
+      return normalizeInteractive(node);
+    case 'container':
+      return normalizeContainer(node);
+    case 'text':
+      return normalizeText(node);
+  }
+}
+
+function normalizeRoot(node: CSTRootNode): Record<string, unknown> {
+  return ordered([
+    ['type', node.type],
+    ['title', node.title],
+    ['url', node.url],
+    ['children', node.children.map(normalizeNode)],
+  ]);
+}
+
+function normalizeInteractive(
+  node: CSTInteractiveNode,
+): Record<string, unknown> {
+  return ordered([
+    ['type', node.type],
+    ['role', node.role],
+    ['ref', node.ref],
+    ['name', node.name],
+    ['value', node.value],
+    ['placeholder', node.placeholder],
+    ['disabled', node.disabled],
+    ['checked', node.checked],
+    ['expanded', node.expanded],
+    ['required', node.required],
+  ]);
+}
+
+function normalizeContainer(node: CSTContainerNode): Record<string, unknown> {
+  return ordered([
+    ['type', node.type],
+    ['role', node.role],
+    ['name', node.name],
+    ['children', node.children.map(normalizeNode)],
+  ]);
+}
+
+function normalizeText(node: CSTTextNode): Record<string, unknown> {
+  return ordered([
+    ['type', node.type],
+    ['content', node.content],
+  ]);
+}
+
+/** Serialize a CST tree to a deterministic JSON string. */
+export function serializeCST(node: CSTNode): string {
+  return JSON.stringify(normalizeNode(node));
+}