acrylblends 0.7.8 → 0.8.0

package/plugins/acryl-door/src/page/driver/driver.ts

@@ -0,0 +1,153 @@
+/**
+ * The in-page driver: takes one {@link UiRequest} and does it, under the rules.
+ *
+ * It owns the ref table, the kill switch, the yield-to-the-user rule and the "driving" activity the indicator
+ * shows. It touches only the document it is given, never another window and never the OS.
+ */
+
+import { UiControlError, MUTATING_OPS, type UiRequest, type UiResult } from '../protocol.ts'
+import { click, pressKey, scroll, selectOption, typeText, wait, type ActionEnvironment } from './actions.ts'
+import { RefTable } from './ref-table.ts'
+import { SnapshotBuilder } from './snapshot.ts'
+
+/** After the user touches the page, agent actions wait this long, so the user always wins. */
+const YIELD_MS = 400
+const MAX_YIELD_MS = 2000
+
+export interface DriverActivity {
+  /** True from the moment a call starts until it ends. */
+  readonly busy: boolean
+  /** What the agent is doing, for the indicator ("click", "type", ...). */
+  readonly doing: string
+  /** True after the user pressed the kill switch. */
+  readonly killed: boolean
+}
+
+export interface UiDriverOptions {
+  readonly document: () => Document
+  /** Milliseconds since the user last used the page (mouse, keyboard); Infinity when never. */
+  readonly msSinceUserInput?: () => number
+  readonly sleep?: (ms: number, signal: AbortSignal) => Promise<void>
+}
+
+function realSleep(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal.aborted) { reject(signal.reason); return }
+    const timer = setTimeout(() => { signal.removeEventListener('abort', onAbort); resolve() }, ms)
+    const onAbort = (): void => { clearTimeout(timer); reject(signal.reason) }
+    signal.addEventListener('abort', onAbort, { once: true })
+  })
+}
+
+export class UiDriver {
+  private readonly refs = new RefTable()
+  private readonly snapshots: SnapshotBuilder
+  private readonly listeners = new Set<(activity: DriverActivity) => void>()
+  private readonly env: ActionEnvironment
+  private readonly stopAll = new Set<AbortController>()
+  private killedFlag = false
+  private disposed = false
+  private busyCount = 0
+  private doing = ''
+  private current: DriverActivity = { busy: false, doing: '', killed: false }
+
+  private readonly options: UiDriverOptions
+
+  constructor(options: UiDriverOptions) {
+    this.options = options
+    this.snapshots = new SnapshotBuilder(this.refs, () => options.document().body)
+    this.env = { refs: this.refs, document: options.document, sleep: options.sleep ?? realSleep }
+  }
+
+  /** The current activity; the same object until something changes, as a store snapshot must be. */
+  get activity(): DriverActivity {
+    return this.current
+  }
+
+  subscribe(listener: (activity: DriverActivity) => void): () => void {
+    this.listeners.add(listener)
+    return () => { this.listeners.delete(listener) }
+  }
+
+  /** The user pressed the kill switch: pending calls settle as `killed` and new ones are refused. */
+  kill(): void {
+    this.killedFlag = true
+    for (const controller of [...this.stopAll]) controller.abort(new UiControlError('killed', 'the user stopped the agent'))
+    this.refs.clear()
+    this.snapshots.clear()
+    this.emit()
+  }
+
+  /** The user allowed the agent to drive again. */
+  resume(): void {
+    this.killedFlag = false
+    this.emit()
+  }
+
+  /** The driver goes away: pending calls settle as `unloaded`; refs are gone. */
+  dispose(): void {
+    this.disposed = true
+    for (const controller of [...this.stopAll]) controller.abort(new UiControlError('unloaded', 'the driver was unloaded'))
+    this.refs.clear()
+    this.snapshots.clear()
+    this.listeners.clear()
+  }
+
+  /** @param signal - the Host cancelled the call. @throws UiControlError. */
+  async handle(request: UiRequest, signal?: AbortSignal): Promise<UiResult> {
+    if (this.disposed) throw new UiControlError('unloaded', 'the driver was unloaded')
+    if (this.killedFlag) throw new UiControlError('killed', 'the user stopped the agent; it cannot drive until they allow it again')
+    const controller = new AbortController()
+    this.stopAll.add(controller)
+    const onCancel = (): void => { controller.abort(new UiControlError('aborted', 'the call was cancelled')) }
+    signal?.addEventListener('abort', onCancel, { once: true })
+    this.busyCount += 1
+    this.doing = request.op
+    this.emit()
+    try {
+      if (MUTATING_OPS.includes(request.op)) await this.yieldToUser(controller.signal)
+      if (controller.signal.aborted) throw controller.signal.reason
+      return await this.run(request, controller.signal)
+    } catch (cause) {
+      if (cause instanceof UiControlError) throw cause
+      if (controller.signal.aborted && controller.signal.reason instanceof UiControlError) throw controller.signal.reason
+      throw new UiControlError('not-actionable', cause instanceof Error ? cause.message : 'the action failed')
+    } finally {
+      signal?.removeEventListener('abort', onCancel)
+      this.stopAll.delete(controller)
+      this.busyCount -= 1
+      if (this.busyCount === 0) this.doing = ''
+      this.emit()
+    }
+  }
+
+  private async run(request: UiRequest, signal: AbortSignal): Promise<UiResult> {
+    switch (request.op) {
+      case 'snapshot': return this.snapshots.snapshot({ ...(request.cursor === undefined ? {} : { cursor: request.cursor }), ...(request.maxNodes === undefined ? {} : { maxNodes: request.maxNodes }) })
+      case 'click': return click(this.env, request)
+      case 'type': return typeText(this.env, request)
+      case 'select': return selectOption(this.env, request)
+      case 'press': return pressKey(this.env, request)
+      case 'scroll': return scroll(this.env, request)
+      case 'wait': return wait(this.env, request, signal)
+    }
+  }
+
+  private async yieldToUser(signal: AbortSignal): Promise<void> {
+    const since = this.options.msSinceUserInput
+    if (since === undefined) return
+    let waited = 0
+    while (since() < YIELD_MS && waited < MAX_YIELD_MS) {
+      await this.env.sleep(100, signal)
+      waited += 100
+    }
+  }
+
+  private emit(): void {
+    const next: DriverActivity = { busy: this.busyCount > 0, doing: this.doing, killed: this.killedFlag }
+    if (next.busy === this.current.busy && next.doing === this.current.doing && next.killed === this.current.killed) return
+    this.current = next
+    const activity = next
+    for (const listener of [...this.listeners]) listener(activity)
+  }
+}

package/plugins/acryl-door/src/page/driver/ref-table.ts

@@ -0,0 +1,78 @@
+/**
+ * Refs: short-lived handles an agent uses to name an element.
+ *
+ * A ref is `<generation>.<n>`. Each snapshot starts a new generation and every older ref becomes stale, so a
+ * ref can never quietly point at a different element after the page changed. Even within a generation, a
+ * ref is honoured only while its element is still in the document and still has the role and name it had when
+ * the ref was issued; a re-rendered or replaced control is a typed `stale-ref`, never a click somewhere else.
+ */
+
+import { UiControlError } from '../protocol.ts'
+import { nameOf, roleOf, TEXT_ROLE, textRunsOf } from './accessibility.ts'
+
+interface Entry {
+  readonly element: WeakRef<Element>
+  readonly role: string
+  readonly name: string
+}
+
+export interface ResolvedRef {
+  readonly element: Element
+  readonly role: string
+  readonly name: string
+}
+
+export class RefTable {
+  private generationNumber = 0
+  private entries = new Map<number, Entry>()
+  private nextIndex = 1
+
+  get generation(): number {
+    return this.generationNumber
+  }
+
+  /** Start a new generation: every earlier ref is now stale. */
+  begin(): number {
+    this.generationNumber += 1
+    this.entries = new Map()
+    this.nextIndex = 1
+    return this.generationNumber
+  }
+
+  /** Forget everything (the driver is going away or was stopped). */
+  clear(): void {
+    this.begin()
+  }
+
+  issue(element: Element, role: string, name: string): string {
+    const index = this.nextIndex
+    this.nextIndex += 1
+    this.entries.set(index, { element: new WeakRef(element), role, name })
+    return `${String(this.generationNumber)}.${String(index)}`
+  }
+
+  /** @throws UiControlError `unknown-ref` or `stale-ref`. */
+  resolve(ref: string): ResolvedRef {
+    const [generationText, indexText] = ref.split('.')
+    const generation = Number(generationText)
+    const index = Number(indexText)
+    if (!Number.isInteger(generation) || !Number.isInteger(index)) throw new UiControlError('unknown-ref', `${ref} is not a ref`)
+    if (generation !== this.generationNumber) {
+      throw new UiControlError('stale-ref', `${ref} is from an earlier snapshot (the latest is ${String(this.generationNumber)}); take a new snapshot`)
+    }
+    const entry = this.entries.get(index)
+    if (entry === undefined) throw new UiControlError('unknown-ref', `${ref} was not in the snapshot`)
+    const element = entry.element.deref()
+    if (element === undefined || !element.isConnected) throw new UiControlError('stale-ref', `${ref} is no longer on the page; take a new snapshot`)
+    // The same node can be reused for something else after a re-render: check it is still what was listed.
+    if (entry.role === TEXT_ROLE) {
+      if (!textRunsOf(element).includes(entry.name)) throw new UiControlError('stale-ref', `${ref} changed since the snapshot ("${entry.name}" is no longer there); take a new snapshot`)
+      return { element, role: entry.role, name: entry.name }
+    }
+    const role = roleOf(element)
+    if (role !== entry.role || nameOf(element, role) !== entry.name) {
+      throw new UiControlError('stale-ref', `${ref} changed since the snapshot ("${entry.name}" is now something else); take a new snapshot`)
+    }
+    return { element, role: entry.role, name: entry.name }
+  }
+}

package/plugins/acryl-door/src/page/driver/sensitivity.ts

@@ -0,0 +1,49 @@
+/**
+ * What the agent may never see or type into, decided by rule and not by trusting the agent:
+ * secrets and payment fields (sensitive), and controls that govern the agent itself (protected).
+ */
+
+/** Field names and labels that mean a secret or a payment detail. */
+const SENSITIVE_WORDS = /pass(?:word|code|phrase)|secret|token|api[-_ ]?key|credential|private[-_ ]?key|\bpin\b|card|cvv|cvc|iban|routing|account[-_ ]?number|ssn|social[-_ ]?security|one[-_ ]?time|otp/i
+/** `autocomplete` values that browsers treat as credentials or payment data. */
+const SENSITIVE_AUTOCOMPLETE = /^(?:cc-|current-password|new-password|one-time-code)/
+
+const label = (el: Element, attr: string): string => el.getAttribute(attr) ?? ''
+
+/** True for a form field (or anything marked) that holds a secret or payment detail. */
+export function isSensitive(el: Element): boolean {
+  if (el.closest('[data-acryl-sensitive]') !== null) return true
+  if (!(el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement)) return false
+  if (el instanceof HTMLInputElement && el.type === 'password') return true
+  const autocomplete = label(el, 'autocomplete').trim().toLowerCase()
+  if (autocomplete.split(/\s+/).some(token => SENSITIVE_AUTOCOMPLETE.test(token))) return true
+  return SENSITIVE_WORDS.test([el.name, el.id, label(el, 'placeholder'), label(el, 'aria-label')].join(' '))
+}
+
+/** Areas whose controls the agent may look at but never operate. */
+const PROTECTED_REGION = /approval|permission|policy|agent control|ui control|acryl-agent-control|acryl-door|kill switch/i
+
+/** The text an ancestor uses to name itself, for region rules. */
+function ancestorName(el: Element): string {
+  const own = label(el, 'aria-label') || label(el, 'data-region') || ''
+  if (own !== '') return own
+  const labelled = label(el, 'aria-labelledby')
+  if (labelled !== '') {
+    return labelled.split(/\s+/).map(id => el.ownerDocument.getElementById(id)?.textContent ?? '').join(' ')
+  }
+  return ''
+}
+
+/**
+ * True for a control the agent must not operate: inside `[data-acryl-no-agent]`, inside a region whose
+ * name says approval, permission, policy or Agent Control, or itself named after switching Agent Control off.
+ */
+export function isProtected(el: Element): boolean {
+  if (el.closest('[data-acryl-no-agent]') !== null) return true
+  let node: Element | null = el
+  for (let depth = 0; node !== null && depth < 12; depth += 1) {
+    if (PROTECTED_REGION.test(ancestorName(node))) return true
+    node = node.parentElement
+  }
+  return false
+}

package/plugins/acryl-door/src/page/driver/snapshot.ts

@@ -0,0 +1,132 @@
+/** Builds a bounded, redacted accessibility snapshot of the page, with refs. */
+
+import { DEFAULT_SNAPSHOT_NODES, MAX_SNAPSHOT_NODES, UiControlError, type UiNode, type UiSnapshot } from '../protocol.ts'
+import { isHidden, levelOf, NAMED_BY_CONTENT, nameOf, roleOf, SHOWN_AS_VALUE, statesOf, TEXT_ROLE, textRun, valueOf } from './accessibility.ts'
+import type { RefTable } from './ref-table.ts'
+import { isProtected, isSensitive } from './sensitivity.ts'
+
+/** Marks the driver's own UI (the indicator), which a snapshot never lists. */
+export const DRIVER_UI_ATTRIBUTE = 'data-acryl-agent-control'
+
+/** The most elements one snapshot considers, however large the page. */
+const MAX_CANDIDATES = 5000
+
+
+interface Candidate {
+  readonly element: Element
+  readonly role: string
+  readonly name: string
+  readonly depth: number
+}
+
+/** True when the element (with layout) is at least partly inside the window; unknown layout counts as inside. */
+function inViewport(element: Element, view: Window | null): boolean {
+  if (view === null) return true
+  const rect = element.getBoundingClientRect()
+  if (rect.width === 0 && rect.height === 0) return true
+  return rect.bottom > 0 && rect.right > 0 && rect.top < view.innerHeight && rect.left < view.innerWidth
+}
+
+/**
+ * Every element with a role, and every run of visible text, in reading order. Text that is already a name (inside a button, a link, a
+ * heading, a label) is not listed again; a run is the text between two child elements, so `<p>Left: <b>24:59</b></p>` reads as
+ * `Left:` and `24:59`.
+ */
+function collect(root: Element): Candidate[] {
+  const found: Candidate[] = []
+  const walk = (element: Element, depth: number, named: boolean): void => {
+    if (found.length >= MAX_CANDIDATES) return
+    if (element.hasAttribute(DRIVER_UI_ATTRIBUTE)) return
+    const tag = element.tagName.toLowerCase()
+    if (tag === 'script' || tag === 'style' || tag === 'template' || tag === 'noscript') return
+    if (isHidden(element)) return
+    // A secret or payment field is not listed, and neither is anything inside a marked area.
+    if (isSensitive(element)) return
+    const role = roleOf(element)
+    let childDepth = depth
+    if (role !== null) {
+      found.push({ element, role, name: nameOf(element, role), depth })
+      childDepth = depth + 1
+    }
+    const inName = named || tag === 'label' || (role !== null && (NAMED_BY_CONTENT.has(role) || SHOWN_AS_VALUE.has(role)))
+    let run = ''
+    const flush = (): void => {
+      const text = textRun(run)
+      run = ''
+      if (text !== '' && !inName && found.length < MAX_CANDIDATES) found.push({ element, role: TEXT_ROLE, name: text, depth: childDepth })
+    }
+    for (const child of Array.from(element.childNodes)) {
+      if (child.nodeType === Node.TEXT_NODE) run += child.textContent ?? ''
+      else if (child instanceof Element) {
+        flush()
+        walk(child, childDepth, inName)
+      }
+    }
+    flush()
+  }
+  walk(root, 0, false)
+  return found
+}
+
+export interface SnapshotOptions {
+  readonly cursor?: number
+  readonly maxNodes?: number
+}
+
+/** Snapshots the page and remembers the full candidate list of the current generation, for paging. */
+export class SnapshotBuilder {
+  private candidates: Candidate[] = []
+
+  private readonly refs: RefTable
+  private readonly root: () => Element
+
+  constructor(refs: RefTable, root: () => Element) {
+    this.refs = refs
+    this.root = root
+  }
+
+  snapshot(options: SnapshotOptions = {}): UiSnapshot {
+    const max = Math.min(options.maxNodes ?? DEFAULT_SNAPSHOT_NODES, MAX_SNAPSHOT_NODES)
+    const root = this.root()
+    const view = root.ownerDocument.defaultView
+    let offset = options.cursor ?? 0
+    if (options.cursor === undefined) {
+      this.refs.begin()
+      const all = collect(root)
+      // What the user can see comes first, so a long page is cut from the bottom, not the middle of the window.
+      this.candidates = [...all.filter(c => inViewport(c.element, view)), ...all.filter(c => !inViewport(c.element, view))]
+      offset = 0
+    } else if (this.candidates.length === 0 || offset > this.candidates.length) {
+      throw new UiControlError('stale-ref', 'that page of the snapshot is gone; take a new snapshot')
+    }
+    const page = this.candidates.slice(offset, offset + max)
+    const nodes: UiNode[] = page.map((candidate) => {
+      const states = statesOf(candidate.element, candidate.role)
+      if (isProtected(candidate.element)) states.push('protected')
+      const value = valueOf(candidate.element, candidate.role)
+      const level = candidate.role === 'heading' ? levelOf(candidate.element) : undefined
+      return {
+        ref: this.refs.issue(candidate.element, candidate.role, candidate.name),
+        role: candidate.role,
+        name: candidate.name,
+        depth: candidate.depth,
+        states,
+        ...(value === undefined ? {} : { value }),
+        ...(level === undefined ? {} : { level }),
+      }
+    })
+    const end = offset + page.length
+    return {
+      generation: this.refs.generation,
+      title: root.ownerDocument.title,
+      nodes,
+      total: this.candidates.length,
+      ...(end < this.candidates.length ? { nextCursor: end } : {}),
+    }
+  }
+
+  /** Drop the remembered page list (with the refs). */
+  clear(): void {
+    this.candidates = []
+  }
+}

package/plugins/acryl-door/src/page/index.ts

@@ -0,0 +1,40 @@
+/**
+ * The door's page module (`main.ts` starts it; `<script type="module" src="/api/acryl-door/page.js"></script>` loads that). It drives this
+ * page's document for the agent (snapshots with refs, clicks, typing), answers the Host over the channel, and shows the indicator with
+ * the kill switch. It touches only this document, never another window and never the OS.
+ */
+
+import { PageChannel } from './channel.ts'
+import { UiDriver } from './driver/driver.ts'
+import { mountIndicator } from './indicator.ts'
+import { UserInputClock } from './user-input.ts'
+
+export const DOOR_BASE = '/api/acryl-door'
+
+/** Start the door on `document`. @returns the disposer. */
+export function startDoor(document: Document, base: string = DOOR_BASE): () => void {
+  const view = document.defaultView
+  if (view === null) throw new Error('acryl-door: the document has no window')
+  const clock = new UserInputClock()
+  const driver = new UiDriver({ document: () => document, msSinceUserInput: clock.msSince })
+  const windowId = typeof view.crypto?.randomUUID === 'function' ? view.crypto.randomUUID() : `w-${String(Date.now())}-${String(Math.random()).slice(2, 8)}`
+  const focused = (): boolean => document.visibilityState === 'visible' && document.hasFocus()
+  const detachInput = clock.attach(document)
+  const removeIndicator = mountIndicator(driver, document)
+  const channel = new PageChannel(driver, { windowId, focused, base })
+  const reportFocus = (): void => { channel.reportFocus(focused()) }
+  view.addEventListener('focus', reportFocus)
+  view.addEventListener('blur', reportFocus)
+  document.addEventListener('visibilitychange', reportFocus)
+  channel.connect()
+  return () => {
+    view.removeEventListener('focus', reportFocus)
+    view.removeEventListener('blur', reportFocus)
+    document.removeEventListener('visibilitychange', reportFocus)
+    channel.dispose()
+    removeIndicator()
+    detachInput()
+    // Pending calls settle as `unloaded` and every ref goes stale.
+    driver.dispose()
+  }
+}

package/plugins/acryl-door/src/page/indicator.ts

@@ -0,0 +1,72 @@
+/**
+ * "The agent is driving" bar with its kill switch, in plain DOM (the React one in `acryl-agent-control` shows the same states). Shown while
+ * a call runs and after the person stopped the agent. It is marked as driver UI (so snapshots skip it) and as off limits to the agent (so
+ * it can never press its own Stop or Allow buttons). Escape stops the agent too, while it is working.
+ */
+
+import type { DriverActivity, UiDriver } from './driver/driver.ts'
+
+const LABELS: Record<string, string> = { snapshot: 'looking at the page', click: 'clicking', type: 'typing', select: 'choosing an option', press: 'pressing a key', scroll: 'scrolling', wait: 'waiting' }
+
+const STYLE_ID = 'acryl-door-indicator-style'
+const STYLES = `
+.acrylDoorBar { position: fixed; z-index: 2147483647; left: 50%; bottom: 16px; transform: translateX(-50%); display: flex; align-items: center; gap: 10px;
+  padding: 8px 12px; border-radius: 8px; font: 13px/1.4 system-ui, sans-serif; color: #fff; background: #1f2937; box-shadow: 0 4px 16px rgba(0,0,0,.25); }
+.acrylDoorBar[data-killed] { background: #7f1d1d; }
+.acrylDoorBar button { font: inherit; color: inherit; background: rgba(255,255,255,.15); border: 1px solid rgba(255,255,255,.35); border-radius: 6px; padding: 2px 10px; cursor: pointer; }
+.acrylDoorDot { width: 8px; height: 8px; border-radius: 50%; background: #34d399; }
+`
+
+/** Show the bar for `driver` in `document`. @returns the disposer, which removes the bar, its style and its listeners. */
+export function mountIndicator(driver: UiDriver, document: Document): () => void {
+  if (document.getElementById(STYLE_ID) === null) {
+    const style = document.createElement('style')
+    style.id = STYLE_ID
+    style.textContent = STYLES
+    document.head.append(style)
+  }
+  const bar = document.createElement('div')
+  bar.className = 'acrylDoorBar'
+  bar.setAttribute('data-acryl-agent-control', '')
+  bar.setAttribute('data-acryl-no-agent', '')
+  bar.setAttribute('role', 'status')
+  bar.setAttribute('aria-live', 'polite')
+
+  const onKey = (event: KeyboardEvent): void => {
+    if (event.key === 'Escape' && event.isTrusted && driver.activity.busy) driver.kill()
+  }
+  document.addEventListener('keydown', onKey, true)
+
+  const render = (activity: DriverActivity): void => {
+    if (!activity.busy && !activity.killed) { bar.remove(); return }
+    const button = document.createElement('button')
+    button.type = 'button'
+    const text = document.createElement('span')
+    if (activity.killed) {
+      bar.setAttribute('data-killed', '')
+      text.textContent = 'Agent stopped. It cannot use the page until you allow it.'
+      button.textContent = 'Allow again'
+      button.addEventListener('click', () => { driver.resume() })
+      bar.replaceChildren(text, button)
+    } else {
+      bar.removeAttribute('data-killed')
+      const dot = document.createElement('span')
+      dot.className = 'acrylDoorDot'
+      dot.setAttribute('aria-hidden', 'true')
+      text.textContent = `Agent is driving: ${LABELS[activity.doing] ?? activity.doing}`
+      button.textContent = 'Stop'
+      button.addEventListener('click', () => { driver.kill() })
+      bar.replaceChildren(dot, text, button)
+    }
+    if (!bar.isConnected) document.body.append(bar)
+  }
+  const unsubscribe = driver.subscribe(render)
+  render(driver.activity)
+
+  return () => {
+    unsubscribe()
+    document.removeEventListener('keydown', onKey, true)
+    bar.remove()
+    document.getElementById(STYLE_ID)?.remove()
+  }
+}

package/plugins/acryl-door/src/page/main.ts

@@ -0,0 +1,7 @@
+/** The page's script: starts the door once on this document, even if the script tag appears twice. */
+
+import { startDoor } from './index.ts'
+
+const flag = '__acrylDoor'
+const host = globalThis as { [flag]?: () => void }
+if (host[flag] === undefined) host[flag] = startDoor(document)