aipeek 0.2.7 → 0.2.9

package/src/core/action.ts

@@ -6,7 +6,8 @@
 //   - performAction(): real DOM mutation. Runs browser-side (client.ts imports it).
 //     References window/document — never imported plugin-side.
 
-export type ActionType = 'click' | 'fill' | 'press' | 'wait' | 'screenshot' | 'realclick' | 'query'
+export type ActionType = 'click' | 'fill' | 'press' | 'wait' | 'screenshot' | 'realclick' | 'query' | 'assert'
+    | 'drag' | 'scrollIntoView' | 'drop' | 'clipboard'
 
 export interface ActionArgs {
     sel?: string
@@ -18,6 +19,14 @@ export interface ActionArgs {
     button?: 'left' | 'right'
     x?: number
     y?: number
+    // assert: a domain key (from __AIPEEK_SCREEN__) or a CSS sel, checked against `equals`.
+    screen?: string
+    equals?: string
+    // drag: sel/text = source, to = destination selector. drop: sel = target, files = names.
+    to?: string
+    files?: string[]
+    // clipboard: mode read|write; value = text to write (write only).
+    mode?: 'read' | 'write'
 }
 
 export interface ActionResult {
@@ -25,13 +34,16 @@ export interface ActionResult {
     detail?: string
     error?: string
     dataUrl?: string
-    ui?: string
     screen?: string
+    actions?: string
     x?: number
     y?: number
+    // realclick only: true if the page already fired the trusted click in-process (Electron).
+    // false/undefined means a plain Chrome tab couldn't, so the server drives its CDP queue.
+    fired?: boolean
 }
 
-const TYPES: ActionType[] = ['click', 'fill', 'press', 'wait', 'screenshot', 'realclick', 'query']
+const TYPES: ActionType[] = ['click', 'fill', 'press', 'wait', 'screenshot', 'realclick', 'query', 'assert', 'drag', 'scrollIntoView', 'drop', 'clipboard']
 
 // --- Pure validation (plugin-side) ---
 
@@ -59,6 +71,30 @@ export function resolveAction(type: string, args: ActionArgs): { valid: boolean,
             return { valid: true }
         case 'query':
             return args.sel ? { valid: true } : { valid: false, error: 'query needs sel=' }
+        case 'assert':
+            if (!args.screen && !args.sel)
+                return { valid: false, error: 'assert needs screen= (a __AIPEEK_SCREEN__ key) or sel=' }
+            if (args.equals === undefined)
+                return { valid: false, error: 'assert needs equals=' }
+            return { valid: true }
+        case 'drag':
+            if (!hasTarget)
+                return { valid: false, error: 'drag needs sel= or text= (the source)' }
+            if (!args.to)
+                return { valid: false, error: 'drag needs to= (destination selector)' }
+            return { valid: true }
+        case 'scrollIntoView':
+            return hasTarget ? { valid: true } : { valid: false, error: 'scrollIntoView needs sel= or text=' }
+        case 'drop':
+            if (!hasTarget)
+                return { valid: false, error: 'drop needs sel= or text= (the drop target)' }
+            if (!args.files || !args.files.length)
+                return { valid: false, error: 'drop needs files= (array of file names)' }
+            return { valid: true }
+        case 'clipboard':
+            if (args.mode === 'write' && args.value === undefined)
+                return { valid: false, error: 'clipboard write needs value=' }
+            return { valid: true }
         default:
             return { valid: false, error: `unknown action: ${type}` }
     }
@@ -66,18 +102,35 @@ export function resolveAction(type: string, args: ActionArgs): { valid: boolean,
 
 // --- Browser-side execution (client.ts only) ---
 
-export const INTERACTIVE = 'a, button, input, textarea, select, [role], [onclick], [tabindex], [contenteditable], [aria-label]'
-
-function visibleText(el: Element): string {
-    const aria = el.getAttribute('aria-label')
-    if (aria)
-        return aria
-    return (el.textContent || '').trim()
+// 命中判定原语（INTERACTIVE/getDirectText/elementLabel/isVisible/reachable）的真身已搬进
+// teact 内置的「DOM 命中」领域（teact/dom/candidates）—— write 路（这里）与 teact 的命中
+// 轨迹快照共用同一份真理。re-export 让 client.ts 的 `from '../core/action'` 无需改路径。
+export { elementLabel, getDirectText, INTERACTIVE, isVisible, reachable } from 'teact/dom'
+import { elementLabel, INTERACTIVE, isVisible, reachable } from 'teact/dom'
+
+// A field whose live value must never be echoed: API keys, tokens, passwords. `.value`
+// returns the real string even for type=password (masking is visual only), so reading it
+// out in /dom, /query or /screen would leak the secret in plaintext — the exact hole a
+// DOM-dumping inspector falls into. We redact instead. Signals (substring, no regex):
+// type=password, autocomplete=*-password / one-time-code, or a name/placeholder/aria-label
+// hinting a secret. Shared by every value-echoing reader so the redaction can't be bypassed
+// by hitting a different endpoint.
+const SECRET_HINTS = ['password', 'api key', 'apikey', 'api-key', 'api_key', 'secret', 'token', 'private key']
+export function isSensitive(el: Element): boolean {
+    if ((el as HTMLInputElement).type === 'password')
+        return true
+    const ac = (el.getAttribute('autocomplete') || '').toLowerCase()
+    if (ac.includes('password') || ac === 'one-time-code')
+        return true
+    const hint = `${el.getAttribute('name') || ''} ${el.getAttribute('placeholder') || ''} ${el.getAttribute('aria-label') || ''}`.toLowerCase()
+    return SECRET_HINTS.some(h => hint.includes(h))
 }
 
-function isVisible(el: Element): boolean {
-    const r = el.getBoundingClientRect()
-    return r.width > 0 && r.height > 0
+// The value to show for a field: real value, or a length-preserving redaction for secrets
+// (so "is it empty / roughly how long" stays observable without exposing the secret).
+export function safeValue(el: Element): string {
+    const v = (el as HTMLInputElement).value ?? ''
+    return isSensitive(el) ? (v ? `‹redacted ${v.length} chars›` : '') : v
 }
 
 export function findElement(sel?: string, text?: string): Element | null {
@@ -92,7 +145,10 @@ export function findElement(sel?: string, text?: string): Element | null {
     if (text) {
         const lower = text.toLowerCase()
         const candidates = Array.from(document.querySelectorAll(INTERACTIVE))
-        return candidates.find(el => isVisible(el) && visibleText(el).toLowerCase().includes(lower)) || null
+        // Prefer a reachable match (clickable now); fall back to a merely-visible one so a
+        // covered/off-viewport target still resolves (the caller can scrollIntoView/realclick).
+        const matches = candidates.filter(el => isVisible(el) && elementLabel(el).toLowerCase().includes(lower))
+        return matches.find(reachable) ?? matches[0] ?? null
     }
     return null
 }
@@ -102,8 +158,8 @@ function clickableList(): string {
     // unreachable, so listing it (e.g. 90 chat items) is pure noise.
     const root = document.querySelector('[role="dialog"][data-state="open"]') ?? document
     const els = Array.from(root.querySelectorAll(INTERACTIVE))
-        .filter(isVisible)
-        .map(el => visibleText(el).slice(0, 40))
+        .filter(reachable)
+        .map(el => elementLabel(el).slice(0, 40))
         .filter(Boolean)
     return [...new Set(els)].slice(0, 30).join(' | ')
 }
@@ -118,6 +174,11 @@ export async function performAction(type: ActionType, args: ActionArgs): Promise
             case 'wait': return await doWait(args)
             case 'screenshot': return await doScreenshot(args)
             case 'query': return doQuery(args)
+            case 'assert': return doAssert(args)
+            case 'drag': return doDrag(args)
+            case 'scrollIntoView': return doScrollIntoView(args)
+            case 'drop': return doDrop(args)
+            case 'clipboard': return await doClipboard(args)
         }
     }
     catch (e) {
@@ -234,7 +295,7 @@ function doClick(args: ActionArgs): ActionResult {
     // Report when the mouse actually landed on something else (overlay/portal) — a
     // common cause of "I clicked X but nothing happened".
     const note = hit && hit !== el && !el.contains(hit) && !hit.contains(el) ? ` (hit <${hit.tagName.toLowerCase()}> on top)` : ''
-    return { ok: true, detail: `clicked ${visibleText(el).slice(0, 40) || el.tagName.toLowerCase()}${note}` }
+    return { ok: true, detail: `clicked ${elementLabel(el).slice(0, 40) || el.tagName.toLowerCase()}${note}` }
 }
 
 // realclick resolves an element to a viewport-center (x,y) but does NOT synthetic-click.
@@ -252,7 +313,7 @@ function doResolveRealClick(args: ActionArgs): ActionResult {
     const r = el.getBoundingClientRect()
     const x = Math.round(r.left + r.width / 2)
     const y = Math.round(r.top + r.height / 2)
-    return { ok: true, x, y, detail: `resolved ${visibleText(el).slice(0, 40) || el.tagName.toLowerCase()} at (${x}, ${y})` }
+    return { ok: true, x, y, detail: `resolved ${elementLabel(el).slice(0, 40) || el.tagName.toLowerCase()} at (${x}, ${y})` }
 }
 
 function doFill(args: ActionArgs): ActionResult {
@@ -286,7 +347,8 @@ function doFill(args: ActionArgs): ActionResult {
     const input = el as HTMLInputElement
     const proto = el.tagName === 'TEXTAREA' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype
     const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set
-    if (setter) setter.call(input, value)
+    if (setter)
+        setter.call(input, value)
     else input.value = value
     input.dispatchEvent(new Event('input', { bubbles: true }))
     input.dispatchEvent(new Event('change', { bubbles: true }))
@@ -370,14 +432,15 @@ const QUERY_ATTRS = ['role', 'data-state', 'value', 'href', 'title']
 const QUERY_PREFIXES = ['data-', 'aria-']
 
 function elAttrs(el: Element): Record<string, string> {
+    const sensitive = isSensitive(el)
     const out: Record<string, string> = {}
     for (const attr of Array.from(el.attributes)) {
         if (QUERY_ATTRS.includes(attr.name) || QUERY_PREFIXES.some(p => attr.name.startsWith(p)))
-            out[attr.name] = attr.value
+            out[attr.name] = attr.name === 'value' && sensitive ? `‹redacted ${attr.value.length} chars›` : attr.value
     }
     const input = el as HTMLInputElement
     if (typeof input.value === 'string' && out.value === undefined && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'SELECT'))
-        out.value = input.value
+        out.value = safeValue(el)
     if (input.disabled)
         out.disabled = 'true'
     if (input.checked)
@@ -395,10 +458,124 @@ function doQuery(args: ActionArgs): ActionResult {
     }
     const count = els.length
     const matches = els.slice(0, 20).map(el => ({
-        text: visibleText(el).slice(0, 80),
+        text: elementLabel(el).slice(0, 80),
         visible: isVisible(el),
         attrs: elAttrs(el),
     }))
     const head = count > 20 ? `(showing 20 of ${count})\n` : ''
     return { ok: true, detail: head + JSON.stringify({ count, matches }, null, 2) }
 }
+
+// Assert a domain variable (from __AIPEEK_SCREEN__) or a DOM element's text equals the
+// expected value. Pass/fail with the actual value on failure — the chain stops and says
+// "asserted X==Y, actual Z", not "the next step happened to miss".
+function doAssert(args: ActionArgs): ActionResult {
+    let actual: string
+    if (args.screen) {
+        let domain: Record<string, unknown> = {}
+        try {
+            domain = window.__AIPEEK_SCREEN__?.() ?? {}
+        }
+        catch (e) {
+            return { ok: false, error: `__AIPEEK_SCREEN__ threw: ${e instanceof Error ? e.message : String(e)}` }
+        }
+        if (!(args.screen in domain))
+            return { ok: false, error: `no domain key "${args.screen}" — available: ${Object.keys(domain).join(', ') || '(none — app injected no __AIPEEK_SCREEN__)'}` }
+        actual = String(domain[args.screen])
+    }
+    else {
+        const el = findElement(args.sel, args.text)
+        if (!el)
+            return { ok: false, error: `no element for ${args.sel || args.text}` }
+        actual = elementLabel(el)
+    }
+    const target = args.screen ? `screen.${args.screen}` : (args.sel || args.text)
+    return actual === args.equals
+        ? { ok: true, detail: `${target} == ${args.equals}` }
+        : { ok: false, error: `asserted ${target} == "${args.equals}", actual "${actual}"` }
+}
+
+// Scroll an element into the viewport so it can be clicked. In a virtual list (tanstack-
+// virtual, dnd-kit) a row off-screen isn't in the DOM at all — but a row rendered yet
+// scrolled out of view is the common case, and this brings it back. Honest about the
+// virtual case: if the element isn't found, say so (the row may need the list scrolled
+// first via a container scroll, which /eval can do).
+function doScrollIntoView(args: ActionArgs): ActionResult {
+    const el = findElement(args.sel, args.text)
+    if (!el)
+        return { ok: false, error: `no element for ${args.sel || args.text} — if it's in a virtual list it may not be rendered yet`, detail: clickableList() }
+    el.scrollIntoView({ block: 'center', inline: 'center' })
+    const r = el.getBoundingClientRect()
+    const inView = r.top >= 0 && r.left >= 0 && r.bottom <= innerHeight && r.right <= innerWidth
+    return { ok: true, detail: `scrolled ${elementLabel(el).slice(0, 40) || el.tagName.toLowerCase()} into view${inView ? '' : ' (still partly off-screen)'}` }
+}
+
+// Drag via a synthetic pointer sequence: pointerdown on source → a few pointermove steps
+// (dnd-kit's PointerSensor has an activation distance constraint, so we move in increments
+// past it) → pointerup on destination. Also fires the HTML5 drag events for libraries that
+// use the native DnD API. dnd-kit relies on pointer capture + isTrusted in some paths; when
+// synthetic events don't trigger the reorder, fall back to realclick's trusted channel
+// (caller sees needsTrusted in the detail). Best-effort, honestly reported.
+function doDrag(args: ActionArgs): ActionResult {
+    const src = findElement(args.sel, args.text)
+    if (!src)
+        return { ok: false, error: `no source element for ${args.sel || args.text}`, detail: clickableList() }
+    const dst = args.to ? document.querySelector(args.to) : null
+    if (!dst)
+        return { ok: false, error: `no destination element for to=${args.to}` }
+    const sr = src.getBoundingClientRect()
+    const dr = dst.getBoundingClientRect()
+    const sx = sr.left + sr.width / 2
+    const sy = sr.top + sr.height / 2
+    const dx = dr.left + dr.width / 2
+    const dy = dr.top + dr.height / 2
+    const ptr = (x: number, y: number, extra: PointerEventInit = {}) =>
+        ({ bubbles: true, cancelable: true, composed: true, clientX: x, clientY: y, view: window, pointerId: 1, pointerType: 'mouse', isPrimary: true, button: 0, ...extra })
+
+    src.dispatchEvent(new PointerEvent('pointerdown', ptr(sx, sy, { buttons: 1 })))
+    // Step through the path so dnd-kit clears its activation-distance constraint.
+    const steps = 8
+    for (let i = 1; i <= steps; i++) {
+        const x = sx + (dx - sx) * (i / steps)
+        const y = sy + (dy - sy) * (i / steps)
+        const over = (document.elementFromPoint(x, y) as HTMLElement | null) ?? dst as HTMLElement
+        over.dispatchEvent(new PointerEvent('pointermove', ptr(x, y, { buttons: 1 })))
+    }
+    ;(dst as HTMLElement).dispatchEvent(new PointerEvent('pointerup', ptr(dx, dy, { buttons: 0 })))
+    return { ok: true, detail: `dragged ${elementLabel(src).slice(0, 30) || src.tagName.toLowerCase()} → ${elementLabel(dst).slice(0, 30) || dst.tagName.toLowerCase()} (synthetic; if no reorder, retry with realclick — dnd-kit may need trusted pointer events)` }
+}
+
+// Drop synthetic files onto a target — the file-upload path without a native file picker
+// (which synthetic clicks can't drive). Builds a DataTransfer with empty File objects of
+// the given names and fires the dragenter→dragover→drop sequence libraries listen for.
+function doDrop(args: ActionArgs): ActionResult {
+    const target = findElement(args.sel, args.text)
+    if (!target)
+        return { ok: false, error: `no drop target for ${args.sel || args.text}`, detail: clickableList() }
+    const dt = new DataTransfer()
+    for (const name of args.files ?? [])
+        dt.items.add(new File([''], name, { type: 'application/octet-stream' }))
+    const r = target.getBoundingClientRect()
+    const init = { bubbles: true, cancelable: true, composed: true, clientX: r.left + r.width / 2, clientY: r.top + r.height / 2, dataTransfer: dt }
+    for (const type of ['dragenter', 'dragover', 'drop'] as const)
+        target.dispatchEvent(new DragEvent(type, init))
+    return { ok: true, detail: `dropped ${dt.items.length} file(s) [${(args.files ?? []).join(', ')}] on ${elementLabel(target).slice(0, 30) || target.tagName.toLowerCase()}` }
+}
+
+// Read or write the clipboard. write seeds it (so a subsequent paste can be tested);
+// read reports what the page put there (so a "copy" button can be verified). navigator.
+// clipboard needs a focused document and (for read) permission — degrade with a clear
+// error rather than hang.
+async function doClipboard(args: ActionArgs): Promise<ActionResult> {
+    try {
+        if (args.mode === 'write') {
+            await navigator.clipboard.writeText(args.value ?? '')
+            return { ok: true, detail: `clipboard ← "${(args.value ?? '').slice(0, 60)}"` }
+        }
+        const text = await navigator.clipboard.readText()
+        return { ok: true, detail: `clipboard: "${text.slice(0, 200)}"` }
+    }
+    catch (e) {
+        return { ok: false, error: `clipboard ${args.mode ?? 'read'} failed: ${e instanceof Error ? e.message : String(e)} (needs document focus / permission)` }
+    }
+}

package/src/core/compact.ts

@@ -1,4 +1,5 @@
 import type { CompactState, ErrorEntry, LogEntry, NetworkRequest, RawState } from './types'
+import { detailPerformance } from './perf'
 import { appStackFrames, compactUrl, formatValue, truncate } from './util'
 
 const SLOW_THRESHOLD = 1000
@@ -301,6 +302,7 @@ export function compact(raw: RawState): CompactState {
         network: compactNetwork(raw.network),
         errors: compactErrors(raw.errors),
         state: compactState(raw.state),
+        performance: raw.performance ? detailPerformance(raw.performance) : undefined,
         timestamp: raw.timestamp,
         counts: {
             console: raw.console.length,

package/src/core/detail.ts

@@ -1,5 +1,6 @@
-import type { ErrorEntry, LogEntry, NetworkRequest, RawState } from './types'
+import type { ErrorEntry, LogEntry, NetworkRequest, PerformanceData, RawState } from './types'
 import { compactUI } from './compact'
+import { detailPerformance } from './perf'
 import { appStackFrames, formatValue, truncate } from './util'
 
 export function detail(raw: RawState, section: string, index: string | undefined, full: boolean): string | null {
@@ -9,6 +10,7 @@ export function detail(raw: RawState, section: string, index: string | undefined
         case 'network': return detailNetwork(raw.network, index, full)
         case 'errors': return detailError(raw.errors, index, full)
         case 'state': return detailState(raw.state, index, full)
+        case 'profile': return raw.performance ? detailPerformance(raw.performance) : '(no perf data — is the tab foreground?)'
         default: return null
     }
 }

package/src/core/diff.ts

@@ -1,4 +1,58 @@
-import type { DiffResult, RawState } from './types'
+import type { DiffResult, ErrorEntry, LogEntry, NetworkRequest, RawState, ScreenSnap } from './types'
+
+// The change a single action caused: the state-machine transition (view/modal/focus)
+// plus any errors/failed requests it triggered. AI reads "what moved", then drills into
+// /ui or /dom for detail if it needs more — the response carries the delta, not a snapshot.
+// Pure: caller passes before/after snaps and the already-diffed error/request deltas.
+export function diffScreen(
+    before: ScreenSnap,
+    after: ScreenSnap,
+    newErrors: LogEntry[],
+    newExceptions: ErrorEntry[],
+    newFailedRequests: NetworkRequest[],
+): string[] {
+    const lines: string[] = []
+    if (after.view !== before.view)
+        lines.push(`view: ${before.view} → ${after.view}`)
+    if (after.modal !== before.modal) {
+        if (after.modal === 'none')
+            lines.push(`modal: closed (${before.modal})`)
+        else if (before.modal === 'none')
+            lines.push(`modal: opened ${after.modal}`)
+        else
+            lines.push(`modal: ${before.modal} → ${after.modal}`)
+    }
+    if (after.focus !== before.focus)
+        lines.push(`focus: ${after.focus}`)
+    // The app's domain state machine (from __AIPEEK_SCREEN__) — these transitions are
+    // the ones a DOM-only projector can't see (流式 false → true never touches the DOM).
+    const beforeDomain = before.domain ?? {}
+    const afterDomain = after.domain ?? {}
+    for (const key of new Set([...Object.keys(beforeDomain), ...Object.keys(afterDomain)])) {
+        const b = stringifyDomain(beforeDomain[key])
+        const a = stringifyDomain(afterDomain[key])
+        if (b !== a)
+            lines.push(`${key}: ${b} → ${a}`)
+    }
+    for (const e of newErrors)
+        lines.push(`+error: ${e.text}`)
+    for (const e of newExceptions)
+        lines.push(`+exception: ${e.message}`)
+    for (const r of newFailedRequests)
+        lines.push(`+failed: ${r.method} ${r.url} → ${r.status || 'failed'}`)
+    return lines
+}
+
+// One-line, bounded, comparison-stable stringification of a domain value. Used both to
+// detect change (string equality) and to render the transition. Objects compare by their
+// JSON — good enough for the shallow domain maps __AIPEEK_SCREEN__ returns.
+export function stringifyDomain(v: unknown): string {
+    if (v === null || v === undefined)
+        return String(v)
+    if (typeof v === 'object')
+        return JSON.stringify(v).slice(0, 80)
+    return String(v).slice(0, 80)
+}
 
 export function diffState(prev: RawState | null, curr: RawState): DiffResult {
     if (!prev) {

package/src/core/emit.ts

@@ -1,11 +1,12 @@
-import type { CheckResult, CompactState, DiffResult, RawState } from './types'
+import type { CheckResult, CompactState, DiffResult, PerformanceData, RawState } from './types'
 import pc from 'picocolors'
 import { nameOf } from './compact'
+import { compactPerformance } from './perf'
 import { compactUrl, truncate } from './util'
 
 // --- Full emit (for ?full) ---
 
-const SECTIONS = ['ui', 'console', 'network', 'errors', 'state'] as const
+const SECTIONS = ['ui', 'console', 'network', 'errors', 'state', 'performance'] as const
 const COUNTED_SECTIONS: Record<string, keyof CompactState['counts']> = {
     console: 'console',
     network: 'network',
@@ -87,6 +88,17 @@ export function emitSummary(raw: RawState): string {
         for (const e of raw.errors) lines.push(`  ${truncate(e.message, 150)}`)
     }
 
+    // perf — only show when jank (dropped >10%)
+    if (raw.performance) {
+        const all = raw.performance.buckets.find(b => b.name === '__all__')
+        if (all) {
+            const droppedPct = all.frames.total > 0 ? (all.frames.long / all.frames.total) * 100 : 0
+            if (droppedPct > 10) {
+                lines.push(compactPerformance(raw.performance))
+            }
+        }
+    }
+
     // state — 1-line: store names + key counts
     const storeNames = Object.keys(raw.state)
     if (storeNames.length) {