aipeek 0.2.7 → 0.2.8

package/src/client/client.ts

@@ -3,18 +3,48 @@
 // The patch code (console/fetch/XHR/errors) is in client-patch.ts,
 // injected as a synchronous inline script before any modules execute.
 
-import type { ActionArgs, ActionType } from '../core/action'
-import type { ErrorEntry, LogEntry, NetworkRequest } from '../core/types'
-import { INTERACTIVE, performAction, runEval, withDialogGuard } from '../core/action'
+import type { ActionArgs, ActionResult, ActionType } from '../core/action'
+import type { ActionEntry, ErrorEntry, LogEntry, NetworkRequest, PerformanceData, PerfBucketData, ScreenSnap } from '../core/types'
+import { elementLabel, getDirectText, INTERACTIVE, isSensitive, performAction, reachable, runEval, safeValue, withDialogGuard } from '../core/action'
+import { diffScreen, diffState, stringifyDomain } from '../core/diff'
+import { formatActions } from '../core/util'
 
 declare global {
-    interface Window { __AIPEEK_STORES__?: Record<string, unknown> }
+    interface Window {
+        __AIPEEK_STORES__?: Record<string, unknown>
+        // The app's own domain projection — collapses the UI to a few domain variables
+        // that don't live in the DOM (是否流式 / 选了哪个模型 / 未读数). Optional; when
+        // absent /screen falls back to the generic view/modal/focus projection unchanged.
+        __AIPEEK_SCREEN__?: () => Record<string, unknown>
+        // Idempotency for shortest-path delivery: the server re-broadcasts an addressed action
+        // while a tab is mid self-heal, so the same action id can arrive more than once. Map id →
+        // its final result; on a repeat we replay the cached result instead of re-executing.
+        __AIPEEK_DONE_ACTIONS__?: Map<number, ActionResult>
+    }
+}
+
+// Internal perf buffer shape (from client-patch.ts) — differs from PerformanceData
+interface PerfStatBuf { total: number, n: number, max: number, samples: number[] }
+interface PerfBucketBuf {
+    components: Record<string, PerfStatBuf>
+    frames: { total: number, long: number, max: number, samples: number[] }
+    lines: Record<string, PerfStatBuf>
+}
+interface PerfBuffer {
+    startedAt: number
+    hiddenFrames: number
+    mobxPatched: boolean
+    buckets: Record<string, PerfBucketBuf>
+    active: string
+    longtasks: { count: number, max: number }
 }
 
 interface AipeekBuffers {
     consoleLogs: LogEntry[]
     networkRequests: NetworkRequest[]
     errorEntries: ErrorEntry[]
+    actionEntries: ActionEntry[]
+    perf: PerfBuffer
 }
 
 // Read buffers from the synchronous patch script
@@ -22,6 +52,71 @@ const buffers = (window as any).__AIPEEK_BUFFERS__ as AipeekBuffers | undefined
 const consoleLogs: LogEntry[] = buffers?.consoleLogs ?? []
 const networkRequests: NetworkRequest[] = buffers?.networkRequests ?? []
 const errorEntries: ErrorEntry[] = buffers?.errorEntries ?? []
+const actionEntries: ActionEntry[] = buffers?.actionEntries ?? []
+const perfBuffer: PerfBuffer | undefined = buffers?.perf
+
+// Project the runtime perf buffer into PerformanceData for collection.
+// Top-20 components by total, capped samples already in buffer.
+function collectPerformance(): PerformanceData | undefined {
+    if (!perfBuffer) return undefined
+    const buckets: PerfBucketData[] = []
+    // __all__ first, then named buckets
+    const names = Object.keys(perfBuffer.buckets).sort((a, b) => (a === '__all__' ? -1 : b === '__all__' ? 1 : a.localeCompare(b)))
+    for (const name of names) {
+        const b = perfBuffer.buckets[name]
+        const comps: Record<string, { total: number, n: number, max: number, samples: number[] }> = {}
+        const sorted = Object.entries(b.components).sort((a, b) => b[1].total - a[1].total).slice(0, 20)
+        for (const [cname, stat] of sorted) {
+            comps[cname] = { total: stat.total, n: stat.n, max: stat.max, samples: [...stat.samples] }
+        }
+        const lines: Record<string, { total: number, n: number, max: number, samples: number[] }> = {}
+        const sortedLines = Object.entries(b.lines ?? {}).sort((a, b) => b[1].total - a[1].total).slice(0, 30)
+        for (const [label, stat] of sortedLines) {
+            lines[label] = { total: stat.total, n: stat.n, max: stat.max, samples: [...stat.samples] }
+        }
+        buckets.push({
+            name,
+            components: comps,
+            frames: { total: b.frames.total, long: b.frames.long, max: b.frames.max, samples: [...b.frames.samples] },
+            lines,
+        })
+    }
+    return {
+        windowMs: Date.now() - perfBuffer.startedAt,
+        hiddenFrames: perfBuffer.hiddenFrames,
+        mobxPatched: perfBuffer.mobxPatched,
+        buckets,
+        longtasks: { ...perfBuffer.longtasks },
+    }
+}
+
+// Reset the perf buffer (clear window, start fresh)
+function resetPerf() {
+    if (!perfBuffer) return
+    perfBuffer.startedAt = Date.now()
+    perfBuffer.hiddenFrames = 0
+    perfBuffer.buckets = {}
+    perfBuffer.longtasks = { count: 0, max: 0 }
+}
+
+// Stable per-tab id so commands can address one tab among many (?tab=). Lives in
+// sessionStorage: one browser tab = one id, survives HMR/refresh, cleared on close.
+// Falls back to an in-memory id if sessionStorage is unavailable (degrades to old
+// broadcast behavior, never throws).
+const TAB_ID = (() => {
+    try {
+        const k = '__aipeek_tab_id__'
+        let id = sessionStorage.getItem(k)
+        if (!id) {
+            id = `t${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`
+            sessionStorage.setItem(k, id)
+        }
+        return id
+    }
+    catch {
+        return `t${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`
+    }
+})()
 
 // --- On-demand collectors ---
 
@@ -228,18 +323,6 @@ function collectDomFallback(): string {
     return lines.join('\n')
 }
 
-function getDirectText(el: Element): string {
-    let text = ''
-    for (const node of el.childNodes) {
-        if (node.nodeType === Node.TEXT_NODE) {
-            const t = node.textContent?.trim()
-            if (t)
-                text += (text ? ' ' : '') + t
-        }
-    }
-    return text
-}
-
 // --- Semantic DOM collector ---
 // UI-as-text: strip Tailwind atomic classes, keep only what carries meaning —
 // tag, role, semantic classes, data-*/aria, text, interactive state, a usable selector.
@@ -396,14 +479,28 @@ function baseState(el: Element): string[] {
     return parts
 }
 
+// Heuristic: does opening this control need a *trusted* event (realclick), not a synthetic
+// one? Synthetic dispatchEvent can't open a Radix portal popup reliably (and never a native
+// right-click menu). aria-haspopup is the W3C marker Radix puts on menu/dropdown/popover/
+// select triggers — the honest, low-false-positive signal. Tag it so the AI picks realclick
+// up front instead of discovering it by a failed synthetic click. The `?` keeps it honest:
+// a hint, not a guarantee (right-click-only handlers carry no DOM attribute, so they can't be
+// detected here — those still surface only on a miss).
+function needsTrusted(el: Element): boolean {
+    const hp = el.getAttribute('aria-haspopup')
+    return hp === 'menu' || hp === 'dialog' || hp === 'listbox' || hp === 'true'
+}
+
 function domState(el: Element): string {
     const parts = baseState(el)
     if (el.getAttribute('aria-selected') === 'true' || el.getAttribute('aria-checked') === 'true')
         parts.push('selected')
     if ((el as HTMLInputElement).value)
-        parts.push(`value="${(el as HTMLInputElement).value.slice(0, 40)}"`)
+        parts.push(`value="${safeValue(el).slice(0, 40)}"`)
     if (el.hasAttribute('contenteditable'))
         parts.push('editable')
+    if (needsTrusted(el))
+        parts.push('needs-trusted?')
     return parts.length ? ` {${parts.join(' ')}}` : ''
 }
 
@@ -447,9 +544,12 @@ export function collectDom(scope?: string, sel?: string): string {
         // data-* minus framework noise. data-insp-path (code-inspector) is the source
         // location — keep it, but compacted to its file:line tail, shown only when the
         // node is itself meaningful (carries role/text/state), not on bare wrappers.
+        // On a secret field, redact data-* values too — an app could mirror the value into
+        // a data attr, and we never want that echoed (same stance as the value redaction).
+        const secret = isSensitive(el)
         const dataAttrs = Array.from(el.attributes)
             .filter(a => a.name.startsWith('data-') && a.name !== 'data-testid' && a.name !== 'data-insp-path')
-            .map(a => a.value ? `${a.name}="${a.value.slice(0, 30)}"` : a.name)
+            .map(a => a.value ? `${a.name}="${secret ? `‹redacted ${a.value.length}›` : a.value.slice(0, 30)}"` : a.name)
             .slice(0, 3)
         const aria = el.getAttribute('aria-label') || ''
         const text = aria || getDirectText(el)
@@ -508,7 +608,9 @@ function collectState(): Record<string, unknown> {
 // — the few variables a human reads off a washing-machine panel to act in O(1).
 // knobs are clipped to what's reachable *now*: when a modal is open, only its
 // subtree counts (the layer beneath is unclickable).
-export function collectScreen(): string {
+// The state-machine projection — just the three vars that move when the UI transitions.
+// Shared by collectScreen (the /screen header) and the action handler's before/after diff.
+export function screenSnap(): ScreenSnap {
     const stores = window.__AIPEEK_STORES__ as Record<string, any> | undefined
     const view = stores?.appUIStore?.mode ?? '(unknown)'
 
@@ -518,22 +620,37 @@ export function collectScreen(): string {
         : ''
     const modal = modalEl ? `${modalTitle || 'untitled'} @${inspPath(modalEl) || 'dialog'}` : 'none'
 
-    // Icon buttons carry no direct text — fall back to title, then any nested text.
-    const labelOf = (el: Element) => (el.getAttribute('aria-label') || getDirectText(el) || el.getAttribute('title') || (el.textContent || '').trim()).trim()
-    const visible = (el: Element) => el.getBoundingClientRect().width > 0
-
     const active = document.activeElement
     const focus = active && active !== document.body
-        ? `${labelOf(active).slice(0, 30) || active.tagName.toLowerCase()} [${active.tagName.toLowerCase()}]${domState(active)}`
+        ? `${elementLabel(active).slice(0, 30) || active.tagName.toLowerCase()} [${active.tagName.toLowerCase()}]${domState(active)}`
         : 'none'
 
+    let domain: Record<string, unknown> = {}
+    try {
+        domain = window.__AIPEEK_SCREEN__?.() ?? {}
+    }
+    catch {
+        domain = { '[error]': '__AIPEEK_SCREEN__ threw' }
+    }
+
+    return { view, modal, focus, domain }
+}
+
+export function collectScreen(): string {
+    const { view, modal, focus, domain } = screenSnap()
+    const domainLines = Object.entries(domain).map(([k, v]) => `  ${k}: ${stringifyDomain(v)}`)
+
+    const modalEl = document.querySelector('[role="dialog"][data-state="open"]')
+
     // Group by source location. A source rendered many times is a *list* (content),
     // not a control — collapse it to one "source ×N" line. The washing-machine
     // panel shows "the laundry", not every garment.
     const root = modalEl ?? document.body
     const bySource = new Map<string, Element[]>()
     for (const el of root.querySelectorAll(INTERACTIVE)) {
-        if (!visible(el))
+        // reachable, not just isVisible: only list knobs a click can actually land on now —
+        // same predicate /click uses, so read and write agree (no "listed but unclickable").
+        if (!reachable(el))
             continue
         const loc = inspPath(el) || '?'
         if (!bySource.has(loc))
@@ -547,7 +664,7 @@ export function collectScreen(): string {
             continue
         }
         for (const el of els) {
-            const label = labelOf(el).slice(0, 40)
+            const label = elementLabel(el).slice(0, 40)
             knobs.push(`  ${label || `<${el.tagName.toLowerCase()}>`} [${el.tagName.toLowerCase()}]${loc !== '?' ? ` ${loc}` : ''}${domState(el)}`)
         }
         if (knobs.length >= 40)
@@ -556,6 +673,7 @@ export function collectScreen(): string {
 
     return [
         `view: ${view}`,
+        ...(domainLines.length ? [`domain:`, ...domainLines] : []),
         `modal: ${modal}`,
         `focus: ${focus}`,
         `knobs (${knobs.length}):`,
@@ -563,6 +681,22 @@ export function collectScreen(): string {
     ].join('\n')
 }
 
+// Called from the action handler after the DOM has settled. The synthetic event aipeek
+// just fired was captured into actionEntries by client-patch; find that entry (latest
+// synthetic) and back-fill its snapshot synchronously off the now-settled DOM — not via
+// client-patch's async 180ms stamp, which may not have run yet (Risk 1). Then render the
+// timeline anchored on it, so the model sees its own action between before/after context.
+function attachTimeline(): string {
+    const cur = [...actionEntries].reverse().find(e => !e.trusted)
+    if (cur) {
+        const stores = window.__AIPEEK_STORES__ as Record<string, any> | undefined
+        cur.view = stores?.appUIStore?.mode ?? '(unknown)'
+        const modalEl = document.querySelector('[role="dialog"][data-state="open"]')
+        cur.modal = modalEl ? (modalEl.querySelector('h1, h2, [id^="radix"]')?.textContent || 'untitled').trim().slice(0, 40) : 'none'
+    }
+    return formatActions([...actionEntries], cur?.ts)
+}
+
 function boundedSnapshot(obj: unknown, depth = 0): unknown {
     if (depth > 3)
         return '[…]'
@@ -621,28 +755,63 @@ function waitForStable(quiet = 150, timeout = 2000): Promise<string> {
 // would answer. So the server sends requireVisible=true first; if it times out with
 // no answer it retries requireVisible=false, and every tab answers (the original
 // race). `skip` collapses that into one guard: hidden tab skips only round one.
-function skip(msg?: { requireVisible?: boolean }) {
+function skip(msg?: { requireVisible?: boolean, tab?: string }) {
+    if (msg?.tab) // addressed by id: only that tab answers, regardless of visibility (incl. background)
+        return msg.tab !== TAB_ID
     return msg?.requireVisible !== false && document.visibilityState !== 'visible'
 }
 
 if (import.meta.hot) {
-    import.meta.hot.on('aipeek:collect', (msg: { requireVisible?: boolean }) => {
+    // 主动报到:补上 roster 缺失的「注册边」。在此之前 roster 只在收到轮询回复时(aipeek:state/
+    // result/dom/screen)才登记一个 tab——于是会话里第一个命令若是 /tabs(纯 roster 读,自身不轮询),
+    // 即使前台开着健康 tab 也必然读到 (no live tabs)。连上即 hello、visibilitychange 重发,roster
+    // 实时为真。TAB_ID 存 sessionStorage 跨自愈 reload 存活,reload 后 hello 自然重放。
+    const hello = () => import.meta.hot!.send('aipeek:hello', {
+        tab: TAB_ID,
+        url: location.href,
+        title: document.title,
+        visible: document.visibilityState === 'visible',
+    })
+    hello()
+    addEventListener('visibilitychange', hello)
+
+    import.meta.hot.on('aipeek:collect', (msg: { requireVisible?: boolean, tab?: string }) => {
         if (skip(msg))
             return
         import.meta.hot!.send('aipeek:state', {
+            tab: TAB_ID,
             url: location.href,
+            title: document.title,
+            visible: document.visibilityState === 'visible',
             ui: collectUI(),
             console: [...consoleLogs],
             network: [...networkRequests],
             errors: [...errorEntries],
+            actions: [...actionEntries],
             state: collectState(),
+            performance: collectPerformance(),
             timestamp: Date.now(),
         })
     })
 
-    import.meta.hot.on('aipeek:action', async (msg: { id: number, type: ActionType, args: ActionArgs, requireVisible?: boolean }) => {
+    import.meta.hot.on('aipeek:action', async (msg: { id: number, type: ActionType, args: ActionArgs, requireVisible?: boolean, tab?: string }) => {
         if (skip(msg))
             return
+        // Idempotency: the server re-broadcasts an addressed action while the tab is mid self-heal,
+        // so the same id can land twice. Execute once per id; on a repeat replay the cached result
+        // (re-settles the server's pendingActions slot if the original reply was lost in teardown).
+        const done = (window.__AIPEEK_DONE_ACTIONS__ ??= new Map<number, ActionResult>())
+        const cached = done.get(msg.id)
+        if (cached) {
+            import.meta.hot!.send('aipeek:result', { tab: TAB_ID, id: msg.id, ...cached })
+            return
+        }
+        // Snapshot the state machine + error/network buffers *before* the action, so the
+        // post-settle diff is exactly what this action caused (see result.screen below).
+        const before = screenSnap()
+        const beforeConsole = [...consoleLogs]
+        const beforeNetwork = [...networkRequests]
+        const beforeErrors = [...errorEntries]
         // Guard against native alert/confirm/prompt freezing the probe (see withDialogGuard).
         const { result, dialogs } = await withDialogGuard(() => performAction(msg.type, msg.args))
         if (dialogs.length)
@@ -652,39 +821,72 @@ if (import.meta.hot) {
         // can reach the main process via electronAPI.invoke('aipeek:input') → sendInputEvent;
         // in a plain Chrome tab it can't (no chrome.debugger from page JS), so leave ui
         // undefined and let the server drive its extension queue.
+        // realclick resolved (x,y) but synthetic events can't open a Radix ContextMenu — fire
+        // a trusted click via Electron's main process if reachable, else leave it for the
+        // server's extension queue. Either path then settles like any other mutating action.
         const electronAPI = (window as { electronAPI?: { invoke: (channel: string, ...args: unknown[]) => Promise<unknown> } }).electronAPI
         if (msg.type === 'realclick' && result.ok && electronAPI) {
             await electronAPI.invoke('aipeek:input', { type: 'click', button: msg.args.button ?? 'left', x: result.x, y: result.y })
-            result.ui = await waitForStable()
-            result.screen = collectScreen()
+            result.fired = true // server: trusted click already fired in-process, skip the extension queue
         }
-        // For mutating actions, settle the DOM then ship both the full UI tree and
-        // the compact screen projection — the caller skips a round-trip to /ui, and
-        // /chain uses the per-step screen so an interaction's every transition shows.
-        else if (result.ok && (msg.type === 'click' || msg.type === 'fill' || msg.type === 'press')) {
-            result.ui = await waitForStable()
-            result.screen = collectScreen()
+
+        // For mutating actions, settle the DOM then ship *only the change this action caused*
+        // (state-machine transition + new errors/failed requests), not a fresh snapshot. AI
+        // reads the delta and drills into /ui or /dom for detail if needed. The timeline still
+        // surfaces concurrent user actions.
+        const mutated = msg.type === 'click' || msg.type === 'fill' || msg.type === 'press'
+            || (msg.type === 'realclick' && !!electronAPI)
+        if (result.ok && mutated) {
+            await waitForStable()
+            const after = screenSnap()
+            const d = diffState(
+                { ui: '', console: beforeConsole, network: beforeNetwork, errors: beforeErrors, state: {}, url: '', timestamp: 0 },
+                { ui: '', console: [...consoleLogs], network: [...networkRequests], errors: [...errorEntries], state: {}, url: '', timestamp: 0 },
+            )
+            const changed = diffScreen(before, after, d.newErrors, d.newExceptions, d.newFailedRequests)
+            result.screen = changed.length ? changed.join('\n') : '(no state change)'
+            result.actions = attachTimeline()
         }
-        import.meta.hot!.send('aipeek:result', { id: msg.id, ...result })
+        done.set(msg.id, result)
+        if (done.size > 64)
+            done.delete(done.keys().next().value!)
+        import.meta.hot!.send('aipeek:result', { tab: TAB_ID, id: msg.id, ...result })
     })
 
     // eval: run server-supplied code in the page with auto-return (see runEval).
-    import.meta.hot.on('aipeek:eval', async (msg: { id: number, code: string, requireVisible?: boolean }) => {
+    import.meta.hot.on('aipeek:eval', async (msg: { id: number, code: string, requireVisible?: boolean, tab?: string }) => {
         if (skip(msg))
             return
         const { ok, value, error } = await runEval(msg.code)
-        import.meta.hot!.send('aipeek:eval-result', { id: msg.id, ok, value, error })
+        import.meta.hot!.send('aipeek:eval-result', { tab: TAB_ID, id: msg.id, ok, value, error })
+    })
+
+    import.meta.hot.on('aipeek:collect-dom', (msg: { scope?: string, sel?: string, requireVisible?: boolean, tab?: string }) => {
+        if (skip(msg))
+            return
+        import.meta.hot!.send('aipeek:dom', { tab: TAB_ID, dom: collectDom(msg?.scope, msg?.sel) })
     })
 
-    import.meta.hot.on('aipeek:collect-dom', (msg: { scope?: string, sel?: string, requireVisible?: boolean }) => {
+    import.meta.hot.on('aipeek:collect-screen', (msg: { requireVisible?: boolean, tab?: string }) => {
         if (skip(msg))
             return
-        import.meta.hot!.send('aipeek:dom', { dom: collectDom(msg?.scope, msg?.sel) })
+        // Ship the rendered text *and* the structured snap + buffers, so the server can serve
+        // /screen?since=<token> by diffing this snap against a stashed prior one (diffScreen).
+        import.meta.hot!.send('aipeek:screen', {
+            tab: TAB_ID,
+            screen: collectScreen(),
+            snap: screenSnap(),
+            console: [...consoleLogs],
+            network: [...networkRequests],
+            errors: [...errorEntries],
+        })
     })
 
-    import.meta.hot.on('aipeek:collect-screen', (msg: { requireVisible?: boolean }) => {
+    // Perf profiler handlers
+    import.meta.hot.on('aipeek:perf-reset', (msg: { requireVisible?: boolean, tab?: string }) => {
         if (skip(msg))
             return
-        import.meta.hot!.send('aipeek:screen', { screen: collectScreen() })
+        resetPerf()
+        import.meta.hot!.send('aipeek:perf-reset-ack', { tab: TAB_ID })
     })
 }