appilot 0.0.1 → 0.1.1

package/dist/domResponder.js

@@ -0,0 +1,355 @@
+/**
+ * In-page DOM responder for the agent runtime.
+ *
+ * Implements the six Web-MCP-aligned DOM tools (plus the `http_fetch`
+ * HTTP-proxy tool) the agent calls via the agent_dom_request SSE round-trip.
+ * See docs/agent/dom-tools.md for the wire format.
+ *
+ * This module is the in-page side of the contract, shared by both surfaces:
+ *   - Chrome extension: invoked from the content-script message listener
+ *     (content.ts) when the background relays an agent_dom_request to the
+ *     active tab.
+ *   - Embeddable widget: invoked directly by the widget's AgentDomBridge
+ *     (services/AgentDomBridge.ts), which intercepts agent_dom_request on the
+ *     widget SSE stream and POSTs the result to /widget/agent/dom-response.
+ *
+ * Element-id correlation: each call to get_page_outline / find_elements /
+ * inspect_element returns ephemeral ids that subsequent calls (inspect /
+ * wait_for) can dereference. Ids are kept in `currentTurnElementMap`; the
+ * map is cleared on each NEW turn (`resetTurn` on the request, or
+ * `resetDomToolTurn()`).
+ *
+ * Authoring controls bridge: when find_elements / inspect_element hit an
+ * element matched by an authored Control's locator, the `matches_authored_control`
+ * field returns the Control's semantic_id. The Controls registry is provided
+ * per turn (controlsForTurn).
+ */
+import { httpFetch } from './httpFetch.js';
+import { invokeClientTool } from './webmcp/registry.js';
+import { computeUniqueSelector } from './dom/uniqueSelector.js';
+import { isSensitiveField } from './dom/sensitiveFields.js';
+const MAX_HITS = 20;
+const HIT_LIMIT_HARD = 50;
+const MAX_TEXT_CHARS = 2000;
+let nextElementId = 1;
+let currentTurnElementMap = new Map();
+let controlsForTurn = [];
+/** Clear the per-turn element-id map. Call at the start of every new turn. */
+export function resetDomToolTurn() {
+    currentTurnElementMap = new Map();
+    nextElementId = 1;
+}
+function freshId() {
+    return `el-${nextElementId++}`;
+}
+function registerElement(el) {
+    for (const [id, existing] of currentTurnElementMap) {
+        if (existing === el)
+            return id;
+    }
+    const id = freshId();
+    currentTurnElementMap.set(id, el);
+    return id;
+}
+function rectOf(el) {
+    const r = el.getBoundingClientRect();
+    return { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) };
+}
+function isVisible(el) {
+    if (!(el instanceof HTMLElement) && !(el instanceof SVGElement))
+        return false;
+    const r = el.getBoundingClientRect();
+    if (r.width === 0 && r.height === 0)
+        return false;
+    const style = window.getComputedStyle(el);
+    if (style.display === 'none' || style.visibility === 'hidden')
+        return false;
+    if (parseFloat(style.opacity || '1') === 0)
+        return false;
+    return true;
+}
+function accessibleName(el) {
+    const ariaLabel = el.getAttribute('aria-label');
+    if (ariaLabel)
+        return ariaLabel.trim();
+    const labelledby = el.getAttribute('aria-labelledby');
+    if (labelledby) {
+        const ref = document.getElementById(labelledby);
+        if (ref)
+            return (ref.textContent || '').trim();
+    }
+    const text = (el.textContent || '').trim();
+    return text.slice(0, 120);
+}
+function roleOf(el) {
+    const explicit = el.getAttribute('role');
+    if (explicit)
+        return explicit;
+    const tag = el.tagName.toLowerCase();
+    if (tag === 'button')
+        return 'button';
+    if (tag === 'a')
+        return 'link';
+    if (tag === 'input') {
+        const type = el.type || 'text';
+        if (type === 'checkbox')
+            return 'checkbox';
+        if (type === 'radio')
+            return 'radio';
+        return 'textbox';
+    }
+    if (tag === 'textarea')
+        return 'textbox';
+    if (tag === 'select')
+        return 'combobox';
+    return tag;
+}
+function matchesAuthoredControl(el) {
+    for (const c of controlsForTurn) {
+        try {
+            const candidate = document.querySelector(c.locator);
+            if (candidate === el)
+                return c.semantic_id;
+        }
+        catch { /* invalid selector */ }
+    }
+    return undefined;
+}
+function elementSummary(el) {
+    const id = registerElement(el);
+    const disableable = el;
+    return {
+        id,
+        tag: el.tagName.toLowerCase(),
+        role: roleOf(el),
+        name: accessibleName(el),
+        text: (el.textContent || '').trim().slice(0, 120),
+        visible: isVisible(el),
+        disabled: 'disabled' in disableable ? !!disableable.disabled : false,
+        rect: rectOf(el),
+        selector: computeUniqueSelector(el),
+    };
+}
+// ── Tool implementations ────────────────────────────────────────────────
+function getPageOutline() {
+    const landmarks = [];
+    for (const el of document.querySelectorAll('[role="navigation"], [role="main"], [role="banner"], [role="contentinfo"], [role="complementary"], header, nav, main, aside, footer')) {
+        landmarks.push({
+            id: registerElement(el),
+            role: roleOf(el),
+            name: accessibleName(el),
+            rect: rectOf(el),
+        });
+    }
+    const headings = [];
+    for (const el of document.querySelectorAll('h1, h2, h3, h4')) {
+        headings.push({
+            id: registerElement(el),
+            level: Number(el.tagName.substring(1)),
+            text: (el.textContent || '').trim().slice(0, 120),
+            rect: rectOf(el),
+        });
+    }
+    return { landmarks, headings };
+}
+function findElements(args) {
+    const selector = typeof args?.selector === 'string' ? args.selector : undefined;
+    const role = typeof args?.role === 'string' ? args.role : undefined;
+    const name = typeof args?.name === 'string' ? args.name : undefined;
+    const visibleOnly = args?.visible_only !== false;
+    const limit = Math.min(Math.max(Number(args?.limit) || MAX_HITS, 1), HIT_LIMIT_HARD);
+    if (!selector && !role && !name) {
+        return { error: 'invalid_args', detail: 'at least one of selector, role, name required' };
+    }
+    let candidates = [];
+    if (selector) {
+        try {
+            candidates = Array.from(document.querySelectorAll(selector));
+        }
+        catch {
+            return { error: 'invalid_selector', detail: selector };
+        }
+    }
+    else {
+        candidates = Array.from(document.querySelectorAll('*'));
+    }
+    const hits = [];
+    const nameLower = name?.toLowerCase();
+    for (const el of candidates) {
+        if (visibleOnly && !isVisible(el))
+            continue;
+        if (role && roleOf(el) !== role)
+            continue;
+        if (nameLower && !accessibleName(el).toLowerCase().includes(nameLower))
+            continue;
+        hits.push(elementSummary(el));
+        if (hits.length >= limit)
+            break;
+    }
+    return { hits, truncated: candidates.length > hits.length };
+}
+function inspectElement(args) {
+    const id = String(args?.id || '');
+    const el = currentTurnElementMap.get(id);
+    if (!el)
+        return { error: 'unknown_id', detail: id };
+    const attributes = {};
+    // A server-rendered `value="..."` on a credential field is the same secret by
+    // another route, so the attribute dump is filtered the same way.
+    const redactValue = isSensitiveField(el);
+    for (const attr of Array.from(el.attributes)) {
+        attributes[attr.name] = redactValue && attr.name === 'value' ? '' : attr.value;
+    }
+    const parentChain = [];
+    let parent = el.parentElement;
+    while (parent && parentChain.length < 8) {
+        parentChain.push({ tag: parent.tagName.toLowerCase(), id: registerElement(parent), role: roleOf(parent) });
+        parent = parent.parentElement;
+    }
+    return {
+        ...elementSummary(el),
+        attributes,
+        parent_chain: parentChain,
+        matches_authored_control: matchesAuthoredControl(el),
+    };
+}
+function readFormState(args) {
+    let forms = [];
+    if (args?.form_id) {
+        const el = currentTurnElementMap.get(String(args.form_id));
+        if (el)
+            forms = [el];
+    }
+    else if (typeof args?.selector === 'string') {
+        try {
+            forms = Array.from(document.querySelectorAll(args.selector));
+        }
+        catch {
+            return { error: 'invalid_selector', detail: args.selector };
+        }
+    }
+    else {
+        forms = Array.from(document.querySelectorAll('form'));
+    }
+    const result = forms.slice(0, 5).map(form => {
+        const fields = [];
+        for (const fieldEl of form.querySelectorAll('input, select, textarea, [contenteditable]')) {
+            const f = fieldEl;
+            // The value is withheld for credential fields, never the field itself:
+            // the agent still needs to know the password input EXISTS and whether it
+            // is filled, so it can reason about the form without reading the secret.
+            const sensitive = isSensitiveField(f);
+            fields.push({
+                name: f.name || f.id || accessibleName(f),
+                label: accessibleName(f),
+                value: sensitive ? '' : (f.value ?? f.textContent ?? ''),
+                value_withheld: sensitive || undefined,
+                filled: sensitive ? !!f.value : undefined,
+                visible: isVisible(f),
+                disabled: !!f.disabled,
+                required: !!f.required,
+                validation_error: f.validationMessage || null,
+                matches_authored_control: matchesAuthoredControl(f) ?? null,
+                selector: computeUniqueSelector(f),
+            });
+        }
+        return { form_id: registerElement(form), fields };
+    });
+    return { forms: result };
+}
+function getVisibleText(args) {
+    let region = null;
+    if (args?.region_id)
+        region = currentTurnElementMap.get(String(args.region_id)) || null;
+    if (!region && typeof args?.selector === 'string') {
+        try {
+            region = document.querySelector(args.selector);
+        }
+        catch {
+            return { error: 'invalid_selector', detail: args.selector };
+        }
+    }
+    if (!region)
+        return { error: 'no_target', detail: 'pass region_id or selector' };
+    const max = Math.min(Math.max(Number(args?.max_chars) || MAX_TEXT_CHARS, 50), 10_000);
+    const text = region.innerText || region.textContent || '';
+    const trimmed = text.trim();
+    return { text: trimmed.slice(0, max), truncated: trimmed.length > max };
+}
+async function waitFor(args) {
+    const event = String(args?.event || '');
+    const target = args?.target || {};
+    const timeoutMs = Math.min(Math.max(Number(args?.timeout_ms) || 5000, 500), 30_000);
+    const t0 = performance.now();
+    const check = () => {
+        let el = null;
+        if (target.id)
+            el = currentTurnElementMap.get(String(target.id)) || null;
+        if (!el && target.selector) {
+            try {
+                el = document.querySelector(target.selector);
+            }
+            catch { /* noop */ }
+        }
+        if (event === 'element_visible')
+            return { matched: !!el && isVisible(el), evidence: el ? { id: registerElement(el) } : undefined };
+        if (event === 'element_disappears')
+            return { matched: !el || !isVisible(el) };
+        if (event === 'text_appears') {
+            const text = (target.text_substring || '').toLowerCase();
+            if (!text)
+                return { matched: false };
+            const body = document.body?.innerText?.toLowerCase() || '';
+            return { matched: body.includes(text), evidence: { text_excerpt: target.text_substring } };
+        }
+        if (event === 'text_disappears') {
+            const text = (target.text_substring || '').toLowerCase();
+            if (!text)
+                return { matched: true };
+            const body = document.body?.innerText?.toLowerCase() || '';
+            return { matched: !body.includes(text) };
+        }
+        if (event === 'value_changes') {
+            // Best-effort: snapshot value first time we're called for this target.
+            // A more robust impl would diff against a snapshot taken at args-receive.
+            return { matched: false };
+        }
+        return { matched: false };
+    };
+    return new Promise(resolve => {
+        const tick = () => {
+            const r = check();
+            const elapsed = performance.now() - t0;
+            if (r.matched)
+                return resolve({ matched: true, elapsed_ms: Math.round(elapsed), evidence: r.evidence });
+            if (elapsed >= timeoutMs)
+                return resolve({ matched: false, elapsed_ms: Math.round(elapsed), reason: 'timeout' });
+            setTimeout(tick, 150);
+        };
+        tick();
+    });
+}
+export async function runDomTool(req) {
+    if (req.resetTurn) {
+        resetDomToolTurn();
+    }
+    if (req.controlsForTurn)
+        controlsForTurn = req.controlsForTurn;
+    switch (req.tool) {
+        case 'get_page_outline': return getPageOutline();
+        case 'find_elements': return findElements(req.args);
+        case 'inspect_element': return inspectElement(req.args);
+        case 'read_form_state': return readFormState(req.args);
+        case 'get_visible_text': return getVisibleText(req.args);
+        case 'wait_for': return waitFor(req.args);
+        case 'http_fetch': return httpFetch(req.args);
+        case 'invoke_client_action': {
+            const action = typeof req.args.action === 'string' ? req.args.action : '';
+            const actionArgs = (req.args.args && typeof req.args.args === 'object')
+                ? req.args.args
+                : {};
+            return invokeClientTool(action, actionArgs);
+        }
+        default: return { error: 'unknown_tool', detail: req.tool };
+    }
+}

package/dist/focusedSessions/pageSdk.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Focused Sessions page SDK: the surface a host app calls to run an
+ * app-initiated, mission-scoped agent conversation and receive its structured
+ * outcome (docs/agent/focused-sessions.md).
+ *
+ *   const handle = await startFocusedSession({ template_id: 'role-play', variables: { role: 'Sam' } });
+ *   handle.onOutcome(outcome => saveEvidence(outcome));
+ *
+ * Architecture mirrors the WebMCP registry: the SDK itself performs NO HTTP
+ * and holds NO credentials. It delegates to a PROVIDER the assistant surface
+ * registers on `window.__APPILOT_SESSIONS__`:
+ *   - the widget registers a direct provider (it shares the page realm and
+ *     owns the widget-key + user-token REST calls), and
+ *   - the extension's MAIN-world script registers a postMessage relay to its
+ *     isolated content script -> background -> JWT REST.
+ * A window-anchored global (not a module singleton) because the host app's
+ * bundle and the assistant surface's bundle are DIFFERENT module graphs; a
+ * module-level variable would give each its own copy.
+ *
+ * Import from `appilot` (same shipping posture as `registerTool`).
+ */
+import type { FocusedSessionEndedEvent, FocusedSessionEndStatus, FocusedSessionOutcomeEvent, FocusedSessionStartRequest, FocusedSessionStartResult } from '../sessions/types.js';
+/** What the assistant surface (widget / extension bridge) implements. */
+export interface FocusedSessionProvider {
+    start(request: FocusedSessionStartRequest): Promise<FocusedSessionStartResult>;
+    end(focusedSessionId: string): Promise<void>;
+    /** Subscribe to outcome events; returns an unsubscribe function. */
+    onOutcome(cb: (event: FocusedSessionOutcomeEvent) => void): () => void;
+    /**
+     * Subscribe to terminal-WITHOUT-outcome events; returns an unsubscribe
+     * function. Optional so older surfaces (the extension MAIN-world bridge)
+     * keep type-checking until they wire it; the page handle degrades to a
+     * never-firing subscription in that case.
+     */
+    onEnded?(cb: (event: FocusedSessionEndedEvent) => void): () => void;
+    /**
+     * Re-attach to a session this identity already started, after the page lost
+     * its handle (a full browser reload). Resolves to null when the session is
+     * no longer running. Optional so older surfaces keep type-checking; without
+     * it `resumeFocusedSession` resolves to null rather than throwing.
+     */
+    resume?(focusedSessionId: string): Promise<FocusedSessionStartResult | null>;
+}
+/**
+ * Called by the assistant surface (widget boot / extension MAIN-world bridge)
+ * to expose the session transport to the page. Last registration wins: when
+ * both the widget and the extension are present on a page, the most recent
+ * surface serves new sessions (in practice hosts embed exactly one).
+ */
+export declare function registerFocusedSessionProvider(provider: FocusedSessionProvider): void;
+/** A running (or finished) focused session, from the host page's side. */
+export interface FocusedSessionPageHandle {
+    focusedSessionId: string;
+    conversationId: number;
+    /** Resolved display name (the session header the assistant surface shows). */
+    name: string;
+    /**
+     * Subscribe to THIS session's outcome. The outcome is AI output: run it
+     * through your app's own review/validation model before treating it as
+     * truth. Returns an unsubscribe function.
+     */
+    onOutcome(cb: (outcome: Record<string, unknown>) => void): () => void;
+    /**
+     * Subscribe to THIS session ending WITHOUT an outcome: abandoned by the
+     * server (turn budget exhausted), expired, failed, or ended by the page
+     * (`end()`). Fires exactly once per session and is mutually exclusive with
+     * `onOutcome`, a completed session fires only `onOutcome`. Use it to stop
+     * waiting: without it a page that waits for the outcome hangs forever when
+     * the session dies outcome-less. Returns an unsubscribe function.
+     */
+    onEnded(cb: (event: {
+        focusedSessionId: string;
+        status: FocusedSessionEndStatus;
+    }) => void): () => void;
+    /** Abandon the session (no outcome will be produced). */
+    end(): Promise<void>;
+}
+/**
+ * Start a focused session. Resolves once the assistant surface has created
+ * the session server-side (mission resolved + conversation bound); rejects
+ * with the surface's typed error message when the template is unknown, a
+ * required variable is missing, or no assistant surface is present.
+ */
+export declare function startFocusedSession(request: FocusedSessionStartRequest): Promise<FocusedSessionPageHandle>;
+/**
+ * The id of the session this tab last started, if it has not reached a terminal
+ * state. Useful for deciding whether to offer a "continue" affordance before
+ * paying for the round-trip that `resumeFocusedSession` makes.
+ */
+export declare function getResumableFocusedSessionId(): string | null;
+/** Forget the remembered session without ending it server-side. */
+export declare function forgetFocusedSession(): void;
+/**
+ * Re-attach to a session this tab started before a reload.
+ *
+ * Resolves to a handle when the session is still running and belongs to the
+ * current identity, and to `null` otherwise: nothing remembered, the assistant
+ * surface cannot resume, the session already ended, or it is not this user's.
+ * Ownership is checked server-side; possessing an id proves nothing.
+ *
+ * ```ts
+ * const handle = await resumeFocusedSession();
+ * if (handle) handle.onOutcome(saveEvidence);
+ * else showActivityFinishedState();
+ * ```
+ */
+export declare function resumeFocusedSession(focusedSessionId?: string): Promise<FocusedSessionPageHandle | null>;