@pipedog/core 0.0.0 → 0.2.0

package/dist/workers/ui/ui_worker.d.ts

@@ -0,0 +1,571 @@
+import type { BusJobRequestMessage, BusJobResponseMessage, BusJobResponseUrgentMessage, BusJobStreamEndMessage, BusJobUpdateMessage, BusJobUpdateUrgentMessage, BusMessage } from "../../bus/messages.ts";
+import { BusUIEventMessage } from "../../bus/ui/messages.ts";
+import { type BaseClassifier, type YesNoResult } from "../../classifiers/base_classifier.ts";
+import { type JobGroup, type JobGroupResponse, JobStatus } from "../../pipeline/job_context.ts";
+import { type LLMContext } from "../../processors/aggregators/llm_context.ts";
+import type { LLMAssistantAggregatorParams } from "../../processors/aggregators/llm_response_universal.ts";
+import type { LLMService } from "../../services/llm_service.ts";
+import type { JobGroupOptions } from "../base_worker.ts";
+import { LLMContextWorker } from "../llm/llm_context_worker.ts";
+type Dict = Record<string, unknown>;
+/** The text the user has selected on the page, and the element it is in. */
+export declare class UISelection {
+    readonly ref: string;
+    readonly text: string;
+    constructor(options: {
+        ref: string;
+        text: string;
+    });
+}
+/** A named element of the snapshot, as the screen questions see it. */
+interface _Element {
+    ref: string;
+    role: string;
+    name: string;
+    state: string[];
+    value: string | null;
+}
+/**
+ * Raised by `UIWorker.act` for an action that is not one of the five
+ * (upstream: `ValueError`).
+ */
+export declare class UnknownScreenActionError extends Error {
+    name: string;
+}
+/** A pending job answer (upstream: an `asyncio.Future`). */
+declare class _PendingAnswer {
+    #private;
+    done(): boolean;
+    setResult(result: {
+        response: Dict | null;
+        status: JobStatus;
+    }): void;
+    wait(): Promise<{
+        response: Dict | null;
+        status: JobStatus;
+    }>;
+}
+export interface UIWorkerOptions {
+    /** Unique name for this worker. */
+    name: string;
+    /** The LLM service. */
+    llm: LLMService;
+    /**
+     * Optional pre-built `LLMContext`. Seeded messages are part of the mutable
+     * history and are cleared on each `keepHistory: false` reset; put durable
+     * instructions in the LLM's `systemInstruction` instead.
+     */
+    context?: LLMContext | null;
+    /**
+     * Answers the small questions about the screen (`shouldRespond`,
+     * `whichElement`). Without one, the `llm` answers them through an
+     * `LLMClassifier`, which costs an LLM call per question; a `JevClassifier`
+     * answers in about a tenth of a second with a calibrated probability.
+     */
+    classifier?: BaseClassifier | null;
+    /**
+     * Optional assistant-aggregator parameters, e.g. to enable context
+     * summarization for `keepHistory: true` workers.
+     */
+    assistantParams?: LLMAssistantAggregatorParams | null;
+    /**
+     * When true (the default), append each UI event to the context as a
+     * `<ui_event>` developer message. Override `renderUiEvent` to change the
+     * content, or set false to disable.
+     */
+    injectEvents?: boolean;
+    /**
+     * When true (the default), append the latest `<ui_state>` snapshot to the
+     * context before every inference (via the LLM's `on_before_process_frame`
+     * hook). Set false to inject manually with `injectUiState()`.
+     */
+    autoInjectUiState?: boolean;
+    /**
+     * When false (the default), the context is cleared at the start of every
+     * job, so each turn sees only the current `<ui_state>` and query -- best for
+     * the stateless-delegate role. When true, history accumulates across jobs
+     * so the LLM can resolve multi-turn references ("the next one", "the Pro
+     * version"), at the cost of more tokens and possible confusion from stale
+     * `<ui_state>` blocks. Use context summarization to prune the history when
+     * it gets too large.
+     */
+    keepHistory?: boolean;
+    /**
+     * Wire-format guide appended to the LLM's `systemInstruction` so it can
+     * parse the `<ui_state>` / `<ui_event>` messages. Defaults to
+     * `UI_STATE_PROMPT_GUIDE`; pass a string to override or null to disable.
+     * Living in `systemInstruction`, it survives context resets.
+     */
+    promptGuide?: string | null;
+}
+/**
+ * LLM worker that reads and drives a client GUI over the RTVI UI channel.
+ *
+ * A `UIWorker` connects an LLM to whatever the user is looking at: it sees the
+ * screen as accessibility snapshots, reacts to the user's UI events, and acts
+ * on the page by sending commands to the client. It is the delegate side of a
+ * voice/UI split -- a voice layer (the main pipeline's LLM, or a separate
+ * `LLMWorker`) handles speech and hands screen-relevant work to this worker.
+ *
+ * Capabilities:
+ *
+ * - See the screen. The latest accessibility snapshot is rendered as
+ *   `<ui_state>` and auto-injected into the LLM context before each
+ *   inference. Code reads it through `snapshot` and the user's `selection`.
+ * - React to UI events, dispatched to `uiEvent(name)` handlers.
+ * - Drive the UI with `sendCommand` and the `scrollTo` / `highlight` /
+ *   `selectText` / `click` / `setInputValue` helpers.
+ * - Decide small things with a classifier, not an LLM turn: whether a UI
+ *   event deserves a comment (`shouldRespond`), which element on screen the
+ *   user means (`whichElement`), whether something is true of the screen
+ *   (`checkScreen`), which elements match a description (`selectElements`),
+ *   and `act` on an element named in words.
+ * - Answer a voice LLM's questions about the screen through the `screen`
+ *   job: find an element, check whether something is true, select the
+ *   elements matching a description, list what is on screen, read the user's
+ *   selection, or click, scroll to, highlight, select or fill an element.
+ *   Every answer is short data and never the page; `screenTools` gives the
+ *   voice LLM the tool that sends it.
+ * - Answer as a delegate. The built-in single-flight `respond` job runs one
+ *   screen-grounded LLM turn and answers with the reply the LLM writes. A
+ *   `tool` that calls `respondToJob` answers instead when it needs to decide
+ *   how the answer reaches the user.
+ * - Surface long work. Every job group this worker dispatches is reported to
+ *   the client as it goes: a card when the group starts, a line per worker's
+ *   progress and completion, and the close when the group completes, whether
+ *   normally, by cancellation or by timeout. The client can cancel a group
+ *   dispatched as cancellable.
+ *
+ * `PipelineWorker` connects a UIWorker to the client automatically when RTVI
+ * is enabled -- no extra wiring. A working worker needs only an LLM; subclass
+ * it to react to UI events or add jobs and tools, and override `renderQuery`
+ * to read a non-default job payload.
+ *
+ * @example
+ * class MyUIWorker extends UIWorker {
+ *   async onNav(message: BusUIEventMessage) {
+ *     const view = (message.payload as Record<string, unknown>).view;
+ *     ...
+ *   }
+ *   static {
+ *     uiEvent("nav_click")(this.prototype.onNav);
+ *   }
+ * }
+ *
+ * const worker = new MyUIWorker({ name: "ui", llm: new OpenAILLMService({ apiKey: "..." }) });
+ *
+ * Note: with client `trackViewport` on (the default), off-screen nodes carry
+ * `[offscreen]` in `<ui_state>`; `scrollTo` before acting on them.
+ */
+export declare class UIWorker extends LLMContextWorker {
+    protected _classifier: BaseClassifier;
+    protected _injectEvents: boolean;
+    protected _autoInjectUiState: boolean;
+    protected _keepHistory: boolean;
+    protected _uiEventHandlers: Map<string, (...args: any[]) => any>;
+    protected _latestSnapshot: Dict | null;
+    protected _currentJob: BusJobRequestMessage | null;
+    protected _pending: _PendingAnswer | null;
+    /** Initialize the UIWorker. */
+    constructor(options: UIWorkerOptions);
+    /** The classifier this worker asks the small questions about the screen. */
+    get classifier(): BaseClassifier;
+    /** The latest accessibility snapshot of the page, or null before the first. */
+    get snapshot(): Dict | null;
+    /** The text the user has selected on the page, or null when nothing is. */
+    get selection(): UISelection | null;
+    /**
+     * Set the classifier up with this worker's task manager, then activate as
+     * usual.
+     *
+     * @param args Optional activation arguments.
+     */
+    onActivated(args: Dict | null): Promise<void>;
+    /** Clean up the classifier along with the worker. */
+    cleanup(): Promise<void>;
+    /**
+     * Send a named UI command to the client.
+     *
+     * Publishes a `BusUICommandMessage`; when RTVI is enabled, `PipelineWorker`
+     * translates it into an `RTVIUICommandFrame` on the pipeline. Client-side
+     * handlers subscribed to `RTVIEvent.UICommand` (or React's
+     * `useUICommandHandler`) dispatch on the command name.
+     *
+     * @param name App-defined command name (e.g. `"toast"`, `"navigate"`, or any
+     *   app-specific name).
+     * @param payload One of:
+     *   - An RTVI `BaseModel` instance (including the built-in command models
+     *     in `processors/frameworks/rtvi/models`). Converted to a plain object
+     *     with `modelDump()`.
+     *   - A plain object, forwarded as-is.
+     *   - null, forwarded as an empty object.
+     */
+    sendCommand(name: string, payload?: unknown): Promise<void>;
+    /**
+     * Send a `scroll_to` UI command to bring an element into view.
+     *
+     * Convenience wrapper around `sendCommand("scroll_to", new ScrollTo({ ref }))`.
+     * These `scrollTo` / `highlight` / `selectText` / `click` / `setInputValue`
+     * helpers are plain methods, not LLM tools: compose them inside a `job`
+     * handler or a custom `tool` body.
+     *
+     * @param ref Snapshot ref (e.g. `"e42"`) from the latest `<ui_state>`.
+     */
+    scrollTo(ref: string): Promise<void>;
+    /**
+     * Send a `highlight` UI command to briefly flash an element.
+     *
+     * @param ref Snapshot ref (e.g. `"e42"`) from the latest `<ui_state>`.
+     */
+    highlight(ref: string): Promise<void>;
+    /**
+     * Send a `select_text` UI command to select an element's text.
+     *
+     * Selects the whole element by default, or the `startOffset`..`endOffset`
+     * character sub-range (over the element's concatenated `textContent`) when
+     * both are given. Used for deixis -- pointing at content via the page's
+     * text selection.
+     *
+     * @param ref Snapshot ref (e.g. `"e42"`) from the latest `<ui_state>`.
+     * @param options.startOffset Optional start character offset of the selection.
+     * @param options.endOffset Optional end character offset (exclusive).
+     */
+    selectText(ref: string, options?: {
+        startOffset?: number | null;
+        endOffset?: number | null;
+    }): Promise<void>;
+    /**
+     * Send a `click` UI command (checkboxes, radios, submit buttons).
+     *
+     * The standard client handler no-ops on `disabled` targets, so the worker
+     * can't bypass affordances meant to be user-controlled.
+     *
+     * @param ref Snapshot ref (e.g. `"e42"`) from the latest `<ui_state>`.
+     */
+    click(ref: string): Promise<void>;
+    /**
+     * Send a `set_input_value` UI command to fill a text input/textarea.
+     *
+     * @param ref Snapshot ref (e.g. `"e42"`) of the input or textarea.
+     * @param value Text to write into the field.
+     * @param options.replace When true (the default), overwrite the field; when
+     *   false, append (e.g. to continue a long answer in a textarea).
+     */
+    setInputValue(ref: string, value: string, options?: {
+        replace?: boolean;
+    }): Promise<void>;
+    /**
+     * Ask the classifier whether a UI event calls for the assistant to speak.
+     *
+     * Most clicks and edits need no comment, and a handler that reacts to
+     * events has to tell the few that do apart without an LLM turn. The
+     * question carries the event and the latest `<ui_state>` snapshot.
+     *
+     * @param message The UI event.
+     * @param criteria What is being checked for, as a yes or no question.
+     * @returns Whether the assistant should respond to the event.
+     * @throws ClassifierError If the classifier could not answer.
+     */
+    shouldRespond(message: BusUIEventMessage, criteria?: string): Promise<boolean>;
+    /**
+     * Ask the classifier which element on screen the user means.
+     *
+     * The candidates are the snapshot's named elements, described by their role
+     * and name. The user's words and the screen are the state.
+     *
+     * @param description What the user said, such as "the blue button".
+     * @param threshold The probability below which no element is returned.
+     * @returns The element's snapshot ref, or null when there is no snapshot,
+     *   no named element, or no confident answer.
+     * @throws ClassifierError If the classifier could not answer.
+     */
+    whichElement(description: string, threshold?: number): Promise<string | null>;
+    /**
+     * Ask the classifier whether something is true of the screen.
+     *
+     * @param criteria What is being checked for, as a yes or no question, such
+     *   as "is anything on the list still unchecked?".
+     * @returns How likely the answer is yes.
+     * @throws ClassifierError If the classifier could not answer.
+     */
+    checkScreen(criteria: string): Promise<YesNoResult>;
+    /**
+     * Ask the classifier which named elements on screen match a description.
+     *
+     * One yes or no question per element, all in one call.
+     *
+     * @param criteria What the elements should be, such as "dairy products".
+     * @returns The matching elements, each as `ref`, `label` and `probability`,
+     *   most likely first. Empty when nothing on screen matches or there is no
+     *   snapshot.
+     * @throws ClassifierError If the classifier could not answer.
+     */
+    selectElements(criteria: string): Promise<{
+        ref: string;
+        label: string;
+        probability: number;
+    }[]>;
+    /**
+     * Find the element the description means and act on it.
+     *
+     * @param action One of `click`, `scroll_to`, `highlight`, `select_text` or
+     *   `set_input_value`.
+     * @param description The element in words, such as "the checkout button".
+     * @param options.value The text to write, for `set_input_value`.
+     * @returns The ref of the element acted on, or null when no element matched
+     *   with enough confidence.
+     * @throws UnknownScreenActionError If `action` is not one of the five.
+     * @throws ClassifierError If the classifier could not answer.
+     */
+    act(action: string, description: string, options?: {
+        value?: string | null;
+    }): Promise<string | null>;
+    /**
+     * The named elements on screen, without their refs.
+     *
+     * @param role Only elements of this role, such as `checkbox`; all when null.
+     * @returns One entry per element: `role`, `name`, its `state` tags and, for
+     *   an input, its `value`.
+     */
+    listElements(role?: string | null): {
+        role: string;
+        name: string;
+        state: string[];
+        value: string | null;
+    }[];
+    /** Dispatch UI events alongside base lifecycle handling. */
+    onBusMessage(message: BusMessage): Promise<void>;
+    /**
+     * The job this worker is currently processing, or null when idle.
+     *
+     * Set when a respond turn starts and cleared when the job completes. Lets
+     * `tool` methods inspect the in-flight job without threading the message
+     * through every call.
+     *
+     * @returns The in-flight `BusJobRequestMessage`, or null when idle.
+     */
+    get currentJob(): BusJobRequestMessage | null;
+    protected _respondJob(message: BusJobRequestMessage): Promise<void>;
+    /**
+     * Answer a question about the screen, or act on it, for a voice LLM.
+     *
+     * The payload names the `action`, its `target` and, for a fill, the
+     * `value`. Every answer is short data and never the page.
+     */
+    protected _screenJob(message: BusJobRequestMessage): Promise<void>;
+    /**
+     * Extract the user's query text from a job request.
+     *
+     * Override to read a different payload shape. The returned string is
+     * appended to the LLM context as a user message before the LLM runs. The
+     * default reads `payload["query"]`.
+     *
+     * @param message The inbound job request.
+     * @returns The query text to feed into the LLM.
+     */
+    renderQuery(message: BusJobRequestMessage): string;
+    /**
+     * Run one LLM turn for a job and respond when the turn answers it.
+     *
+     * Body of the built-in `respond` job. Records the in-flight job, clears the
+     * context when `keepHistory` is false, appends the rendered query, and runs
+     * the LLM (the current `<ui_state>` is injected by the
+     * `on_before_process_frame` hook). Then blocks until the LLM's reply ends
+     * the turn or a `tool` calls `respondToJob`, which chooses how the answer
+     * is delivered, and sends the job response.
+     *
+     * Spanning the full round-trip is what makes the job single-flight
+     * (`job({ ..., sequential: true })`).
+     *
+     * @param message The inbound job request.
+     */
+    protected _runLlmTurn(message: BusJobRequestMessage): Promise<void>;
+    /**
+     * Clear the LLM conversation history.
+     *
+     * Replaces all messages in the running context with an empty list via
+     * `LLMMessagesUpdateFrame`. The system prompt (`systemInstruction`) is
+     * unaffected, but messages seeded via `context` live in the same mutable
+     * list and ARE cleared. `keepHistory: false` workers reset automatically
+     * per job; `keepHistory: true` workers call this to deliberately start
+     * over.
+     */
+    protected _resetContext(): Promise<void>;
+    /**
+     * Complete the in-flight job with the worker's answer.
+     *
+     * The reply the LLM writes completes the job on its own; call this from a
+     * `tool` to answer with something else. The job responds with
+     * `{"answer": answer}` for the requester's voice LLM to phrase, and a falsy
+     * `answer` completes the turn silently. No-op when no job is in flight or
+     * it was already answered.
+     *
+     * @param answer The worker's answer, handed to the requester's voice LLM to
+     *   phrase.
+     * @param options.status Completion status. Defaults to `JobStatus.COMPLETED`.
+     */
+    respondToJob(answer?: string | null, options?: {
+        status?: JobStatus;
+    }): Promise<void>;
+    /**
+     * Dispatch a job group and announce it to the client.
+     *
+     * @param workerNames Names of the workers to send the job to.
+     * @param options Everything `BaseWorker.createJobGroupAndRequestJob` takes,
+     *   forwarded unchanged.
+     * @returns The created `JobGroup`.
+     */
+    createJobGroupAndRequestJob(workerNames: string[], options?: JobGroupOptions): Promise<JobGroup>;
+    /**
+     * Cancel a running job group and complete its client card.
+     *
+     * @param jobId The job identifier to cancel.
+     * @param options.reason Optional human-readable reason for cancellation.
+     */
+    cancelJobGroup(jobId: string, options?: {
+        reason?: string | null;
+    }): Promise<void>;
+    /** Forward a worker's progress update to the client. */
+    onJobUpdate(message: BusJobUpdateMessage | BusJobUpdateUrgentMessage): Promise<void>;
+    /**
+     * Forward a worker's response to the client as its terminal envelope.
+     *
+     * Runs before the group is torn down, so on an error status (with
+     * `cancelOnError`) the client learns which worker failed before the card
+     * closes.
+     */
+    onJobResponse(message: BusJobResponseMessage | BusJobResponseUrgentMessage): Promise<void>;
+    /**
+     * Forward a worker's stream end as its terminal envelope.
+     *
+     * A worker may finish by ending its stream instead of responding; the
+     * client is told it completed, with the final stream data as the response
+     * payload.
+     */
+    onJobStreamEnd(message: BusJobStreamEndMessage): Promise<void>;
+    /** Complete the client's card for a group whose workers all finished. */
+    onJobCompleted(result: JobGroupResponse): Promise<void>;
+    /**
+     * Render the latest accessibility snapshot as a `<ui_state>` block.
+     *
+     * Produces Playwright-MCP-style indented text with stable element refs.
+     * Apps inject the output via `injectUiState()` when they want the LLM to
+     * see what's on screen.
+     *
+     * When the snapshot carries a current text selection, a nested
+     * `<selection ref="...">...</selection>` block is appended inside
+     * `<ui_state>` so the LLM can resolve deictic references ("this
+     * paragraph", "what I selected") against on-page content.
+     *
+     * Override to customize the rendered form.
+     *
+     * @returns The `<ui_state>` block, or an empty string if no snapshot has
+     *   been received yet.
+     */
+    renderUiState(): string;
+    /**
+     * Append the latest `<ui_state>` block to the LLM context.
+     *
+     * No-op when no snapshot has been received. Frame has `runLlm: false` —
+     * the snapshot is context, not a user turn.
+     */
+    injectUiState(): Promise<void>;
+    /**
+     * Render a UI event as a string for LLM context injection.
+     *
+     * Override to customize the injected content. The default wraps the event
+     * in a single `<ui_event>` XML tag with a `name` attribute and a
+     * JSON-encoded payload as inner text.
+     *
+     * @param message The UI event to render.
+     * @returns A string to append to the LLM context as a developer message.
+     */
+    renderUiEvent(message: BusUIEventMessage): string;
+    /**
+     * Inject the event into LLM context, then dispatch to the handler.
+     *
+     * Injection runs synchronously first so the `<ui_event>` developer message
+     * lands in the context before any side effects the handler triggers. The
+     * matching `uiEvent` handler then runs in its own task so the bus
+     * dispatcher isn't held open while the handler awaits downstream work (job
+     * requests, network calls). Events with no registered handler are a no-op
+     * after injection.
+     */
+    protected _handleUiEvent(message: BusUIEventMessage): Promise<void>;
+    /**
+     * The element the description means: `ref`, `label` and `confidence`.
+     *
+     * The candidates are the snapshot's named elements, described by role and
+     * name; `ref` and `label` are null when there are none.
+     */
+    protected _find(description: string): Promise<{
+        ref: string | null;
+        label: string | null;
+        confidence: number;
+    }>;
+    /** The snapshot's named elements, breadth first. */
+    protected _namedElements(): _Element[];
+    /** The string a job's payload carries under `key`, empty when missing. */
+    protected _text(message: BusJobRequestMessage, key: string): string;
+    /** The answer to one screen action, as the `screen` job returns it. */
+    protected _screen(action: string, target: string, value: string | null): Promise<Dict>;
+    /** Publish one worker's terminal envelope. */
+    protected _sendJobCompleted(options: {
+        jobId: string;
+        workerName: string;
+        status: string;
+        response: Dict | null;
+    }): Promise<void>;
+    /**
+     * Publish the envelope that closes the client's card.
+     *
+     * Reached once per group: a group either completes with every worker
+     * having finished, or is cancelled, and the two paths are exclusive.
+     */
+    protected _sendGroupCompleted(jobId: string): Promise<void>;
+    /**
+     * Translate the client's cancel event into a cancel request.
+     *
+     * Hands the request to `BaseWorker.requestCancelJobGroup`, which refuses it
+     * for a group that is unknown or was dispatched as non-cancellable.
+     */
+    protected _handleCancelJobEvent(message: BusUIEventMessage): Promise<void>;
+}
+/**
+ * Whether the context's last message is the user's turn.
+ *
+ * Distinguishes a fresh user-turn inference (tail is the user message) from
+ * the follow-up inference the LLM runs after a tool result (tail is the tool
+ * result / assistant output), so the `<ui_state>` snapshot is injected once
+ * per turn rather than again on each tool round.
+ */
+export declare function _isUserTurn(context: LLMContext): boolean;
+/**
+ * Render one A11yNode object as Playwright-MCP-style indented text.
+ *
+ * Format per node:
+ *
+ *     - role "name" [level=N] [cols=N] [rows=N] [state1] [state2] [ref=eN]:
+ *
+ * Trailing `:` when the node has children. `name`, `level`, grid dims, and
+ * state tags are emitted only when present on the node.
+ */
+export declare function _renderNode(node: Dict, options: {
+    depth: number;
+    lines: string[];
+}): void;
+/**
+ * Render an `A11ySelection` object as a `<selection>` block.
+ *
+ * Emitted at the root of `<ui_state>` (no leading indent) so the LLM can spot
+ * it without parsing the tree:
+ *
+ *     <selection ref="e42">
+ *     the actual selected text
+ *     </selection>
+ *
+ * No-op when the selection lacks a `ref` or `text`.
+ */
+export declare function _renderSelection(selection: Dict, lines: string[]): void;
+export {};
+//# sourceMappingURL=ui_worker.d.ts.map