@pipedog/core 0.0.0 → 0.2.0

package/dist/workers/ui/ui_job_context.js

@@ -0,0 +1,17 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/workers/ui/ui_job_context.py
+/**
+ * User-facing job group context (compatibility shim).
+ *
+ * Porting notes:
+ * - Upstream this module holds only `UIJobGroupContext`, a deprecated alias
+ *   (since 1.8.0) for `JobGroupContext` with the historical constructor
+ *   signature. Deprecated upstream APIs are not ported, so this module is
+ *   intentionally empty: a group dispatched by a `UIWorker` is client-visible,
+ *   so use `JobGroupContext` (`worker.jobGroup(...)`) instead.
+ */
+export {};
+//# sourceMappingURL=ui_job_context.js.map

package/dist/workers/ui/ui_prompts.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Canonical prompt fragments describing the UI worker wire format.
+ *
+ * Apps concatenate these constants into their system prompt so the LLM
+ * understands the `<ui_state>` and `<ui_event>` developer messages the SDK
+ * injects on its behalf.
+ *
+ * @example
+ * const systemPrompt = `
+ * You are a voice-driven music player agent.
+ * ...app-specific tool and behavior instructions...
+ *
+ * ${UI_STATE_PROMPT_GUIDE}
+ * `;
+ *
+ * The SDK updates the guide alongside the wire format, so apps get new tags
+ * and semantics automatically on the next release.
+ */
+/** The `<ui_state>` / `<ui_event>` wire-format guide for the LLM. */
+export declare const UI_STATE_PROMPT_GUIDE: string;
+//# sourceMappingURL=ui_prompts.d.ts.map

package/dist/workers/ui/ui_prompts.js

@@ -0,0 +1,41 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/workers/ui/ui_prompts.py
+/**
+ * Canonical prompt fragments describing the UI worker wire format.
+ *
+ * Apps concatenate these constants into their system prompt so the LLM
+ * understands the `<ui_state>` and `<ui_event>` developer messages the SDK
+ * injects on its behalf.
+ *
+ * @example
+ * const systemPrompt = `
+ * You are a voice-driven music player agent.
+ * ...app-specific tool and behavior instructions...
+ *
+ * ${UI_STATE_PROMPT_GUIDE}
+ * `;
+ *
+ * The SDK updates the guide alongside the wire format, so apps get new tags
+ * and semantics automatically on the next release.
+ */
+/** The `<ui_state>` / `<ui_event>` wire-format guide for the LLM. */
+export const UI_STATE_PROMPT_GUIDE = "## UI context\n" +
+    "\n" +
+    "Your developer context includes two kinds of SDK-managed messages:\n" +
+    "\n" +
+    '- ``<ui_event name="..." >payload</ui_event>``: an event the user just triggered on the client (click, tab switch, navigation, etc.). The payload is JSON for that event.\n' +
+    '- ``<ui_state>...</ui_state>``: an accessibility snapshot of the current screen, injected at the start of every turn. Indented tree in Playwright-MCP style. Each line is ``- role "name" [state] [ref=eN]`` with children nested one level deeper. A line can also carry ``= "value"`` (an element\'s current value, e.g. text already typed into an input) and ``[level=N]`` (heading depth).\n' +
+    "\n" +
+    "State tags include ``[focused]``, ``[selected]``, ``[disabled]``, and ``[offscreen]``. A node tagged ``[offscreen]`` exists on the page but is not currently in the user's viewport; only visible (non-offscreen) nodes count for position-based references.\n" +
+    "\n" +
+    'Grids carry a ``[cols=N]`` tag. Their cells are listed in reading order (left-to-right, top-to-bottom); with N columns, cell K sits at row ``ceil(K/N)``, column ``((K-1) mod N) + 1``. Example with ``[cols=8]`` and 16 children: "top right" is cell 8, "bottom left" is cell 9.\n' +
+    "\n" +
+    'Resolve position references ("top right", "the first one", "the third new release") against the most recent ``<ui_state>`` tree. Sibling order matches reading order on screen (top-to-bottom, left-to-right within each region).\n' +
+    "\n" +
+    'When the user has text selected on the page, the snapshot ends with a ``<selection ref="eN">selected text</selection>`` block inside ``<ui_state>``. Treat the selection as the deictic referent for "this", "that", "what I selected", and similar phrases. The ``ref`` identifies the closest enclosing element that has a ref in the tree; the inner text is the actual selected content (truncated if very long). Text inside ``<input>`` or ``<textarea>`` selections is faithful to ``selectionStart``/``selectionEnd`` on the element.\n' +
+    "\n" +
+    "Refs (``e42``) are stable handles for acting on elements: pass the ``ref`` from the most recent ``<ui_state>`` to any tool that operates on a node. The same element keeps its ref across snapshots while it stays on the page, so you can refer back to it across turns. Always resolve refs against the latest snapshot, and bring an ``[offscreen]`` element into view before acting on it.";
+//# sourceMappingURL=ui_prompts.js.map

package/dist/workers/ui/ui_tools.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Tools for a voice LLM to work with a `UIWorker`.
+ *
+ * Ships `screenTools`, the tool a voice LLM uses to ask a `UIWorker` about
+ * the screen, or act on it.
+ *
+ * Porting notes:
+ * - `ReplyToolMixin` is deprecated upstream (since 1.12.0, use
+ *   `screen_tools`) and is not ported.
+ * - The tool's description and parameter types, which upstream reads from
+ *   the docstring and type hints, are attached as direct-function metadata.
+ */
+import { type DirectFunction } from "../../adapters/schemas/direct_function.ts";
+/**
+ * The tools that let a voice LLM ask a `UIWorker` about the screen, or act on
+ * it.
+ *
+ * One tool, `screen(action, target, value)`, sends the worker's `screen` job
+ * and returns its answer as data, so the voice LLM learns what it asked and
+ * nothing more of the page. Hand them to the voice LLM's context:
+ *
+ * @example
+ * const context = new LLMContext({ tools: screenTools("ui") });
+ *
+ * @param worker The name of the `UIWorker` to ask.
+ * @param options.timeout Seconds to wait for an answer. Defaults to 30.
+ * @returns The tools, as functions for an `LLMContext`.
+ */
+export declare function screenTools(worker: string, options?: {
+    timeout?: number;
+}): DirectFunction[];
+//# sourceMappingURL=ui_tools.d.ts.map

package/dist/workers/ui/ui_tools.js

@@ -0,0 +1,112 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/workers/ui/ui_tools.py
+/**
+ * Tools for a voice LLM to work with a `UIWorker`.
+ *
+ * Ships `screenTools`, the tool a voice LLM uses to ask a `UIWorker` about
+ * the screen, or act on it.
+ *
+ * Porting notes:
+ * - `ReplyToolMixin` is deprecated upstream (since 1.12.0, use
+ *   `screen_tools`) and is not ported.
+ * - The tool's description and parameter types, which upstream reads from
+ *   the docstring and type hints, are attached as direct-function metadata.
+ */
+import { toolOptions } from "../../adapters/schemas/direct_function.js";
+import { JobError, JobParams } from "../../pipeline/job_context.js";
+import { logger } from "../../utils/logger.js";
+const SCREEN_DESCRIPTION = `Ask about what is on the user's screen right now, or act on it.
+
+Actions, with what \`\`target\`\` means for each:
+
+- "find": which element a description refers to, such as "the
+  checkout button". Returns its label and how confident the match is,
+  or no label when nothing fits.
+- "check": whether something is true of the screen, such as "is
+  anything on the list still unchecked?". Returns yes or no with a
+  probability.
+- "select": which elements match a description, such as "dairy
+  products". Returns the matching labels, most likely first.
+- "list": the named elements on screen, with their state and values,
+  optionally only those of one role such as "checkbox" or "textbox".
+  Use it to see what is on the page and which inputs are filled.
+- "selection": the text the user has selected on the page, or no text
+  when nothing is selected. Nothing else can tell; call it whenever
+  the user says "this", "that" or "what I selected".
+- "click", "scroll_to", "highlight", "select_text", "fill": do that to
+  the element a description refers to. "fill" writes \`\`value\`\` into
+  it. Returns whether it was done and the label of the element.`;
+/**
+ * The tools that let a voice LLM ask a `UIWorker` about the screen, or act on
+ * it.
+ *
+ * One tool, `screen(action, target, value)`, sends the worker's `screen` job
+ * and returns its answer as data, so the voice LLM learns what it asked and
+ * nothing more of the page. Hand them to the voice LLM's context:
+ *
+ * @example
+ * const context = new LLMContext({ tools: screenTools("ui") });
+ *
+ * @param worker The name of the `UIWorker` to ask.
+ * @param options.timeout Seconds to wait for an answer. Defaults to 30.
+ * @returns The tools, as functions for an `LLMContext`.
+ */
+export function screenTools(worker, options = {}) {
+    const timeout = options.timeout ?? 30.0;
+    /**
+     * @param params Framework-provided tool invocation context.
+     * @param action One of "find", "check", "select", "list", "selection",
+     *   "click", "scroll_to", "highlight", "select_text" or "fill".
+     * @param target The description, criteria or role the action needs; none
+     *   for "selection".
+     * @param value The text to write, only for "fill".
+     */
+    async function screen(params, action, target = null, value = null) {
+        const payload = { action };
+        if (target !== null && target !== undefined) {
+            payload.target = target;
+        }
+        if (value !== null && value !== undefined) {
+            payload.value = value;
+        }
+        const t = params.pipelineWorker.job(worker, {
+            params: new JobParams({ name: "screen", payload, timeout }),
+        });
+        try {
+            await t.run(async () => { });
+        }
+        catch (e) {
+            if (e instanceof JobError) {
+                logger.warning(`screen job on ${worker} failed: ${e.message}`);
+                await params.resultCallback({ error: e.message });
+                return;
+            }
+            throw e;
+        }
+        await params.resultCallback(t.response);
+    }
+    const tool = screen;
+    tool.description = SCREEN_DESCRIPTION;
+    // Upstream reads each parameter's description from the docstring's
+    // ``Args`` section; here they are attached to the JSON schemas directly.
+    tool.parameters = {
+        action: {
+            type: "string",
+            description: 'One of "find", "check", "select", "list", "selection",\n' +
+                '"click", "scroll_to", "highlight", "select_text" or "fill".',
+        },
+        target: {
+            anyOf: [{ type: "string" }, { type: "null" }],
+            description: 'The description, criteria or role the action needs; none\nfor "selection".',
+        },
+        value: {
+            anyOf: [{ type: "string" }, { type: "null" }],
+            description: 'The text to write, only for "fill".',
+        },
+    };
+    return [toolOptions(tool, { cancelOnInterruption: false, timeoutSecs: timeout })];
+}
+//# sourceMappingURL=ui_tools.js.map