@ixo/oracle-runtime-workers 0.2.0

package/src/plugin-api/ucan-errors.ts

@@ -0,0 +1,16 @@
+/**
+ * Thrown by the runtime's UCAN adapter when an invocation cannot be minted for
+ * an EXPECTED reason — the oracle has no signing key loaded, or the user has
+ * no cached delegation. It is deliberately distinct from a plain `Error`
+ * (which signals an UNEXPECTED mint/sign failure) so plugin auth helpers can
+ * classify the two without matching error-message substrings: an
+ * `instanceof UcanMintUnavailableError` check demotes the expected case to a
+ * non-error log and degrades silently, while any other throw is a real
+ * failure.
+ */
+export class UcanMintUnavailableError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'UcanMintUnavailableError';
+  }
+}

package/src/plugins/agui/agui-agent.ts

@@ -0,0 +1,115 @@
+/**
+ * The AG-UI sub-agent — the Workers port of the Node runtime's
+ * `createAguiSubAgent`. One tool per client-declared AG-UI action; every
+ * action call is forwarded into the main chat so the client renders the
+ * matching component events.
+ */
+import type { PluginSubAgent, PluginTool } from '../../plugin-api/types';
+
+const formatToolDocs = (tools: PluginTool[]): string => {
+  if (!tools.length) return '- No AG-UI tools configured.';
+  return tools
+    .map((t) => {
+      const description = t.description?.trim() ?? 'No description provided.';
+      return `- \`${t.name}\`: ${description}`;
+    })
+    .join('\n');
+};
+
+const buildAguiPrompt = (toolsDoc: string): string =>
+  `
+You are the AG-UI Agent — a specialized sub-agent that generates interactive UI
+components in the user's browser by calling AG-UI (Agent Generated UI) tools.
+
+## 🚨 THE ONLY RULE THAT MATTERS
+
+Your response counts as a failure if you say things like "I've created the table" or "The visualization is ready" without actually invoking an AG-UI tool. You MUST call a tool. Words alone do nothing — the UI only changes when a tool is invoked.
+
+## What are AG-UI Tools?
+
+AG-UI tools dynamically generate interactive components (tables, charts, forms,
+etc.) that render directly in the client's browser. They execute instantly
+without backend processing.
+
+## Available AG-UI Tools
+
+${toolsDoc}
+
+## Worked Example — fruits table
+
+\`\`\`
+Task: "Create a data table with these 5 fruits: [{name: 'Apple', color: 'Red', price: 1.5}, ...]"
+
+Your action: call create_data_table with
+  {
+    id: "fruits_table",
+    title: "Fruits",
+    data: [{name: "Apple", color: "Red", price: 1.5}, ...],
+    columns: [
+      {key: "name", label: "Fruit"},
+      {key: "color", label: "Color"},
+      {key: "price", label: "Avg Price"}
+    ]
+  }
+
+Your message: "Here's the fruits table."
+\`\`\`
+
+## Message Output Rules (after the tool call)
+
+Your message to the main agent should ONLY be a short natural-language
+confirmation — NEVER the data, JSON, or a text rendition of the UI.
+
+DO:
+- Call the AG-UI tool FIRST.
+- Then add one short sentence like "Here's the fruits table" or "Rendered the revenue chart."
+
+DON'T:
+- Output data as markdown tables in your message.
+- Display JSON or raw rows in your message.
+- Recreate the table/chart/list as text — the canvas already shows it.
+- Reply "I've created …" without actually having called the tool.
+
+## Schema Compliance
+
+- STRICTLY follow each tool's schema.
+- All required fields must be present and correctly typed.
+- Extract parameters verbatim from the task — do not substitute, guess, or reformat values.
+- Validation errors cause the tool to fail silently — double-check before calling.
+
+## Task Discipline
+
+- You are a one-shot sub-agent invoked by the main agent. Your single task
+  message is ALL the context you have. Do not assume prior conversation state.
+- If the task is unclear or missing critical details, STOP immediately and
+  return a clear message explaining what's missing. Do NOT guess.
+- Never loop or retry the same failing approach. If the first attempt fails,
+  stop and return a clear error — the main agent will re-invoke you if needed.
+
+## Workflow
+
+1. Parse the task. Identify which tool to use and which parameters to pass.
+2. Extract parameters verbatim from the task.
+3. **CALL THE TOOL.** This is not optional.
+4. Add one short confirmation sentence.
+`.trim();
+
+const buildAguiDescription = (tools: PluginTool[]): string => {
+  const names = tools.map((t) => t.name).join(', ') || 'no configured tools';
+  return `Specialized AG-UI Agent that generates interactive UI components (tables, charts, forms) in the user's browser. Available tools: (${names}). Give it ONE self-contained task including every value the component needs.`;
+};
+
+export function createAguiSubAgent(tools: PluginTool[]): PluginSubAgent {
+  return {
+    name: 'AG-UI Agent',
+    description: buildAguiDescription(tools),
+    systemPrompt: buildAguiPrompt(formatToolDocs(tools)),
+    tools,
+    model: 'subagent',
+    middlewares: [],
+    // AG-UI action invocations must surface in the main chat so the UI can
+    // render the corresponding component events. Forward every action this
+    // sub-agent exposes — the tool list IS the action list.
+    forwardTools: true,
+  };
+}

package/src/plugins/agui/agui.plugin.ts

@@ -0,0 +1,148 @@
+/**
+ * AG-UI plugin — the Workers port of the Node runtime's `AGUIPlugin`.
+ *
+ * The client declares its AG-UI actions in the turn body (`agActions[]` →
+ * `state.agActions`, each `{ name, description, schema, hasRender? }`); the
+ * plugin builds one tool per action and exposes them through the AG-UI
+ * sub-agent (`call_ag-ui_agent`). Calling an action emits `action_call` over
+ * the realtime channel (`ctx.frontend.callAgAction`) and waits for the
+ * client's `action_call_result` — the SSE stream shows the same call as an
+ * `action_call` event from the sub-agent's forwarded tool call.
+ */
+import { z } from 'zod';
+import type { AgAction } from '../../core/state';
+import { clientSchemaToZod } from '../../core/utils';
+import { OraclePlugin } from '../../plugin-api/oracle-plugin';
+import { tool } from '../../plugin-api/tool-helper';
+import type {
+  PluginManifest,
+  PluginSubAgent,
+  PluginTool,
+  RuntimeContext,
+} from '../../plugin-api/types';
+import { logActionToMatrix } from '../portal/action-log';
+import { createAguiSubAgent } from './agui-agent';
+
+const manifest: PluginManifest = {
+  title: 'AG-UI',
+  summary:
+    "Renders interactive UI components (tables, charts, forms) in the user's browser via AG-UI actions.",
+  whenToUse: [
+    'User asks for an interactive table, chart, or form to be rendered.',
+    'A response is best shown as a structured UI component rather than plain text.',
+  ],
+  whenNotToUse: [
+    'No AG-UI actions are declared on this request (sub-agent is not built).',
+    "A plain text answer is sufficient — don't render UI just because you can.",
+  ],
+  examples: [
+    {
+      user: 'Show me the results as a table.',
+      thought:
+        'Structured display — delegate to call_ag-ui_agent with the rows.',
+      tool: 'call_ag-ui_agent',
+    },
+  ],
+  tags: ['agui', 'ui', 'portal', 'copilot'],
+  category: 'ui',
+  visibility: 'on-demand',
+  stability: 'stable',
+};
+
+export const AG_ACTION_TIMEOUT_MS = 15_000;
+
+const AG_ACTION_SHAPE = z.object({
+  name: z.string(),
+  description: z.string(),
+  schema: z.record(z.string(), z.unknown()),
+  hasRender: z.boolean().optional(),
+});
+
+const ARGS_RECORD_SHAPE = z.record(z.string(), z.unknown());
+
+export function parseAgActions(value: unknown): AgAction[] {
+  if (!Array.isArray(value)) return [];
+  const out: AgAction[] = [];
+  for (const entry of value) {
+    const parsed = AG_ACTION_SHAPE.safeParse(entry);
+    if (parsed.success) out.push(parsed.data);
+  }
+  return out;
+}
+
+function parseArgs(input: unknown): Record<string, unknown> {
+  const parsed = ARGS_RECORD_SHAPE.safeParse(input ?? {});
+  return parsed.success ? parsed.data : {};
+}
+
+export function buildActionTool(action: AgAction): PluginTool | null {
+  const schema = clientSchemaToZod(action.schema, action.name);
+  if (!schema) return null;
+  return tool(
+    async (input, ctx: RuntimeContext) => {
+      const sessionId = ctx.session.id;
+      if (!sessionId) {
+        throw new Error('sessionId is required for AG-UI actions');
+      }
+      const frontend = ctx.frontend;
+      if (!frontend) {
+        throw new Error(
+          `AG-UI action ${action.name} needs the realtime channel, which this host does not provide.`,
+        );
+      }
+      if (!frontend.hasClient(sessionId)) {
+        throw new Error(
+          `No browser is connected to session ${sessionId}, so AG-UI action ${action.name} cannot run — the client must open the realtime (socket.io) channel for this session first.`,
+        );
+      }
+      const args = parseArgs(input);
+      const requestId = ctx.session.requestId;
+      const toolCallId = `ag_${requestId || 'noreq'}_${crypto
+        .randomUUID()
+        .slice(0, 8)}`;
+      const result = await frontend.callAgAction({
+        sessionId,
+        toolCallId,
+        toolName: action.name,
+        args,
+        timeoutMs: AG_ACTION_TIMEOUT_MS,
+      });
+      logActionToMatrix(ctx, {
+        name: action.name,
+        args,
+        result,
+        success: true,
+      });
+      return JSON.stringify(result);
+    },
+    {
+      name: action.name,
+      description: action.description,
+      schema,
+    },
+  );
+}
+
+function readAgActions(rtCtx: RuntimeContext): AgAction[] {
+  return parseAgActions(rtCtx.history.state.agActions);
+}
+
+export class AGUIPlugin extends OraclePlugin {
+  readonly name = 'agui';
+
+  readonly version = '1.0.0';
+
+  readonly manifest = manifest;
+
+  override async getRequestSubAgents(
+    rtCtx: RuntimeContext,
+  ): Promise<PluginSubAgent[]> {
+    const actions = readAgActions(rtCtx);
+    if (actions.length === 0) return [];
+    const tools = actions
+      .map(buildActionTool)
+      .filter((t): t is PluginTool => t !== null);
+    if (tools.length === 0) return [];
+    return [createAguiSubAgent(tools)];
+  }
+}

package/src/plugins/agui/index.ts

@@ -0,0 +1,2 @@
+export { AGUIPlugin, parseAgActions } from './agui.plugin';
+export { createAguiSubAgent } from './agui-agent';

package/src/plugins/attachments/attachments.plugin.ts

@@ -0,0 +1,54 @@
+/**
+ * Attachments plugin — the agent-facing half of attachment payload
+ * retention (`src/attachments/retention.ts`), Workers-only: the Node runtime
+ * keeps every attachment inline for the life of a session.
+ *
+ * Contributes one request-time tool, `view_attachment`, and only on turns
+ * whose session already has an offloaded payload (the host surface says so),
+ * so sessions without attachments never see the tool.
+ */
+import { OraclePlugin } from '../../plugin-api/oracle-plugin';
+import type {
+  PluginManifest,
+  PluginTool,
+  RuntimeContext,
+} from '../../plugin-api/types';
+import { createViewAttachmentTool } from './view-attachment-tool';
+
+const manifest: PluginManifest = {
+  title: 'Attachments',
+  summary:
+    'Fetch a file the user attached earlier in this conversation again once its bytes are no longer inline.',
+  whenToUse: [
+    'The user refers to an image or file from earlier and the message that carried it now shows an "[attachment offloaded]" note instead of the file.',
+    'You need to read, describe or compare an earlier attachment precisely rather than from memory.',
+  ],
+  whenNotToUse: [
+    'The attachment is still inline in a recent message — read it directly.',
+    'The user wants a NEW file processed — it arrives with their message, no tool call needed.',
+  ],
+  examples: [
+    {
+      user: 'What colour was the logo in the image I sent you earlier?',
+      thought:
+        'The earlier message only lists the image with a ref — fetch it again before answering.',
+      tool: 'view_attachment',
+      args: { ref: '$abc123:example.org' },
+    },
+  ],
+  visibility: 'always',
+  stability: 'stable',
+  category: 'core',
+};
+
+export class AttachmentsPlugin extends OraclePlugin {
+  readonly name = 'attachments';
+
+  readonly version = '1.0.0';
+
+  readonly manifest = manifest;
+
+  override getRequestTools(rtCtx: RuntimeContext): PluginTool[] {
+    return rtCtx.attachments?.offloaded ? [createViewAttachmentTool()] : [];
+  }
+}

package/src/plugins/attachments/index.ts

@@ -0,0 +1,6 @@
+export { AttachmentsPlugin } from './attachments.plugin';
+export {
+  createViewAttachmentTool,
+  VIEW_ATTACHMENT_DESCRIPTION,
+  viewAttachmentSchema,
+} from './view-attachment-tool';

package/src/plugins/attachments/view-attachment-tool.ts

@@ -0,0 +1,91 @@
+/**
+ * `view_attachment` — fetch an attachment of this session again after its
+ * inline payload was offloaded (see `src/attachments/retention.ts`).
+ *
+ * Two result shapes, decided by the host surface with the same routing as
+ * the turn pipeline:
+ *  - text (helper-model description / transcription / document text, or a
+ *    decoded plain-text file): returned as the tool output;
+ *  - native (the current model reads the modality): the bytes go back to the
+ *    model as a content block. A tool message cannot carry one on the
+ *    OpenAI-compatible wire, so the tool returns a LangGraph `Command` that
+ *    appends its tool message AND a human message holding the block — the
+ *    same proven shape a fresh attachment turn uses. That message is tagged
+ *    `lc_source: attachment_view`: transcripts hide it, retention treats it
+ *    as part of the current user turn and offloads it again later.
+ */
+import { HumanMessage, ToolMessage } from '@langchain/core/messages';
+import { Command } from '@langchain/langgraph';
+import { z } from 'zod';
+import {
+  ATTACHMENT_VIEW_SOURCE,
+  buildUserMessageContent,
+  VIEW_ATTACHMENT_TOOL_NAME,
+} from '../../attachments';
+import { tool } from '../../plugin-api/tool-helper';
+import type { PluginTool, RuntimeContext } from '../../plugin-api/types';
+
+export const viewAttachmentSchema = z.object({
+  ref: z
+    .string()
+    .min(1)
+    .describe(
+      'The attachment reference exactly as listed in the "[attachment offloaded]" note of the message it belongs to (a Matrix event id such as "$abc…" or an mxc:// URI).',
+    ),
+});
+
+export const VIEW_ATTACHMENT_DESCRIPTION =
+  'Look at an attachment from earlier in this conversation again. Older messages keep only a note listing their ' +
+  'attachments ("[attachment offloaded] … ref: …") instead of the file itself; pass that ref to get the file back. ' +
+  'Images and documents the current model can read are re-attached as-is; anything else comes back as text ' +
+  '(a description, transcription or the document text). Use it whenever the user refers to a file that is no ' +
+  'longer inline — do not guess its contents from memory.';
+
+export function createViewAttachmentTool(): PluginTool {
+  return tool(
+    async (rawArgs: unknown, ctx: RuntimeContext) => {
+      const { ref } = viewAttachmentSchema.parse(rawArgs);
+      if (!ctx.attachments) {
+        throw new Error(
+          'attachment access is not available on this host — the file cannot be fetched again',
+        );
+      }
+      const { meta, view } = await ctx.attachments.view(ref);
+      if (view.kind === 'text') return view.text;
+      if (!ctx.toolCallId) {
+        throw new Error(
+          `"${meta.filename}" was fetched but cannot be re-attached outside a tool call`,
+        );
+      }
+      const label = `Re-attached "${meta.filename}" (${meta.mimetype}) from earlier in this conversation.`;
+      const timestamp = new Date().toISOString();
+      const msgFromMatrixRoom = ctx.session.client === 'matrix';
+      return new Command({
+        update: {
+          messages: [
+            new ToolMessage({
+              content: `${label} It follows in the next message.`,
+              tool_call_id: ctx.toolCallId,
+            }),
+            new HumanMessage({
+              content: buildUserMessageContent(label, [view.native]),
+              additional_kwargs: {
+                lc_source: ATTACHMENT_VIEW_SOURCE,
+                timestamp,
+                msgFromMatrixRoom,
+                attachment: meta,
+                attachments: [meta],
+              },
+            }),
+          ],
+        },
+      });
+    },
+    {
+      name: VIEW_ATTACHMENT_TOOL_NAME,
+      description: VIEW_ATTACHMENT_DESCRIPTION,
+      schema: viewAttachmentSchema,
+      visibility: 'always',
+    },
+  );
+}

package/src/plugins/base64.ts

@@ -0,0 +1,26 @@
+/**
+ * Base64 helpers over the Web platform primitives (`btoa`/`atob`) — the
+ * Workers runtime has no `Buffer`. Chunked so multi-megabyte payloads never
+ * hit the argument-count limit of `String.fromCharCode(...bytes)`.
+ */
+
+const CHUNK = 0x8000;
+
+/** Encode raw bytes as standard base64. */
+export function bytesToBase64(bytes: Uint8Array): string {
+  let binary = '';
+  for (let i = 0; i < bytes.length; i += CHUNK) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
+  }
+  return btoa(binary);
+}
+
+/** Decode standard base64 to raw bytes. Throws on malformed input. */
+export function base64ToBytes(b64: string): Uint8Array {
+  const binary = atob(b64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i += 1) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return bytes;
+}