@ixo/oracle-runtime-workers 0.35.1 → 0.36.0

package/src/do/run-coordinator.ts

@@ -19,6 +19,7 @@ import {
 } from './run-buffer';
 import {
   attemptSeqBase,
+  decideOrphanRecovery,
   decideRecovery,
   patchRunRecord,
   runHasCheckpointed,
@@ -813,7 +814,9 @@ export class RunCoordinator {
    * boot is a new instance, so anything active that this coordinator does
    * not hold is an orphan: queued runs are re-opened (and started when
    * their session is idle); running ones get a recovery attempt scheduled
-   * with the backoff, or are closed as interrupted after the cap.
+   * with the backoff, or are closed as interrupted after either cap
+   * (`decideOrphanRecovery`: consecutive attempts without progress, or
+   * recoveries in all).
    */
   async recoverOrphans(): Promise<void> {
     const active = await this.host.store.listActive();
@@ -826,7 +829,12 @@ export class RunCoordinator {
         continue;
       }
       const current = await this.host.checkpointIdOf(record.sessionId);
-      const decision = decideRecovery(record, current, now, this.host.config);
+      const decision = decideOrphanRecovery(
+        record,
+        current,
+        now,
+        this.host.config,
+      );
       // Everything packed so far: the cursor continues after it (the row's
       // `lastSeq` is only written when a run ends) and the continuation note
       // shows the model what the user already read.
@@ -844,7 +852,9 @@ export class RunCoordinator {
       const startSeq = Math.max(lastPacked, attemptSeqBase(generation));
       if (decision.action === 'interrupt') {
         this.host.log.warn(
-          `[runs] ${record.runId}: ${decision.attempts} recovery attempt(s) without progress; closing as interrupted`,
+          decision.reason === 'total'
+            ? `[runs] ${record.runId}: recovered ${record.generation} time(s) in all (cap ${this.host.config.recoveryTotalAttempts}); closing as interrupted`
+            : `[runs] ${record.runId}: ${decision.attempts} recovery attempt(s) without progress; closing as interrupted`,
         );
         const updatedAt = await this.host.store.update(record.runId, {
           generation,

package/src/do/run-store.ts

@@ -7,7 +7,8 @@
  *     closed when the run ends. It carries what a later incarnation of the
  *     object needs to resume the run after a reset (the request), how many
  *     recovery attempts were spent without progress, and the checkpoint id
- *     seen at the last attempt (progress resets the counter).
+ *     seen at the last attempt (progress resets the counter), and how many
+ *     recoveries it got in all (`generation`, never reset).
  *   - `turn_run_segments`: the run's output, packed by `RunBuffer` into one
  *     row per flush, so a client can re-join after any cursor and a recovery
  *     can restore exactly what the user had already seen. Deleted at cutover.
@@ -117,6 +118,7 @@ export interface RunRecord {
    * are numbered from `attemptSeqBase(g)`, so a client's cursor from an
    * earlier attempt — even one pointing at frames that were never packed
    * before the reset — is below every frame a later attempt produces.
+   * Also the count `decideOrphanRecovery` caps (`recoveryTotalAttempts`).
    */
   generation: number;
   /** When the next recovery attempt is due (ms since epoch), while `recovering`. */
@@ -204,6 +206,14 @@ export const DEFAULT_RECOVERY_DELAYS_MS: readonly number[] = [
   5_000, 15_000, 30_000, 60_000,
 ];
 export const DEFAULT_RECOVERY_ATTEMPTS = 4;
+/**
+ * Recoveries a run gets in all, progress or not (`generation` is never
+ * reset). Each one is a restart of the object while the run was active; a
+ * turn that resets the isolate every time it makes a step (one checkpoint,
+ * then the next model call exceeds the memory limit) would otherwise be
+ * resumed for as long as each attempt checkpoints once.
+ */
+export const DEFAULT_RECOVERY_TOTAL_ATTEMPTS = 8;
 export const DEFAULT_KEEPALIVE_MS = 20_000;
 export const DEFAULT_SEGMENT_FLUSH_MS = 2_000;
 export const DEFAULT_SEGMENT_BYTES = 16 * 1024;
@@ -213,6 +223,8 @@ export interface RunDurabilityConfig {
   segmentFlushMs: number;
   segmentBytes: number;
   recoveryAttempts: number;
+  /** Recoveries in all, progress or not (`DEFAULT_RECOVERY_TOTAL_ATTEMPTS`). */
+  recoveryTotalAttempts: number;
   recoveryDelaysMs: readonly number[];
   multitaskDefault: 'interrupt' | 'enqueue';
 }
@@ -244,6 +256,11 @@ export function runDurabilityConfig(
       DEFAULT_RECOVERY_ATTEMPTS,
       0,
     ),
+    recoveryTotalAttempts: int(
+      'RUN_RECOVERY_TOTAL_ATTEMPTS',
+      DEFAULT_RECOVERY_TOTAL_ATTEMPTS,
+      0,
+    ),
     recoveryDelaysMs: delays.length > 0 ? delays : DEFAULT_RECOVERY_DELAYS_MS,
     multitaskDefault: multitask === 'enqueue' ? 'enqueue' : 'interrupt',
   };
@@ -277,7 +294,16 @@ export function runHasCheckpointed(
 
 export type RecoveryDecision =
   | { action: 'schedule'; at: number; attempts: number }
-  | { action: 'interrupt'; attempts: number };
+  | {
+      action: 'interrupt';
+      attempts: number;
+      /**
+       * `no-progress`: `recoveryAttempts` consecutive attempts left the
+       * checkpoint where it was. `total`: the run was recovered
+       * `recoveryTotalAttempts` times in all (`decideOrphanRecovery`).
+       */
+      reason: 'no-progress' | 'total';
+    };
 
 /**
  * What to do with an orphaned run found at boot (or an attempt that just
@@ -294,12 +320,36 @@ export function decideRecovery(
   const progressed = checkpointAdvanced(run.checkpointId, currentCheckpointId);
   const spent = progressed ? 0 : run.attempts;
   if (spent >= config.recoveryAttempts)
-    return { action: 'interrupt', attempts: spent };
+    return { action: 'interrupt', attempts: spent, reason: 'no-progress' };
   const delays = config.recoveryDelaysMs;
   const delay = delays[Math.min(spent, delays.length - 1)] ?? 0;
   return { action: 'schedule', at: now + delay, attempts: spent + 1 };
 }
 
+/**
+ * What to do with a run a previous incarnation left running or recovering:
+ * `decideRecovery`, once the run is still within its recoveries in all.
+ * `generation` counts every recovery the run got and is never reset, so a
+ * run that checkpoints once per attempt and then resets the object again
+ * (progress each time, so `attempts` stays at 1) is closed after
+ * `recoveryTotalAttempts` recoveries instead of being resumed until its
+ * client gives up. A deferral (`RunAttemptDeferred`) is not a restart and
+ * is bounded by `decideRecovery` alone.
+ */
+export function decideOrphanRecovery(
+  run: Pick<RunRecord, 'attempts' | 'checkpointId' | 'generation'>,
+  currentCheckpointId: string | null,
+  now: number,
+  config: Pick<
+    RunDurabilityConfig,
+    'recoveryAttempts' | 'recoveryTotalAttempts' | 'recoveryDelaysMs'
+  >,
+): RecoveryDecision {
+  if (run.generation >= config.recoveryTotalAttempts)
+    return { action: 'interrupt', attempts: run.attempts, reason: 'total' };
+  return decideRecovery(run, currentCheckpointId, now, config);
+}
+
 type RunRow = {
   run_id: string;
   session_id: string;

package/src/do/transcript.ts

@@ -16,7 +16,8 @@ import {
 import { SUMMARY_PREFIX } from '../core/middlewares/summarization';
 import { ChannelOrigin, ChannelReplyReference } from '../channels/contract';
 import { CREATE_ARTIFACT_TOOL, createArtifactParts } from '../artifacts/tool';
-import { planText } from '../delivery/schema';
+import { planText, planTextForModel } from '../delivery/schema';
+import { SEND_FILE_TOOL, sentFile } from '../plugins/sandbox/send-file';
 import type {
   ThreadMessageAnchor,
   ThreadMessageRow,
@@ -166,6 +167,7 @@ export async function transformTranscript(
   // The first listed reply that made each tool call: a tool result folds
   // into it (looked up, not searched, so long transcripts stay linear).
   const callers = new Map<string, MessageDto>();
+  const filesOnly = new Set<MessageDto>();
   for (const message of messages) {
     const toolMsg = message.type === 'tool' ? (message as ToolMessage) : null;
     const kw = (message.additional_kwargs ?? {}) as Record<string, unknown>;
@@ -248,6 +250,20 @@ export async function transformTranscript(
               contentToText(toolMsg.content),
             ),
           });
+        // A reply of files alone lists them by name: the link has expired.
+        if (
+          call?.name === SEND_FILE_TOOL &&
+          (filesOnly.has(el) || !el.content.trim())
+        ) {
+          const sent = sentFile(toolMsg.artifact);
+          if (sent) {
+            const named = planTextForModel({
+              parts: [{ kind: 'file', file: sent.file }],
+            });
+            el.content = el.content ? `${el.content}\n\n${named}` : named;
+            filesOnly.add(el);
+          }
+        }
         el.toolCalls = el.toolCalls?.map((t) =>
           t.id === toolCallId
             ? {

package/src/do/user-oracle-do.ts

@@ -404,6 +404,7 @@ import { buildCreateArtifactTool } from '../artifacts/tool';
 import { resolveDeliveryProfile } from '../delivery/profile';
 import {
   draftReplyPlan,
+  failedRunFiles,
   materializeReplyPlan,
   turnSteps,
 } from '../delivery/plan';
@@ -6623,18 +6624,26 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
           completed && delivery.kind === 'chat'
             ? await this.replyPlanOf(live, req, capture, delivery)
             : undefined;
+        // `send_file` always needs another model step, which can fail. A
+        // gateway reads a plan only off a `finished` run, so a run that failed
+        // after sending files finishes with them; the row keeps the error.
+        const filesPlan =
+          outcome.status === 'failed' && delivery.kind === 'chat'
+            ? await this.filesOfFailedRun(live, sessionId, delivery)
+            : null;
         const text =
           (live.continuation ?? '') +
           (completed ? lastAiText(capture) : outcome.fullText);
         return {
           status:
-            outcome.status === 'completed'
+            outcome.status === 'completed' || filesPlan
               ? 'finished'
               : outcome.status === 'aborted'
                 ? 'aborted'
                 : 'failed',
           text,
           ...(plan ? { plan } : {}),
+          ...(filesPlan ? { plan: filesPlan } : {}),
           ...(messageId ? { messageId } : {}),
           toolCalls,
           ...(outcome.status === 'failed' ? { error: outcome.error } : {}),
@@ -6662,10 +6671,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       delivery: Extract<DeliveryProfile, { kind: 'chat' }>,
     ): Promise<ReplyPlan> {
       const artifacts = this.artifacts?.canShare ? this.artifacts : null;
-      const { steps, toolResults } = turnSteps(capture);
+      const { steps, toolResults, toolArtifacts } = turnSteps(capture);
       const draft = draftReplyPlan({
         steps,
         toolResults,
+        toolArtifacts,
         continuation: live.continuation,
         limits: delivery.limits,
         canSpill: artifacts !== null,
@@ -6690,11 +6700,49 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       });
       await this.runStore!.setPlan(live.runId, JSON.stringify(plan));
       console.log(
-        `[delivery] run ${live.runId}: ${delivery.surface} reply in ${plan.parts.length} parts (${plan.parts.filter((p) => p.kind === 'artifact').length} artefacts)`,
+        `[delivery] run ${live.runId}: ${delivery.surface} reply in ${plan.parts.length} parts (${plan.parts.filter((p) => p.kind === 'artifact').length} artefacts, ${plan.parts.filter((p) => p.kind === 'file').length} files)`,
       );
       return plan;
     }
 
+    /**
+     * The plan of a chat run that failed after sending files, or null. It
+     * reads the checkpoint, not `capture`: without the graph's own end,
+     * `capture` is whatever node ended last with messages, and nothing when
+     * an attempt failed early, which would lose an earlier attempt's files.
+     * A read failure leaves the run failed, as it was.
+     */
+    private async filesOfFailedRun(
+      live: LiveRun,
+      sessionId: string,
+      delivery: Extract<DeliveryProfile, { kind: 'chat' }>,
+    ): Promise<ReplyPlan | null> {
+      try {
+        const tuple = await this.saver!.getTuple({
+          configurable: { thread_id: sessionId },
+        });
+        const channel: unknown = tuple?.checkpoint.channel_values.messages;
+        const marks = await this.runStore!.listMarks(live.runId);
+        const plan = await failedRunFiles(
+          Array.isArray(channel) ? channel.filter(isBaseMessage) : [],
+          new Set(marks.map((mark) => mark.toolCallId)),
+          delivery.limits,
+        );
+        if (plan) {
+          await this.runStore!.setPlan(live.runId, JSON.stringify(plan));
+          console.warn(
+            `[delivery] run ${live.runId}: failed after sending ${plan.parts.filter((p) => p.kind === 'file').length} files; delivering them`,
+          );
+        }
+        return plan;
+      } catch (error) {
+        console.warn(
+          `[delivery] run ${live.runId}: could not read the files of a failed run: ${error instanceof Error ? error.name : 'unknown'}`,
+        );
+        return null;
+      }
+    }
+
     private async runDirectRead(
       live: LiveRun,
       req: TurnRequest,

package/src/matrix/reply-parts.ts

@@ -4,7 +4,8 @@
  * plain `m.text`, exactly as before chat delivery existed. With a plan it is
  * one `m.text` per part, in the thread, rendered to HTML with raw HTML from
  * the model escaped. Each part has its own transaction id, so a replayed
- * turn is deduplicated part by part.
+ * turn is deduplicated part by part. A file is a link line: rooms have no
+ * gateway to fetch it into an attachment, so it works until the link expires.
  */
 import { Marked } from 'marked';
 import type { TurnResult } from '../do/contracts';
@@ -60,6 +61,13 @@ export function replyPartContent(part: ReplyPart): {
       body: part.text,
       formattedBody: renderRoomMarkdown(part.text),
     };
+  if (part.kind === 'file') {
+    const { filename, url, caption } = part.file;
+    return {
+      body: [caption, filename, url].filter(Boolean).join('\n'),
+      formattedBody: `<p><strong>${escapeHtml(filename)}</strong>${caption ? `<br>${escapeHtml(caption)}` : ''}<br><a href="${escapeHtml(url)}">Open file</a></p>`,
+    };
+  }
   const { title, url } =
     part.kind === 'action'
       ? part.action

package/src/plugins/ixo-chain/names/client.ts

@@ -193,8 +193,12 @@ function pause(baseUrl: string, retryAfterMs: number | undefined): number {
 
 /* ── one POST ─────────────────────────────────────────────────────────── */
 
-/** A POST's own signal: no caller's abort reaches it. */
-const NEVER_ABORTED = new AbortController().signal;
+/**
+ * A POST's own signal: no caller's abort reaches it. Made on first use:
+ * workerd refuses `new AbortController()` in global scope, so a module-level
+ * one stopped `wrangler dev` from loading the Worker.
+ */
+let neverAborted: AbortSignal | undefined;
 
 /**
  * Asks the worker for `ids` (at most {@link MAX_BATCH_IDS}); the answers it
@@ -214,7 +218,7 @@ async function post(
     blocksyncUrl: '',
     user: `names|${target.baseUrl}|${target.clientId}`,
     deadline: Date.now() + POST_TIMEOUT_MS,
-    signal: NEVER_ABORTED,
+    signal: (neverAborted ??= new AbortController().signal),
     fetch: fetcher,
   };
   try {

package/src/plugins/sandbox/index.ts

@@ -25,6 +25,7 @@ export {
   createSandboxWriteBlobTool,
   type CreateSandboxWriteBlobToolParams,
 } from './sandbox-write-blob';
+export { buildSendFileTool, SEND_FILE_TOOL } from './send-file';
 export {
   SandboxPlugin,
   type SandboxMcpClientFactory,

package/src/plugins/sandbox/sandbox.plugin.ts

@@ -1,6 +1,7 @@
 import { type ClientConfig } from '@langchain/mcp-adapters';
 import { withCallTimeout } from '../mcp-call-timeout';
 import { z } from 'zod';
+import { deliversAttachments } from '../../delivery/prompt';
 import { OraclePlugin, type PluginEnv } from '../../plugin-api/oracle-plugin';
 import type {
   PluginManifest,
@@ -17,6 +18,13 @@ import {
   type SandboxAuthBuilder,
 } from './sandbox-mcp';
 import { createSandboxWriteBlobTool } from './sandbox-write-blob';
+import {
+  buildSendFileTool,
+  CHAT_URL_NOTE,
+  PRESIGNED_URL_TOOL,
+  stripArtifactUrls,
+  URL_RESULT_TOOLS,
+} from './send-file';
 import { SANDBOX_CAPABILITY } from '../delegated-capabilities';
 
 /**
@@ -225,6 +233,11 @@ const SANDBOX_TOOL_DEFS_TTL_MS = 5 * 60 * 1000;
  *
  * Tools are discovered per request via {@link getRequestTools} because the
  * MCP client has to be authenticated as the in-flight user.
+ *
+ * On a chat surface the model cannot hold a signed link: the upstream
+ * `artifact_get_presigned_url` is replaced by `send_file`, and the URL fields
+ * of the other `artifact_*` lookups are removed. Portal and every other
+ * surface get the upstream tools unchanged.
  */
 export class SandboxPlugin extends OraclePlugin {
   /**
@@ -554,8 +567,8 @@ export class SandboxPlugin extends OraclePlugin {
   }
 
   /**
-   * Shared tail of both paths: visibility filter, PluginTool mapping, and
-   * the synthetic `sandbox_write_blob` companion.
+   * Shared tail of both paths: visibility filter, chat-surface file delivery,
+   * PluginTool mapping, and the synthetic `sandbox_write_blob` companion.
    */
   private toPluginTools(
     upstream: SandboxMcpTool[],
@@ -567,12 +580,32 @@ export class SandboxPlugin extends OraclePlugin {
           (t) => !t.name.startsWith(ORACLE_MANAGEMENT_TOOL_PREFIX),
         );
 
-    const upstreamTools: PluginTool[] = filtered.map((tool) => ({
-      name: tool.name,
-      description: tool.description,
-      schema: tool.schema,
-      handler: async (args) => tool.invoke(args),
-    }));
+    const surface = rtCtx.session.surface;
+    const chat = surface?.kind === 'chat';
+    const upstreamTools: PluginTool[] = filtered
+      .filter((tool) => !chat || tool.name !== PRESIGNED_URL_TOOL)
+      .map((tool) =>
+        chat && URL_RESULT_TOOLS.has(tool.name)
+          ? {
+              name: tool.name,
+              description: `${tool.description} ${CHAT_URL_NOTE}`,
+              schema: tool.schema,
+              handler: async (args) =>
+                stripArtifactUrls(await tool.invoke(args)),
+            }
+          : {
+              name: tool.name,
+              description: tool.description,
+              schema: tool.schema,
+              handler: async (args) => tool.invoke(args),
+            },
+      );
+
+    const presignTool = filtered.find((t) => t.name === PRESIGNED_URL_TOOL);
+    if (chat && presignTool)
+      upstreamTools.push(
+        buildSendFileTool(presignTool, deliversAttachments(surface.surface)),
+      );
 
     // sandbox_write_blob — companion to mint_invocation. Synthetic wrapper
     // that takes a server-stored blobId + a sandbox path, looks the value up