@ixo/oracle-runtime-workers 0.13.0 → 0.16.0

package/src/plugins/vfs/vfs-tools.ts

@@ -3,6 +3,7 @@ import { tool } from '../../plugin-api/tool-helper';
 import type { PluginTool, RuntimeContext } from '../../plugin-api/types';
 import { vfsBearer } from './vfs-auth';
 import {
+  hasGlobWildcard,
   VfsClient,
   type VfsBatchItemResult,
   type VfsSearchHit,
@@ -191,6 +192,31 @@ async function withConflictRetry<T>(op: () => Promise<T>): Promise<T> {
   }
 }
 
+/** Appended wherever a path resolved to no file — folders never resolve. */
+const FILES_ONLY_NOTE =
+  'Paths must name files exactly — folders and wildcard patterns are not accepted (find files with `vfs_glob`).';
+
+/** The answer for a move/delete source that is not a file. */
+function notAFileMessage(path: string, verb: string): string {
+  return `No file at \`${path}\` to ${verb}. ${FILES_ONLY_NOTE}`;
+}
+
+/**
+ * Map `items` through `fn` with at most `width` calls in flight, preserving
+ * order. Each batch settles before the next starts.
+ */
+async function mapInBatches<T, R>(
+  items: readonly T[],
+  width: number,
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const out: R[] = [];
+  for (let i = 0; i < items.length; i += width) {
+    out.push(...(await Promise.all(items.slice(i, i + width).map(fn))));
+  }
+  return out;
+}
+
 export function invalidArgs(err: z.ZodError): string {
   return `Invalid arguments: ${err.issues
     .map((i) => `${i.path.join('.') || '(root)'}: ${i.message}`)
@@ -291,16 +317,32 @@ const editSchema = z.object({
 });
 
 const moveSchema = z.object({
-  from: z.string().describe('Absolute source path.'),
+  from: z.string().describe('Absolute path of the file to move.'),
   to: z.string().describe('Absolute destination path.'),
 });
 
+/**
+ * Most paths one `vfs_delete` call accepts. Each path costs one `/glob`
+ * lookup (plus, on a cold delegation cache, a UCAN-store fetch), so the cap
+ * keeps a call well inside the per-invocation subrequest budget.
+ */
+export const VFS_DELETE_MAX_PATHS = 50;
+
+/**
+ * Path lookups `vfs_delete` keeps in flight at once — the Workers limit on
+ * simultaneous open connections per invocation. Requests beyond it would
+ * queue inside `fetch` while their timeout is already running.
+ */
+export const VFS_PATH_LOOKUP_CONCURRENCY = 6;
+
 const deleteSchema = z.object({
   paths: z
     .array(z.string())
     .min(1)
-    .max(1000)
-    .describe('Absolute paths to move to trash.'),
+    .max(VFS_DELETE_MAX_PATHS)
+    .describe(
+      `Absolute paths of the files to move to trash (at most ${VFS_DELETE_MAX_PATHS} per call).`,
+    ),
 });
 
 const shareSchema = z.object({
@@ -529,13 +571,7 @@ export function createVfsTools(deps: CreateVfsToolsDeps): PluginTool[] {
         withConflictRetry(async () => {
           // The worker addresses moves by file id, not source path — resolve it.
           const stat = await c.statByPath(from);
-          if (!stat) {
-            throw new VfsHttpError({
-              status: 404,
-              message: `No such file at ${from}`,
-              raw: '',
-            });
-          }
+          if (!stat) return notAFileMessage(from, 'move');
           const results = await c.move([{ id: stat.id, destinationPath: to }]);
           const failed = results.find((r) => !r.ok);
           if (failed) {
@@ -548,7 +584,7 @@ export function createVfsTools(deps: CreateVfsToolsDeps): PluginTool[] {
     {
       name: 'vfs_move',
       description:
-        'Move or rename a file or folder. Confirm destructive actions with the user first.',
+        'Move or rename one file (`from` must be the exact path of a file; wildcards and folders are not accepted). To move a folder, find its files with `vfs_glob` and move each. Confirm destructive actions with the user first.',
       schema: moveSchema,
     },
   );
@@ -561,33 +597,32 @@ export function createVfsTools(deps: CreateVfsToolsDeps): PluginTool[] {
         const err = validatePath(p);
         if (err) return `Invalid path "${p}": ${err}`;
       }
+      const paths = [...new Set(parsed.data.paths)];
       const c = client(ctx);
       return guard(ctx, {}, async () => {
-        const resolved = await Promise.all(
-          parsed.data.paths.map(async (p) => ({
-            path: p,
-            stat: await c.statByPath(p),
-          })),
+        const resolved = await mapInBatches(
+          paths,
+          VFS_PATH_LOOKUP_CONCURRENCY,
+          async (p) => ({ path: p, stat: await c.statByPath(p) }),
         );
         const missing = resolved
           .filter((x) => x.stat === null)
           .map((x) => x.path);
         const ids = resolved.flatMap((x) => (x.stat ? [x.stat.id] : []));
         if (ids.length === 0) {
-          return `No such file(s): ${parsed.data.paths.join(', ')}.`;
+          return `No such file(s): ${paths.join(', ')}. ${FILES_ONLY_NOTE}`;
         }
         const results = await c.trash(ids);
         let summary = summarizeBatch(results, 'Moved to trash', ids.length);
         if (missing.length > 0) {
-          summary += `\nNot found (skipped): ${missing.join(', ')}.`;
+          summary += `\nNot found (skipped): ${missing.join(', ')}. ${FILES_ONLY_NOTE}`;
         }
         return summary;
       });
     },
     {
       name: 'vfs_delete',
-      description:
-        'Move a file or folder to trash (recoverable). Confirm destructive actions with the user first. Deletes go to trash, not permanent.',
+      description: `Move files to trash (recoverable), up to ${VFS_DELETE_MAX_PATHS} per call. Each path must be the exact path of a file; wildcards and folders are not accepted — find a folder's files with \`vfs_glob\` and pass them. Confirm destructive actions with the user first. Deletes go to trash, not permanent.`,
       schema: deleteSchema,
     },
   );
@@ -603,6 +638,11 @@ export function createVfsTools(deps: CreateVfsToolsDeps): PluginTool[] {
       const c = client(ctx);
       return guard(ctx, { path }, async () => {
         const stat = await c.statByPath(path);
+        // A path that is not a file is published as a folder; a wildcard
+        // pattern is neither, so it is refused rather than sent on.
+        if (!stat && hasGlobWildcard(path)) {
+          return `No file at \`${path}\`. \`*\` and \`?\` are not wildcards here — give the exact path of one file or folder (find it with \`vfs_glob\`).`;
+        }
         const result = stat
           ? await c.setFilePublic(stat.id, pub)
           : await c.setFolderPublic(path, pub);

package/src/realtime/frontend-call-registry.ts

@@ -1,111 +1,191 @@
 /**
- * Pending frontend (browser-tool / AG-UI action) calls awaiting a result
- * from the user's browser — the Workers counterpart of the Node runtime's
- * `callFrontendTool`, which parks a promise on `rootEventEmitter` until the
- * socket delivers `browser_tool_result` / `action_call_result` for the same
- * `toolCallId`.
+ * Frontend (browser-tool / AG-UI action) invocations of one user object:
+ * the calls waiting for an answer from the user's browser, and a bounded
+ * record of the ones already finished.
  *
- * Settlement rules are Node's, verbatim:
- *   - `error` set                      → reject(Error(error))
- *   - AG-UI and `result.success === false` → reject(Error(result.error ?? 'Action failed'))
- *   - otherwise                        → resolve(result)
- *   - no result within `timeoutMs`     → reject("<kind> timeout after <ms>ms: <tool>")
+ * The contract (see docs/frontend-bridge.md):
+ *   - every invocation has its own id (`frontendInvocationId`), and an id is
+ *     issued once — never reused while it is pending or remembered;
+ *   - an invocation is sent to ONE socket (`dispatched`), and only a result
+ *     arriving on that socket — same engine.io sid, same session, same
+ *     authenticated DID — settles it. Another tab of the same session, a
+ *     reconnected tab (a new sid) or another session cannot;
+ *   - a settled invocation is remembered (bounded by count and age), so a
+ *     replayed or late result is recognised and rejected as such;
+ *   - pending invocations are never evicted to make room; a full table
+ *     refuses new calls instead;
+ *   - an invocation whose answer cannot arrive any more — its deadline
+ *     passed, or its socket went — resolves with `FRONTEND_OUTCOME_UNKNOWN`:
+ *     the browser may have performed the write, so it is neither a success
+ *     nor a failure, and it is never sent again.
  *
- * Purely in-memory: a pending call does not survive an object restart, and
- * neither does the turn that made it.
+ * Settlement rules for a matching result:
+ *   - `error` set                                   → reject(Error(error))
+ *   - AG-UI, `result.success === false`, and not an
+ *     unknown outcome the client reported            → reject(Error(result.error ?? 'Action failed'))
+ *   - otherwise                                     → resolve(result)
+ *
+ * Purely in-memory, which is exact on Workers: a pending call holds a timer,
+ * so its object cannot hibernate before the call ends, and an object restart
+ * ends the turn that made it (a resumed durable run does not re-run a write
+ * that had started — see tool-marks). Finished-call records are lost when
+ * the object hibernates; a result for a forgotten id is then rejected as
+ * never issued, which is the same outcome.
  */
+import {
+  frontendOutcomeUnknown,
+  reportsUnknownOutcome,
+} from '@ixo/common/ai/frontend-bridge';
 
 export type FrontendCallKind = 'browser' | 'agui';
 
+/** The socket an invocation was sent to, as its attachment identifies it. */
+export interface FrontendExecutor {
+  /** engine.io sid of the socket (stable across hibernation, new on reconnect). */
+  sid: string;
+  sessionId: string;
+  /** The DID the socket's CONNECT authenticated. */
+  userDid: string;
+}
+
 export interface FrontendCallResult {
+  /** The invocation id the client echoes back. */
   toolCallId: string;
+  /** The socket the result arrived on (never the client's claims). */
+  from: FrontendExecutor;
   result?: unknown;
   error?: string;
 }
 
+/** Why a result settled nothing (the call, if any, keeps waiting). */
+export type FrontendResultRejection =
+  /** No such invocation was issued (or its record has expired). */
+  | 'not-issued'
+  /** The invocation already ended: a duplicate, replayed or late result. */
+  | 'already-settled'
+  /** The result came from a socket other than the one the call was sent to. */
+  | 'wrong-socket'
+  /** A browser-tool result for an AG-UI invocation, or the reverse. */
+  | 'wrong-kind';
+
+export type SettleOutcome =
+  | { settled: true }
+  | { settled: false; reason: FrontendResultRejection };
+
 export interface PendingFrontendCall {
   kind: FrontendCallKind;
+  /** The invocation id (what the browser received as `toolCallId`). */
   toolCallId: string;
   toolName: string;
   sessionId: string;
   startedAt: number;
+  /** sid of the socket executing it (unset only before it is sent). */
+  executorSid?: string;
 }
 
-interface Pending extends PendingFrontendCall {
+interface Pending extends Omit<PendingFrontendCall, 'executorSid'> {
+  executor?: FrontendExecutor;
   resolve: (value: unknown) => void;
   reject: (error: Error) => void;
   timer: ReturnType<typeof setTimeout>;
 }
 
-const KIND_LABEL: Record<FrontendCallKind, string> = {
+/** Finished invocations kept to recognise replays: at most this many… */
+export const MAX_COMPLETED_FRONTEND_CALLS = 1024;
+/** …for at most this long. */
+export const COMPLETED_FRONTEND_CALL_TTL_MS = 30 * 60_000;
+/** Calls in flight at once for one user; beyond it new calls are refused. */
+export const MAX_PENDING_FRONTEND_CALLS = 256;
+
+export const FRONTEND_CALL_LABEL: Record<FrontendCallKind, string> = {
   browser: 'Browser tool',
   agui: 'AG-UI action',
 };
 
+export interface FrontendCallRegistryOptions {
+  now?: () => number;
+  maxPending?: number;
+  maxCompleted?: number;
+  completedTtlMs?: number;
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null;
 }
 
-function readSuccessFlag(result: unknown): {
-  success?: boolean;
-  error?: string;
-} {
-  if (!isRecord(result)) return {};
-  const record = result;
-  return {
-    ...(typeof record.success === 'boolean' ? { success: record.success } : {}),
-    ...(typeof record.error === 'string' ? { error: record.error } : {}),
-  };
+function sameExecutor(a: FrontendExecutor, b: FrontendExecutor): boolean {
+  return (
+    a.sid === b.sid && a.sessionId === b.sessionId && a.userDid === b.userDid
+  );
 }
 
 export class FrontendCallRegistry {
   private readonly pending = new Map<string, Pending>();
 
-  private key(kind: FrontendCallKind, toolCallId: string): string {
-    return `${kind}:${toolCallId}`;
+  /** Invocation id → when it finished; insertion order is completion order. */
+  private readonly completed = new Map<string, number>();
+
+  private readonly now: () => number;
+
+  private readonly maxPending: number;
+
+  private readonly maxCompleted: number;
+
+  private readonly completedTtlMs: number;
+
+  constructor(options: FrontendCallRegistryOptions = {}) {
+    this.now = options.now ?? (() => Date.now());
+    this.maxPending = options.maxPending ?? MAX_PENDING_FRONTEND_CALLS;
+    this.maxCompleted = options.maxCompleted ?? MAX_COMPLETED_FRONTEND_CALLS;
+    this.completedTtlMs =
+      options.completedTtlMs ?? COMPLETED_FRONTEND_CALL_TTL_MS;
   }
 
   /**
-   * Park a call until `settle` delivers its result or `timeoutMs` elapses.
-   * `signal` (the turn's abort signal) rejects early so an aborted turn does
-   * not keep a browser waiting.
+   * Register an invocation BEFORE it is sent, so an immediate answer cannot
+   * be missed, and return the promise its result settles. Throws when the id
+   * was already issued or the table is full — the call is then not sent.
+   * `signal` (the turn's abort signal) ends it early: rejected as not sent
+   * before `dispatched`, an unknown outcome after.
    */
-  wait(
-    call: Omit<PendingFrontendCall, 'startedAt'>,
+  open(
+    call: Omit<PendingFrontendCall, 'startedAt' | 'executorSid'>,
     opts: { timeoutMs: number; signal?: AbortSignal },
   ): Promise<unknown> {
-    const key = this.key(call.kind, call.toolCallId);
-    const existing = this.pending.get(key);
-    if (existing) {
-      this.finish(key);
-      existing.reject(
-        new Error(`${KIND_LABEL[call.kind]} ${call.toolCallId} was superseded`),
+    const id = call.toolCallId;
+    const label = FRONTEND_CALL_LABEL[call.kind];
+    this.prune();
+    if (this.pending.has(id) || this.completed.has(id))
+      throw new Error(`${label} invocation ${id} was already issued`);
+    if (this.pending.size >= this.maxPending)
+      throw new Error(
+        `${label} ${call.toolName} was not sent: ${this.maxPending} frontend calls are already in flight`,
       );
-    }
     return new Promise<unknown>((resolve, reject) => {
       const timer = setTimeout(() => {
-        this.finish(key);
-        reject(
-          new Error(
-            `${KIND_LABEL[call.kind]} timeout after ${opts.timeoutMs}ms: ${call.toolName}`,
-          ),
-        );
+        if (this.complete(id)) resolve(frontendOutcomeUnknown(id));
       }, opts.timeoutMs);
       const entry: Pending = {
         ...call,
-        startedAt: Date.now(),
+        startedAt: this.now(),
         resolve,
         reject,
         timer,
       };
-      this.pending.set(key, entry);
+      this.pending.set(id, entry);
       if (opts.signal) {
+        // The turn ended (Stop, deadline). Before the call was sent it is a
+        // definite failure; once sent, the browser may already have run it.
         const onAbort = () => {
-          if (this.pending.get(key) !== entry) return;
-          this.finish(key);
-          reject(
-            new Error(`${KIND_LABEL[call.kind]} ${call.toolName} aborted`),
-          );
+          if (this.pending.get(id) !== entry) return;
+          this.complete(id);
+          if (entry.executor) resolve(frontendOutcomeUnknown(id));
+          else
+            reject(
+              new Error(
+                `${label} ${call.toolName} was not sent: the turn was aborted`,
+              ),
+            );
         };
         if (opts.signal.aborted) onAbort();
         else opts.signal.addEventListener('abort', onAbort, { once: true });
@@ -113,40 +193,88 @@ export class FrontendCallRegistry {
     });
   }
 
+  /** Whether `toolCallId` is still waiting (an already-aborted call is not). */
+  isPending(toolCallId: string): boolean {
+    return this.pending.has(toolCallId);
+  }
+
+  /** Bind a pending invocation to the one socket it was sent to. */
+  dispatched(toolCallId: string, executor: FrontendExecutor): void {
+    const entry = this.pending.get(toolCallId);
+    if (entry) entry.executor = executor;
+  }
+
+  /** End an invocation that never reached a browser: a definite failure. */
+  fail(toolCallId: string, error: Error): void {
+    const entry = this.pending.get(toolCallId);
+    if (!entry || !this.complete(toolCallId)) return;
+    entry.reject(error);
+  }
+
+  /** Deliver a result that arrived on a socket. */
+  settle(kind: FrontendCallKind, data: FrontendCallResult): SettleOutcome {
+    const id = data.toolCallId;
+    const entry = this.pending.get(id);
+    if (!entry) {
+      this.prune();
+      return {
+        settled: false,
+        reason: this.completed.has(id) ? 'already-settled' : 'not-issued',
+      };
+    }
+    if (entry.kind !== kind) return { settled: false, reason: 'wrong-kind' };
+    if (!entry.executor || !sameExecutor(entry.executor, data.from))
+      return { settled: false, reason: 'wrong-socket' };
+    this.complete(id);
+    const result = data.result;
+    if (data.error) entry.reject(new Error(data.error));
+    else if (
+      kind === 'agui' &&
+      isRecord(result) &&
+      result.success === false &&
+      !reportsUnknownOutcome(result)
+    )
+      entry.reject(
+        new Error(
+          typeof result.error === 'string' && result.error
+            ? result.error
+            : 'Action failed',
+        ),
+      );
+    else entry.resolve(result);
+    return { settled: true };
+  }
+
   /**
-   * Deliver a result from the socket. Returns false when nothing was waiting
-   * for that id (late, duplicate or unknown result — logged by the caller).
+   * The socket `sid` is gone (closed, errored, dropped by the heartbeat).
+   * Its pending invocations can never be answered — a reconnect is a new
+   * socket — and are never re-sent elsewhere: they resolve with an unknown
+   * outcome now instead of at their deadline. Returns how many there were.
    */
-  settle(kind: FrontendCallKind, data: FrontendCallResult): boolean {
-    const key = this.key(kind, data.toolCallId);
-    const entry = this.pending.get(key);
-    if (!entry) return false;
-    this.finish(key);
-    if (data.error) {
-      entry.reject(new Error(data.error));
-      return true;
+  executorGone(sid: string): number {
+    let ended = 0;
+    for (const [id, entry] of [...this.pending]) {
+      if (entry.executor?.sid !== sid) continue;
+      this.complete(id);
+      entry.resolve(frontendOutcomeUnknown(id));
+      ended += 1;
     }
-    if (kind === 'agui') {
-      const flags = readSuccessFlag(data.result);
-      if (flags.success === false) {
-        entry.reject(new Error(flags.error || 'Action failed'));
-        return true;
-      }
-    }
-    entry.resolve(data.result);
-    return true;
+    return ended;
   }
 
   /** Pending calls, oldest first (diagnostics / status). */
   list(): PendingFrontendCall[] {
     return [...this.pending.values()]
-      .map(({ kind, toolCallId, toolName, sessionId, startedAt }) => ({
-        kind,
-        toolCallId,
-        toolName,
-        sessionId,
-        startedAt,
-      }))
+      .map(
+        ({ kind, toolCallId, toolName, sessionId, startedAt, executor }) => ({
+          kind,
+          toolCallId,
+          toolName,
+          sessionId,
+          startedAt,
+          ...(executor ? { executorSid: executor.sid } : {}),
+        }),
+      )
       .sort((a, b) => a.startedAt - b.startedAt);
   }
 
@@ -154,10 +282,29 @@ export class FrontendCallRegistry {
     return this.pending.size;
   }
 
-  private finish(key: string): void {
-    const entry = this.pending.get(key);
-    if (!entry) return;
+  /** Finished invocations currently remembered. */
+  get completedCount(): number {
+    this.prune();
+    return this.completed.size;
+  }
+
+  /** Move a pending invocation to the finished records; false if it was not pending. */
+  private complete(id: string): boolean {
+    const entry = this.pending.get(id);
+    if (!entry) return false;
     clearTimeout(entry.timer);
-    this.pending.delete(key);
+    this.pending.delete(id);
+    this.completed.set(id, this.now());
+    this.prune();
+    return true;
+  }
+
+  /** Drop finished records past their age, then the oldest beyond the bound. */
+  private prune(): void {
+    const cutoff = this.now() - this.completedTtlMs;
+    for (const [id, at] of this.completed) {
+      if (at > cutoff && this.completed.size <= this.maxCompleted) break;
+      this.completed.delete(id);
+    }
   }
 }