@ixo/oracle-runtime-workers 0.37.2 → 0.39.0

package/src/attachments/route.ts

@@ -2,26 +2,59 @@ import type { ModelInputCapabilities } from '../core/llm';
 import type { AttachmentKind } from './classify';
 
 /**
- * How an attachment reaches the model (verbatim port of the Node rule):
+ * How an attachment reaches the model:
  * - `parse-local`   — read to text ourselves (free). Plain-text files only.
  * - `send-native`   — attach the raw bytes to the user message; the selected
  *                     model reads it directly. Only when the model accepts that
- *                     modality.
+ *                     modality and, for a document, the file is not declared
+ *                     too large (the caller checks the downloaded length).
  * - `model-extract` — the helper (vision) model turns it into text first. Used
- *                     for anything the selected model can't accept natively.
+ *                     for anything the selected model can't accept natively,
+ *                     and for documents over {@link NATIVE_DOCUMENT_MAX_BYTES}.
  */
 export type AttachmentStrategy =
   | 'parse-local'
   | 'send-native'
   | 'model-extract';
 
+/**
+ * The largest document sent to the model as a native file block (20 KiB).
+ * The block is the file's base64, which the runtime counts as tokens at
+ * chars/4: 20,480 bytes are ~27k base64 chars ≈ 7k tokens by that estimate.
+ * Providers charge a native PDF per page, so this keeps a native document
+ * to a few pages. A larger document
+ * always goes through the extract lane: its text from the helper model (cut
+ * at 50k chars), a sandbox copy and a 500-char inline excerpt with the
+ * sandbox path.
+ */
+export const NATIVE_DOCUMENT_MAX_BYTES = 20_480;
+
 /**
  * Decide, for one attachment, how to get it to the selected model — the single
  * cost-aware, capability-aware routing rule. Pure and total.
+ *
+ * Documents are routed by SIZE (`opts.size`, the declared byte length), the
+ * same for every source (Portal, Matrix, chat channels): a native file block
+ * is replayed to the model on every later turn and held in the user's
+ * checkpoint, so its cost grows with the file, not with where it came from.
+ * A document is a native candidate when the model reads files AND its
+ * declared size is unknown or at most {@link NATIVE_DOCUMENT_MAX_BYTES}; only
+ * a KNOWN size over the limit goes straight to `model-extract`, which skips
+ * the native download. For a candidate the downloaded byte length decides
+ * (the pipeline and `view.ts`): at most the limit is sent natively, more goes
+ * to the extract lane with the bytes already loaded. So an unsized small
+ * file still goes natively, and a declared size that is wrong cannot slip a
+ * large one through.
+ *
+ * Images go native whenever the model reads images, whatever their size:
+ * they are downscaled before the model sees them (`resize.ts`) when the
+ * oracle binds Cloudflare Images. Audio, video, text and unknown files are
+ * routed by capability alone.
  */
 export function routeAttachment(
   kind: AttachmentKind,
   caps: ModelInputCapabilities,
+  opts: { size?: number } = {},
 ): AttachmentStrategy {
   switch (kind) {
     case 'text':
@@ -29,7 +62,9 @@ export function routeAttachment(
     case 'image':
       return caps.image ? 'send-native' : 'model-extract';
     case 'document':
-      return caps.file ? 'send-native' : 'model-extract';
+      return caps.file && mayFitNativeDocument(opts.size)
+        ? 'send-native'
+        : 'model-extract';
     case 'audio':
       return caps.audio ? 'send-native' : 'model-extract';
     case 'video':
@@ -39,3 +74,11 @@ export function routeAttachment(
       return 'model-extract';
   }
 }
+
+/**
+ * Whether a document declared at `size` bytes may go as a native file block:
+ * unknown counts as maybe, and the downloaded length decides.
+ */
+function mayFitNativeDocument(size: number | undefined): boolean {
+  return size === undefined || size <= NATIVE_DOCUMENT_MAX_BYTES;
+}

package/src/attachments/sandbox-archive.ts

@@ -5,6 +5,11 @@
  * to `<SANDBOX_MCP_URL without /mcp>/artifacts/upload`, authenticated with
  * the same `Authorization: Bearer <ucan>` + `X-Auth-Type: ucan` the sandbox
  * MCP client sends.
+ *
+ * The endpoint parses the whole multipart body (`Request.formData()`) and
+ * refuses a file over 25 MB with `413`, so the pipeline never reads a file
+ * over `MAX_FILE_SIZE` only to archive it, and a streamed body would buy
+ * nothing: every archived file is one the turn already holds in memory.
  */
 import { sanitizeSandboxPath } from './sanitize';
 
@@ -55,11 +60,17 @@ export async function uploadToSandbox(
     );
   }
   const raw: unknown = await response.json();
-  const result = raw as { path?: string; url?: string; previewUrl?: string };
+  const field = (name: string): string | undefined => {
+    const value: unknown =
+      typeof raw === 'object' && raw !== null ? Reflect.get(raw, name) : null;
+    return typeof value === 'string' && value ? value : undefined;
+  };
+  const url = field('url');
+  const previewUrl = field('previewUrl');
   return {
-    path: typeof result.path === 'string' ? result.path : safePath,
-    ...(result.url ? { url: result.url } : {}),
-    ...(result.previewUrl ? { previewUrl: result.previewUrl } : {}),
+    path: field('path') ?? safePath,
+    ...(url ? { url } : {}),
+    ...(previewUrl ? { previewUrl } : {}),
   };
 }
 

package/src/attachments/sanitize.ts

@@ -17,18 +17,28 @@ function cutUnits(value: string, max: number): string {
 }
 
 /**
- * Strip control characters and bracket sequences from a filename to prevent
- * prompt injection when interpolated into LLM context. The archive writes
- * under this exact name.
+ * `value` without what can disguise text: Unicode controls (Cc), format
+ * characters (Cf: bidi overrides such as U+202E, zero-width joiners), line
+ * and paragraph separators (Zl, Zp), and `[` / `]` (they would forge a
+ * `[source: …]` note).
+ */
+function visibleText(value: string): string {
+  return value.replace(/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}[\]]/gu, '');
+}
+
+/**
+ * Clean a client-supplied filename before it reaches LLM context, a log
+ * line, a room event or an archive path (the archive writes under this exact
+ * name): `visibleText`, then only the last path segment is kept (`/` and
+ * `\` both separate) and any remaining `..` becomes `_`, cut to 255 UTF-16
+ * units without splitting a surrogate pair.
  */
 export function sanitizeAttachmentFilename(filename: string): string {
-  return cutUnits(
-    filename
-      // eslint-disable-next-line no-control-regex
-      .replace(/[\x00-\x1f\x7f]/g, '')
-      .replace(/[[\]]/g, ''),
-    255,
+  const visible = visibleText(filename);
+  const basename = visible.slice(
+    Math.max(visible.lastIndexOf('/'), visible.lastIndexOf('\\')) + 1,
   );
+  return cutUnits(basename.replace(/\.\.+/g, '_'), 255);
 }
 
 /** Longest filename / mimetype / event id / mxc URI written into a hidden note. */
@@ -39,12 +49,13 @@ export const NOTE_URI_MAX = 512;
 
 /**
  * Clean a client-supplied value (filename, mimetype, media event id, mxc
- * URI) for the hidden notes the model reads: the filename cleaner plus
- * quotes removed (the notes wrap names and refs in `"…"`), cut to `max`
- * UTF-16 units without splitting a surrogate pair.
+ * URI) for the hidden notes the model reads: `visibleText` (no path
+ * handling: a mimetype or a URI keeps its slashes) plus quotes removed
+ * (the notes wrap names and refs in `"…"`), cut to `max` UTF-16 units
+ * without splitting a surrogate pair.
  */
 export function sanitizeNoteField(value: string, max: number): string {
-  return cutUnits(sanitizeAttachmentFilename(value).replace(/"/g, ''), max);
+  return cutUnits(visibleText(value).replace(/"/g, ''), max);
 }
 
 /**

package/src/attachments/view.ts

@@ -1,11 +1,18 @@
 /**
  * Re-fetch one attachment of a session on demand — the `view_attachment`
- * lane behind payload retention (`retention.ts`). The same routing as the
- * turn pipeline applies: when the current model reads the modality the bytes
- * come back as a native content block, otherwise the helper (`vision` role)
- * model turns them into text; plain text is decoded locally. Text results
- * are cached per session (`deps.cache`); native views are not (they are the
- * bytes themselves).
+ * lane: attachment bytes are never stored with the conversation (the
+ * checkpointer keeps a placeholder, `src/sqlite/inline-payloads.ts`), so a
+ * file of an earlier turn is read again from its source. The same routing as
+ * the turn pipeline applies (`route.ts`: capability, and size for a
+ * document): when the current model reads the modality the bytes come back
+ * as a native content block (an image downscaled when `deps.images` is
+ * bound, see `resize.ts`), otherwise the helper (`vision` role) model turns
+ * them into text; plain text is decoded locally. A document whose downloaded
+ * length is over `NATIVE_DOCUMENT_MAX_BYTES`, or an image the resize step
+ * could not make fit (`dropped`), comes back as text from the bytes already
+ * loaded. Text results are cached per session (`deps.cache`); native views
+ * are not (they are the bytes themselves). Log lines name a file by its
+ * cleaned name (`sanitizeAttachmentFilename`).
  */
 import type { ChannelMediaSource } from './channel';
 import { classifyAttachment } from './classify';
@@ -20,10 +27,12 @@ import {
 import { categorizeFile, isPlainTextType, verifyMagicBytes } from './magic';
 import { extractPdfText } from './pdf-text';
 import { attachmentMetas, attachmentRef } from './retention';
-import { routeAttachment } from './route';
+import { resizeForModel } from './resize';
+import { NATIVE_DOCUMENT_MAX_BYTES, routeAttachment } from './route';
 import {
   NOTE_FILENAME_MAX,
   NOTE_MIMETYPE_MAX,
+  sanitizeAttachmentFilename,
   sanitizeNoteField,
 } from './sanitize';
 import {
@@ -47,6 +56,8 @@ export interface ViewAttachmentDeps {
   /** Where `channel:` references are read (the owner object's links). */
   channel?: ChannelMediaSource;
   extraction: ExtractionProvider | null;
+  /** Cloudflare Images: downscales native images for the model; absent = sent as they are. */
+  images?: ImagesBinding;
   /** Capabilities of the model that will read the result. */
   caps: ModelInputCapabilities;
   roomId?: string;
@@ -129,7 +140,10 @@ export async function viewAttachment(
 ): Promise<AttachmentView> {
   const attachment = toInput(meta);
   const kind = classifyAttachment(attachment);
-  const strategy = routeAttachment(kind, deps.caps);
+  const strategy = routeAttachment(kind, deps.caps, { size: meta.size });
+  const name = sanitizeAttachmentFilename(attachment.filename);
+  // Downloaded for a native view but unfit for one: read as text instead.
+  let preloaded: { bytes: Uint8Array; mimetype: string } | undefined;
   if (strategy === 'send-native') {
     const { bytes, mimetype, sniffed } = await loadAttachmentBytes(
       attachment,
@@ -138,7 +152,7 @@ export async function viewAttachment(
       downloadOptions(deps),
     );
     deps.logger.log(
-      `[attachments] view "${attachment.filename}" (${mimetype}) kind=${kind} → ${strategy} (${bytes.length} bytes)`,
+      `[attachments] view "${name}" (${mimetype}) kind=${kind} size=${meta.size ?? 'unknown'} → ${strategy} (${bytes.length} bytes)`,
     );
     const loaded = classifyAttachment({
       mimetype,
@@ -149,15 +163,37 @@ export async function viewAttachment(
       throw new Error(
         `File content mismatch: claimed ${attachment.mimetype} but detected ${mimetype}`,
       );
-    return {
-      kind: 'native',
-      native: {
-        kind: loaded === 'image' ? 'image' : 'file',
-        mimeType: mimetype,
-        base64: bytesToBase64(bytes),
-        filename: attachment.filename,
-      },
+    let sent: { bytes: Uint8Array; mimetype: string } | null = {
+      bytes,
+      mimetype,
     };
+    if (loaded === 'document' && bytes.length > NATIVE_DOCUMENT_MAX_BYTES) {
+      deps.logger.log(
+        `[attachments] view "${name}" is ${bytes.length} bytes, over the ${NATIVE_DOCUMENT_MAX_BYTES}-byte native document limit → model-extract`,
+      );
+      sent = null;
+    } else if (loaded === 'image') {
+      const resized = await resizeForModel(bytes, mimetype, deps.images, {
+        logger: deps.logger,
+      });
+      if (resized.dropped) {
+        deps.logger.log(
+          `[attachments] view "${name}" cannot go to the model as an image (${resized.dropped}) → model-extract`,
+        );
+        sent = null;
+      } else sent = resized;
+    }
+    if (sent)
+      return {
+        kind: 'native',
+        native: {
+          kind: loaded === 'image' ? 'image' : 'file',
+          mimeType: sent.mimetype,
+          base64: bytesToBase64(sent.bytes),
+          filename: name,
+        },
+      };
+    preloaded = { bytes, mimetype };
   }
 
   const category = categorizeFile(attachment.mimetype);
@@ -178,22 +214,28 @@ export async function viewAttachment(
     const cached = await deps.cache?.get(cacheKey.ref, cacheKey.model);
     if (cached !== undefined) {
       deps.logger.log(
-        `[attachments] view "${attachment.filename}" → cached text (${cacheKey.model})`,
+        `[attachments] view "${name}" → cached text (${cacheKey.model})`,
       );
       return { kind: 'text', text: cached };
     }
   }
-  const { bytes, mimetype } = await loadAttachmentBytes(
-    attachment,
-    deps.roomId,
-    deps.source,
-    downloadOptions(deps),
-  );
-  deps.logger.log(
-    `[attachments] view "${attachment.filename}" (${mimetype}) kind=${kind} → ${strategy} (${bytes.length} bytes)`,
-  );
-  verifyMagicBytes(bytes, category, attachment, (m) =>
-    deps.logger.warn(`[attachments] ${m}`),
+  const { bytes, mimetype } =
+    preloaded ??
+    (await loadAttachmentBytes(
+      attachment,
+      deps.roomId,
+      deps.source,
+      downloadOptions(deps),
+    ));
+  if (!preloaded)
+    deps.logger.log(
+      `[attachments] view "${name}" (${mimetype}) kind=${kind} size=${meta.size ?? 'unknown'} → ${strategy} (${bytes.length} bytes)`,
+    );
+  verifyMagicBytes(
+    bytes,
+    category,
+    { mimetype: attachment.mimetype, filename: name },
+    (m) => deps.logger.warn(`[attachments] ${m}`),
   );
   const { text } = await extractText(
     bytes,
@@ -237,6 +279,7 @@ export async function extractAttachmentText(
   opts: { maxChars: number },
 ): Promise<AttachmentTextExtraction> {
   const attachment = toInput(meta);
+  const name = sanitizeAttachmentFilename(attachment.filename);
   const category = categorizeFile(attachment.mimetype);
   if (category === 'unsupported') {
     throw new Error(
@@ -276,7 +319,7 @@ export async function extractAttachmentText(
       const hit = deps.ingestCache?.get(ref, key);
       if (!hit) continue;
       deps.logger.log(
-        `[attachments] ingest "${attachment.filename}" → cached text (${key}, memory)`,
+        `[attachments] ingest "${name}" → cached text (${key}, memory)`,
       );
       const capped = capText(hit.text, opts.maxChars);
       return result(
@@ -294,9 +337,7 @@ export async function extractAttachmentText(
     for (const [key, method] of lookups) {
       const cached = await cache.get(ref, key);
       if (cached === undefined) continue;
-      deps.logger.log(
-        `[attachments] ingest "${attachment.filename}" → cached text (${key})`,
-      );
+      deps.logger.log(`[attachments] ingest "${name}" → cached text (${key})`);
       const capped = capText(cached, opts.maxChars);
       return result(
         capped.text,
@@ -325,11 +366,14 @@ export async function extractAttachmentText(
     deps.source,
     downloadOptions(deps),
   );
-  verifyMagicBytes(bytes, category, attachment, (m) =>
-    deps.logger.warn(`[attachments] ${m}`),
+  verifyMagicBytes(
+    bytes,
+    category,
+    { mimetype: attachment.mimetype, filename: name },
+    (m) => deps.logger.warn(`[attachments] ${m}`),
   );
   deps.logger.log(
-    `[attachments] ingest "${attachment.filename}" (${mimetype}) (${bytes.length} bytes)`,
+    `[attachments] ingest "${name}" (${mimetype}) (${bytes.length} bytes)`,
   );
   const store = async (
     key: string,
@@ -378,13 +422,13 @@ export async function extractAttachmentText(
         );
       }
       deps.logger.log(
-        `[attachments] ingest "${attachment.filename}": no text layer (${pdf.pages} page(s)), using the helper model`,
+        `[attachments] ingest "${name}": no text layer (${pdf.pages} page(s)), using the helper model`,
       );
     } catch (error) {
       // A cancelled turn stops here; it must not start a model call.
       if (deps.signal?.aborted) throw error;
       deps.logger.warn(
-        `[attachments] ingest "${attachment.filename}": local PDF parse failed (${error instanceof Error ? error.message : String(error)}), using the helper model`,
+        `[attachments] ingest "${name}": local PDF parse failed (${error instanceof Error ? error.message : String(error)}), using the helper model`,
       );
     }
   }

package/src/channels/replies.ts

@@ -1,4 +1,5 @@
 import type { BaseMessage, MessageContent } from '@langchain/core/messages';
+import { TURN_GUIDE_NOTE_KWARG } from '../core/operating-guides';
 import {
   stripTurnTimeNote,
   TURN_TIME_NOTE_KWARG,
@@ -9,7 +10,7 @@ import type { ChannelFileNotice, ChannelReplyContext } from './contract';
  * `additional_kwargs` key holding the exact quoted-reply note a channel
  * message was sent with, so readers can take it off the stored content again
  * (see `userContent`). The note sits between the time note and what the
- * person typed: `time note \n\n reply note \n\n message`.
+ * person typed: `time note \n\n guide note \n\n reply note \n\n message`.
  */
 export const CHANNEL_REPLY_NOTE_KWARG = 'channel_reply_note';
 
@@ -84,8 +85,8 @@ export function isDeliveryNoticeMessage(message: BaseMessage): boolean {
 }
 
 /**
- * The content as the person sent it: without the runtime's time note,
- * without the quoted-reply note and without a delivery notice. Every reader
+ * The content as the person sent it: without the runtime's time and guide
+ * notes, without the quoted-reply note and without a delivery notice. Every reader
  * that treats a human message as the person's own words (the transcript,
  * memory indexing, the evidence a proposal is checked against) goes through
  * here, so quoted text or a gateway's notice is never taken for something
@@ -97,7 +98,10 @@ export function userContent(
 ): MessageContent {
   return stripTurnTimeNote(
     stripTurnTimeNote(
-      stripTurnTimeNote(content, kwargs?.[TURN_TIME_NOTE_KWARG]),
+      stripTurnTimeNote(
+        stripTurnTimeNote(content, kwargs?.[TURN_TIME_NOTE_KWARG]),
+        kwargs?.[TURN_GUIDE_NOTE_KWARG],
+      ),
       kwargs?.[CHANNEL_REPLY_NOTE_KWARG],
     ),
     kwargs?.[CHANNEL_DELIVERY_NOTICE_KWARG],

package/src/core/capability-expiry.ts

@@ -0,0 +1,62 @@
+/**
+ * How long an on-demand capability stays loaded on a thread.
+ *
+ * A loaded plugin's tools and operating guide go out with every model call,
+ * so a plugin the conversation has moved on from is paid for on every call.
+ * Taking it away changes the tool list and the system prompt, which costs a
+ * full cache miss on a warm thread: the request's whole prefix is read again
+ * at the uncached price, far more than the plugin's tools cost at the cached
+ * one. On a thread that has gone quiet long enough for the provider's cache
+ * to expire the miss has already happened, so the runtime only ever unloads
+ * a plugin on such a turn, and only one the last `maxIdleTurns` turns did not
+ * touch. The model can load it again: the capability gate's refusal says how.
+ *
+ * Use is read off the thread itself: a call to one of the plugin's tools, or
+ * a `load_capability` naming it. Nothing is stored, so a summary that
+ * condensed a plugin's last use makes it idle, which is what it is.
+ */
+import { AIMessage, type BaseMessage } from '@langchain/core/messages';
+import { z } from 'zod';
+import { LOAD_CAPABILITY_TOOL_NAME } from './meta-tools';
+import { turnOpenings } from './middlewares/turn-boundary';
+
+const loadArgs = z.object({ names: z.array(z.string()) });
+
+/** The plugins one model response touched. */
+function pluginsTouched(
+  message: AIMessage,
+  pluginOf: (toolName: string) => string | undefined,
+): string[] {
+  return (message.tool_calls ?? []).flatMap((call) => {
+    if (call.name === LOAD_CAPABILITY_TOOL_NAME) {
+      const args = loadArgs.safeParse(call.args);
+      return args.success ? args.data.names : [];
+    }
+    const plugin = pluginOf(call.name);
+    return plugin ? [plugin] : [];
+  });
+}
+
+/**
+ * Which of the `loaded` plugins no turn among the last `maxIdleTurns` of the
+ * thread touched. `messages` are the finished turns, the current one not yet
+ * among them; a thread with fewer turns than that unloads nothing.
+ */
+export function idlePlugins(input: {
+  messages: readonly BaseMessage[];
+  loaded: readonly string[];
+  pluginOf: (toolName: string) => string | undefined;
+  maxIdleTurns: number;
+}): string[] {
+  const { messages, loaded, pluginOf, maxIdleTurns } = input;
+  if (maxIdleTurns <= 0) return [];
+  const openings = turnOpenings(messages);
+  const recent = openings[openings.length - maxIdleTurns];
+  if (recent === undefined) return [];
+  const touched = new Set<string>();
+  for (const message of messages.slice(recent))
+    if (AIMessage.isInstance(message))
+      for (const plugin of pluginsTouched(message, pluginOf))
+        touched.add(plugin);
+  return loaded.filter((plugin) => !touched.has(plugin));
+}