@ixo/oracle-runtime-workers 0.25.0 → 0.27.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ixo/oracle-runtime-workers",
-  "version": "0.25.0",
+  "version": "0.27.0",
   "description": "QiForge oracle runtime for Cloudflare Workers — Durable Object per user (SQLite in WASM over DO storage), Matrix E2EE gateway Durable Object, Hono HTTP/SSE shell.",
   "private": false,
   "publishConfig": {
@@ -58,14 +58,15 @@
     "matrix-js-sdk": "^42.2.0",
     "multiformats": "14.0.4",
     "superjson": "1.13.3",
+    "unpdf": "1.8.1",
     "wa-sqlite": "1.0.0",
     "yjs": "13.6.32",
     "zod": "^4.4.3",
     "@ixo/impactxclient-sdk": "^3.0.0",
     "protobufjs": "^7.6.5",
     "@ixo/topic-protocol": "1.0.0-rc.8",
-    "@ixo/common": "1.7.1",
     "@ixo/ucan": "2.3.0",
+    "@ixo/common": "1.7.1",
     "@ixo/ixo-transaction": "0.2.0"
   },
   "devDependencies": {

package/src/attachments/extract.ts

@@ -9,7 +9,8 @@
  * Reduced vs Node: there is no local PDF/office parser on workerd
  * (`loadFileFromBuffer` is Node-only), so every non-plain-text document goes
  * straight to the helper model's `file` lane instead of "local parse, then
- * model on failure".
+ * model on failure". (Document ingestion parses PDFs locally first:
+ * `pdf-text.ts`.)
  */
 import { bytesToBase64 } from '../plugins/base64';
 import { validateUrlTarget } from './download';
@@ -36,15 +37,36 @@ export interface ExtractionUsage {
 export interface ExtractionResult {
   content: string;
   usage?: ExtractionUsage;
+  /** True when the provider stopped the reply at its output-token limit. */
+  truncated?: boolean;
 }
 
 export interface ExtractOptions {
   fetchImpl?: typeof fetch;
   signal?: AbortSignal;
   timeoutMs?: number;
+  /**
+   * Raw mode (document ingestion): `extractText` returns the text itself —
+   * no `[Content of …]` header, no `[...truncated]` marker — cut at
+   * `maxChars` with `truncated` reporting whether anything was cut. Unset
+   * keeps the chat-attachment behaviour.
+   */
+  raw?: { maxChars: number };
+  /**
+   * `max_tokens` for the completion, sent when the provider accepts it (a
+   * 400 that names it is retried once without). Set by raw mode.
+   */
+  maxTokens?: number;
 }
 
 export const MAX_TEXT_LENGTH = 50_000;
+/**
+ * Largest scanned or binary file handed to the helper model (inline base64
+ * in the request body); also the Files plugin's `MAX_VISION_BYTES`.
+ */
+export const MAX_MODEL_LANE_BYTES = 10 * 1024 * 1024;
+/** Upper bound on the `max_tokens` raw mode asks for. */
+export const MAX_RAW_COMPLETION_TOKENS = 32_768;
 const AI_PROCESS_TIMEOUT_MS = 120_000;
 const MAX_ERROR_BODY_LENGTH = 1024;
 
@@ -70,6 +92,18 @@ export function truncateText(text: string): string {
   return `${text.slice(0, MAX_TEXT_LENGTH)}\n\n[...truncated]`;
 }
 
+/** Cut `text` to at most `maxChars` UTF-16 units without splitting a surrogate pair. */
+export function capText(
+  text: string,
+  maxChars: number,
+): { text: string; truncated: boolean } {
+  if (text.length <= maxChars) return { text, truncated: false };
+  let end = maxChars;
+  const last = text.charCodeAt(end - 1);
+  if (last >= 0xd800 && last <= 0xdbff) end -= 1;
+  return { text: text.slice(0, end), truncated: true };
+}
+
 export function getAudioFormat(
   mimetype: string,
 ): 'mp3' | 'wav' | 'ogg' | 'flac' | 'webm' | 'mp4' | 'aac' {
@@ -84,7 +118,10 @@ export function getAudioFormat(
 }
 
 interface ChatCompletion {
-  choices?: Array<{ message?: { content?: string | null } }>;
+  choices?: Array<{
+    message?: { content?: unknown };
+    finish_reason?: string | null;
+  }>;
   usage?: {
     prompt_tokens?: number;
     completion_tokens?: number;
@@ -94,6 +131,27 @@ interface ChatCompletion {
   model?: string;
 }
 
+/** The text of a reply: a string as is, a block array reduced to its text blocks. */
+export function replyText(content: unknown): string {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  const parts: string[] = [];
+  for (const block of content) {
+    if (typeof block === 'string') parts.push(block);
+    else if (
+      typeof block === 'object' &&
+      block !== null &&
+      'type' in block &&
+      block.type === 'text' &&
+      'text' in block &&
+      typeof block.text === 'string'
+    ) {
+      parts.push(block.text);
+    }
+  }
+  return parts.join('');
+}
+
 async function chatCompletion(
   provider: ExtractionProvider,
   contentParts: Array<Record<string, unknown>>,
@@ -106,9 +164,8 @@ async function chatCompletion(
   if (opts.signal?.aborted) controller.abort();
   const timer = setTimeout(onAbort, opts.timeoutMs ?? AI_PROCESS_TIMEOUT_MS);
   try {
-    const response = await fetchImpl(
-      `${provider.baseURL.replace(/\/+$/, '')}/chat/completions`,
-      {
+    const send = (withMaxTokens: boolean): Promise<Response> =>
+      fetchImpl(`${provider.baseURL.replace(/\/+$/, '')}/chat/completions`, {
         method: 'POST',
         headers: {
           Authorization: `Bearer ${provider.apiKey}`,
@@ -118,20 +175,34 @@ async function chatCompletion(
         body: JSON.stringify({
           model: provider.model,
           messages: [{ role: 'user', content: contentParts }],
+          ...(withMaxTokens && opts.maxTokens !== undefined
+            ? { max_tokens: opts.maxTokens }
+            : {}),
         }),
         signal: controller.signal,
-      },
-    );
+      });
+    let response = await send(true);
     if (!response.ok) {
       const errorText = await response.text().catch(() => '');
-      throw new Error(
-        `AI processing failed (${response.status}): ${errorText.slice(0, MAX_ERROR_BODY_LENGTH)}`,
-      );
+      // A provider that rejects `max_tokens` gets the request without it.
+      if (
+        opts.maxTokens !== undefined &&
+        response.status === 400 &&
+        /max_(completion_)?tokens/i.test(errorText)
+      ) {
+        response = await send(false);
+      } else {
+        throw new Error(
+          `AI processing failed (${response.status}): ${errorText.slice(0, MAX_ERROR_BODY_LENGTH)}`,
+        );
+      }
     }
     const raw: unknown = await response.json();
     const result = raw as ChatCompletion;
+    const choice = result.choices?.[0];
     return {
-      content: result.choices?.[0]?.message?.content ?? '',
+      content: replyText(choice?.message?.content),
+      ...(choice?.finish_reason === 'length' ? { truncated: true } : {}),
       ...(result.usage
         ? {
             usage: {
@@ -197,6 +268,14 @@ export async function aiProcess(
   return chatCompletion(provider, parts, opts);
 }
 
+/**
+ * The `max_tokens` that matches a character cap: about three characters per
+ * token, bounded by what providers commonly accept.
+ */
+export function rawMaxTokens(maxChars: number): number {
+  return Math.min(Math.ceil(maxChars / 3), MAX_RAW_COMPLETION_TOKENS);
+}
+
 /**
  * Text for one downloaded attachment by category — the Node
  * `processDocument` / `processImage` / `processAudio` / `processVideo`
@@ -208,9 +287,14 @@ export async function extractText(
   category: Exclude<FileCategory, 'unsupported'>,
   provider: ExtractionProvider | null,
   opts: ExtractOptions = {},
-): Promise<{ text: string; usage?: ExtractionUsage }> {
+): Promise<{ text: string; usage?: ExtractionUsage; truncated?: boolean }> {
   const safeName = sanitizeAttachmentFilename(attachment.filename);
   if (category === 'document' && isPlainTextType(attachment.mimetype)) {
+    if (opts.raw) {
+      // At most 3 UTF-8 bytes per UTF-16 unit: decoding past this adds nothing.
+      const head = bytes.subarray(0, opts.raw.maxChars * 3 + 4);
+      return capText(new TextDecoder().decode(head), opts.raw.maxChars);
+    }
     const text = new TextDecoder().decode(bytes);
     return { text: formatContent('Content', safeName, truncateText(text)) };
   }
@@ -225,14 +309,24 @@ export async function extractText(
       : category === 'audio'
         ? 'Transcription'
         : 'Description';
-  const { content, usage } = await aiProcess(
+  const { content, usage, truncated } = await aiProcess(
     bytes,
     attachment.mimetype,
     category,
     attachment.filename,
     provider,
-    opts,
+    opts.raw && opts.maxTokens === undefined
+      ? { ...opts, maxTokens: rawMaxTokens(opts.raw.maxChars) }
+      : opts,
   );
+  if (opts.raw) {
+    const capped = capText(content, opts.raw.maxChars);
+    return {
+      text: capped.text,
+      truncated: capped.truncated || truncated === true,
+      ...(usage ? { usage } : {}),
+    };
+  }
   const body = category === 'document' ? truncateText(content) : content;
   return {
     text: formatContent(label, safeName, body),

package/src/attachments/index.ts

@@ -20,6 +20,7 @@ export {
 export {
   aiProcess,
   aiProcessFromUrl,
+  capText,
   extractText,
   formatContent,
   getAudioFormat,
@@ -39,6 +40,7 @@ export {
   type FileCategory,
 } from './magic';
 export {
+  nativeAttachmentNote,
   prepareAttachments,
   SANDBOX_TRUNCATE_LIMIT,
   type AttachmentPipelineDeps,
@@ -46,6 +48,11 @@ export {
   type PrepareAttachmentsParams,
   type PreparedAttachments,
 } from './pipeline';
+export {
+  extractPdfText,
+  MIN_CHARS_PER_PAGE,
+  type PdfTextResult,
+} from './pdf-text';
 export { routeAttachment, type AttachmentStrategy } from './route';
 export {
   buildAnalysisMarkdown,
@@ -81,14 +88,22 @@ export {
   type AttachmentRetentionResult,
 } from './retention';
 export {
+  createAttachmentViewSurface,
+  extractAttachmentText,
+  findPersistedAttachmentMeta,
   viewAttachment,
+  type AttachmentMessageStore,
+  type AttachmentTextExtraction,
   type AttachmentView,
   type AttachmentViewSurface,
   type ViewAttachmentDeps,
 } from './view';
 export {
   AttachmentTextCacheStore,
+  IngestTextLru,
   LOCAL_TEXT_EXTRACTION,
   VIEW_CACHE_MAX_CHARS,
   type AttachmentTextCache,
+  type IngestedText,
+  type IngestTextScope,
 } from './view-cache';

package/src/attachments/pdf-text.ts

@@ -0,0 +1,99 @@
+/**
+ * Local PDF text extraction (pdf.js through `unpdf`'s serverless build, which
+ * runs on Workers with no DOM or canvas). Used by document ingestion, where
+ * the text must be the document's own words rather than a model's rendering.
+ * A PDF without a text layer (a scan) is reported as `scanned` so the caller
+ * can fall back to the helper model.
+ *
+ * Pages are read one at a time and reading stops as soon as the caller's
+ * character cap is reached, so a thousand-page file costs what the first
+ * pages cost, and the document is always destroyed (pdf.js otherwise keeps
+ * its worker-side state, fonts and page caches, alive).
+ */
+import { getDocumentProxy } from 'unpdf';
+
+import { capText } from './extract';
+
+/** Below this many non-whitespace characters per page on average, a PDF is treated as scanned. */
+export const MIN_CHARS_PER_PAGE = 50;
+
+/** Pages read at most, whatever the character cap. */
+export const DEFAULT_MAX_PDF_PAGES = 2000;
+
+export interface PdfTextOptions {
+  /** Stop reading once the text passes this many UTF-16 units. */
+  maxChars: number;
+  /** Stop after this many pages (default {@link DEFAULT_MAX_PDF_PAGES}). */
+  maxPages?: number;
+  /** Checked between pages; an aborted signal throws its reason. */
+  signal?: AbortSignal;
+}
+
+export type PdfTextResult =
+  | {
+      kind: 'text';
+      /** At most `maxChars` units, as `--- page N ---` blocks. */
+      text: string;
+      /** Pages in the document. */
+      pages: number;
+      /** Pages actually read. */
+      pagesRead: number;
+      /** True when pages were left unread or the text was cut at `maxChars`. */
+      truncated: boolean;
+    }
+  | { kind: 'scanned'; pages: number };
+
+function pageBlock(pageNumber: number, text: string): string {
+  return `--- page ${pageNumber} ---\n${text.trim()}`;
+}
+
+export async function extractPdfText(
+  bytes: Uint8Array,
+  opts: PdfTextOptions,
+): Promise<PdfTextResult> {
+  const maxPages = opts.maxPages ?? DEFAULT_MAX_PDF_PAGES;
+  opts.signal?.throwIfAborted();
+  // pdf.js takes ownership of (and detaches) the buffer it is handed.
+  const pdf = await getDocumentProxy(new Uint8Array(bytes));
+  try {
+    const totalPages = pdf.numPages;
+    const blocks: string[] = [];
+    let length = 0;
+    let visible = 0;
+    let pagesRead = 0;
+    while (pagesRead < totalPages && pagesRead < maxPages) {
+      opts.signal?.throwIfAborted();
+      const page = await pdf.getPage(pagesRead + 1);
+      let pageText: string;
+      try {
+        const content = await page.getTextContent();
+        pageText = content.items
+          .map((item) =>
+            'str' in item ? item.str + (item.hasEOL ? '\n' : '') : '',
+          )
+          .join('');
+      } finally {
+        page.cleanup();
+      }
+      pagesRead += 1;
+      visible += pageText.replace(/\s/g, '').length;
+      const block = pageBlock(pagesRead, pageText);
+      blocks.push(block);
+      length += block.length + 2;
+      if (length > opts.maxChars) break;
+    }
+    if (totalPages === 0 || visible < pagesRead * MIN_CHARS_PER_PAGE) {
+      return { kind: 'scanned', pages: totalPages };
+    }
+    const capped = capText(blocks.join('\n\n'), opts.maxChars);
+    return {
+      kind: 'text',
+      text: capped.text,
+      pages: totalPages,
+      pagesRead,
+      truncated: capped.truncated || pagesRead < totalPages,
+    };
+  } finally {
+    await pdf.loadingTask.destroy();
+  }
+}

package/src/attachments/pipeline.ts

@@ -58,7 +58,11 @@ import {
   type SandboxUploadConfig,
 } from './sandbox-archive';
 import {
+  NOTE_EVENT_ID_MAX,
+  NOTE_MIMETYPE_MAX,
+  NOTE_URI_MAX,
   sanitizeAttachmentFilename,
+  sanitizeNoteField,
   sanitizeSandboxPath,
   SANDBOX_OUTPUT_PREFIX,
 } from './sanitize';
@@ -91,6 +95,13 @@ export interface PreparedAttachments {
   metas: AttachmentMeta[];
   /** Extraction-lane results, to append as hidden context messages. */
   extracted: ExtractedAttachment[];
+  /**
+   * One line per file sent natively, to append as hidden context messages
+   * like `extracted`: the model reads the file itself but would otherwise
+   * never learn the reference that tools (`view_attachment`, document
+   * ingestion) take.
+   */
+  nativeNotes: ExtractedAttachment[];
   usage: { cost: number; promptTokens: number; completionTokens: number };
 }
 
@@ -115,12 +126,34 @@ function metaFor(attachment: AttachmentInput): AttachmentMeta {
   };
 }
 
+/**
+ * The line that names an attachment's reference to the model, for both lanes:
+ * `[source: eventId="…"]` (the media event id) or `[source: url="…"]` (the
+ * mxc / http URI), the same two values `attachmentRef` accepts.
+ */
 function sourceRef(attachment: AttachmentInput): string {
-  if (attachment.eventId) return `[source: eventId="${attachment.eventId}"]`;
-  if (attachment.mxcUri) return `[source: url="${attachment.mxcUri}"]`;
+  // A reference the cleaner would alter is not the reference tools match on,
+  // so such a value is left out rather than shown changed.
+  const clean = (value: string, max: number): string => {
+    const cleaned = sanitizeNoteField(value, max);
+    return cleaned === value ? cleaned : '';
+  };
+  const eventId = attachment.eventId
+    ? clean(attachment.eventId, NOTE_EVENT_ID_MAX)
+    : '';
+  if (eventId) return `[source: eventId="${eventId}"]`;
+  const url = attachment.mxcUri ? clean(attachment.mxcUri, NOTE_URI_MAX) : '';
+  if (url) return `[source: url="${url}"]`;
   return '';
 }
 
+/** The note for a file sent natively: its source line, name and type. */
+export function nativeAttachmentNote(attachment: AttachmentInput): string {
+  const ref = sourceRef(attachment);
+  if (!ref) return '';
+  return `${ref} "${sanitizeAttachmentFilename(attachment.filename)}" (${sanitizeNoteField(attachment.mimetype, NOTE_MIMETYPE_MAX)}) is attached to this message.`;
+}
+
 function errorMessage(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }
@@ -320,7 +353,13 @@ export async function prepareAttachments(
   const metas = params.attachments.map(metaFor);
   const usage = { cost: 0, promptTokens: 0, completionTokens: 0 };
   if (params.attachments.length === 0) {
-    return { content: params.text, metas, extracted: [], usage };
+    return {
+      content: params.text,
+      metas,
+      extracted: [],
+      nativeNotes: [],
+      usage,
+    };
   }
 
   const reportedTotal = params.attachments.reduce(
@@ -354,6 +393,7 @@ export async function prepareAttachments(
 
   const budget = new DownloadBudget();
   const natives: NativeAttachment[] = [];
+  const nativeNotes: ExtractedAttachment[] = [];
   for (const attachment of nativeCandidates) {
     try {
       const { bytes, mimetype, sniffed } = await budget.load(
@@ -382,6 +422,8 @@ export async function prepareAttachments(
         base64: bytesToBase64(bytes),
         filename: attachment.filename,
       });
+      const note = nativeAttachmentNote(attachment);
+      if (note) nativeNotes.push({ text: note, meta: metaFor(attachment) });
       deps.logger.log(
         `[attachments] NATIVE → "${attachment.filename}" sent directly to ${params.model} (${bytes.length} bytes, ${kind})`,
       );
@@ -449,7 +491,7 @@ export async function prepareAttachments(
         deps.logger.error(
           `[attachments] failed to process ${attachment.filename}: ${errorMessage(error)}`,
         );
-        text = `[File "${sanitizeAttachmentFilename(attachment.filename)}" (${attachment.mimetype}) failed to process: ${errorMessage(error)}]`;
+        text = `[File "${sanitizeAttachmentFilename(attachment.filename)}" (${sanitizeNoteField(attachment.mimetype, NOTE_MIMETYPE_MAX)}) failed to process: ${errorMessage(error)}]`;
       }
       const ref = sourceRef(attachment);
       extracted.push({ text: ref ? `${ref}\n${text}` : text, meta });
@@ -460,6 +502,7 @@ export async function prepareAttachments(
     content: buildUserMessageContent(params.text, natives),
     metas,
     extracted,
+    nativeNotes,
     usage,
   };
 }

package/src/attachments/sanitize.ts

@@ -21,6 +21,20 @@ export function sanitizeAttachmentFilename(filename: string): string {
   );
 }
 
+/** Longest mimetype / event id / mxc URI written into a hidden note. */
+export const NOTE_MIMETYPE_MAX = 100;
+export const NOTE_EVENT_ID_MAX = 255;
+export const NOTE_URI_MAX = 512;
+
+/**
+ * Clean a client-supplied value (mimetype, media event id, mxc URI) for the
+ * hidden notes the model reads: the filename cleaner plus quotes removed
+ * (the notes wrap refs in `"…"`), cut to `max` characters.
+ */
+export function sanitizeNoteField(value: string, max: number): string {
+  return sanitizeAttachmentFilename(value).replace(/"/g, '').slice(0, max);
+}
+
 /**
  * Sanitize a filename/path for the sandbox upload endpoint: only
  * alphanumerics, dots, dashes, underscores and slashes, and no `..` segments

package/src/attachments/view-cache.ts

@@ -140,3 +140,98 @@ export class AttachmentTextCacheStore {
     return changes;
   }
 }
+
+/** What {@link IngestTextLru} keeps per extraction. */
+export interface IngestedText {
+  text: string;
+  /** True when the extraction itself was cut (cap reached, reply at its limit). */
+  truncated: boolean;
+  /** Size of the original file. */
+  bytes: number;
+  mimetype: string;
+  /** Page count of a PDF's text layer. */
+  pages?: number;
+}
+
+/** One session's view of an {@link IngestTextLru}. */
+export interface IngestTextScope {
+  get(ref: string, key: string): IngestedText | undefined;
+  put(ref: string, key: string, value: IngestedText): void;
+}
+
+/** The longest text the ingestion LRU keeps per entry (characters). */
+export const INGEST_LRU_MAX_CHARS = 1_000_000;
+/** Entries the ingestion LRU keeps (per Durable Object). */
+export const INGEST_LRU_MAX_ENTRIES = 4;
+/** How long an ingestion LRU entry is served. */
+export const INGEST_LRU_TTL_MS = 10 * 60 * 1000;
+
+/**
+ * Text extracted for document ingestion, in memory only: a few large
+ * entries (up to a megabyte of text each, far over the 64 KB the SQLite
+ * cache keeps) so the second call for the same file (confirm, retry,
+ * `allow_truncation`) costs no download and no parse. Least recently used
+ * entries go first; entries expire after the TTL.
+ */
+export class IngestTextLru {
+  private readonly entries = new Map<
+    string,
+    { value: IngestedText; at: number }
+  >();
+
+  constructor(
+    private readonly opts: {
+      maxEntries?: number;
+      maxChars?: number;
+      ttlMs?: number;
+      now?: () => number;
+    } = {},
+  ) {}
+
+  private now(): number {
+    return (this.opts.now ?? Date.now)();
+  }
+
+  private get(key: string): IngestedText | undefined {
+    const entry = this.entries.get(key);
+    if (!entry) return undefined;
+    if (this.now() - entry.at >= (this.opts.ttlMs ?? INGEST_LRU_TTL_MS)) {
+      this.entries.delete(key);
+      return undefined;
+    }
+    // Re-insert: Map iteration order is the recency order.
+    this.entries.delete(key);
+    this.entries.set(key, entry);
+    return entry.value;
+  }
+
+  private put(key: string, value: IngestedText): void {
+    if (value.text.length > (this.opts.maxChars ?? INGEST_LRU_MAX_CHARS)) {
+      return;
+    }
+    this.entries.delete(key);
+    this.entries.set(key, { value, at: this.now() });
+    const max = this.opts.maxEntries ?? INGEST_LRU_MAX_ENTRIES;
+    while (this.entries.size > max) {
+      const oldest = this.entries.keys().next();
+      if (oldest.done) break;
+      this.entries.delete(oldest.value);
+    }
+  }
+
+  get size(): number {
+    return this.entries.size;
+  }
+
+  forSession(sessionId: string): IngestTextScope {
+    const keyOf = (ref: string, key: string) =>
+      JSON.stringify([sessionId, ref, key]);
+    return {
+      get: (ref, key) =>
+        isCacheable(ref) ? this.get(keyOf(ref, key)) : undefined,
+      put: (ref, key, value) => {
+        if (isCacheable(ref)) this.put(keyOf(ref, key), value);
+      },
+    };
+  }
+}