@ixo/oracle-runtime-workers 0.24.0 → 0.26.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ixo/oracle-runtime-workers",
-  "version": "0.24.0",
+  "version": "0.26.0",
   "description": "QiForge oracle runtime for Cloudflare Workers — Durable Object per user (SQLite in WASM over DO storage), Matrix E2EE gateway Durable Object, Hono HTTP/SSE shell.",
   "private": false,
   "publishConfig": {
@@ -33,7 +33,7 @@
     "!src/plugins/pod-creator/test-fixtures.ts"
   ],
   "dependencies": {
-    "@ixo/action-protocol": "0.4.1",
+    "@ixo/action-protocol": "0.5.0",
     "@blocknote/server-util": "0.29.1",
     "@composio/core": "0.10.0",
     "@composio/langchain": "0.9.2",
@@ -58,6 +58,7 @@
     "matrix-js-sdk": "^42.2.0",
     "multiformats": "14.0.4",
     "superjson": "1.13.3",
+    "unpdf": "1.8.1",
     "wa-sqlite": "1.0.0",
     "yjs": "13.6.32",
     "zod": "^4.4.3",
@@ -71,6 +72,7 @@
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "^0.22.0",
     "@cloudflare/workers-types": "^5.20260825.1",
+    "@ixo/action-protocol-041": "npm:@ixo/action-protocol@0.4.1",
     "@modelcontextprotocol/sdk": "1.30.0",
     "happy-dom": "^19.0.0",
     "typescript": "^5.9.3",

package/src/action-proposals/store.ts

@@ -76,12 +76,6 @@ export const ACTION_PROPOSAL_UNRESOLVED_RETENTION_MS = 180 * 24 * 60 * 60_000;
  * still possible, so the link stays used up.
  */
 export const ACTION_PROPOSAL_EXECUTION_WINDOW_MS = 60 * 60_000;
-/** Cards one agent run may prepare; more is a looping run, refused. */
-export const ACTION_PROPOSALS_PER_RUN = 8;
-/** Cards still awaiting the owner (live `prepared`, `claimed`, hash-less `unknown`) per owner. */
-export const ACTION_PROPOSALS_OPEN_PER_OWNER = 16;
-/** A `prepared` card counts as open this long after creation (its TTL plus clock skew). */
-export const OPEN_PREPARED_MS = 20 * 60_000;
 
 /**
  * A card's claim time from its lifecycle JSON, NULL when the JSON is not
@@ -256,11 +250,6 @@ export class ActionProposalStore {
       }
       if (Date.parse(subject.expiresAt) <= this.now())
         throw new ActionProposalConflict('Proposal expired');
-      await this.assertCapacity(
-        subject.ownerDid,
-        subject.origin.sessionId,
-        subject.origin.runId,
-      );
       await this.db.run(
         `INSERT INTO action_proposals (id,owner_did,invocation_id,session_id,run_id,subject,digest,lifecycle,status,created_at) VALUES (?,?,?,?,?,?,?,?,?,?)`,
         [
@@ -293,35 +282,6 @@ export class ActionProposalStore {
     });
   }
 
-  /**
-   * A looping or injected run must not mint cards without bound: at most
-   * `ACTION_PROPOSALS_PER_RUN` per run and `ACTION_PROPOSALS_OPEN_PER_OWNER`
-   * still awaiting the owner. A resumed tool call never reaches here (it
-   * returns its original card above).
-   */
-  private async assertCapacity(
-    ownerDid: string,
-    sessionId: string,
-    runId: string,
-  ): Promise<void> {
-    const run = await this.db.get<{ n: number }>(
-      `SELECT COUNT(*) AS n FROM action_proposals WHERE owner_did=? AND session_id=? AND run_id=?`,
-      [ownerDid, sessionId, runId],
-    );
-    if ((run?.n ?? 0) >= ACTION_PROPOSALS_PER_RUN)
-      throw new ActionProposalConflict(
-        'Too many action cards in this request; finish the open ones first',
-      );
-    const open = await this.db.get<{ n: number }>(
-      `SELECT COUNT(*) AS n FROM action_proposals WHERE owner_did=? AND ((status='prepared' AND created_at>?) OR status='claimed' OR (status='unknown' AND transaction_hash IS NULL))`,
-      [ownerDid, this.now() - OPEN_PREPARED_MS],
-    );
-    if ((open?.n ?? 0) >= ACTION_PROPOSALS_OPEN_PER_OWNER)
-      throw new ActionProposalConflict(
-        'Too many action cards are open; finish or decline them first',
-      );
-  }
-
   async get(ownerDid: string, id: string): Promise<ActionProposal | null> {
     const row = await this.db.get<{
       subject: string;

package/src/action-proposals/topic-store.ts

@@ -24,9 +24,6 @@ import {
   ActionProposalConflict,
   ACTION_PROPOSAL_RETENTION_MS,
   ACTION_PROPOSAL_UNRESOLVED_RETENTION_MS,
-  ACTION_PROPOSALS_OPEN_PER_OWNER,
-  ACTION_PROPOSALS_PER_RUN,
-  OPEN_PREPARED_MS,
   RECEIPT_MAX_ATTEMPTS,
   RECEIPT_RETRY_BASE_MS,
   RECEIPT_RETRY_MAX_MS,
@@ -163,11 +160,6 @@ export class TopicProposalStore {
       }
       if (Date.parse(subject.expiresAt) <= this.now())
         throw new ActionProposalConflict('Proposal expired');
-      await this.assertCapacity(
-        subject.ownerDid,
-        subject.origin.sessionId,
-        subject.origin.runId,
-      );
       const steps: TopicStepId[] =
         subject.action.name === 'create_topic'
           ? ['anchor', 'body', 'operation', 'snapshot']
@@ -201,40 +193,6 @@ export class TopicProposalStore {
       return TopicProposalV2Schema.parse({ ...subject, digest, lifecycle });
     });
   }
-  /**
-   * At most `ACTION_PROPOSALS_PER_RUN` cards per run and
-   * `ACTION_PROPOSALS_OPEN_PER_OWNER` open ones (prepared in the last 20
-   * minutes, or claimed / unknown in the last two hours: the claim window
-   * plus the execution window). The owner object runs this before the
-   * authority read a refused card would waste; `create` runs it again.
-   */
-  async assertCapacity(
-    owner: string,
-    sessionId: string,
-    runId: string,
-  ): Promise<void> {
-    const now = this.now();
-    const run = await this.db.get<{ n: number }>(
-      `SELECT COUNT(*) AS n FROM topic_action_proposals WHERE owner_did=? AND session_id=? AND run_id=?`,
-      [owner, sessionId, runId],
-    );
-    if ((run?.n ?? 0) >= ACTION_PROPOSALS_PER_RUN)
-      throw new ActionProposalConflict(
-        'Too many action cards in this request; finish the open ones first',
-      );
-    const open = await this.db.get<{ n: number }>(
-      `SELECT COUNT(*) AS n FROM topic_action_proposals WHERE owner_did=? AND ((status='prepared' AND created_at>?) OR (status IN ('claimed','unknown') AND created_at>?))`,
-      [
-        owner,
-        now - OPEN_PREPARED_MS,
-        now - OPEN_PREPARED_MS - TOPIC_EXECUTION_WINDOW_MS - 40 * 60_000,
-      ],
-    );
-    if ((open?.n ?? 0) >= ACTION_PROPOSALS_OPEN_PER_OWNER)
-      throw new ActionProposalConflict(
-        'Too many action cards are open; finish or decline them first',
-      );
-  }
   async get(owner: string, id: string): Promise<TopicActionProposal | null> {
     const row = await this.db.get<{
       subject: string;

package/src/attachments/extract.ts

@@ -9,7 +9,8 @@
  * Reduced vs Node: there is no local PDF/office parser on workerd
  * (`loadFileFromBuffer` is Node-only), so every non-plain-text document goes
  * straight to the helper model's `file` lane instead of "local parse, then
- * model on failure".
+ * model on failure". (Document ingestion parses PDFs locally first:
+ * `pdf-text.ts`.)
  */
 import { bytesToBase64 } from '../plugins/base64';
 import { validateUrlTarget } from './download';
@@ -36,15 +37,36 @@ export interface ExtractionUsage {
 export interface ExtractionResult {
   content: string;
   usage?: ExtractionUsage;
+  /** True when the provider stopped the reply at its output-token limit. */
+  truncated?: boolean;
 }
 
 export interface ExtractOptions {
   fetchImpl?: typeof fetch;
   signal?: AbortSignal;
   timeoutMs?: number;
+  /**
+   * Raw mode (document ingestion): `extractText` returns the text itself —
+   * no `[Content of …]` header, no `[...truncated]` marker — cut at
+   * `maxChars` with `truncated` reporting whether anything was cut. Unset
+   * keeps the chat-attachment behaviour.
+   */
+  raw?: { maxChars: number };
+  /**
+   * `max_tokens` for the completion, sent when the provider accepts it (a
+   * 400 that names it is retried once without). Set by raw mode.
+   */
+  maxTokens?: number;
 }
 
 export const MAX_TEXT_LENGTH = 50_000;
+/**
+ * Largest scanned or binary file handed to the helper model (inline base64
+ * in the request body); also the Files plugin's `MAX_VISION_BYTES`.
+ */
+export const MAX_MODEL_LANE_BYTES = 10 * 1024 * 1024;
+/** Upper bound on the `max_tokens` raw mode asks for. */
+export const MAX_RAW_COMPLETION_TOKENS = 32_768;
 const AI_PROCESS_TIMEOUT_MS = 120_000;
 const MAX_ERROR_BODY_LENGTH = 1024;
 
@@ -70,6 +92,18 @@ export function truncateText(text: string): string {
   return `${text.slice(0, MAX_TEXT_LENGTH)}\n\n[...truncated]`;
 }
 
+/** Cut `text` to at most `maxChars` UTF-16 units without splitting a surrogate pair. */
+export function capText(
+  text: string,
+  maxChars: number,
+): { text: string; truncated: boolean } {
+  if (text.length <= maxChars) return { text, truncated: false };
+  let end = maxChars;
+  const last = text.charCodeAt(end - 1);
+  if (last >= 0xd800 && last <= 0xdbff) end -= 1;
+  return { text: text.slice(0, end), truncated: true };
+}
+
 export function getAudioFormat(
   mimetype: string,
 ): 'mp3' | 'wav' | 'ogg' | 'flac' | 'webm' | 'mp4' | 'aac' {
@@ -84,7 +118,10 @@ export function getAudioFormat(
 }
 
 interface ChatCompletion {
-  choices?: Array<{ message?: { content?: string | null } }>;
+  choices?: Array<{
+    message?: { content?: unknown };
+    finish_reason?: string | null;
+  }>;
   usage?: {
     prompt_tokens?: number;
     completion_tokens?: number;
@@ -94,6 +131,27 @@ interface ChatCompletion {
   model?: string;
 }
 
+/** The text of a reply: a string as is, a block array reduced to its text blocks. */
+export function replyText(content: unknown): string {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  const parts: string[] = [];
+  for (const block of content) {
+    if (typeof block === 'string') parts.push(block);
+    else if (
+      typeof block === 'object' &&
+      block !== null &&
+      'type' in block &&
+      block.type === 'text' &&
+      'text' in block &&
+      typeof block.text === 'string'
+    ) {
+      parts.push(block.text);
+    }
+  }
+  return parts.join('');
+}
+
 async function chatCompletion(
   provider: ExtractionProvider,
   contentParts: Array<Record<string, unknown>>,
@@ -106,9 +164,8 @@ async function chatCompletion(
   if (opts.signal?.aborted) controller.abort();
   const timer = setTimeout(onAbort, opts.timeoutMs ?? AI_PROCESS_TIMEOUT_MS);
   try {
-    const response = await fetchImpl(
-      `${provider.baseURL.replace(/\/+$/, '')}/chat/completions`,
-      {
+    const send = (withMaxTokens: boolean): Promise<Response> =>
+      fetchImpl(`${provider.baseURL.replace(/\/+$/, '')}/chat/completions`, {
         method: 'POST',
         headers: {
           Authorization: `Bearer ${provider.apiKey}`,
@@ -118,20 +175,34 @@ async function chatCompletion(
         body: JSON.stringify({
           model: provider.model,
           messages: [{ role: 'user', content: contentParts }],
+          ...(withMaxTokens && opts.maxTokens !== undefined
+            ? { max_tokens: opts.maxTokens }
+            : {}),
         }),
         signal: controller.signal,
-      },
-    );
+      });
+    let response = await send(true);
     if (!response.ok) {
       const errorText = await response.text().catch(() => '');
-      throw new Error(
-        `AI processing failed (${response.status}): ${errorText.slice(0, MAX_ERROR_BODY_LENGTH)}`,
-      );
+      // A provider that rejects `max_tokens` gets the request without it.
+      if (
+        opts.maxTokens !== undefined &&
+        response.status === 400 &&
+        /max_(completion_)?tokens/i.test(errorText)
+      ) {
+        response = await send(false);
+      } else {
+        throw new Error(
+          `AI processing failed (${response.status}): ${errorText.slice(0, MAX_ERROR_BODY_LENGTH)}`,
+        );
+      }
     }
     const raw: unknown = await response.json();
     const result = raw as ChatCompletion;
+    const choice = result.choices?.[0];
     return {
-      content: result.choices?.[0]?.message?.content ?? '',
+      content: replyText(choice?.message?.content),
+      ...(choice?.finish_reason === 'length' ? { truncated: true } : {}),
       ...(result.usage
         ? {
             usage: {
@@ -197,6 +268,14 @@ export async function aiProcess(
   return chatCompletion(provider, parts, opts);
 }
 
+/**
+ * The `max_tokens` that matches a character cap: about three characters per
+ * token, bounded by what providers commonly accept.
+ */
+export function rawMaxTokens(maxChars: number): number {
+  return Math.min(Math.ceil(maxChars / 3), MAX_RAW_COMPLETION_TOKENS);
+}
+
 /**
  * Text for one downloaded attachment by category — the Node
  * `processDocument` / `processImage` / `processAudio` / `processVideo`
@@ -208,9 +287,14 @@ export async function extractText(
   category: Exclude<FileCategory, 'unsupported'>,
   provider: ExtractionProvider | null,
   opts: ExtractOptions = {},
-): Promise<{ text: string; usage?: ExtractionUsage }> {
+): Promise<{ text: string; usage?: ExtractionUsage; truncated?: boolean }> {
   const safeName = sanitizeAttachmentFilename(attachment.filename);
   if (category === 'document' && isPlainTextType(attachment.mimetype)) {
+    if (opts.raw) {
+      // At most 3 UTF-8 bytes per UTF-16 unit: decoding past this adds nothing.
+      const head = bytes.subarray(0, opts.raw.maxChars * 3 + 4);
+      return capText(new TextDecoder().decode(head), opts.raw.maxChars);
+    }
     const text = new TextDecoder().decode(bytes);
     return { text: formatContent('Content', safeName, truncateText(text)) };
   }
@@ -225,14 +309,24 @@ export async function extractText(
       : category === 'audio'
         ? 'Transcription'
         : 'Description';
-  const { content, usage } = await aiProcess(
+  const { content, usage, truncated } = await aiProcess(
     bytes,
     attachment.mimetype,
     category,
     attachment.filename,
     provider,
-    opts,
+    opts.raw && opts.maxTokens === undefined
+      ? { ...opts, maxTokens: rawMaxTokens(opts.raw.maxChars) }
+      : opts,
   );
+  if (opts.raw) {
+    const capped = capText(content, opts.raw.maxChars);
+    return {
+      text: capped.text,
+      truncated: capped.truncated || truncated === true,
+      ...(usage ? { usage } : {}),
+    };
+  }
   const body = category === 'document' ? truncateText(content) : content;
   return {
     text: formatContent(label, safeName, body),

package/src/attachments/index.ts

@@ -20,6 +20,7 @@ export {
 export {
   aiProcess,
   aiProcessFromUrl,
+  capText,
   extractText,
   formatContent,
   getAudioFormat,
@@ -39,6 +40,7 @@ export {
   type FileCategory,
 } from './magic';
 export {
+  nativeAttachmentNote,
   prepareAttachments,
   SANDBOX_TRUNCATE_LIMIT,
   type AttachmentPipelineDeps,
@@ -46,6 +48,11 @@ export {
   type PrepareAttachmentsParams,
   type PreparedAttachments,
 } from './pipeline';
+export {
+  extractPdfText,
+  MIN_CHARS_PER_PAGE,
+  type PdfTextResult,
+} from './pdf-text';
 export { routeAttachment, type AttachmentStrategy } from './route';
 export {
   buildAnalysisMarkdown,
@@ -81,14 +88,22 @@ export {
   type AttachmentRetentionResult,
 } from './retention';
 export {
+  createAttachmentViewSurface,
+  extractAttachmentText,
+  findPersistedAttachmentMeta,
   viewAttachment,
+  type AttachmentMessageStore,
+  type AttachmentTextExtraction,
   type AttachmentView,
   type AttachmentViewSurface,
   type ViewAttachmentDeps,
 } from './view';
 export {
   AttachmentTextCacheStore,
+  IngestTextLru,
   LOCAL_TEXT_EXTRACTION,
   VIEW_CACHE_MAX_CHARS,
   type AttachmentTextCache,
+  type IngestedText,
+  type IngestTextScope,
 } from './view-cache';

package/src/attachments/pdf-text.ts

@@ -0,0 +1,99 @@
+/**
+ * Local PDF text extraction (pdf.js through `unpdf`'s serverless build, which
+ * runs on Workers with no DOM or canvas). Used by document ingestion, where
+ * the text must be the document's own words rather than a model's rendering.
+ * A PDF without a text layer (a scan) is reported as `scanned` so the caller
+ * can fall back to the helper model.
+ *
+ * Pages are read one at a time and reading stops as soon as the caller's
+ * character cap is reached, so a thousand-page file costs what the first
+ * pages cost, and the document is always destroyed (pdf.js otherwise keeps
+ * its worker-side state, fonts and page caches, alive).
+ */
+import { getDocumentProxy } from 'unpdf';
+
+import { capText } from './extract';
+
+/** Below this many non-whitespace characters per page on average, a PDF is treated as scanned. */
+export const MIN_CHARS_PER_PAGE = 50;
+
+/** Pages read at most, whatever the character cap. */
+export const DEFAULT_MAX_PDF_PAGES = 2000;
+
+export interface PdfTextOptions {
+  /** Stop reading once the text passes this many UTF-16 units. */
+  maxChars: number;
+  /** Stop after this many pages (default {@link DEFAULT_MAX_PDF_PAGES}). */
+  maxPages?: number;
+  /** Checked between pages; an aborted signal throws its reason. */
+  signal?: AbortSignal;
+}
+
+export type PdfTextResult =
+  | {
+      kind: 'text';
+      /** At most `maxChars` units, as `--- page N ---` blocks. */
+      text: string;
+      /** Pages in the document. */
+      pages: number;
+      /** Pages actually read. */
+      pagesRead: number;
+      /** True when pages were left unread or the text was cut at `maxChars`. */
+      truncated: boolean;
+    }
+  | { kind: 'scanned'; pages: number };
+
+function pageBlock(pageNumber: number, text: string): string {
+  return `--- page ${pageNumber} ---\n${text.trim()}`;
+}
+
+export async function extractPdfText(
+  bytes: Uint8Array,
+  opts: PdfTextOptions,
+): Promise<PdfTextResult> {
+  const maxPages = opts.maxPages ?? DEFAULT_MAX_PDF_PAGES;
+  opts.signal?.throwIfAborted();
+  // pdf.js takes ownership of (and detaches) the buffer it is handed.
+  const pdf = await getDocumentProxy(new Uint8Array(bytes));
+  try {
+    const totalPages = pdf.numPages;
+    const blocks: string[] = [];
+    let length = 0;
+    let visible = 0;
+    let pagesRead = 0;
+    while (pagesRead < totalPages && pagesRead < maxPages) {
+      opts.signal?.throwIfAborted();
+      const page = await pdf.getPage(pagesRead + 1);
+      let pageText: string;
+      try {
+        const content = await page.getTextContent();
+        pageText = content.items
+          .map((item) =>
+            'str' in item ? item.str + (item.hasEOL ? '\n' : '') : '',
+          )
+          .join('');
+      } finally {
+        page.cleanup();
+      }
+      pagesRead += 1;
+      visible += pageText.replace(/\s/g, '').length;
+      const block = pageBlock(pagesRead, pageText);
+      blocks.push(block);
+      length += block.length + 2;
+      if (length > opts.maxChars) break;
+    }
+    if (totalPages === 0 || visible < pagesRead * MIN_CHARS_PER_PAGE) {
+      return { kind: 'scanned', pages: totalPages };
+    }
+    const capped = capText(blocks.join('\n\n'), opts.maxChars);
+    return {
+      kind: 'text',
+      text: capped.text,
+      pages: totalPages,
+      pagesRead,
+      truncated: capped.truncated || pagesRead < totalPages,
+    };
+  } finally {
+    await pdf.loadingTask.destroy();
+  }
+}

package/src/attachments/pipeline.ts

@@ -58,7 +58,11 @@ import {
   type SandboxUploadConfig,
 } from './sandbox-archive';
 import {
+  NOTE_EVENT_ID_MAX,
+  NOTE_MIMETYPE_MAX,
+  NOTE_URI_MAX,
   sanitizeAttachmentFilename,
+  sanitizeNoteField,
   sanitizeSandboxPath,
   SANDBOX_OUTPUT_PREFIX,
 } from './sanitize';
@@ -91,6 +95,13 @@ export interface PreparedAttachments {
   metas: AttachmentMeta[];
   /** Extraction-lane results, to append as hidden context messages. */
   extracted: ExtractedAttachment[];
+  /**
+   * One line per file sent natively, to append as hidden context messages
+   * like `extracted`: the model reads the file itself but would otherwise
+   * never learn the reference that tools (`view_attachment`, document
+   * ingestion) take.
+   */
+  nativeNotes: ExtractedAttachment[];
   usage: { cost: number; promptTokens: number; completionTokens: number };
 }
 
@@ -115,12 +126,34 @@ function metaFor(attachment: AttachmentInput): AttachmentMeta {
   };
 }
 
+/**
+ * The line that names an attachment's reference to the model, for both lanes:
+ * `[source: eventId="…"]` (the media event id) or `[source: url="…"]` (the
+ * mxc / http URI), the same two values `attachmentRef` accepts.
+ */
 function sourceRef(attachment: AttachmentInput): string {
-  if (attachment.eventId) return `[source: eventId="${attachment.eventId}"]`;
-  if (attachment.mxcUri) return `[source: url="${attachment.mxcUri}"]`;
+  // A reference the cleaner would alter is not the reference tools match on,
+  // so such a value is left out rather than shown changed.
+  const clean = (value: string, max: number): string => {
+    const cleaned = sanitizeNoteField(value, max);
+    return cleaned === value ? cleaned : '';
+  };
+  const eventId = attachment.eventId
+    ? clean(attachment.eventId, NOTE_EVENT_ID_MAX)
+    : '';
+  if (eventId) return `[source: eventId="${eventId}"]`;
+  const url = attachment.mxcUri ? clean(attachment.mxcUri, NOTE_URI_MAX) : '';
+  if (url) return `[source: url="${url}"]`;
   return '';
 }
 
+/** The note for a file sent natively: its source line, name and type. */
+export function nativeAttachmentNote(attachment: AttachmentInput): string {
+  const ref = sourceRef(attachment);
+  if (!ref) return '';
+  return `${ref} "${sanitizeAttachmentFilename(attachment.filename)}" (${sanitizeNoteField(attachment.mimetype, NOTE_MIMETYPE_MAX)}) is attached to this message.`;
+}
+
 function errorMessage(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }
@@ -320,7 +353,13 @@ export async function prepareAttachments(
   const metas = params.attachments.map(metaFor);
   const usage = { cost: 0, promptTokens: 0, completionTokens: 0 };
   if (params.attachments.length === 0) {
-    return { content: params.text, metas, extracted: [], usage };
+    return {
+      content: params.text,
+      metas,
+      extracted: [],
+      nativeNotes: [],
+      usage,
+    };
   }
 
   const reportedTotal = params.attachments.reduce(
@@ -354,6 +393,7 @@ export async function prepareAttachments(
 
   const budget = new DownloadBudget();
   const natives: NativeAttachment[] = [];
+  const nativeNotes: ExtractedAttachment[] = [];
   for (const attachment of nativeCandidates) {
     try {
       const { bytes, mimetype, sniffed } = await budget.load(
@@ -382,6 +422,8 @@ export async function prepareAttachments(
         base64: bytesToBase64(bytes),
         filename: attachment.filename,
       });
+      const note = nativeAttachmentNote(attachment);
+      if (note) nativeNotes.push({ text: note, meta: metaFor(attachment) });
       deps.logger.log(
         `[attachments] NATIVE → "${attachment.filename}" sent directly to ${params.model} (${bytes.length} bytes, ${kind})`,
       );
@@ -449,7 +491,7 @@ export async function prepareAttachments(
         deps.logger.error(
           `[attachments] failed to process ${attachment.filename}: ${errorMessage(error)}`,
         );
-        text = `[File "${sanitizeAttachmentFilename(attachment.filename)}" (${attachment.mimetype}) failed to process: ${errorMessage(error)}]`;
+        text = `[File "${sanitizeAttachmentFilename(attachment.filename)}" (${sanitizeNoteField(attachment.mimetype, NOTE_MIMETYPE_MAX)}) failed to process: ${errorMessage(error)}]`;
       }
       const ref = sourceRef(attachment);
       extracted.push({ text: ref ? `${ref}\n${text}` : text, meta });
@@ -460,6 +502,7 @@ export async function prepareAttachments(
     content: buildUserMessageContent(params.text, natives),
     metas,
     extracted,
+    nativeNotes,
     usage,
   };
 }

package/src/attachments/sanitize.ts

@@ -21,6 +21,20 @@ export function sanitizeAttachmentFilename(filename: string): string {
   );
 }
 
+/** Longest mimetype / event id / mxc URI written into a hidden note. */
+export const NOTE_MIMETYPE_MAX = 100;
+export const NOTE_EVENT_ID_MAX = 255;
+export const NOTE_URI_MAX = 512;
+
+/**
+ * Clean a client-supplied value (mimetype, media event id, mxc URI) for the
+ * hidden notes the model reads: the filename cleaner plus quotes removed
+ * (the notes wrap refs in `"…"`), cut to `max` characters.
+ */
+export function sanitizeNoteField(value: string, max: number): string {
+  return sanitizeAttachmentFilename(value).replace(/"/g, '').slice(0, max);
+}
+
 /**
  * Sanitize a filename/path for the sandbox upload endpoint: only
  * alphanumerics, dots, dashes, underscores and slashes, and no `..` segments