@ixo/oracle-runtime-workers 0.25.0 → 0.26.0

package/src/attachments/view.ts

@@ -10,15 +10,26 @@
 import { classifyAttachment } from './classify';
 import type { NativeAttachment } from './content-blocks';
 import { loadAttachmentBytes, type MatrixMediaSource } from './download';
-import { extractText, type ExtractionProvider } from './extract';
+import {
+  capText,
+  extractText,
+  MAX_MODEL_LANE_BYTES,
+  type ExtractionProvider,
+} from './extract';
 import { categorizeFile, isPlainTextType, verifyMagicBytes } from './magic';
-import { attachmentRef } from './retention';
+import { extractPdfText } from './pdf-text';
+import { attachmentMetas, attachmentRef } from './retention';
 import { routeAttachment } from './route';
 import { sanitizeAttachmentFilename } from './sanitize';
-import { LOCAL_TEXT_EXTRACTION, type AttachmentTextCache } from './view-cache';
+import {
+  LOCAL_TEXT_EXTRACTION,
+  type AttachmentTextCache,
+  type IngestTextScope,
+} from './view-cache';
 import type { AttachmentInput } from './types';
 import type { ModelInputCapabilities } from '../core/llm';
 import type { AttachmentMeta } from '../sqlite/serialization';
+import type { BaseMessage } from '@langchain/core/messages';
 import type { Logger } from '../plugin-api/types';
 import { bytesToBase64 } from '../plugins/base64';
 
@@ -40,6 +51,11 @@ export interface ViewAttachmentDeps {
    * a hit costs no download and no helper-model call.
    */
   cache?: AttachmentTextCache;
+  /**
+   * The session's in-memory cache of ingestion text (`IngestTextLru`): holds
+   * texts far over the SQLite cache's per-entry limit for a few minutes.
+   */
+  ingestCache?: IngestTextScope;
 }
 
 /** What the `view_attachment` tool sees on the host: session-scoped access. */
@@ -52,6 +68,32 @@ export interface AttachmentViewSurface {
    * not belong to an attachment of this session.
    */
   view(ref: string): Promise<{ meta: AttachmentMeta; view: AttachmentView }>;
+  /**
+   * The text of one of the session's attachments for ingestion into
+   * knowledge (never a native block): decoded locally for plain text, parsed
+   * locally for PDFs with a text layer, otherwise extracted by the helper
+   * model. Unwrapped and capped at `maxChars`. Resolves `null` when the
+   * reference does not belong to an attachment of this session; rejects when
+   * the file cannot be downloaded or read.
+   */
+  extractText(
+    ref: string,
+    opts: { maxChars: number },
+  ): Promise<AttachmentTextExtraction | null>;
+}
+
+/** What {@link AttachmentViewSurface.extractText} returns. */
+export interface AttachmentTextExtraction {
+  text: string;
+  filename: string;
+  mimetype: string;
+  /** Size of the original file when known, else of the extracted text. */
+  bytes: number;
+  /** True when `text` was cut at the caller's `maxChars`. */
+  truncated: boolean;
+  method: 'text' | 'pdf' | 'model';
+  /** Page count, set when the text came from a PDF's text layer. */
+  pages?: number;
 }
 
 function toInput(meta: AttachmentMeta): AttachmentInput {
@@ -146,3 +188,266 @@ export async function viewAttachment(
   if (cacheKey) await deps.cache?.put(cacheKey.ref, cacheKey.model, text);
   return { kind: 'text', text };
 }
+
+const PDF_MIME = 'application/pdf';
+
+/** Text-cache keys of ingestion: distinct from the `view_attachment` ones. */
+const INGEST_TEXT_KEY = 'ingest:local-text';
+const INGEST_PDF_KEY = 'ingest:local-pdf';
+
+function megabytes(bytes: number): string {
+  return (bytes / 1024 / 1024).toFixed(1);
+}
+
+/** The refusal for a scanned or binary file over the helper-model limit. */
+function modelLaneTooLarge(filename: string, bytes: number): Error {
+  return new Error(
+    `"${sanitizeAttachmentFilename(filename)}" cannot be read: scanned or binary files up to ${megabytes(MAX_MODEL_LANE_BYTES)} MB can be processed; this one is ${megabytes(bytes)} MB. PDFs with a text layer and text files can be larger.`,
+  );
+}
+
+/**
+ * The ingestion counterpart of {@link viewAttachment}: always text, raw (no
+ * `[Content of …]` header) and capped at `maxChars`. Results are kept in the
+ * in-memory ingestion cache (any size up to its limit, cut or not) and, when
+ * small and not cut, in the session's SQLite cache under `ingest:` keys; a
+ * cached text is re-capped to the caller's limit. The helper-model lane
+ * (scans, images, Office files) refuses files over `MAX_MODEL_LANE_BYTES`.
+ */
+export async function extractAttachmentText(
+  meta: AttachmentMeta,
+  deps: ViewAttachmentDeps,
+  opts: { maxChars: number },
+): Promise<AttachmentTextExtraction> {
+  const attachment = toInput(meta);
+  const category = categorizeFile(attachment.mimetype);
+  if (category === 'unsupported') {
+    throw new Error(
+      `"${sanitizeAttachmentFilename(attachment.filename)}" (${attachment.mimetype}) is not a supported file type`,
+    );
+  }
+  const plain = category === 'document' && isPlainTextType(attachment.mimetype);
+  const ref = attachmentRef(meta);
+  const modelKey = deps.extraction ? `ingest:${deps.extraction.model}` : '';
+  const result = (
+    text: string,
+    method: AttachmentTextExtraction['method'],
+    mimetype: string,
+    bytes: number,
+    truncated: boolean,
+    pages?: number,
+  ): AttachmentTextExtraction => ({
+    text,
+    filename: attachment.filename,
+    mimetype,
+    bytes,
+    truncated,
+    method,
+    ...(pages !== undefined ? { pages } : {}),
+  });
+
+  const lookups: Array<[string, AttachmentTextExtraction['method']]> = plain
+    ? [[INGEST_TEXT_KEY, 'text']]
+    : [
+        ...(attachment.mimetype === PDF_MIME
+          ? [[INGEST_PDF_KEY, 'pdf'] as [string, 'pdf']]
+          : []),
+        ...(modelKey ? [[modelKey, 'model'] as [string, 'model']] : []),
+      ];
+  if (ref) {
+    for (const [key, method] of lookups) {
+      const hit = deps.ingestCache?.get(ref, key);
+      if (!hit) continue;
+      deps.logger.log(
+        `[attachments] ingest "${attachment.filename}" → cached text (${key}, memory)`,
+      );
+      const capped = capText(hit.text, opts.maxChars);
+      return result(
+        capped.text,
+        method,
+        hit.mimetype,
+        hit.bytes,
+        capped.truncated || hit.truncated,
+        hit.pages,
+      );
+    }
+  }
+  const cache = deps.cache;
+  if (cache && ref) {
+    for (const [key, method] of lookups) {
+      const cached = await cache.get(ref, key);
+      if (cached === undefined) continue;
+      deps.logger.log(
+        `[attachments] ingest "${attachment.filename}" → cached text (${key})`,
+      );
+      const capped = capText(cached, opts.maxChars);
+      return result(
+        capped.text,
+        method,
+        attachment.mimetype,
+        meta.size ?? new TextEncoder().encode(cached).length,
+        capped.truncated,
+      );
+    }
+  }
+
+  // The helper model takes the bytes inline: refuse a big scan or binary
+  // before downloading it. (A PDF may still have a text layer, so it is
+  // checked once its bytes are in hand.)
+  if (
+    !plain &&
+    attachment.mimetype !== PDF_MIME &&
+    (meta.size ?? 0) > MAX_MODEL_LANE_BYTES
+  ) {
+    throw modelLaneTooLarge(attachment.filename, meta.size ?? 0);
+  }
+
+  const { bytes, mimetype } = await loadAttachmentBytes(
+    attachment,
+    deps.roomId,
+    deps.source,
+    { fetchImpl: deps.fetchImpl, signal: deps.signal },
+  );
+  verifyMagicBytes(bytes, category, attachment, (m) =>
+    deps.logger.warn(`[attachments] ${m}`),
+  );
+  deps.logger.log(
+    `[attachments] ingest "${attachment.filename}" (${mimetype}) (${bytes.length} bytes)`,
+  );
+  const store = async (
+    key: string,
+    value: { text: string; truncated: boolean; pages?: number },
+  ) => {
+    if (!ref || !key) return;
+    deps.ingestCache?.put(ref, key, {
+      ...value,
+      bytes: bytes.length,
+      mimetype,
+    });
+    if (cache && !value.truncated) await cache.put(ref, key, value.text);
+  };
+
+  if (plain) {
+    const { text, truncated } = await extractText(
+      bytes,
+      attachment,
+      category,
+      null,
+      { raw: opts },
+    );
+    await store(INGEST_TEXT_KEY, { text, truncated: truncated ?? false });
+    return result(text, 'text', mimetype, bytes.length, truncated ?? false);
+  }
+
+  if (mimetype === PDF_MIME) {
+    try {
+      const pdf = await extractPdfText(bytes, {
+        maxChars: opts.maxChars,
+        signal: deps.signal,
+      });
+      if (pdf.kind === 'text') {
+        await store(INGEST_PDF_KEY, {
+          text: pdf.text,
+          truncated: pdf.truncated,
+          pages: pdf.pages,
+        });
+        return result(
+          pdf.text,
+          'pdf',
+          mimetype,
+          bytes.length,
+          pdf.truncated,
+          pdf.pages,
+        );
+      }
+      deps.logger.log(
+        `[attachments] ingest "${attachment.filename}": no text layer (${pdf.pages} page(s)), using the helper model`,
+      );
+    } catch (error) {
+      // A cancelled turn stops here; it must not start a model call.
+      if (deps.signal?.aborted) throw error;
+      deps.logger.warn(
+        `[attachments] ingest "${attachment.filename}": local PDF parse failed (${error instanceof Error ? error.message : String(error)}), using the helper model`,
+      );
+    }
+  }
+
+  if (bytes.length > MAX_MODEL_LANE_BYTES) {
+    throw modelLaneTooLarge(attachment.filename, bytes.length);
+  }
+  const { text, truncated } = await extractText(
+    bytes,
+    attachment,
+    category,
+    deps.extraction,
+    { fetchImpl: deps.fetchImpl, signal: deps.signal, raw: opts },
+  );
+  await store(modelKey, { text, truncated: truncated ?? false });
+  return result(text, 'model', mimetype, bytes.length, truncated ?? false);
+}
+
+/** The message sources {@link findPersistedAttachmentMeta} reads. */
+export interface AttachmentMessageStore {
+  /** Messages whose stored text contains `contentContains`, oldest first. */
+  listThreadMessagesMatching(
+    threadId: string,
+    filter: { contentContains: string },
+  ): Promise<BaseMessage[]>;
+  listThreadMessages(threadId: string): Promise<BaseMessage[]>;
+}
+
+/**
+ * The attachment of a session with the given reference, from earlier turns.
+ * The reference is quoted in the text of the person's message (placeholder),
+ * of the hidden extraction text and of the hidden note of a natively sent
+ * file, and all of those carry the attachment metadata, so a SQL pre-filter
+ * on the reference usually inflates one or two small rows; only when none of
+ * them holds it (an inline attachment of an old message) is the whole session
+ * loaded.
+ */
+export async function findPersistedAttachmentMeta(
+  store: AttachmentMessageStore,
+  sessionId: string,
+  ref: string,
+): Promise<AttachmentMeta | undefined> {
+  const find = (messages: BaseMessage[]) =>
+    messages
+      .filter((m) => m.type === 'human' || m.type === 'ai')
+      .flatMap((m) => attachmentMetas(m))
+      .find((meta) => attachmentRef(meta) === ref);
+  const candidates = await store.listThreadMessagesMatching(sessionId, {
+    contentContains: ref,
+  });
+  return find(candidates) ?? find(await store.listThreadMessages(sessionId));
+}
+
+/**
+ * The session's attachment surface. A reference resolves first against the
+ * attachments of the turn now running (`turnMetas`: their message may not be
+ * persisted yet when a tool runs), then against the persisted session via
+ * `findPersisted`.
+ */
+export function createAttachmentViewSurface(input: {
+  offloaded: boolean;
+  turnMetas: readonly AttachmentMeta[];
+  findPersisted: (ref: string) => Promise<AttachmentMeta | undefined>;
+  deps: () => ViewAttachmentDeps;
+}): AttachmentViewSurface {
+  const findMeta = async (ref: string) =>
+    input.turnMetas.find((meta) => attachmentRef(meta) === ref) ??
+    (await input.findPersisted(ref));
+  return {
+    offloaded: input.offloaded,
+    view: async (ref) => {
+      const meta = await findMeta(ref);
+      if (!meta) {
+        throw new Error(`no attachment with ref "${ref}" in this conversation`);
+      }
+      return { meta, view: await viewAttachment(meta, input.deps()) };
+    },
+    extractText: async (ref, opts) => {
+      const meta = await findMeta(ref);
+      return meta ? extractAttachmentText(meta, input.deps(), opts) : null;
+    },
+  };
+}

package/src/do/turn-messages.ts

@@ -0,0 +1,54 @@
+/**
+ * Ordering of the messages a turn feeds the graph: hidden attachment context
+ * and the person's message.
+ */
+
+import {
+  AIMessage,
+  type BaseMessage,
+  type HumanMessage,
+} from '@langchain/core/messages';
+
+import type { ExtractedAttachment } from '../attachments';
+
+export interface TurnMessageParts {
+  human: HumanMessage;
+  /** Extraction-lane text: follows the HumanMessage (as on Node). */
+  extracted: readonly ExtractedAttachment[];
+  /**
+   * One line per natively-sent file naming its reference. These go BEFORE the
+   * HumanMessage so the turn never ends on an assistant message (providers
+   * reject or mis-handle an assistant-last conversation).
+   */
+  nativeNotes: readonly ExtractedAttachment[];
+  matrixRoom: boolean;
+  now?: () => Date;
+}
+
+function hiddenNote(
+  item: ExtractedAttachment,
+  matrixRoom: boolean,
+  now: () => Date,
+): AIMessage {
+  return new AIMessage({
+    content: item.text,
+    additional_kwargs: {
+      msgFromMatrixRoom: matrixRoom,
+      timestamp: now().toISOString(),
+      attachment: item.meta,
+    },
+  });
+}
+
+/**
+ * Native notes, then the HumanMessage, then the extraction-lane text. The
+ * hidden messages carry an `attachment` kwarg, which the transcript hides.
+ */
+export function assembleTurnMessages(parts: TurnMessageParts): BaseMessage[] {
+  const now = parts.now ?? (() => new Date());
+  return [
+    ...parts.nativeNotes.map((n) => hiddenNote(n, parts.matrixRoom, now)),
+    parts.human,
+    ...parts.extracted.map((n) => hiddenNote(n, parts.matrixRoom, now)),
+  ];
+}

package/src/do/user-oracle-do.ts

@@ -111,16 +111,17 @@ import {
   applyAttachmentRetention,
   ATTACHMENT_PAYLOAD_TURNS,
   ATTACHMENT_PLACEHOLDER_PREFIX,
-  attachmentMetas,
-  attachmentRef,
+  createAttachmentViewSurface,
+  findPersistedAttachmentMeta,
+  IngestTextLru,
   INLINE_PAYLOAD_NEEDLE,
   isAttachmentViewMessage,
   offloadInlinePayloads,
   parseAttachmentInputs,
   prepareAttachments,
-  viewAttachment,
   type AttachmentViewSurface,
   type MatrixMediaSource,
+  type ViewAttachmentDeps,
   type SandboxUploadConfig,
   DOWNLOAD_TIMEOUT_MS,
   MAX_FILE_SIZE,
@@ -199,6 +200,7 @@ import {
 } from '../sqlite/cache-config';
 import { DoSqliteDatabase, type DoSqliteOpenOptions } from '../sqlite/database';
 import type { TierFlushResult } from '../sqlite/do-vfs';
+import type { AttachmentMeta } from '../sqlite/serialization';
 import {
   DEFAULT_TIER_HOT_BUDGET_BYTES,
   type PageTierOptions,
@@ -272,6 +274,7 @@ import { decideBootDirty } from './boot-dirty';
 import { decideFlush } from './flush-schedule';
 import { mirrorTxnId, mirrorEventContent, RoomMirror } from './room-mirror';
 import { prefixSpeaker } from './speaker-prefix';
+import { assembleTurnMessages } from './turn-messages';
 import {
   type ObservedMessage,
   summarizeObservedMessages,
@@ -1040,6 +1043,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
     private lastActionPruneAt: number | null = null;
     /** Text `view_attachment` extracted, per session (`attachments/view-cache.ts`). */
     private attachmentTextCache: AttachmentTextCacheStore | null = null;
+    /** Large texts of document ingestion, in memory (a few, minutes). */
+    private readonly ingestTextLru = new IngestTextLru();
     /**
      * Per-model context windows; learned limits persist in this object's KV
      * storage (`ctxwin:<model>`), so a provider's rejection is remembered
@@ -6877,54 +6882,34 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       ambientLlm: LlmAdapter;
       platform: OpenRouterLlmAdapter;
       signal: AbortSignal;
+      /** Attachments of the turn now running (not necessarily persisted yet). */
+      turnMetas: readonly AttachmentMeta[];
     }): Promise<AttachmentViewSurface> {
       const saver = this.saver!;
       const offloaded = await saver.hasThreadMessageMatching(input.sessionId, {
         messageType: 'human',
         contentContains: ATTACHMENT_PLACEHOLDER_PREFIX,
       });
-      const findMeta = async (ref: string) => {
-        // The placeholder quotes the ref, so a cheap pre-filter usually
-        // finds the message; a still-inline attachment needs the full scan.
-        const candidates = await saver.listThreadMessagesMatching(
-          input.sessionId,
-          { messageType: 'human', contentContains: ref },
-        );
-        const scan =
-          candidates.length > 0
-            ? candidates
-            : await saver.listThreadMessages(input.sessionId);
-        return scan
-          .filter((m) => m.type === 'human')
-          .flatMap((m) => attachmentMetas(m))
-          .find((meta) => attachmentRef(meta) === ref);
-      };
-      return {
+      const findMeta = (ref: string) =>
+        findPersistedAttachmentMeta(saver, input.sessionId, ref);
+      const viewDeps = (): ViewAttachmentDeps => ({
+        source: this.matrixMediaSource(),
+        extraction: this.extractionProvider(input.ambientLlm, input.platform),
+        caps: getModelCapabilities(input.model),
+        roomId: input.roomId,
+        signal: input.signal,
+        logger: console,
+        ...(this.attachmentTextCache
+          ? { cache: this.attachmentTextCache.forSession(input.sessionId) }
+          : {}),
+        ingestCache: this.ingestTextLru.forSession(input.sessionId),
+      });
+      return createAttachmentViewSurface({
         offloaded,
-        view: async (ref) => {
-          const meta = await findMeta(ref);
-          if (!meta) {
-            throw new Error(
-              `no attachment with ref "${ref}" in this conversation`,
-            );
-          }
-          const view = await viewAttachment(meta, {
-            source: this.matrixMediaSource(),
-            extraction: this.extractionProvider(
-              input.ambientLlm,
-              input.platform,
-            ),
-            caps: getModelCapabilities(input.model),
-            roomId: input.roomId,
-            signal: input.signal,
-            logger: console,
-            ...(this.attachmentTextCache
-              ? { cache: this.attachmentTextCache.forSession(input.sessionId) }
-              : {}),
-          });
-          return { meta, view };
-        },
-      };
+        turnMetas: input.turnMetas,
+        findPersisted: findMeta,
+        deps: viewDeps,
+      });
     }
 
     /**
@@ -7300,6 +7285,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
             ambientLlm: baseAmbient.llm,
             platform: core.llm,
             signal: abortController.signal,
+            turnMetas: prepared?.metas ?? [],
           });
 
       // Request metadata (editor room, space, session run, entity) → state,
@@ -7697,36 +7683,30 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         ? undefined
         : renderTurnTimeNote(turnAt, req.identity.timezone);
       const content = timeNote ? withTurnTimeNote(spoken, timeNote) : spoken;
+      const human = new HumanMessage({
+        content,
+        additional_kwargs: {
+          timestamp: turnAt.toISOString(),
+          ...(timeNote ? { [TURN_TIME_NOTE_KWARG]: timeNote } : {}),
+          oracleName: core.identity.name,
+          msgFromMatrixRoom: req.client === 'matrix',
+          ...speakerKwargs,
+          ...(req.channel
+            ? { 'org.ixo.qi.origin': channelOrigin(req.channel) }
+            : {}),
+          ...attachmentKwargs,
+        },
+      });
       const stateInput = {
-        messages: [
-          new HumanMessage({
-            content,
-            additional_kwargs: {
-              timestamp: turnAt.toISOString(),
-              ...(timeNote ? { [TURN_TIME_NOTE_KWARG]: timeNote } : {}),
-              oracleName: core.identity.name,
-              msgFromMatrixRoom: req.client === 'matrix',
-              ...speakerKwargs,
-              ...(req.channel
-                ? { 'org.ixo.qi.origin': channelOrigin(req.channel) }
-                : {}),
-              ...attachmentKwargs,
-            },
-          }),
-          // Extraction-lane text rides as hidden context messages (the
-          // transcript hides AI messages that carry an `attachment`), as on Node.
-          ...(prepared?.extracted ?? []).map(
-            (item) =>
-              new AIMessage({
-                content: item.text,
-                additional_kwargs: {
-                  msgFromMatrixRoom: req.client === 'matrix',
-                  timestamp: new Date().toISOString(),
-                  attachment: item.meta,
-                },
-              }),
-          ),
-        ],
+        // Hidden attachment context (the transcript hides AI messages that
+        // carry an `attachment`): native-lane notes precede the person's
+        // message, extraction-lane text follows it, as on Node.
+        messages: assembleTurnMessages({
+          human,
+          extracted: prepared?.extracted ?? [],
+          nativeNotes: prepared?.nativeNotes ?? [],
+          matrixRoom: req.client === 'matrix',
+        }),
         config: { did: req.identity.userDid },
         client: req.client,
         // First contact's gathered context is checkpointed with the turn,

package/src/memory/user-context-invalidation.ts

@@ -19,6 +19,10 @@ import {
   MEMORY_DELETE_EDGE_MCP_NAME,
   MEMORY_DELETE_EPISODE_MCP_NAME,
 } from '../plugins/memory/memory-tools';
+import {
+  ADD_ORACLE_DOCUMENT_TOOL_NAME,
+  DELETE_ORACLE_DOCUMENT_TOOL_NAME,
+} from '../plugins/memory/oracle-document-tools';
 
 /** The memory engine tools that write or delete what the prefetch reads. */
 export const MEMORY_WRITE_TOOL_NAMES: readonly string[] = [
@@ -27,6 +31,8 @@ export const MEMORY_WRITE_TOOL_NAMES: readonly string[] = [
   MEMORY_DELETE_EPISODE_MCP_NAME,
   MEMORY_DELETE_EDGE_MCP_NAME,
   MEMORY_CLEAR_MCP_NAME,
+  ADD_ORACLE_DOCUMENT_TOOL_NAME,
+  DELETE_ORACLE_DOCUMENT_TOOL_NAME,
 ];
 
 export interface UserContextInvalidationOptions {