nexus-ai-pro 2.2.0 → 2.4.0

package/dist/protocols/ag-ui.js

@@ -0,0 +1,180 @@
+import { answerOf, asText, messageOf, protocolId, textOf, toolCallOf, } from './shared.js';
+export function agUiMessages(messages) {
+    const converted = [];
+    for (const message of messages) {
+        if (!['user', 'assistant', 'system', 'developer', 'tool'].includes(message.role))
+            continue;
+        const content = textOf(message.content);
+        converted.push(messageOf(message.role, content, {
+            ...(message.toolCalls?.length ? { toolCalls: message.toolCalls } : {}),
+            ...(message.role === 'tool' && message.toolCallId ? { toolCallId: message.toolCallId } : {}),
+        }));
+    }
+    return converted;
+}
+export async function* agUiEvents(graph, run, options = {}) {
+    const queue = [];
+    let wake;
+    let done = false;
+    const push = (event) => {
+        queue.push({ ...event, timestamp: Date.now() });
+        wake?.();
+    };
+    const translate = translator(push);
+    push({
+        type: 'RUN_STARTED',
+        threadId: run.threadId,
+        runId: run.runId,
+        ...(run.parentRunId ? { parentRunId: run.parentRunId } : {}),
+    });
+    const runOptions = {
+        threadId: run.threadId,
+        onEvent: translate.event,
+        ...(options.signal ? { signal: options.signal } : {}),
+        ...(options.principal ? { principal: options.principal } : {}),
+    };
+    const finished = (async () => {
+        try {
+            let result;
+            if (run.resume?.length) {
+                const answer = options.answer ??
+                    ((entry) => (entry.status === 'cancelled' ? false : (entry.payload ?? true)));
+                result = await graph.resumeInterruptsWith(run.threadId, Object.fromEntries(run.resume.map((entry) => [entry.interruptId, answer(entry)])), runOptions);
+            }
+            else {
+                const messages = agUiMessages(run.messages);
+                result = await graph.invoke(options.input ? options.input(run, messages) : { messages }, runOptions);
+            }
+            translate.close();
+            if (options.stateSnapshot !== false)
+                push({ type: 'STATE_SNAPSHOT', snapshot: result.state });
+            push(finishedEvent(run, result));
+        }
+        catch (error) {
+            translate.close();
+            push({
+                type: 'RUN_ERROR',
+                message: error instanceof Error ? error.message : String(error),
+                ...(error?.code ? { code: String(error.code) } : {}),
+            });
+        }
+        finally {
+            done = true;
+            wake?.();
+        }
+    })();
+    while (true) {
+        if (queue.length > 0) {
+            yield queue.shift();
+            continue;
+        }
+        if (done)
+            break;
+        await new Promise((resolve) => {
+            wake = resolve;
+        });
+        wake = undefined;
+    }
+    await finished;
+}
+export function agUiHandler(graph, options = {}) {
+    return async (request) => {
+        if (request.method !== 'POST')
+            return new Response('AG-UI runs are POSTed', { status: 405, headers: { allow: 'POST' } });
+        let run;
+        try {
+            run = (await request.json());
+        }
+        catch {
+            return new Response('The body is not a RunAgentInput', { status: 400 });
+        }
+        if (typeof run?.threadId !== 'string' || (!Array.isArray(run.messages) && !run.resume)) {
+            return new Response('A RunAgentInput needs a threadId and messages', { status: 400 });
+        }
+        run = { ...run, runId: run.runId || protocolId('run'), messages: run.messages ?? [] };
+        const caller = await options.principal?.(request);
+        const encoder = new TextEncoder();
+        const body = new ReadableStream({
+            async start(controller) {
+                const { principal: _resolve, ...rest } = options;
+                for await (const event of agUiEvents(graph, run, {
+                    ...rest,
+                    signal: request.signal,
+                    ...(caller ? { principal: caller } : {}),
+                })) {
+                    controller.enqueue(encoder.encode(encodeAgUiEvent(event)));
+                }
+                controller.close();
+            },
+        });
+        return new Response(body, {
+            headers: { 'content-type': 'text/event-stream', 'cache-control': 'no-cache', connection: 'keep-alive' },
+        });
+    };
+}
+export function encodeAgUiEvent(event) {
+    return `data: ${JSON.stringify(event)}\n\n`;
+}
+function translator(push) {
+    let open;
+    const closeMessage = () => {
+        if (open)
+            push({ type: 'TEXT_MESSAGE_END', messageId: open });
+        open = undefined;
+    };
+    return {
+        event(event) {
+            if (event.type === 'task_start' && !event.namespace)
+                push({ type: 'STEP_STARTED', stepName: event.node });
+            else if (event.type === 'task_end' && !event.namespace) {
+                closeMessage();
+                push({ type: 'STEP_FINISHED', stepName: event.node });
+            }
+            else if (event.type === 'message' && event.chunk.kind !== 'reasoning' && event.chunk.content) {
+                const messageId = event.chunk.messageId ?? `${event.taskId}:${event.step}`;
+                if (open !== messageId) {
+                    closeMessage();
+                    open = messageId;
+                    push({ type: 'TEXT_MESSAGE_START', messageId, role: 'assistant' });
+                }
+                push({ type: 'TEXT_MESSAGE_CONTENT', messageId, delta: event.chunk.content });
+            }
+            else if (event.type === 'tool') {
+                closeMessage();
+                const tool = event.tool;
+                if (tool.phase === 'start') {
+                    push({ type: 'TOOL_CALL_START', toolCallId: tool.id, toolCallName: tool.name });
+                    push({ type: 'TOOL_CALL_ARGS', toolCallId: tool.id, delta: asText(tool.args ?? {}) });
+                    push({ type: 'TOOL_CALL_END', toolCallId: tool.id });
+                }
+                else if (tool.phase === 'result' || tool.phase === 'error') {
+                    push({
+                        type: 'TOOL_CALL_RESULT',
+                        messageId: `${tool.id}:result`,
+                        toolCallId: tool.id,
+                        role: 'tool',
+                        content: tool.phase === 'result' ? asText(tool.result) : (tool.error ?? 'The tool failed'),
+                    });
+                }
+            }
+        },
+        close: closeMessage,
+    };
+}
+function finishedEvent(run, result) {
+    const base = { type: 'RUN_FINISHED', threadId: run.threadId, runId: run.runId };
+    if (result.status === 'awaiting_input' && result.interrupts?.length) {
+        return { ...base, outcome: { type: 'interrupt', interrupts: result.interrupts.map(agUiInterrupt) } };
+    }
+    return { ...base, outcome: { type: 'success' }, result: answerOf(result.state) };
+}
+function agUiInterrupt(interrupt) {
+    const call = toolCallOf(interrupt);
+    return {
+        id: interrupt.id,
+        reason: call ? 'tool_call' : 'input_required',
+        ...(interrupt.reason ? { message: interrupt.reason } : {}),
+        ...(call ? { toolCallId: call.id } : {}),
+        ...(interrupt.payload === undefined ? {} : { metadata: { payload: interrupt.payload } }),
+    };
+}

package/dist/protocols/shared.d.ts

@@ -0,0 +1,80 @@
+/**
+ * What the protocol adapters share: the part of a compiled graph they drive, and the conversions
+ * between a protocol's messages and the graph's. Structural, so a protocol entry point never imports
+ * the graph runtime: an agent or any compiled graph already satisfies it.
+ */
+import type { GraphEvent } from '../types/graph.js';
+import type { Message } from '../types/messages.js';
+import type { Principal } from '../types/principal.js';
+/** Run options a protocol passes to a graph. */
+export interface ProtocolRunOptions {
+    /** The thread: a protocol's conversation, context, or session id. */
+    threadId?: string;
+    /** Stops the run, as a protocol's cancel does. */
+    signal?: AbortSignal;
+    /** Receives the run's fine-grained events, which the protocol translates as they happen. */
+    onEvent?: (event: GraphEvent) => void;
+    /** Who the run is for, when the protocol's transport authenticated the caller. */
+    principal?: Principal;
+}
+/** A question a paused run is waiting on. */
+export interface ProtocolInterrupt {
+    /** Its id, which an answer names. */
+    id: string;
+    /** The question. */
+    reason?: string;
+    /** What it is about, such as the tool call awaiting approval. */
+    payload?: unknown;
+}
+/** What a run ended with. */
+export interface ProtocolResult {
+    /** The thread it ran on. */
+    threadId: string;
+    /** `completed`, `awaiting_input`, or another graph status. */
+    status: string;
+    /** The state it ended with. */
+    state: unknown;
+    /** Every question a paused run asked. */
+    interrupts?: ProtocolInterrupt[];
+}
+/** The part of a compiled graph the protocols drive. An agent from `createAgent()` satisfies it. */
+export interface ProtocolGraph {
+    /** Runs an input. */
+    invoke(input: unknown, options?: ProtocolRunOptions): Promise<ProtocolResult>;
+    /** Answers a paused run's questions, by interrupt id, and continues it. */
+    resumeInterruptsWith(threadId: string, answers: Record<string, unknown>, options?: ProtocolRunOptions): Promise<ProtocolResult>;
+}
+/** A role a protocol message may carry. */
+export type ProtocolRole = 'user' | 'assistant' | 'system' | 'developer' | 'tool' | 'agent';
+/** The final answer a run produced: an agent's `answer`, or its state as JSON. */
+export declare function answerOf(state: unknown): string;
+/**
+ * The conversation a finished run leaves, to send whole with the next turn: an agent's transcript,
+ * tool calls and results included, or the fallback for a graph that keeps none.
+ */
+export declare function transcriptOf(state: unknown, fallback: Message[]): Message[];
+/** The text of a protocol message's content, for content as a string or a list of text parts. */
+export declare function textOf(content: unknown): string;
+/** A Nexus message from a protocol role and text. A developer message is a system message. */
+export declare function messageOf(role: ProtocolRole, content: string, extra?: Partial<Message>): Message;
+/** A short random id, for runs and messages the protocol creates. */
+export declare function protocolId(prefix: string): string;
+/** Whether an interrupt asks to approve a tool call, as an agent's approvals do. */
+export declare function toolCallOf(interrupt: ProtocolInterrupt): {
+    id: string;
+    name: string;
+    args: unknown;
+} | undefined;
+/**
+ * Words answering an approval, as an approval: yes, ok, allow, approve, or confirm allows, and
+ * anything else refuses, with the words as the reason. A protocol that carries only text needs this
+ * so that "no" never reads as consent.
+ */
+export declare function approvalOf(text: string): {
+    approved: true;
+} | {
+    approved: false;
+    reason?: string;
+};
+/** A value as the text a protocol's text field carries. */
+export declare function asText(value: unknown): string;

package/dist/protocols/shared.js

@@ -0,0 +1,49 @@
+export function answerOf(state) {
+    const answer = state?.answer;
+    if (typeof answer === 'string')
+        return answer;
+    return state === undefined ? '' : JSON.stringify(state);
+}
+export function transcriptOf(state, fallback) {
+    const messages = state?.messages;
+    return Array.isArray(messages) && messages.every((item) => typeof item?.role === 'string')
+        ? messages
+        : fallback;
+}
+export function textOf(content) {
+    if (typeof content === 'string')
+        return content;
+    if (Array.isArray(content)) {
+        return content
+            .map((part) => {
+            const value = part;
+            return typeof value.text === 'string' ? value.text : '';
+        })
+            .filter(Boolean)
+            .join('\n');
+    }
+    return '';
+}
+export function messageOf(role, content, extra = {}) {
+    const mapped = role === 'developer' ? 'system' : role === 'agent' ? 'assistant' : role;
+    return { role: mapped, content, ...extra };
+}
+export function protocolId(prefix) {
+    return `${prefix}-${globalThis.crypto.randomUUID()}`;
+}
+export function toolCallOf(interrupt) {
+    const payload = interrupt.payload;
+    return typeof payload?.id === 'string' && typeof payload.name === 'string'
+        ? { id: payload.id, name: payload.name, args: payload.args }
+        : undefined;
+}
+export function approvalOf(text) {
+    const words = text.trim();
+    if (/^(y|yes|ok|okay|allow|allowed|approve|approved|confirm|confirmed|go ahead)\b/i.test(words)) {
+        return { approved: true };
+    }
+    return { approved: false, ...(words ? { reason: words } : {}) };
+}
+export function asText(value) {
+    return typeof value === 'string' ? value : JSON.stringify(value ?? null);
+}

package/dist/providers/anthropic.js

@@ -163,6 +163,7 @@ export class AnthropicProvider extends BaseProvider {
             this.client = new Anthropic({
                 apiKey: this.config.apiKey,
                 baseURL: this.config.baseUrl,
+                ...(this.config.fetch ? { fetch: this.config.fetch } : {}),
             });
         }
         return this.client;

package/dist/providers/cohere.js

@@ -14,7 +14,7 @@ export class CohereProvider extends BaseProvider {
         try {
             this.throwIfAborted(request);
             const startTime = Date.now();
-            const response = await fetch(`${this.baseUrl()}/v2/chat`, {
+            const response = await (this.config.fetch ?? fetch)(`${this.baseUrl()}/v2/chat`, {
                 method: 'POST',
                 headers: {
                     Authorization: `Bearer ${this.config.apiKey}`,
@@ -78,7 +78,7 @@ export class CohereProvider extends BaseProvider {
         }, request.signal);
     }
     async healthCheck() {
-        const response = await fetch(`${this.baseUrl()}/v2/models`, {
+        const response = await (this.config.fetch ?? fetch)(`${this.baseUrl()}/v2/models`, {
             headers: { Authorization: `Bearer ${this.config.apiKey}` },
         });
         return response.ok;

package/dist/providers/google.js

@@ -1,3 +1,4 @@
+import { decodeBase64 } from '../utils/base64.js';
 import { BaseProvider } from './base.js';
 import { buildMeta } from '../core/usage.js';
 import { generateRequestId } from '../utils/ids.js';
@@ -119,7 +120,7 @@ export class GoogleProvider extends BaseProvider {
         const endpoint = stream ? 'streamGenerateContent' : 'generateContent';
         const url = `${this.baseUrl()}/models/${encodeURIComponent(model)}:${endpoint}?key=${encodeURIComponent(this.config.apiKey)}${stream ? '&alt=sse' : ''}`;
         const body = JSON.stringify(this.createBody(request));
-        const response = await fetch(url, {
+        const response = await (this.config.fetch ?? fetch)(url, {
             method: 'POST',
             headers: { 'content-type': 'application/json' },
             body,
@@ -310,13 +311,3 @@ export class GoogleProvider extends BaseProvider {
         return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
     }
 }
-function decodeBase64(data) {
-    const buffer = globalThis.Buffer;
-    if (buffer)
-        return new Uint8Array(buffer.from(data, 'base64'));
-    const binary = atob(data);
-    const bytes = new Uint8Array(binary.length);
-    for (let index = 0; index < binary.length; index += 1)
-        bytes[index] = binary.charCodeAt(index);
-    return bytes;
-}

package/dist/providers/openai.js

@@ -25,6 +25,7 @@ export class OpenAIProvider extends BaseProvider {
                 organization: this.config.organization,
                 defaultHeaders: this.config.defaultHeaders,
                 defaultQuery: this.config.defaultQuery,
+                ...(this.config.fetch ? { fetch: this.config.fetch } : {}),
             });
         }
         return this.client;

package/dist/rag/elasticsearch.d.ts

@@ -0,0 +1,71 @@
+import type { RagChunk } from '../hallucination/rag.js';
+import type { VectorSearchResult } from '../hallucination/retrieval.js';
+import type { RetrieveOptions, SparseRetriever } from './retrievers.js';
+/** Options for the Elasticsearch keyword index. */
+export interface ElasticsearchKeywordIndexOptions {
+    /** The cluster, such as `http://localhost:9200`, an Elastic Cloud endpoint, or an OpenSearch domain. */
+    url: string;
+    /** The index chunks are kept in. */
+    index: string;
+    /** Sent as `Authorization: ApiKey <key>`. */
+    apiKey?: string;
+    /** Sent as basic authentication, for a cluster with users instead of API keys. */
+    username?: string;
+    /** The password for `username`. */
+    password?: string;
+    /** The analyzer `migrate()` gives the `content` field, such as `english`. Defaults to the cluster's standard one. */
+    analyzer?: string;
+    /**
+     * When a write becomes searchable. `wait_for` (the default) returns once a search would see it, as
+     * the vector stores do; `false` returns sooner and leaves it to the index's refresh interval.
+     */
+    refresh?: 'wait_for' | boolean;
+    /** Replaces the global `fetch`, for a proxy, retries, signed requests, or tests. */
+    fetch?: typeof globalThis.fetch;
+    /** Headers added to every request. */
+    headers?: Record<string, string>;
+    /** Aborts a request that takes longer, in milliseconds. Defaults to 30 seconds. */
+    timeoutMs?: number;
+}
+/** Raised when the cluster answers with an error. */
+export declare class ElasticsearchError extends Error {
+    /** The HTTP status the cluster answered with. */
+    readonly status: number;
+    /** The cluster's response body, as text. */
+    readonly body?: string | undefined;
+    constructor(message: string, 
+    /** The HTTP status the cluster answered with. */
+    status: number, 
+    /** The cluster's response body, as text. */
+    body?: string | undefined);
+}
+/**
+ * Keyword search in Elasticsearch or OpenSearch, through their REST API.
+ *
+ * No client library is a dependency: the index speaks HTTP through `fetch`, as the vector stores do,
+ * so it runs wherever `fetch` does. Scores are the engine's own BM25, ranked in the cluster, so only
+ * the best matches come back. A `SparseRetriever`, so `hybridRetriever()` fuses it with any vector
+ * store. A metadata filter is an exact match on top-level fields, as in the vector stores.
+ */
+export declare class ElasticsearchKeywordIndex implements SparseRetriever {
+    private readonly options;
+    private readonly base;
+    private readonly fetchImpl;
+    constructor(options: ElasticsearchKeywordIndexOptions);
+    /**
+     * Creates the index unless it exists: `content` as analyzed text, `id` and `source` as keywords,
+     * and every string under `metadata` as a keyword, so a filter matches it exactly. Never runs
+     * implicitly.
+     */
+    migrate(): Promise<void>;
+    /** Adds chunks, or replaces those whose id exists, in one bulk request. */
+    add(chunks: readonly RagChunk[]): Promise<void>;
+    /** Removes chunks by id. An id that is not held is ignored. */
+    delete(ids: readonly string[]): Promise<void>;
+    /** The chunks that best match a query's terms, best first, scored by the engine's BM25. */
+    retrieve(query: string, options?: RetrieveOptions): Promise<VectorSearchResult[]>;
+    /** Chunks held. */
+    size(): Promise<number>;
+    private bulk;
+    private request;
+}

package/dist/rag/elasticsearch.js

@@ -0,0 +1,130 @@
+import { linkSignals } from '../utils/signals.js';
+export class ElasticsearchError extends Error {
+    status;
+    body;
+    constructor(message, status, body) {
+        super(message);
+        this.status = status;
+        this.body = body;
+        this.name = 'ElasticsearchError';
+    }
+}
+export class ElasticsearchKeywordIndex {
+    options;
+    base;
+    fetchImpl;
+    constructor(options) {
+        this.options = options;
+        if (!/^[a-z0-9][a-z0-9._-]*$/.test(options.index)) {
+            throw new RangeError(`"${options.index}" is not a valid index name: use lowercase letters, digits, ".", "_", and "-"`);
+        }
+        this.base = options.url.replace(/\/+$/, '');
+        this.fetchImpl = options.fetch ?? globalThis.fetch;
+    }
+    async migrate() {
+        const existing = await this.request('HEAD', `/${this.options.index}`, undefined, { allowNotFound: true });
+        if (existing !== undefined)
+            return;
+        await this.request('PUT', `/${this.options.index}`, {
+            mappings: {
+                dynamic_templates: [
+                    {
+                        metadata_strings: { path_match: 'metadata.*', match_mapping_type: 'string', mapping: { type: 'keyword' } },
+                    },
+                ],
+                properties: {
+                    id: { type: 'keyword' },
+                    content: { type: 'text', ...(this.options.analyzer ? { analyzer: this.options.analyzer } : {}) },
+                    source: { type: 'keyword' },
+                    metadata: { type: 'object', dynamic: true },
+                },
+            },
+        });
+    }
+    async add(chunks) {
+        if (chunks.length === 0)
+            return;
+        const lines = chunks.flatMap((chunk) => [
+            { index: { _index: this.options.index, _id: chunk.id } },
+            { id: chunk.id, content: chunk.content, source: chunk.source ?? null, metadata: chunk.metadata ?? null },
+        ]);
+        await this.bulk(lines);
+    }
+    async delete(ids) {
+        if (ids.length === 0)
+            return;
+        await this.bulk(ids.map((id) => ({ delete: { _index: this.options.index, _id: id } })));
+    }
+    async retrieve(query, options = {}) {
+        const filter = Object.entries(options.filter ?? {});
+        const clauses = [];
+        for (const [key, value] of filter) {
+            if (value === null)
+                clauses.push({ bool: { must_not: { exists: { field: `metadata.${key}` } } } });
+            else if (['string', 'number', 'boolean'].includes(typeof value))
+                clauses.push({ term: { [`metadata.${key}`]: value } });
+            else
+                return [];
+        }
+        const result = (await this.request('POST', `/${this.options.index}/_search`, {
+            size: options.topK ?? 5,
+            query: { bool: { must: [{ match: { content: { query, operator: 'or' } } }], filter: clauses } },
+            _source: ['id', 'content', 'source', 'metadata'],
+        }, { signal: options.signal }));
+        return (result.hits?.hits ?? []).map((hit) => ({
+            id: hit._source?.id ?? hit._id,
+            content: hit._source?.content ?? '',
+            ...(hit._source?.source ? { source: hit._source.source } : {}),
+            ...(hit._source?.metadata ? { metadata: hit._source.metadata } : {}),
+            score: hit._score ?? 0,
+        }));
+    }
+    async size() {
+        const result = (await this.request('GET', `/${this.options.index}/_count`));
+        return result.count ?? 0;
+    }
+    async bulk(lines) {
+        const refresh = this.options.refresh ?? 'wait_for';
+        const result = (await this.request('POST', `/_bulk${refresh === false ? '' : `?refresh=${refresh === true ? 'true' : 'wait_for'}`}`, `${lines.map((line) => JSON.stringify(line)).join('\n')}\n`));
+        if (!result.errors)
+            return;
+        const failed = (result.items ?? [])
+            .map((item) => Object.entries(item)[0])
+            .map((entry) => (entry?.[1]?.error && !(entry[0] === 'delete' && entry[1].status === 404) ? entry[1] : undefined))
+            .find((outcome) => outcome !== undefined);
+        if (failed)
+            throw new ElasticsearchError(`A bulk write failed: ${failed.error?.reason ?? 'unknown reason'}`, failed.status ?? 500);
+    }
+    async request(method, path, body, options = {}) {
+        const timeout = AbortSignal.timeout(this.options.timeoutMs ?? 30_000);
+        const linked = linkSignals([timeout, options.signal]);
+        const ndjson = typeof body === 'string';
+        const authorization = this.options.apiKey
+            ? `ApiKey ${this.options.apiKey}`
+            : this.options.username !== undefined
+                ? `Basic ${btoa(`${this.options.username}:${this.options.password ?? ''}`)}`
+                : undefined;
+        try {
+            const response = await this.fetchImpl(`${this.base}${path}`, {
+                method,
+                headers: {
+                    ...(body === undefined ? {} : { 'content-type': ndjson ? 'application/x-ndjson' : 'application/json' }),
+                    ...(authorization ? { authorization } : {}),
+                    ...this.options.headers,
+                },
+                ...(body === undefined ? {} : { body: ndjson ? body : JSON.stringify(body) }),
+                signal: linked.signal,
+            });
+            if (response.status === 404 && options.allowNotFound)
+                return undefined;
+            const text = method === 'HEAD' ? '' : await response.text();
+            if (!response.ok) {
+                throw new ElasticsearchError(`Elasticsearch ${method} ${path} failed with ${response.status}`, response.status, text);
+            }
+            return text ? JSON.parse(text) : {};
+        }
+        finally {
+            linked.dispose();
+        }
+    }
+}

package/dist/rag/file-ingestion.js

@@ -1,4 +1,4 @@
-import { scanUploads } from '../security/upload-scanner.js';
+import { isTextual, scanUploads, textOf, } from '../security/upload-scanner.js';
 import { ingestDocuments } from './ingestion.js';
 export async function ingestFilesAfterScan(files, options = {}) {
     const scan = scanUploads(files, options.scan);
@@ -10,19 +10,30 @@ export async function ingestFilesAfterScan(files, options = {}) {
     for (const file of files) {
         const extractor = options.extractors?.find((item) => item.supports(file));
         if (extractor) {
+            const text = await extractor.extract(file);
+            if (options.scan?.scanTextContent !== false) {
+                const extracted = scanUploads([{ name: file.name, mimeType: 'text/plain', content: text }], {
+                    ...(options.scan?.forbiddenPatterns ? { forbiddenPatterns: options.scan.forbiddenPatterns } : {}),
+                    blockedExtensions: [],
+                });
+                if (!extracted.ok) {
+                    throw new Error(`Upload scan failed on extracted text: ${extracted.findings.map((finding) => `${finding.fileName}: ${finding.message}`).join('; ')}`);
+                }
+            }
             documents.push({
                 id: file.name,
                 source: file.name,
-                text: await extractor.extract(file),
+                text,
                 metadata: { mimeType: file.mimeType, sizeBytes: file.sizeBytes },
             });
             continue;
         }
-        if (typeof file.content === 'string' && isTextMime(file.mimeType)) {
+        const text = isTextual(file.mimeType) ? textOf(file.content, file.mimeType) : undefined;
+        if (text !== undefined) {
             documents.push({
                 id: file.name,
                 source: file.name,
-                text: file.content,
+                text,
                 metadata: { mimeType: file.mimeType, sizeBytes: file.sizeBytes },
             });
             continue;
@@ -48,8 +59,3 @@ export function createOcrExtractor(ocrImage, mimeTypes = ['image/png', 'image/jp
         extract: ocrImage,
     };
 }
-function isTextMime(mimeType) {
-    if (!mimeType)
-        return true;
-    return (mimeType.startsWith('text/') || ['application/json', 'application/xml', 'application/markdown'].includes(mimeType));
-}