nexus-ai-pro 2.2.0 → 2.4.0

package/dist-cjs/protocols/shared.d.ts

@@ -0,0 +1,80 @@
+/**
+ * What the protocol adapters share: the part of a compiled graph they drive, and the conversions
+ * between a protocol's messages and the graph's. Structural, so a protocol entry point never imports
+ * the graph runtime: an agent or any compiled graph already satisfies it.
+ */
+import type { GraphEvent } from '../types/graph.js';
+import type { Message } from '../types/messages.js';
+import type { Principal } from '../types/principal.js';
+/** Run options a protocol passes to a graph. */
+export interface ProtocolRunOptions {
+    /** The thread: a protocol's conversation, context, or session id. */
+    threadId?: string;
+    /** Stops the run, as a protocol's cancel does. */
+    signal?: AbortSignal;
+    /** Receives the run's fine-grained events, which the protocol translates as they happen. */
+    onEvent?: (event: GraphEvent) => void;
+    /** Who the run is for, when the protocol's transport authenticated the caller. */
+    principal?: Principal;
+}
+/** A question a paused run is waiting on. */
+export interface ProtocolInterrupt {
+    /** Its id, which an answer names. */
+    id: string;
+    /** The question. */
+    reason?: string;
+    /** What it is about, such as the tool call awaiting approval. */
+    payload?: unknown;
+}
+/** What a run ended with. */
+export interface ProtocolResult {
+    /** The thread it ran on. */
+    threadId: string;
+    /** `completed`, `awaiting_input`, or another graph status. */
+    status: string;
+    /** The state it ended with. */
+    state: unknown;
+    /** Every question a paused run asked. */
+    interrupts?: ProtocolInterrupt[];
+}
+/** The part of a compiled graph the protocols drive. An agent from `createAgent()` satisfies it. */
+export interface ProtocolGraph {
+    /** Runs an input. */
+    invoke(input: unknown, options?: ProtocolRunOptions): Promise<ProtocolResult>;
+    /** Answers a paused run's questions, by interrupt id, and continues it. */
+    resumeInterruptsWith(threadId: string, answers: Record<string, unknown>, options?: ProtocolRunOptions): Promise<ProtocolResult>;
+}
+/** A role a protocol message may carry. */
+export type ProtocolRole = 'user' | 'assistant' | 'system' | 'developer' | 'tool' | 'agent';
+/** The final answer a run produced: an agent's `answer`, or its state as JSON. */
+export declare function answerOf(state: unknown): string;
+/**
+ * The conversation a finished run leaves, to send whole with the next turn: an agent's transcript,
+ * tool calls and results included, or the fallback for a graph that keeps none.
+ */
+export declare function transcriptOf(state: unknown, fallback: Message[]): Message[];
+/** The text of a protocol message's content, for content as a string or a list of text parts. */
+export declare function textOf(content: unknown): string;
+/** A Nexus message from a protocol role and text. A developer message is a system message. */
+export declare function messageOf(role: ProtocolRole, content: string, extra?: Partial<Message>): Message;
+/** A short random id, for runs and messages the protocol creates. */
+export declare function protocolId(prefix: string): string;
+/** Whether an interrupt asks to approve a tool call, as an agent's approvals do. */
+export declare function toolCallOf(interrupt: ProtocolInterrupt): {
+    id: string;
+    name: string;
+    args: unknown;
+} | undefined;
+/**
+ * Words answering an approval, as an approval: yes, ok, allow, approve, or confirm allows, and
+ * anything else refuses, with the words as the reason. A protocol that carries only text needs this
+ * so that "no" never reads as consent.
+ */
+export declare function approvalOf(text: string): {
+    approved: true;
+} | {
+    approved: false;
+    reason?: string;
+};
+/** A value as the text a protocol's text field carries. */
+export declare function asText(value: unknown): string;

package/dist-cjs/protocols/shared.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.answerOf = answerOf;
+exports.transcriptOf = transcriptOf;
+exports.textOf = textOf;
+exports.messageOf = messageOf;
+exports.protocolId = protocolId;
+exports.toolCallOf = toolCallOf;
+exports.approvalOf = approvalOf;
+exports.asText = asText;
+function answerOf(state) {
+    const answer = state?.answer;
+    if (typeof answer === 'string')
+        return answer;
+    return state === undefined ? '' : JSON.stringify(state);
+}
+function transcriptOf(state, fallback) {
+    const messages = state?.messages;
+    return Array.isArray(messages) && messages.every((item) => typeof item?.role === 'string')
+        ? messages
+        : fallback;
+}
+function textOf(content) {
+    if (typeof content === 'string')
+        return content;
+    if (Array.isArray(content)) {
+        return content
+            .map((part) => {
+            const value = part;
+            return typeof value.text === 'string' ? value.text : '';
+        })
+            .filter(Boolean)
+            .join('\n');
+    }
+    return '';
+}
+function messageOf(role, content, extra = {}) {
+    const mapped = role === 'developer' ? 'system' : role === 'agent' ? 'assistant' : role;
+    return { role: mapped, content, ...extra };
+}
+function protocolId(prefix) {
+    return `${prefix}-${globalThis.crypto.randomUUID()}`;
+}
+function toolCallOf(interrupt) {
+    const payload = interrupt.payload;
+    return typeof payload?.id === 'string' && typeof payload.name === 'string'
+        ? { id: payload.id, name: payload.name, args: payload.args }
+        : undefined;
+}
+function approvalOf(text) {
+    const words = text.trim();
+    if (/^(y|yes|ok|okay|allow|allowed|approve|approved|confirm|confirmed|go ahead)\b/i.test(words)) {
+        return { approved: true };
+    }
+    return { approved: false, ...(words ? { reason: words } : {}) };
+}
+function asText(value) {
+    return typeof value === 'string' ? value : JSON.stringify(value ?? null);
+}

package/dist-cjs/providers/anthropic.js

@@ -199,6 +199,7 @@ class AnthropicProvider extends base_js_1.BaseProvider {
             this.client = new Anthropic({
                 apiKey: this.config.apiKey,
                 baseURL: this.config.baseUrl,
+                ...(this.config.fetch ? { fetch: this.config.fetch } : {}),
             });
         }
         return this.client;

package/dist-cjs/providers/cohere.js

@@ -17,7 +17,7 @@ class CohereProvider extends base_js_1.BaseProvider {
         try {
             this.throwIfAborted(request);
             const startTime = Date.now();
-            const response = await fetch(`${this.baseUrl()}/v2/chat`, {
+            const response = await (this.config.fetch ?? fetch)(`${this.baseUrl()}/v2/chat`, {
                 method: 'POST',
                 headers: {
                     Authorization: `Bearer ${this.config.apiKey}`,
@@ -81,7 +81,7 @@ class CohereProvider extends base_js_1.BaseProvider {
         }, request.signal);
     }
     async healthCheck() {
-        const response = await fetch(`${this.baseUrl()}/v2/models`, {
+        const response = await (this.config.fetch ?? fetch)(`${this.baseUrl()}/v2/models`, {
             headers: { Authorization: `Bearer ${this.config.apiKey}` },
         });
         return response.ok;

package/dist-cjs/providers/google.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GoogleProvider = void 0;
+const base64_js_1 = require("../utils/base64.js");
 const base_js_1 = require("./base.js");
 const usage_js_1 = require("../core/usage.js");
 const ids_js_1 = require("../utils/ids.js");
@@ -122,7 +123,7 @@ class GoogleProvider extends base_js_1.BaseProvider {
         const endpoint = stream ? 'streamGenerateContent' : 'generateContent';
         const url = `${this.baseUrl()}/models/${encodeURIComponent(model)}:${endpoint}?key=${encodeURIComponent(this.config.apiKey)}${stream ? '&alt=sse' : ''}`;
         const body = JSON.stringify(this.createBody(request));
-        const response = await fetch(url, {
+        const response = await (this.config.fetch ?? fetch)(url, {
             method: 'POST',
             headers: { 'content-type': 'application/json' },
             body,
@@ -277,7 +278,7 @@ class GoogleProvider extends base_js_1.BaseProvider {
         return (content?.parts || [])
             .filter((part) => part.inlineData?.data && !part.thought)
             .map((part) => ({
-            location: { kind: 'bytes', data: decodeBase64(part.inlineData?.data ?? '') },
+            location: { kind: 'bytes', data: (0, base64_js_1.decodeBase64)(part.inlineData?.data ?? '') },
             mimeType: part.inlineData?.mimeType || 'application/octet-stream',
         }));
     }
@@ -314,13 +315,3 @@ class GoogleProvider extends base_js_1.BaseProvider {
     }
 }
 exports.GoogleProvider = GoogleProvider;
-function decodeBase64(data) {
-    const buffer = globalThis.Buffer;
-    if (buffer)
-        return new Uint8Array(buffer.from(data, 'base64'));
-    const binary = atob(data);
-    const bytes = new Uint8Array(binary.length);
-    for (let index = 0; index < binary.length; index += 1)
-        bytes[index] = binary.charCodeAt(index);
-    return bytes;
-}

package/dist-cjs/providers/openai.js

@@ -61,6 +61,7 @@ class OpenAIProvider extends base_js_1.BaseProvider {
                 organization: this.config.organization,
                 defaultHeaders: this.config.defaultHeaders,
                 defaultQuery: this.config.defaultQuery,
+                ...(this.config.fetch ? { fetch: this.config.fetch } : {}),
             });
         }
         return this.client;

package/dist-cjs/rag/elasticsearch.d.ts

@@ -0,0 +1,71 @@
+import type { RagChunk } from '../hallucination/rag.js';
+import type { VectorSearchResult } from '../hallucination/retrieval.js';
+import type { RetrieveOptions, SparseRetriever } from './retrievers.js';
+/** Options for the Elasticsearch keyword index. */
+export interface ElasticsearchKeywordIndexOptions {
+    /** The cluster, such as `http://localhost:9200`, an Elastic Cloud endpoint, or an OpenSearch domain. */
+    url: string;
+    /** The index chunks are kept in. */
+    index: string;
+    /** Sent as `Authorization: ApiKey <key>`. */
+    apiKey?: string;
+    /** Sent as basic authentication, for a cluster with users instead of API keys. */
+    username?: string;
+    /** The password for `username`. */
+    password?: string;
+    /** The analyzer `migrate()` gives the `content` field, such as `english`. Defaults to the cluster's standard one. */
+    analyzer?: string;
+    /**
+     * When a write becomes searchable. `wait_for` (the default) returns once a search would see it, as
+     * the vector stores do; `false` returns sooner and leaves it to the index's refresh interval.
+     */
+    refresh?: 'wait_for' | boolean;
+    /** Replaces the global `fetch`, for a proxy, retries, signed requests, or tests. */
+    fetch?: typeof globalThis.fetch;
+    /** Headers added to every request. */
+    headers?: Record<string, string>;
+    /** Aborts a request that takes longer, in milliseconds. Defaults to 30 seconds. */
+    timeoutMs?: number;
+}
+/** Raised when the cluster answers with an error. */
+export declare class ElasticsearchError extends Error {
+    /** The HTTP status the cluster answered with. */
+    readonly status: number;
+    /** The cluster's response body, as text. */
+    readonly body?: string | undefined;
+    constructor(message: string, 
+    /** The HTTP status the cluster answered with. */
+    status: number, 
+    /** The cluster's response body, as text. */
+    body?: string | undefined);
+}
+/**
+ * Keyword search in Elasticsearch or OpenSearch, through their REST API.
+ *
+ * No client library is a dependency: the index speaks HTTP through `fetch`, as the vector stores do,
+ * so it runs wherever `fetch` does. Scores are the engine's own BM25, ranked in the cluster, so only
+ * the best matches come back. A `SparseRetriever`, so `hybridRetriever()` fuses it with any vector
+ * store. A metadata filter is an exact match on top-level fields, as in the vector stores.
+ */
+export declare class ElasticsearchKeywordIndex implements SparseRetriever {
+    private readonly options;
+    private readonly base;
+    private readonly fetchImpl;
+    constructor(options: ElasticsearchKeywordIndexOptions);
+    /**
+     * Creates the index unless it exists: `content` as analyzed text, `id` and `source` as keywords,
+     * and every string under `metadata` as a keyword, so a filter matches it exactly. Never runs
+     * implicitly.
+     */
+    migrate(): Promise<void>;
+    /** Adds chunks, or replaces those whose id exists, in one bulk request. */
+    add(chunks: readonly RagChunk[]): Promise<void>;
+    /** Removes chunks by id. An id that is not held is ignored. */
+    delete(ids: readonly string[]): Promise<void>;
+    /** The chunks that best match a query's terms, best first, scored by the engine's BM25. */
+    retrieve(query: string, options?: RetrieveOptions): Promise<VectorSearchResult[]>;
+    /** Chunks held. */
+    size(): Promise<number>;
+    private bulk;
+    private request;
+}

package/dist-cjs/rag/elasticsearch.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ElasticsearchKeywordIndex = exports.ElasticsearchError = void 0;
+const signals_js_1 = require("../utils/signals.js");
+class ElasticsearchError extends Error {
+    status;
+    body;
+    constructor(message, status, body) {
+        super(message);
+        this.status = status;
+        this.body = body;
+        this.name = 'ElasticsearchError';
+    }
+}
+exports.ElasticsearchError = ElasticsearchError;
+class ElasticsearchKeywordIndex {
+    options;
+    base;
+    fetchImpl;
+    constructor(options) {
+        this.options = options;
+        if (!/^[a-z0-9][a-z0-9._-]*$/.test(options.index)) {
+            throw new RangeError(`"${options.index}" is not a valid index name: use lowercase letters, digits, ".", "_", and "-"`);
+        }
+        this.base = options.url.replace(/\/+$/, '');
+        this.fetchImpl = options.fetch ?? globalThis.fetch;
+    }
+    async migrate() {
+        const existing = await this.request('HEAD', `/${this.options.index}`, undefined, { allowNotFound: true });
+        if (existing !== undefined)
+            return;
+        await this.request('PUT', `/${this.options.index}`, {
+            mappings: {
+                dynamic_templates: [
+                    {
+                        metadata_strings: { path_match: 'metadata.*', match_mapping_type: 'string', mapping: { type: 'keyword' } },
+                    },
+                ],
+                properties: {
+                    id: { type: 'keyword' },
+                    content: { type: 'text', ...(this.options.analyzer ? { analyzer: this.options.analyzer } : {}) },
+                    source: { type: 'keyword' },
+                    metadata: { type: 'object', dynamic: true },
+                },
+            },
+        });
+    }
+    async add(chunks) {
+        if (chunks.length === 0)
+            return;
+        const lines = chunks.flatMap((chunk) => [
+            { index: { _index: this.options.index, _id: chunk.id } },
+            { id: chunk.id, content: chunk.content, source: chunk.source ?? null, metadata: chunk.metadata ?? null },
+        ]);
+        await this.bulk(lines);
+    }
+    async delete(ids) {
+        if (ids.length === 0)
+            return;
+        await this.bulk(ids.map((id) => ({ delete: { _index: this.options.index, _id: id } })));
+    }
+    async retrieve(query, options = {}) {
+        const filter = Object.entries(options.filter ?? {});
+        const clauses = [];
+        for (const [key, value] of filter) {
+            if (value === null)
+                clauses.push({ bool: { must_not: { exists: { field: `metadata.${key}` } } } });
+            else if (['string', 'number', 'boolean'].includes(typeof value))
+                clauses.push({ term: { [`metadata.${key}`]: value } });
+            else
+                return [];
+        }
+        const result = (await this.request('POST', `/${this.options.index}/_search`, {
+            size: options.topK ?? 5,
+            query: { bool: { must: [{ match: { content: { query, operator: 'or' } } }], filter: clauses } },
+            _source: ['id', 'content', 'source', 'metadata'],
+        }, { signal: options.signal }));
+        return (result.hits?.hits ?? []).map((hit) => ({
+            id: hit._source?.id ?? hit._id,
+            content: hit._source?.content ?? '',
+            ...(hit._source?.source ? { source: hit._source.source } : {}),
+            ...(hit._source?.metadata ? { metadata: hit._source.metadata } : {}),
+            score: hit._score ?? 0,
+        }));
+    }
+    async size() {
+        const result = (await this.request('GET', `/${this.options.index}/_count`));
+        return result.count ?? 0;
+    }
+    async bulk(lines) {
+        const refresh = this.options.refresh ?? 'wait_for';
+        const result = (await this.request('POST', `/_bulk${refresh === false ? '' : `?refresh=${refresh === true ? 'true' : 'wait_for'}`}`, `${lines.map((line) => JSON.stringify(line)).join('\n')}\n`));
+        if (!result.errors)
+            return;
+        const failed = (result.items ?? [])
+            .map((item) => Object.entries(item)[0])
+            .map((entry) => (entry?.[1]?.error && !(entry[0] === 'delete' && entry[1].status === 404) ? entry[1] : undefined))
+            .find((outcome) => outcome !== undefined);
+        if (failed)
+            throw new ElasticsearchError(`A bulk write failed: ${failed.error?.reason ?? 'unknown reason'}`, failed.status ?? 500);
+    }
+    async request(method, path, body, options = {}) {
+        const timeout = AbortSignal.timeout(this.options.timeoutMs ?? 30_000);
+        const linked = (0, signals_js_1.linkSignals)([timeout, options.signal]);
+        const ndjson = typeof body === 'string';
+        const authorization = this.options.apiKey
+            ? `ApiKey ${this.options.apiKey}`
+            : this.options.username !== undefined
+                ? `Basic ${btoa(`${this.options.username}:${this.options.password ?? ''}`)}`
+                : undefined;
+        try {
+            const response = await this.fetchImpl(`${this.base}${path}`, {
+                method,
+                headers: {
+                    ...(body === undefined ? {} : { 'content-type': ndjson ? 'application/x-ndjson' : 'application/json' }),
+                    ...(authorization ? { authorization } : {}),
+                    ...this.options.headers,
+                },
+                ...(body === undefined ? {} : { body: ndjson ? body : JSON.stringify(body) }),
+                signal: linked.signal,
+            });
+            if (response.status === 404 && options.allowNotFound)
+                return undefined;
+            const text = method === 'HEAD' ? '' : await response.text();
+            if (!response.ok) {
+                throw new ElasticsearchError(`Elasticsearch ${method} ${path} failed with ${response.status}`, response.status, text);
+            }
+            return text ? JSON.parse(text) : {};
+        }
+        finally {
+            linked.dispose();
+        }
+    }
+}
+exports.ElasticsearchKeywordIndex = ElasticsearchKeywordIndex;

package/dist-cjs/rag/file-ingestion.js

@@ -15,19 +15,30 @@ async function ingestFilesAfterScan(files, options = {}) {
     for (const file of files) {
         const extractor = options.extractors?.find((item) => item.supports(file));
         if (extractor) {
+            const text = await extractor.extract(file);
+            if (options.scan?.scanTextContent !== false) {
+                const extracted = (0, upload_scanner_js_1.scanUploads)([{ name: file.name, mimeType: 'text/plain', content: text }], {
+                    ...(options.scan?.forbiddenPatterns ? { forbiddenPatterns: options.scan.forbiddenPatterns } : {}),
+                    blockedExtensions: [],
+                });
+                if (!extracted.ok) {
+                    throw new Error(`Upload scan failed on extracted text: ${extracted.findings.map((finding) => `${finding.fileName}: ${finding.message}`).join('; ')}`);
+                }
+            }
             documents.push({
                 id: file.name,
                 source: file.name,
-                text: await extractor.extract(file),
+                text,
                 metadata: { mimeType: file.mimeType, sizeBytes: file.sizeBytes },
             });
             continue;
         }
-        if (typeof file.content === 'string' && isTextMime(file.mimeType)) {
+        const text = (0, upload_scanner_js_1.isTextual)(file.mimeType) ? (0, upload_scanner_js_1.textOf)(file.content, file.mimeType) : undefined;
+        if (text !== undefined) {
             documents.push({
                 id: file.name,
                 source: file.name,
-                text: file.content,
+                text,
                 metadata: { mimeType: file.mimeType, sizeBytes: file.sizeBytes },
             });
             continue;
@@ -53,8 +64,3 @@ function createOcrExtractor(ocrImage, mimeTypes = ['image/png', 'image/jpeg', 'i
         extract: ocrImage,
     };
 }
-function isTextMime(mimeType) {
-    if (!mimeType)
-        return true;
-    return (mimeType.startsWith('text/') || ['application/json', 'application/xml', 'application/markdown'].includes(mimeType));
-}

package/dist-cjs/rag/pipeline.d.ts

@@ -0,0 +1,140 @@
+/**
+ * Durable ingestion: keeps a vector store, and any keyword indexes beside it, in step with a corpus
+ * that changes.
+ *
+ * A manifest in any `Store` records, for every document, the version of its content, the chunking,
+ * the embedding model, and each chunk's version. A run reads the corpus, embeds only the chunks that
+ * are new or changed, deletes the chunks a document no longer has and the documents the corpus no
+ * longer has, and writes every change idempotently, before the manifest records it. A run that dies
+ * halfway leaves a manifest that says exactly what finished; the next run skips it, and inside a
+ * durable operation it starts at the document it stopped on.
+ */
+import type { EmbeddingProvider, VectorStore } from '../hallucination/retrieval.js';
+import type { OperationContext, OperationExecutor } from '../types/operations.js';
+import type { Store } from '../types/store.js';
+import { type DocumentSource, type IngestionOptions } from './ingestion.js';
+import type { SparseRetriever } from './retrievers.js';
+/** A source of documents: a loader, an array, or any iterable, read once per run. */
+export type IngestionSource = AsyncIterable<DocumentSource> | Iterable<DocumentSource>;
+/** Options for `createIngestionPipeline()`. */
+export interface IngestionPipelineOptions {
+    /** Names the pipeline. Its manifest is kept under this name, so two pipelines never share one. */
+    name: string;
+    /**
+     * Where the manifest is kept: any `Store`, so it lives in memory, Redis, Postgres, or SQLite. It
+     * holds one item per document, and a `MemoryStore` keeps 10,000 items unless given a larger
+     * `maxItems`: a manifest that forgets a document re-embeds it.
+     */
+    manifest: Store;
+    /** The vector store chunks are embedded into. */
+    vectors: VectorStore;
+    /**
+     * Keyword indexes kept in step, such as an `ElasticsearchKeywordIndex`. A `PostgresKeywordIndex`
+     * that is `shared` needs no place here: the vector store's writes already reach it.
+     */
+    keywords?: readonly SparseRetriever[];
+    /** Embeds chunk texts. Called only for chunks that are new or changed. */
+    embed: EmbeddingProvider;
+    /** The embedding model, recorded with every document: a different one re-embeds everything. */
+    embeddingModel: string;
+    /** How documents are split. Different options re-chunk every document. */
+    chunking?: IngestionOptions;
+    /** Documents read, embedded, and recorded together. Defaults to 64. */
+    batchSize?: number;
+}
+/** Options for one run. */
+export interface IngestionRunOptions {
+    /** Stops the run between batches. Everything recorded so far stays recorded. */
+    signal?: AbortSignal;
+    /**
+     * Removes the documents the manifest has and this run did not see, with their chunks. Defaults to
+     * true: the source is the whole corpus. Turn it off when a run loads only part of it.
+     */
+    deleteMissing?: boolean;
+    /** Called after each batch, with the run so far. */
+    onProgress?: (progress: IngestionReport) => void;
+    /**
+     * The durable operation the run belongs to, when it runs inside an `OperationRunner`. After every
+     * batch the run records how far it got with `heartbeat()`, and a later attempt reads it from
+     * `previousHeartbeat` and skips straight past what was finished.
+     */
+    operation?: Pick<OperationContext, 'heartbeat' | 'previousHeartbeat'>;
+}
+/** What a run did. */
+export interface IngestionReport {
+    /** Documents read. */
+    documents: number;
+    /** Documents whose content, chunking, and model were all as recorded, so nothing was done. */
+    unchanged: number;
+    /** Documents the manifest did not have. */
+    added: number;
+    /** Documents that changed. */
+    updated: number;
+    /** Documents removed because the corpus no longer has them. */
+    deleted: number;
+    /** Chunks embedded and written. */
+    chunksEmbedded: number;
+    /** Chunks of changed documents whose text did not change, so their embedding was kept. */
+    chunksKept: number;
+    /** Chunks removed. */
+    chunksDeleted: number;
+    /** Documents a resumed run skipped because an earlier attempt had finished them. */
+    resumed: number;
+}
+/** What the manifest records for one document. */
+export interface IngestedDocument {
+    /** The document's id. */
+    id: string;
+    /** A hash of its text, source, and metadata. */
+    version: string;
+    /** A hash of the chunking options it was split with. */
+    chunking: string;
+    /** The embedding model its chunks were embedded with. */
+    model: string;
+    /** Each chunk's id, with the hash of its text: the chunk's version. */
+    chunks: Record<string, string>;
+    /** When it was last written, as ISO-8601. */
+    updatedAt: string;
+}
+/** A pipeline: run it as often as the corpus changes. */
+export interface IngestionPipeline {
+    /** Brings the stores in step with the documents a source yields. */
+    run(source: IngestionSource | readonly IngestionSource[], options?: IngestionRunOptions): Promise<IngestionReport>;
+    /** What the manifest records for a document, or `undefined`. */
+    document(id: string): Promise<IngestedDocument | undefined>;
+    /** Removes documents by id, with their chunks, from every store and the manifest. */
+    remove(ids: readonly string[]): Promise<number>;
+}
+/**
+ * Builds a pipeline that keeps the stores in step with a corpus, re-embedding only what changed.
+ *
+ * Documents need an `id` (or a `source`), which is what ties a document to its earlier version. A
+ * chunk's id is its document's id and a hash of its text, so a chunk whose text did not change keeps
+ * its embedding wherever it moved. Chunks are cut by length, within each Markdown section when
+ * `splitOnMarkdownHeadings` is on, so an edit re-embeds the chunks of its section from the edit on,
+ * and a section added or removed leaves the others alone. One run at a time per pipeline; an
+ * operation's idempotency key enforces that across workers.
+ *
+ * ```ts
+ * const pipeline = createIngestionPipeline({
+ *   name: 'help-center',
+ *   manifest: store,
+ *   vectors: new PostgresVectorStore(pool, { dimensions: 1536 }),
+ *   embed,
+ *   embeddingModel: 'text-embedding-3-small',
+ * });
+ * const report = await pipeline.run(loadDirectory('./docs'));
+ * ```
+ */
+export declare function createIngestionPipeline(options: IngestionPipelineOptions): IngestionPipeline;
+/**
+ * The pipeline as the executor of a durable operation: submit it to an `OperationRunner`, and an
+ * attempt that dies halfway is recovered by another worker, which starts at the document the first
+ * one stopped on.
+ *
+ * ```ts
+ * const runner = new OperationRunner({ store: new RedisOperationStore(redis), retry: { maxAttempts: 5 } });
+ * await runner.submit(ingestionExecutor(pipeline, () => loadDirectory('./docs')), { idempotencyKey: 'help-center' });
+ * ```
+ */
+export declare function ingestionExecutor(pipeline: IngestionPipeline, source: () => IngestionSource | readonly IngestionSource[], options?: Omit<IngestionRunOptions, 'operation' | 'signal'>): OperationExecutor<IngestionReport>;