nexus-ai-pro 2.1.0 → 2.3.0

package/dist-cjs/postgres/vectors.d.ts

@@ -1,5 +1,7 @@
 import { type EmbeddingProvider, type VectorDocument, type VectorSearchOptions, type VectorSearchResult, type VectorStore } from '../hallucination/retrieval.js';
+import type { MigrationResult, SchemaMigration } from '../utils/schema-migrations.js';
 import { type PostgresLikeClient } from './client.js';
+import type { PostgresMigrateOptions } from './migrations.js';
 /** Options for the pgvector store. */
 export interface PostgresVectorStoreOptions {
     /** Width of every vector. Must match the embedding function's output. */
@@ -15,6 +17,10 @@ export interface PostgresVectorStoreOptions {
      */
     index?: 'hnsw' | 'none';
 }
+/**
+ * The versioned schema, which `migrate()` and `nexus db migrate` apply. Version 1 is the 2.0 schema.
+ */
+export declare function vectorStoreMigrations(options: Omit<PostgresVectorStoreOptions, 'embed'>): SchemaMigration[];
 /** The schema, as statements: the pgvector extension, the table, and its index. */
 export declare function vectorStoreMigration(options: Omit<PostgresVectorStoreOptions, 'embed'>): string[];
 /**
@@ -30,8 +36,11 @@ export declare class PostgresVectorStore implements VectorStore {
     private readonly table;
     private readonly embed;
     constructor(client: PostgresLikeClient, options: PostgresVectorStoreOptions);
-    /** Creates the extension, table, and index. Never runs implicitly. */
-    migrate(): Promise<void>;
+    /**
+     * Applies this store's pending migrations, recorded in `nexus_schema_migrations`, under a lock.
+     * Never runs implicitly. Safe on a database an older release created: what exists is kept.
+     */
+    migrate(options?: PostgresMigrateOptions): Promise<MigrationResult>;
     /** Adds chunks, or replaces those whose id exists, embedding those without a vector in one batch. */
     add(documents: VectorDocument[]): Promise<void>;
     /** The chunks most similar to a query, best first. */

package/dist-cjs/postgres/vectors.js

@@ -1,11 +1,55 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PostgresVectorStore = void 0;
+exports.vectorStoreMigrations = vectorStoreMigrations;
 exports.vectorStoreMigration = vectorStoreMigration;
 const retrieval_js_1 = require("../hallucination/retrieval.js");
 const client_js_1 = require("./client.js");
 const sql_js_1 = require("./sql.js");
 const DEFAULT_TABLE = 'nexus_vectors';
+function vectorStoreMigrations(options) {
+    return [
+        {
+            component: `vectors:${options.table ?? DEFAULT_TABLE}`,
+            version: 1,
+            name: 'create the vectors table',
+            statements: vectorStoreMigration(options),
+        },
+    ];
+}
 function vectorStoreMigration(options) {
     const dimensions = options.dimensions;
     if (!(Number.isInteger(dimensions) && dimensions > 0))
@@ -42,8 +86,9 @@ class PostgresVectorStore {
         this.table = (0, client_js_1.quoteTable)(options.table ?? DEFAULT_TABLE);
         this.embed = options.embed ?? ((texts) => (0, retrieval_js_1.createHashEmbeddings)(texts, options.dimensions));
     }
-    async migrate() {
-        await (0, client_js_1.runStatements)(this.client, vectorStoreMigration(this.options));
+    async migrate(options = {}) {
+        const { applyPostgresMigrations } = await Promise.resolve().then(() => __importStar(require('./migrations.js')));
+        return applyPostgresMigrations(this.client, vectorStoreMigrations(this.options), options);
     }
     async add(documents) {
         if (documents.length === 0)

package/dist-cjs/rag/elasticsearch.d.ts

@@ -0,0 +1,71 @@
+import type { RagChunk } from '../hallucination/rag.js';
+import type { VectorSearchResult } from '../hallucination/retrieval.js';
+import type { RetrieveOptions, SparseRetriever } from './retrievers.js';
+/** Options for the Elasticsearch keyword index. */
+export interface ElasticsearchKeywordIndexOptions {
+    /** The cluster, such as `http://localhost:9200`, an Elastic Cloud endpoint, or an OpenSearch domain. */
+    url: string;
+    /** The index chunks are kept in. */
+    index: string;
+    /** Sent as `Authorization: ApiKey <key>`. */
+    apiKey?: string;
+    /** Sent as basic authentication, for a cluster with users instead of API keys. */
+    username?: string;
+    /** The password for `username`. */
+    password?: string;
+    /** The analyzer `migrate()` gives the `content` field, such as `english`. Defaults to the cluster's standard one. */
+    analyzer?: string;
+    /**
+     * When a write becomes searchable. `wait_for` (the default) returns once a search would see it, as
+     * the vector stores do; `false` returns sooner and leaves it to the index's refresh interval.
+     */
+    refresh?: 'wait_for' | boolean;
+    /** Replaces the global `fetch`, for a proxy, retries, signed requests, or tests. */
+    fetch?: typeof globalThis.fetch;
+    /** Headers added to every request. */
+    headers?: Record<string, string>;
+    /** Aborts a request that takes longer, in milliseconds. Defaults to 30 seconds. */
+    timeoutMs?: number;
+}
+/** Raised when the cluster answers with an error. */
+export declare class ElasticsearchError extends Error {
+    /** The HTTP status the cluster answered with. */
+    readonly status: number;
+    /** The cluster's response body, as text. */
+    readonly body?: string | undefined;
+    constructor(message: string, 
+    /** The HTTP status the cluster answered with. */
+    status: number, 
+    /** The cluster's response body, as text. */
+    body?: string | undefined);
+}
+/**
+ * Keyword search in Elasticsearch or OpenSearch, through their REST API.
+ *
+ * No client library is a dependency: the index speaks HTTP through `fetch`, as the vector stores do,
+ * so it runs wherever `fetch` does. Scores are the engine's own BM25, ranked in the cluster, so only
+ * the best matches come back. A `SparseRetriever`, so `hybridRetriever()` fuses it with any vector
+ * store. A metadata filter is an exact match on top-level fields, as in the vector stores.
+ */
+export declare class ElasticsearchKeywordIndex implements SparseRetriever {
+    private readonly options;
+    private readonly base;
+    private readonly fetchImpl;
+    constructor(options: ElasticsearchKeywordIndexOptions);
+    /**
+     * Creates the index unless it exists: `content` as analyzed text, `id` and `source` as keywords,
+     * and every string under `metadata` as a keyword, so a filter matches it exactly. Never runs
+     * implicitly.
+     */
+    migrate(): Promise<void>;
+    /** Adds chunks, or replaces those whose id exists, in one bulk request. */
+    add(chunks: readonly RagChunk[]): Promise<void>;
+    /** Removes chunks by id. An id that is not held is ignored. */
+    delete(ids: readonly string[]): Promise<void>;
+    /** The chunks that best match a query's terms, best first, scored by the engine's BM25. */
+    retrieve(query: string, options?: RetrieveOptions): Promise<VectorSearchResult[]>;
+    /** Chunks held. */
+    size(): Promise<number>;
+    private bulk;
+    private request;
+}

package/dist-cjs/rag/elasticsearch.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ElasticsearchKeywordIndex = exports.ElasticsearchError = void 0;
+const signals_js_1 = require("../utils/signals.js");
+class ElasticsearchError extends Error {
+    status;
+    body;
+    constructor(message, status, body) {
+        super(message);
+        this.status = status;
+        this.body = body;
+        this.name = 'ElasticsearchError';
+    }
+}
+exports.ElasticsearchError = ElasticsearchError;
+class ElasticsearchKeywordIndex {
+    options;
+    base;
+    fetchImpl;
+    constructor(options) {
+        this.options = options;
+        if (!/^[a-z0-9][a-z0-9._-]*$/.test(options.index)) {
+            throw new RangeError(`"${options.index}" is not a valid index name: use lowercase letters, digits, ".", "_", and "-"`);
+        }
+        this.base = options.url.replace(/\/+$/, '');
+        this.fetchImpl = options.fetch ?? globalThis.fetch;
+    }
+    async migrate() {
+        const existing = await this.request('HEAD', `/${this.options.index}`, undefined, { allowNotFound: true });
+        if (existing !== undefined)
+            return;
+        await this.request('PUT', `/${this.options.index}`, {
+            mappings: {
+                dynamic_templates: [
+                    {
+                        metadata_strings: { path_match: 'metadata.*', match_mapping_type: 'string', mapping: { type: 'keyword' } },
+                    },
+                ],
+                properties: {
+                    id: { type: 'keyword' },
+                    content: { type: 'text', ...(this.options.analyzer ? { analyzer: this.options.analyzer } : {}) },
+                    source: { type: 'keyword' },
+                    metadata: { type: 'object', dynamic: true },
+                },
+            },
+        });
+    }
+    async add(chunks) {
+        if (chunks.length === 0)
+            return;
+        const lines = chunks.flatMap((chunk) => [
+            { index: { _index: this.options.index, _id: chunk.id } },
+            { id: chunk.id, content: chunk.content, source: chunk.source ?? null, metadata: chunk.metadata ?? null },
+        ]);
+        await this.bulk(lines);
+    }
+    async delete(ids) {
+        if (ids.length === 0)
+            return;
+        await this.bulk(ids.map((id) => ({ delete: { _index: this.options.index, _id: id } })));
+    }
+    async retrieve(query, options = {}) {
+        const filter = Object.entries(options.filter ?? {});
+        const clauses = [];
+        for (const [key, value] of filter) {
+            if (value === null)
+                clauses.push({ bool: { must_not: { exists: { field: `metadata.${key}` } } } });
+            else if (['string', 'number', 'boolean'].includes(typeof value))
+                clauses.push({ term: { [`metadata.${key}`]: value } });
+            else
+                return [];
+        }
+        const result = (await this.request('POST', `/${this.options.index}/_search`, {
+            size: options.topK ?? 5,
+            query: { bool: { must: [{ match: { content: { query, operator: 'or' } } }], filter: clauses } },
+            _source: ['id', 'content', 'source', 'metadata'],
+        }, { signal: options.signal }));
+        return (result.hits?.hits ?? []).map((hit) => ({
+            id: hit._source?.id ?? hit._id,
+            content: hit._source?.content ?? '',
+            ...(hit._source?.source ? { source: hit._source.source } : {}),
+            ...(hit._source?.metadata ? { metadata: hit._source.metadata } : {}),
+            score: hit._score ?? 0,
+        }));
+    }
+    async size() {
+        const result = (await this.request('GET', `/${this.options.index}/_count`));
+        return result.count ?? 0;
+    }
+    async bulk(lines) {
+        const refresh = this.options.refresh ?? 'wait_for';
+        const result = (await this.request('POST', `/_bulk${refresh === false ? '' : `?refresh=${refresh === true ? 'true' : 'wait_for'}`}`, `${lines.map((line) => JSON.stringify(line)).join('\n')}\n`));
+        if (!result.errors)
+            return;
+        const failed = (result.items ?? [])
+            .map((item) => Object.entries(item)[0])
+            .map((entry) => (entry?.[1]?.error && !(entry[0] === 'delete' && entry[1].status === 404) ? entry[1] : undefined))
+            .find((outcome) => outcome !== undefined);
+        if (failed)
+            throw new ElasticsearchError(`A bulk write failed: ${failed.error?.reason ?? 'unknown reason'}`, failed.status ?? 500);
+    }
+    async request(method, path, body, options = {}) {
+        const timeout = AbortSignal.timeout(this.options.timeoutMs ?? 30_000);
+        const linked = (0, signals_js_1.linkSignals)([timeout, options.signal]);
+        const ndjson = typeof body === 'string';
+        const authorization = this.options.apiKey
+            ? `ApiKey ${this.options.apiKey}`
+            : this.options.username !== undefined
+                ? `Basic ${btoa(`${this.options.username}:${this.options.password ?? ''}`)}`
+                : undefined;
+        try {
+            const response = await this.fetchImpl(`${this.base}${path}`, {
+                method,
+                headers: {
+                    ...(body === undefined ? {} : { 'content-type': ndjson ? 'application/x-ndjson' : 'application/json' }),
+                    ...(authorization ? { authorization } : {}),
+                    ...this.options.headers,
+                },
+                ...(body === undefined ? {} : { body: ndjson ? body : JSON.stringify(body) }),
+                signal: linked.signal,
+            });
+            if (response.status === 404 && options.allowNotFound)
+                return undefined;
+            const text = method === 'HEAD' ? '' : await response.text();
+            if (!response.ok) {
+                throw new ElasticsearchError(`Elasticsearch ${method} ${path} failed with ${response.status}`, response.status, text);
+            }
+            return text ? JSON.parse(text) : {};
+        }
+        finally {
+            linked.dispose();
+        }
+    }
+}
+exports.ElasticsearchKeywordIndex = ElasticsearchKeywordIndex;

package/dist-cjs/rag/pipeline.d.ts

@@ -0,0 +1,140 @@
+/**
+ * Durable ingestion: keeps a vector store, and any keyword indexes beside it, in step with a corpus
+ * that changes.
+ *
+ * A manifest in any `Store` records, for every document, the version of its content, the chunking,
+ * the embedding model, and each chunk's version. A run reads the corpus, embeds only the chunks that
+ * are new or changed, deletes the chunks a document no longer has and the documents the corpus no
+ * longer has, and writes every change idempotently, before the manifest records it. A run that dies
+ * halfway leaves a manifest that says exactly what finished; the next run skips it, and inside a
+ * durable operation it starts at the document it stopped on.
+ */
+import type { EmbeddingProvider, VectorStore } from '../hallucination/retrieval.js';
+import type { OperationContext, OperationExecutor } from '../types/operations.js';
+import type { Store } from '../types/store.js';
+import { type DocumentSource, type IngestionOptions } from './ingestion.js';
+import type { SparseRetriever } from './retrievers.js';
+/** A source of documents: a loader, an array, or any iterable, read once per run. */
+export type IngestionSource = AsyncIterable<DocumentSource> | Iterable<DocumentSource>;
+/** Options for `createIngestionPipeline()`. */
+export interface IngestionPipelineOptions {
+    /** Names the pipeline. Its manifest is kept under this name, so two pipelines never share one. */
+    name: string;
+    /**
+     * Where the manifest is kept: any `Store`, so it lives in memory, Redis, Postgres, or SQLite. It
+     * holds one item per document, and a `MemoryStore` keeps 10,000 items unless given a larger
+     * `maxItems`: a manifest that forgets a document re-embeds it.
+     */
+    manifest: Store;
+    /** The vector store chunks are embedded into. */
+    vectors: VectorStore;
+    /**
+     * Keyword indexes kept in step, such as an `ElasticsearchKeywordIndex`. A `PostgresKeywordIndex`
+     * that is `shared` needs no place here: the vector store's writes already reach it.
+     */
+    keywords?: readonly SparseRetriever[];
+    /** Embeds chunk texts. Called only for chunks that are new or changed. */
+    embed: EmbeddingProvider;
+    /** The embedding model, recorded with every document: a different one re-embeds everything. */
+    embeddingModel: string;
+    /** How documents are split. Different options re-chunk every document. */
+    chunking?: IngestionOptions;
+    /** Documents read, embedded, and recorded together. Defaults to 64. */
+    batchSize?: number;
+}
+/** Options for one run. */
+export interface IngestionRunOptions {
+    /** Stops the run between batches. Everything recorded so far stays recorded. */
+    signal?: AbortSignal;
+    /**
+     * Removes the documents the manifest has and this run did not see, with their chunks. Defaults to
+     * true: the source is the whole corpus. Turn it off when a run loads only part of it.
+     */
+    deleteMissing?: boolean;
+    /** Called after each batch, with the run so far. */
+    onProgress?: (progress: IngestionReport) => void;
+    /**
+     * The durable operation the run belongs to, when it runs inside an `OperationRunner`. After every
+     * batch the run records how far it got with `heartbeat()`, and a later attempt reads it from
+     * `previousHeartbeat` and skips straight past what was finished.
+     */
+    operation?: Pick<OperationContext, 'heartbeat' | 'previousHeartbeat'>;
+}
+/** What a run did. */
+export interface IngestionReport {
+    /** Documents read. */
+    documents: number;
+    /** Documents whose content, chunking, and model were all as recorded, so nothing was done. */
+    unchanged: number;
+    /** Documents the manifest did not have. */
+    added: number;
+    /** Documents that changed. */
+    updated: number;
+    /** Documents removed because the corpus no longer has them. */
+    deleted: number;
+    /** Chunks embedded and written. */
+    chunksEmbedded: number;
+    /** Chunks of changed documents whose text did not change, so their embedding was kept. */
+    chunksKept: number;
+    /** Chunks removed. */
+    chunksDeleted: number;
+    /** Documents a resumed run skipped because an earlier attempt had finished them. */
+    resumed: number;
+}
+/** What the manifest records for one document. */
+export interface IngestedDocument {
+    /** The document's id. */
+    id: string;
+    /** A hash of its text, source, and metadata. */
+    version: string;
+    /** A hash of the chunking options it was split with. */
+    chunking: string;
+    /** The embedding model its chunks were embedded with. */
+    model: string;
+    /** Each chunk's id, with the hash of its text: the chunk's version. */
+    chunks: Record<string, string>;
+    /** When it was last written, as ISO-8601. */
+    updatedAt: string;
+}
+/** A pipeline: run it as often as the corpus changes. */
+export interface IngestionPipeline {
+    /** Brings the stores in step with the documents a source yields. */
+    run(source: IngestionSource | readonly IngestionSource[], options?: IngestionRunOptions): Promise<IngestionReport>;
+    /** What the manifest records for a document, or `undefined`. */
+    document(id: string): Promise<IngestedDocument | undefined>;
+    /** Removes documents by id, with their chunks, from every store and the manifest. */
+    remove(ids: readonly string[]): Promise<number>;
+}
+/**
+ * Builds a pipeline that keeps the stores in step with a corpus, re-embedding only what changed.
+ *
+ * Documents need an `id` (or a `source`), which is what ties a document to its earlier version. A
+ * chunk's id is its document's id and a hash of its text, so a chunk whose text did not change keeps
+ * its embedding wherever it moved. Chunks are cut by length, within each Markdown section when
+ * `splitOnMarkdownHeadings` is on, so an edit re-embeds the chunks of its section from the edit on,
+ * and a section added or removed leaves the others alone. One run at a time per pipeline; an
+ * operation's idempotency key enforces that across workers.
+ *
+ * ```ts
+ * const pipeline = createIngestionPipeline({
+ *   name: 'help-center',
+ *   manifest: store,
+ *   vectors: new PostgresVectorStore(pool, { dimensions: 1536 }),
+ *   embed,
+ *   embeddingModel: 'text-embedding-3-small',
+ * });
+ * const report = await pipeline.run(loadDirectory('./docs'));
+ * ```
+ */
+export declare function createIngestionPipeline(options: IngestionPipelineOptions): IngestionPipeline;
+/**
+ * The pipeline as the executor of a durable operation: submit it to an `OperationRunner`, and an
+ * attempt that dies halfway is recovered by another worker, which starts at the document the first
+ * one stopped on.
+ *
+ * ```ts
+ * const runner = new OperationRunner({ store: new RedisOperationStore(redis), retry: { maxAttempts: 5 } });
+ * await runner.submit(ingestionExecutor(pipeline, () => loadDirectory('./docs')), { idempotencyKey: 'help-center' });
+ * ```
+ */
+export declare function ingestionExecutor(pipeline: IngestionPipeline, source: () => IngestionSource | readonly IngestionSource[], options?: Omit<IngestionRunOptions, 'operation' | 'signal'>): OperationExecutor<IngestionReport>;

package/dist-cjs/rag/pipeline.js

@@ -0,0 +1,191 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createIngestionPipeline = createIngestionPipeline;
+exports.ingestionExecutor = ingestionExecutor;
+const ingestion_js_1 = require("./ingestion.js");
+function createIngestionPipeline(options) {
+    const batchSize = options.batchSize ?? 64;
+    if (!(Number.isInteger(batchSize) && batchSize > 0))
+        throw new RangeError('batchSize must be a positive integer');
+    const namespace = ['nexus', 'ingestion', options.name, 'documents'];
+    const keywords = options.keywords ?? [];
+    const chunkingVersion = stableJson(options.chunking ?? {});
+    const read = async (id) => (await options.manifest.get(namespace, id))?.value;
+    const removeChunks = async (ids) => {
+        if (ids.length === 0)
+            return;
+        await Promise.all([options.vectors.delete(ids), ...keywords.map((index) => index.delete(ids))]);
+    };
+    const remove = async (ids) => {
+        let chunks = 0;
+        for (const id of ids) {
+            const recorded = await read(id);
+            if (!recorded)
+                continue;
+            const chunkIds = Object.keys(recorded.chunks);
+            await removeChunks(chunkIds);
+            await options.manifest.delete(namespace, id);
+            chunks += chunkIds.length;
+        }
+        return chunks;
+    };
+    return {
+        document: read,
+        async remove(ids) {
+            await remove(ids);
+            return ids.length;
+        },
+        async run(source, runOptions = {}) {
+            const report = {
+                documents: 0,
+                unchanged: 0,
+                added: 0,
+                updated: 0,
+                deleted: 0,
+                chunksEmbedded: 0,
+                chunksKept: 0,
+                chunksDeleted: 0,
+                resumed: 0,
+            };
+            const previous = runOptions.operation?.previousHeartbeat;
+            const skip = previous?.pipeline === options.name ? previous.documents : 0;
+            const seen = new Set();
+            let batch = [];
+            const flush = async () => {
+                if (batch.length === 0)
+                    return;
+                const current = batch;
+                batch = [];
+                const versions = await Promise.all(current.map(({ document }) => hashText(stableJson([document.text, document.source ?? null, document.metadata ?? null]))));
+                const recorded = await Promise.all(current.map(({ id }) => read(id)));
+                const writes = [];
+                const stale = [];
+                const toEmbed = [];
+                for (const [index, { document, id }] of current.entries()) {
+                    const version = versions[index];
+                    const before = recorded[index];
+                    if (before &&
+                        before.version === version &&
+                        before.chunking === chunkingVersion &&
+                        before.model === options.embeddingModel) {
+                        report.unchanged += 1;
+                        continue;
+                    }
+                    if (before)
+                        report.updated += 1;
+                    else
+                        report.added += 1;
+                    const split = (0, ingestion_js_1.ingestDocuments)([{ ...document, id }], options.chunking).chunks;
+                    const hashes = await Promise.all(split.map((chunk) => hashText(stableJson([chunk.content, document.metadata ?? null]))));
+                    const used = new Map();
+                    const chunks = split.map((chunk, position) => {
+                        const base = `${id}#${hashes[position].slice(0, 12)}`;
+                        const count = (used.get(base) ?? 0) + 1;
+                        used.set(base, count);
+                        return { ...chunk, id: count === 1 ? base : `${base}-${count}` };
+                    });
+                    const reusable = before && before.model === options.embeddingModel ? before.chunks : {};
+                    for (const [position, chunk] of chunks.entries()) {
+                        if (reusable[chunk.id] === hashes[position])
+                            report.chunksKept += 1;
+                        else
+                            toEmbed.push(chunk);
+                    }
+                    const kept = new Set(chunks.map((chunk) => chunk.id));
+                    if (before)
+                        stale.push(...Object.keys(before.chunks).filter((chunkId) => !kept.has(chunkId) || !reusable[chunkId]));
+                    writes.push({ id, version, chunks, hashes });
+                }
+                if (toEmbed.length > 0) {
+                    const vectors = await options.embed(toEmbed.map((chunk) => chunk.content));
+                    if (vectors.length !== toEmbed.length) {
+                        throw new RangeError(`The embedding function returned ${vectors.length} vectors for ${toEmbed.length} texts`);
+                    }
+                    await Promise.all([
+                        options.vectors.add(toEmbed.map((chunk, index) => ({ ...chunk, embedding: vectors[index] }))),
+                        ...keywords.map((index) => index.add(toEmbed)),
+                    ]);
+                    report.chunksEmbedded += toEmbed.length;
+                }
+                const embeddedIds = new Set(toEmbed.map((chunk) => chunk.id));
+                const removable = stale.filter((chunkId) => !embeddedIds.has(chunkId));
+                await removeChunks(removable);
+                report.chunksDeleted += removable.length;
+                const updatedAt = new Date().toISOString();
+                await Promise.all(writes.map(({ id, version, chunks, hashes }) => options.manifest.put(namespace, id, {
+                    id,
+                    version,
+                    chunking: chunkingVersion,
+                    model: options.embeddingModel,
+                    chunks: Object.fromEntries(chunks.map((chunk, index) => [chunk.id, hashes[index]])),
+                    updatedAt,
+                })));
+                await runOptions.operation?.heartbeat({
+                    pipeline: options.name,
+                    documents: report.documents,
+                });
+                runOptions.onProgress?.({ ...report });
+            };
+            const sources = isSourceList(source) ? source : [source];
+            for (const loader of sources) {
+                for await (const document of loader) {
+                    runOptions.signal?.throwIfAborted();
+                    const id = document.id ?? document.source;
+                    if (!id)
+                        throw new TypeError('Durable ingestion needs a document id, or a source to use as one');
+                    seen.add(id);
+                    report.documents += 1;
+                    if (report.documents <= skip) {
+                        report.resumed += 1;
+                        continue;
+                    }
+                    batch.push({ document, id });
+                    if (batch.length >= batchSize)
+                        await flush();
+                }
+            }
+            runOptions.signal?.throwIfAborted();
+            await flush();
+            if (runOptions.deleteMissing !== false) {
+                const missing = [];
+                for (let offset = 0;; offset += 500) {
+                    const page = await options.manifest.search(namespace, { limit: 500, offset });
+                    for (const item of page)
+                        if (!seen.has(item.key))
+                            missing.push(item.key);
+                    if (page.length < 500)
+                        break;
+                }
+                for (const id of missing) {
+                    runOptions.signal?.throwIfAborted();
+                    report.chunksDeleted += await remove([id]);
+                    report.deleted += 1;
+                }
+                if (missing.length > 0)
+                    runOptions.onProgress?.({ ...report });
+            }
+            return report;
+        },
+    };
+}
+function ingestionExecutor(pipeline, source, options = {}) {
+    return (context) => pipeline.run(source(), { ...options, signal: context.signal, operation: context });
+}
+async function hashText(text) {
+    const digest = await globalThis.crypto.subtle.digest('SHA-256', new TextEncoder().encode(text));
+    let hex = '';
+    for (const byte of new Uint8Array(digest).subarray(0, 16))
+        hex += byte.toString(16).padStart(2, '0');
+    return hex;
+}
+function stableJson(value) {
+    return JSON.stringify(value, (_key, entry) => entry && typeof entry === 'object' && !Array.isArray(entry)
+        ? Object.fromEntries(Object.entries(entry).sort(([a], [b]) => a.localeCompare(b)))
+        : entry);
+}
+function isSourceList(value) {
+    if (!Array.isArray(value))
+        return false;
+    const first = value[0];
+    return first !== undefined && typeof first === 'object' && first !== null && !('text' in first);
+}