nexus-ai-pro 2.2.0 → 2.3.0

package/dist/graph/functional.d.ts

@@ -1,4 +1,5 @@
 import type { OperationLifecycleLike } from '../types/lifecycle.js';
+import type { Principal } from '../types/principal.js';
 import { GraphError } from './errors.js';
 import type { GraphCheckpoint, GraphCheckpointer, GraphDescription, GraphEvent, GraphStatus, InterruptRequest, PendingInterrupt, RetryPolicy, RunControlLike } from '../types/graph.js';
 import type { Store } from '../types/store.js';
@@ -86,6 +87,8 @@ export interface WorkflowContext {
     readonly threadId: string;
     /** The tenant the run is for, when it has one. */
     readonly tenantId?: string;
+    /** Who the run is for, when it was started with a principal: the caller the server authenticated. */
+    readonly principal?: Readonly<Principal>;
     /** Aborted when the run is cancelled. */
     readonly signal: AbortSignal;
     /** Long-term memory, when the workflow was created with a store. In a tenant's run, the tenant's view of it. */
@@ -162,6 +165,12 @@ export interface WorkflowRunOptions {
      * without one carries on the thread's.
      */
     tenantId?: string;
+    /**
+     * Who the run is for, read as `context.principal`. Its `tenantId` is the run's tenant when
+     * `tenantId` is not given, and a run whose two tenants differ is refused; its `userId` is recorded
+     * on every checkpoint as `metadata.userId`.
+     */
+    principal?: Principal;
 }
 /** One event of a streamed workflow run. */
 export type WorkflowEvent<O = unknown> = {

package/dist/graph/functional.js

@@ -2,6 +2,7 @@ import { MemoryGraphCheckpointer } from './checkpointer.js';
 import { withLifecycle } from '../lifecycle/run.js';
 import { migrateCheckpoint, toCheckpoint } from './checkpoint-migration.js';
 import { tenantStore } from '../store/tenant.js';
+import { withPrincipal } from '../utils/tenant.js';
 import { GraphDrainedError, GraphError, GraphNotInterruptedError, GraphThreadNotFoundError } from './errors.js';
 export class WorkflowStepTimeoutError extends GraphError {
     stepName;
@@ -52,9 +53,10 @@ export class Workflow {
         const threadId = runOptions.threadId ?? generatedId();
         return this.execute(threadId, runOptions, async () => {
             const latest = await this.latest(threadId);
-            if (latest && runOptions.tenantId !== undefined) {
+            const tenantId = runOptions.tenantId ?? runOptions.principal?.tenantId;
+            if (latest && tenantId !== undefined) {
                 const owner = latest.metadata?.tenantId;
-                if (typeof owner === 'string' && owner !== runOptions.tenantId) {
+                if (typeof owner === 'string' && owner !== tenantId) {
                     throw new GraphError(`Thread "${threadId}" cannot be used by this tenant; choose another id`, 'GRAPH_VALIDATION_ERROR');
                 }
             }
@@ -148,7 +150,21 @@ export class Workflow {
     }
     execute(threadId, runOptions, start) {
         const queue = new EventQueue();
-        void withLifecycle(this.options.lifecycle, { family: 'graph', operation: `workflow.${this.name}`, metadata: { threadId, graph: this.name } }, runOptions.signal ? { signal: runOptions.signal } : {}, () => this.run(threadId, runOptions, start, queue)).then(() => queue.close(), (error) => queue.fail(error));
+        let options;
+        try {
+            options = withPrincipal(runOptions);
+        }
+        catch (error) {
+            queue.fail(error);
+            return queue;
+        }
+        const metadata = {
+            threadId,
+            graph: this.name,
+            ...(options.tenantId !== undefined ? { tenantId: options.tenantId } : {}),
+            ...(options.principal?.userId !== undefined ? { userId: options.principal.userId } : {}),
+        };
+        void withLifecycle(this.options.lifecycle, { family: 'graph', operation: `workflow.${this.name}`, metadata }, options.signal ? { signal: options.signal } : {}, () => this.run(threadId, options, start, queue)).then(() => queue.close(), (error) => queue.fail(error));
         return queue;
     }
     async run(threadId, runOptions, start, queue) {
@@ -203,6 +219,7 @@ export class Workflow {
         const context = {
             threadId,
             ...(tenantId === undefined ? {} : { tenantId }),
+            ...(runOptions.principal ? { principal: runOptions.principal } : {}),
             signal: controller.signal,
             ...(this.options.store
                 ? { store: tenantId === undefined ? this.options.store : tenantStore(this.options.store, tenantId) }

package/dist/graph/graph.js

@@ -1,6 +1,7 @@
 import { randomBytes } from 'node:crypto';
 import { Command, END, Send, START } from '../types/graph.js';
 import { tenantStore } from '../store/tenant.js';
+import { withPrincipal } from '../utils/tenant.js';
 import { linkSignals } from '../utils/signals.js';
 import { MemoryGraphCheckpointer } from './checkpointer.js';
 import { migrateCheckpoint, toCheckpoint } from './checkpoint-migration.js';
@@ -152,9 +153,10 @@ export class CompiledGraph {
         const threadId = resolveThreadId(runOptions.threadId);
         return this.run(threadId, runOptions, async () => {
             this.assertInput(input);
-            if (runOptions.tenantId !== undefined) {
+            const tenantId = runOptions.tenantId ?? runOptions.principal?.tenantId;
+            if (tenantId !== undefined) {
                 const existing = await this.checkpointer()?.get(threadId);
-                if (existing && !ownedBy(existing, runOptions.tenantId)) {
+                if (existing && !ownedBy(existing, tenantId)) {
                     throw new GraphValidationError(`Thread "${threadId}" cannot be used by this tenant; choose another id`);
                 }
             }
@@ -266,6 +268,8 @@ export class CompiledGraph {
             const runOptions = {
                 threadId: `${context.threadId}:${context.taskId}`,
                 signal: context.signal,
+                ...(context.tenantId !== undefined ? { tenantId: context.tenantId } : {}),
+                ...(context.principal ? { principal: context.principal } : {}),
                 ...(forward ? { onEvent: forward, subgraphEvents: true } : {}),
             };
             const paused = await this.state(runOptions.threadId);
@@ -309,7 +313,7 @@ export class CompiledGraph {
     describe() {
         const nodes = [...this.nodes.entries()].map(([id, definition]) => {
             const subgraph = definition.fn[SUBGRAPH];
-            const { ends, defer, timeoutMs, cache, idempotent } = definition.options;
+            const { ends, defer, timeoutMs, cache, idempotent, effects, interrupts } = definition.options;
             const maxAttempts = this.retryPolicy(definition).maxAttempts;
             const timeout = this.timeoutOf(definition);
             return {
@@ -322,6 +326,8 @@ export class CompiledGraph {
                 ...(this.errorHandler(definition) ? { onError: true } : {}),
                 ...(idempotent ? { idempotent: true } : {}),
                 ...(cache ? { cache: true } : {}),
+                ...(effects?.length ? { effects: [...effects] } : {}),
+                ...(interrupts ? { interrupts: true } : {}),
                 ...(subgraph ? { subgraph: subgraph.describe() } : {}),
             };
         });
@@ -358,6 +364,14 @@ export class CompiledGraph {
             durability: this.options.durability ?? 'sync',
             checkpointer: this.options.checkpointer === false ? 'none' : this.options.checkpointer === undefined ? 'memory' : 'custom',
             maxConcurrency: this.options.maxConcurrency ?? DEFAULT_MAX_CONCURRENCY,
+            ...(this.options.tools?.length
+                ? {
+                    tools: this.options.tools.map((tool) => ({
+                        ...tool,
+                        ...(tool.capabilities ? { capabilities: [...tool.capabilities] } : {}),
+                    })),
+                }
+                : {}),
         };
     }
     async updateState(threadId, update, options = {}) {
@@ -418,6 +432,7 @@ export class CompiledGraph {
         return forkId;
     }
     async *run(threadId, runOptions, seed) {
+        runOptions = withPrincipal(runOptions);
         const lifecycle = this.options.lifecycle;
         if (!lifecycle) {
             yield* this.runSteps(threadId, runOptions, seed);
@@ -426,7 +441,12 @@ export class CompiledGraph {
         const ticket = await lifecycle.start({
             family: this.options.lifecycleFamily ?? 'graph',
             operation: this.options.name ? `${this.options.lifecycleFamily ?? 'graph'}.${this.options.name}` : 'graph.run',
-            metadata: { threadId, ...(this.options.name ? { graph: this.options.name } : {}) },
+            metadata: {
+                threadId,
+                ...(this.options.name ? { graph: this.options.name } : {}),
+                ...(runOptions.tenantId !== undefined ? { tenantId: runOptions.tenantId } : {}),
+                ...(runOptions.principal?.userId !== undefined ? { userId: runOptions.principal.userId } : {}),
+            },
         }, runOptions.signal ? { signal: runOptions.signal } : {});
         let last;
         let settled = false;
@@ -850,6 +870,7 @@ export class CompiledGraph {
             step,
             threadId,
             ...(runOptions.tenantId !== undefined ? { tenantId: runOptions.tenantId } : {}),
+            ...(runOptions.principal ? { principal: runOptions.principal } : {}),
             taskId: task.id,
             input: task.input,
             attempt,

package/dist/graph/lint.d.ts

@@ -2,7 +2,7 @@ import type { GraphDescription } from '../types/graph.js';
 /** How much a lint finding matters: `error` fails `nexus graph lint`, the others only report. */
 export type GraphLintSeverity = 'error' | 'warning' | 'info';
 /** The rules `lintGraph()` checks. */
-export type GraphLintCode = 'UNBOUNDED_CYCLE' | 'RETRY_WITHOUT_IDEMPOTENCY' | 'RETRY_WITHOUT_TIMEOUT' | 'DEFERRED_DURABILITY_WITH_SIDE_EFFECTS' | 'MEMORY_CHECKPOINTER' | 'NO_CHECKPOINTER' | 'UNBOUNDED_CONCURRENCY' | 'DYNAMIC_ROUTE';
+export type GraphLintCode = 'UNBOUNDED_CYCLE' | 'RETRY_WITHOUT_IDEMPOTENCY' | 'RETRY_WITHOUT_TIMEOUT' | 'DEFERRED_DURABILITY_WITH_SIDE_EFFECTS' | 'MEMORY_CHECKPOINTER' | 'NO_CHECKPOINTER' | 'UNBOUNDED_CONCURRENCY' | 'DYNAMIC_ROUTE' | 'SENSITIVE_TOOL_WITHOUT_APPROVAL' | 'UNDECLARED_TOOL_CAPABILITIES' | 'SIDE_EFFECT_BEFORE_INTERRUPT';
 /** One design problem `lintGraph()` found. */
 export interface GraphLintFinding {
     /** Which rule found it. */
@@ -43,6 +43,9 @@ export interface GraphLintOptions {
  * | `NO_CHECKPOINTER` | info | No checkpoints, so no interrupt, resume, or recovery |
  * | `UNBOUNDED_CONCURRENCY` | warning | A fan-out with no limit on tasks at once |
  * | `DYNAMIC_ROUTE` | info | A router with no mapping or `ends`, so routes cannot be checked |
+ * | `SENSITIVE_TOOL_WITHOUT_APPROVAL` | warning, or error when `deployed` | A tool that writes, runs commands or code, or reaches the network, with no approval and no permission policy |
+ * | `UNDECLARED_TOOL_CAPABILITIES` | info | A tool with no capabilities and no approval, whose effects cannot be judged |
+ * | `SIDE_EFFECT_BEFORE_INTERRUPT` | warning | A node that declares effects and interrupts, so its effects repeat when it runs again on the answer |
  *
  * @example
  * ```ts

package/dist/graph/lint.js

@@ -1,3 +1,4 @@
+import { isSensitiveCapability } from '../agent/capabilities.js';
 export function lintGraph(graph, options = {}) {
     const description = 'describe' in graph && typeof graph.describe === 'function' ? graph.describe() : graph;
     const findings = [];
@@ -39,9 +40,43 @@ function lintDescription(description, prefix, options, findings, top) {
                 fix: 'Set `timeout: { runMs }`, or `idleMs` with `context.heartbeat()` for long work.',
             });
         }
+        if (node.effects?.length && node.interrupts && !node.idempotent) {
+            findings.push({
+                code: 'SIDE_EFFECT_BEFORE_INTERRUPT',
+                severity: 'warning',
+                node: at(node.id),
+                message: `"${at(node.id)}" declares effects (${node.effects.join(', ')}) and may interrupt, so when the answer comes the node runs again from the top and repeats whatever it did before asking.`,
+                fix: 'Ask in a node of its own before the effect, or give the effect an idempotency key and declare the node idempotent.',
+            });
+        }
         if (node.subgraph)
             lintDescription(node.subgraph, `${at(node.id)}/`, options, findings, false);
     }
+    for (const tool of description.tools ?? []) {
+        if (tool.approval !== 'none')
+            continue;
+        const sensitive = (tool.capabilities ?? []).filter((capability) => isSensitiveCapability(capability));
+        if (sensitive.length > 0 || tool.dynamic) {
+            findings.push({
+                code: 'SENSITIVE_TOOL_WITHOUT_APPROVAL',
+                severity: options.deployed ? 'error' : 'warning',
+                node: `tool:${tool.name}`,
+                message: sensitive.length
+                    ? `Tool "${tool.name}" can ${sensitive.join(', ')}, and nothing approves or limits its calls.`
+                    : `Tool "${tool.name}" computes its capabilities per call, and nothing approves or limits its calls.`,
+                fix: 'Give the agent a permission policy that grants what the tool may do, or approve its calls with interruptOn.',
+            });
+        }
+        else if (tool.capabilities === undefined) {
+            findings.push({
+                code: 'UNDECLARED_TOOL_CAPABILITIES',
+                severity: 'info',
+                node: `tool:${tool.name}`,
+                message: `Tool "${tool.name}" declares no capabilities, so what it does cannot be judged.`,
+                fix: 'Declare its capabilities, or `capabilities: []` for a tool that changes nothing.',
+            });
+        }
+    }
     if (!top)
         return;
     if ((description.durability === 'async' || description.durability === 'exit') && sideEffecting.length > 0) {

package/dist/postgres/fulltext.d.ts

@@ -0,0 +1,67 @@
+import type { RagChunk } from '../hallucination/rag.js';
+import type { VectorSearchResult } from '../hallucination/retrieval.js';
+import type { RetrieveOptions, SparseRetriever } from '../rag/retrievers.js';
+import type { MigrationResult, SchemaMigration } from '../utils/schema-migrations.js';
+import { type PostgresLikeClient } from './client.js';
+import type { PostgresMigrateOptions } from './migrations.js';
+/** Options for `PostgresKeywordIndex`. */
+export interface PostgresKeywordIndexOptions {
+    /** Table name, optionally schema-qualified. Defaults to `nexus_keywords`. */
+    table?: string;
+    /**
+     * Searches the table of a `PostgresVectorStore` instead of keeping chunks of its own. `migrate()`
+     * adds a generated search column and its index to that table, the store's writes keep it current,
+     * and `add()` and `delete()` leave writing to the store. One copy of every chunk, and keyword and
+     * vector search can never disagree about what exists.
+     */
+    shared?: boolean;
+    /** The text search configuration: `english`, `simple`, `german`, and so on. Defaults to `english`. */
+    language?: string;
+    /**
+     * `any` (the default) finds chunks with any of the query's terms, ranked by how many and how close,
+     * as keyword search for retrieval should. `all` needs every term, read as a web search query, with
+     * quoted phrases and `-` for exclusion.
+     */
+    match?: 'any' | 'all';
+}
+/**
+ * The versioned schema, which `migrate()` applies. Version 1 creates the table with a generated
+ * `tsvector` column and a GIN index, or, `shared`, adds the column and the index to a vector table.
+ */
+export declare function keywordIndexMigrations(options?: PostgresKeywordIndexOptions): SchemaMigration[];
+/**
+ * Keyword search in Postgres, with its full-text search: stemmed terms, a GIN index, and
+ * `ts_rank_cd` ranking, so only the best matches leave the database however large the table.
+ *
+ * A `SparseRetriever`, so `hybridRetriever()` fuses it with `PostgresVectorStore`. With `shared`, it
+ * searches the vector store's own table, and the hybrid search runs over one copy of the data. A
+ * metadata filter runs in SQL, with the same meaning as the vector store's.
+ *
+ * ```ts
+ * const vectors = new PostgresVectorStore(client, { dimensions: 1536, embed });
+ * const keywords = new PostgresKeywordIndex(client, { table: 'nexus_vectors', shared: true });
+ * await vectors.migrate();
+ * await keywords.migrate();
+ * const retriever = hybridRetriever([vectorRetriever(vectors), keywords]);
+ * ```
+ */
+export declare class PostgresKeywordIndex implements SparseRetriever {
+    private readonly client;
+    private readonly options;
+    private readonly table;
+    private readonly language;
+    constructor(client: PostgresLikeClient, options?: PostgresKeywordIndexOptions);
+    /**
+     * Applies this index's pending migrations, recorded in `nexus_schema_migrations`, under a lock.
+     * Never runs implicitly. With `shared`, run the vector store's `migrate()` first.
+     */
+    migrate(options?: PostgresMigrateOptions): Promise<MigrationResult>;
+    /** Adds chunks, or replaces those whose id exists. With `shared`, does nothing: the vector store writes. */
+    add(chunks: readonly RagChunk[]): Promise<void>;
+    /** Removes chunks by id. With `shared`, does nothing: the vector store deletes. */
+    delete(ids: readonly string[]): Promise<void>;
+    /** The chunks that best match a query's terms, best first, scored from 0 to 1. */
+    retrieve(query: string, options?: RetrieveOptions): Promise<VectorSearchResult[]>;
+    /** Chunks held. */
+    size(): Promise<number>;
+}

package/dist/postgres/fulltext.js

@@ -0,0 +1,91 @@
+import { fromJson, quoteDerived, quoteTable } from './client.js';
+import { SqlParams, textArray } from './sql.js';
+const DEFAULT_TABLE = 'nexus_keywords';
+function languageOf(options) {
+    const language = options.language ?? 'english';
+    if (!/^[a-z_]{1,63}$/.test(language))
+        throw new RangeError(`"${language}" is not a text search configuration name`);
+    return language;
+}
+export function keywordIndexMigrations(options = {}) {
+    const name = options.table ?? DEFAULT_TABLE;
+    const table = quoteTable(name);
+    const generated = `fts tsvector GENERATED ALWAYS AS (to_tsvector('${languageOf(options)}'::regconfig, content)) STORED`;
+    return [
+        {
+            component: `keywords:${name}`,
+            version: 1,
+            name: options.shared ? 'add full-text search to a vector table' : 'create the keyword table',
+            statements: [
+                options.shared
+                    ? `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${generated}`
+                    : `CREATE TABLE IF NOT EXISTS ${table} (
+  id text PRIMARY KEY,
+  content text NOT NULL,
+  source text,
+  metadata jsonb,
+  ${generated}
+)`,
+                `CREATE INDEX IF NOT EXISTS ${quoteDerived(name, 'fts')} ON ${table} USING gin (fts)`,
+            ],
+        },
+    ];
+}
+export class PostgresKeywordIndex {
+    client;
+    options;
+    table;
+    language;
+    constructor(client, options = {}) {
+        this.client = client;
+        this.options = options;
+        this.table = quoteTable(options.table ?? DEFAULT_TABLE);
+        this.language = languageOf(options);
+    }
+    async migrate(options = {}) {
+        const { applyPostgresMigrations } = await import('./migrations.js');
+        return applyPostgresMigrations(this.client, keywordIndexMigrations(this.options), options);
+    }
+    async add(chunks) {
+        if (this.options.shared || chunks.length === 0)
+            return;
+        const params = new SqlParams();
+        const rows = chunks.map((chunk) => `(${params.add(chunk.id)}, ${params.add(chunk.content)}, ${params.add(chunk.source ?? null)}, ${params.add(chunk.metadata === undefined ? null : JSON.stringify(chunk.metadata))}::jsonb)`);
+        await this.client.query(`INSERT INTO ${this.table} (id, content, source, metadata) VALUES ${rows.join(', ')}
+       ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content, source = EXCLUDED.source, metadata = EXCLUDED.metadata`, params.values);
+    }
+    async delete(ids) {
+        if (this.options.shared || ids.length === 0)
+            return;
+        await this.client.query(`DELETE FROM ${this.table} WHERE id = ANY($1::text[])`, [textArray(ids)]);
+    }
+    async retrieve(query, options = {}) {
+        options.signal?.throwIfAborted();
+        const params = new SqlParams();
+        const text = params.add(query);
+        const language = `'${this.language}'::regconfig`;
+        const tsquery = this.options.match === 'all'
+            ? `websearch_to_tsquery(${language}, ${text})`
+            : `NULLIF(replace(plainto_tsquery(${language}, ${text})::text, ' & ', ' | '), '')::tsquery`;
+        const conditions = ['fts @@ q.query'];
+        if (options.filter && Object.keys(options.filter).length > 0) {
+            conditions.push(`metadata @> ${params.add(JSON.stringify(options.filter))}::jsonb`);
+        }
+        const limit = params.add(options.topK ?? 5);
+        const { rows } = await this.client.query(`SELECT id, content, source, metadata::text AS metadata, ts_rank_cd(fts, q.query, 32) AS score
+       FROM ${this.table}, (SELECT ${tsquery} AS query) AS q
+       WHERE ${conditions.join(' AND ')}
+       ORDER BY score DESC, id LIMIT ${limit}`, params.values);
+        return rows.map((row) => ({
+            id: row.id,
+            content: row.content,
+            ...(row.source === null ? {} : { source: row.source }),
+            ...(row.metadata === null ? {} : { metadata: fromJson(row.metadata) }),
+            score: Number(row.score),
+        }));
+    }
+    async size() {
+        const { rows } = await this.client.query(`SELECT count(*)::int AS count FROM ${this.table}`);
+        return Number(rows[0]?.count ?? 0);
+    }
+}

package/dist/rag/elasticsearch.d.ts

@@ -0,0 +1,71 @@
+import type { RagChunk } from '../hallucination/rag.js';
+import type { VectorSearchResult } from '../hallucination/retrieval.js';
+import type { RetrieveOptions, SparseRetriever } from './retrievers.js';
+/** Options for the Elasticsearch keyword index. */
+export interface ElasticsearchKeywordIndexOptions {
+    /** The cluster, such as `http://localhost:9200`, an Elastic Cloud endpoint, or an OpenSearch domain. */
+    url: string;
+    /** The index chunks are kept in. */
+    index: string;
+    /** Sent as `Authorization: ApiKey <key>`. */
+    apiKey?: string;
+    /** Sent as basic authentication, for a cluster with users instead of API keys. */
+    username?: string;
+    /** The password for `username`. */
+    password?: string;
+    /** The analyzer `migrate()` gives the `content` field, such as `english`. Defaults to the cluster's standard one. */
+    analyzer?: string;
+    /**
+     * When a write becomes searchable. `wait_for` (the default) returns once a search would see it, as
+     * the vector stores do; `false` returns sooner and leaves it to the index's refresh interval.
+     */
+    refresh?: 'wait_for' | boolean;
+    /** Replaces the global `fetch`, for a proxy, retries, signed requests, or tests. */
+    fetch?: typeof globalThis.fetch;
+    /** Headers added to every request. */
+    headers?: Record<string, string>;
+    /** Aborts a request that takes longer, in milliseconds. Defaults to 30 seconds. */
+    timeoutMs?: number;
+}
+/** Raised when the cluster answers with an error. */
+export declare class ElasticsearchError extends Error {
+    /** The HTTP status the cluster answered with. */
+    readonly status: number;
+    /** The cluster's response body, as text. */
+    readonly body?: string | undefined;
+    constructor(message: string, 
+    /** The HTTP status the cluster answered with. */
+    status: number, 
+    /** The cluster's response body, as text. */
+    body?: string | undefined);
+}
+/**
+ * Keyword search in Elasticsearch or OpenSearch, through their REST API.
+ *
+ * No client library is a dependency: the index speaks HTTP through `fetch`, as the vector stores do,
+ * so it runs wherever `fetch` does. Scores are the engine's own BM25, ranked in the cluster, so only
+ * the best matches come back. A `SparseRetriever`, so `hybridRetriever()` fuses it with any vector
+ * store. A metadata filter is an exact match on top-level fields, as in the vector stores.
+ */
+export declare class ElasticsearchKeywordIndex implements SparseRetriever {
+    private readonly options;
+    private readonly base;
+    private readonly fetchImpl;
+    constructor(options: ElasticsearchKeywordIndexOptions);
+    /**
+     * Creates the index unless it exists: `content` as analyzed text, `id` and `source` as keywords,
+     * and every string under `metadata` as a keyword, so a filter matches it exactly. Never runs
+     * implicitly.
+     */
+    migrate(): Promise<void>;
+    /** Adds chunks, or replaces those whose id exists, in one bulk request. */
+    add(chunks: readonly RagChunk[]): Promise<void>;
+    /** Removes chunks by id. An id that is not held is ignored. */
+    delete(ids: readonly string[]): Promise<void>;
+    /** The chunks that best match a query's terms, best first, scored by the engine's BM25. */
+    retrieve(query: string, options?: RetrieveOptions): Promise<VectorSearchResult[]>;
+    /** Chunks held. */
+    size(): Promise<number>;
+    private bulk;
+    private request;
+}

package/dist/rag/elasticsearch.js

@@ -0,0 +1,130 @@
+import { linkSignals } from '../utils/signals.js';
+export class ElasticsearchError extends Error {
+    status;
+    body;
+    constructor(message, status, body) {
+        super(message);
+        this.status = status;
+        this.body = body;
+        this.name = 'ElasticsearchError';
+    }
+}
+export class ElasticsearchKeywordIndex {
+    options;
+    base;
+    fetchImpl;
+    constructor(options) {
+        this.options = options;
+        if (!/^[a-z0-9][a-z0-9._-]*$/.test(options.index)) {
+            throw new RangeError(`"${options.index}" is not a valid index name: use lowercase letters, digits, ".", "_", and "-"`);
+        }
+        this.base = options.url.replace(/\/+$/, '');
+        this.fetchImpl = options.fetch ?? globalThis.fetch;
+    }
+    async migrate() {
+        const existing = await this.request('HEAD', `/${this.options.index}`, undefined, { allowNotFound: true });
+        if (existing !== undefined)
+            return;
+        await this.request('PUT', `/${this.options.index}`, {
+            mappings: {
+                dynamic_templates: [
+                    {
+                        metadata_strings: { path_match: 'metadata.*', match_mapping_type: 'string', mapping: { type: 'keyword' } },
+                    },
+                ],
+                properties: {
+                    id: { type: 'keyword' },
+                    content: { type: 'text', ...(this.options.analyzer ? { analyzer: this.options.analyzer } : {}) },
+                    source: { type: 'keyword' },
+                    metadata: { type: 'object', dynamic: true },
+                },
+            },
+        });
+    }
+    async add(chunks) {
+        if (chunks.length === 0)
+            return;
+        const lines = chunks.flatMap((chunk) => [
+            { index: { _index: this.options.index, _id: chunk.id } },
+            { id: chunk.id, content: chunk.content, source: chunk.source ?? null, metadata: chunk.metadata ?? null },
+        ]);
+        await this.bulk(lines);
+    }
+    async delete(ids) {
+        if (ids.length === 0)
+            return;
+        await this.bulk(ids.map((id) => ({ delete: { _index: this.options.index, _id: id } })));
+    }
+    async retrieve(query, options = {}) {
+        const filter = Object.entries(options.filter ?? {});
+        const clauses = [];
+        for (const [key, value] of filter) {
+            if (value === null)
+                clauses.push({ bool: { must_not: { exists: { field: `metadata.${key}` } } } });
+            else if (['string', 'number', 'boolean'].includes(typeof value))
+                clauses.push({ term: { [`metadata.${key}`]: value } });
+            else
+                return [];
+        }
+        const result = (await this.request('POST', `/${this.options.index}/_search`, {
+            size: options.topK ?? 5,
+            query: { bool: { must: [{ match: { content: { query, operator: 'or' } } }], filter: clauses } },
+            _source: ['id', 'content', 'source', 'metadata'],
+        }, { signal: options.signal }));
+        return (result.hits?.hits ?? []).map((hit) => ({
+            id: hit._source?.id ?? hit._id,
+            content: hit._source?.content ?? '',
+            ...(hit._source?.source ? { source: hit._source.source } : {}),
+            ...(hit._source?.metadata ? { metadata: hit._source.metadata } : {}),
+            score: hit._score ?? 0,
+        }));
+    }
+    async size() {
+        const result = (await this.request('GET', `/${this.options.index}/_count`));
+        return result.count ?? 0;
+    }
+    async bulk(lines) {
+        const refresh = this.options.refresh ?? 'wait_for';
+        const result = (await this.request('POST', `/_bulk${refresh === false ? '' : `?refresh=${refresh === true ? 'true' : 'wait_for'}`}`, `${lines.map((line) => JSON.stringify(line)).join('\n')}\n`));
+        if (!result.errors)
+            return;
+        const failed = (result.items ?? [])
+            .map((item) => Object.entries(item)[0])
+            .map((entry) => (entry?.[1]?.error && !(entry[0] === 'delete' && entry[1].status === 404) ? entry[1] : undefined))
+            .find((outcome) => outcome !== undefined);
+        if (failed)
+            throw new ElasticsearchError(`A bulk write failed: ${failed.error?.reason ?? 'unknown reason'}`, failed.status ?? 500);
+    }
+    async request(method, path, body, options = {}) {
+        const timeout = AbortSignal.timeout(this.options.timeoutMs ?? 30_000);
+        const linked = linkSignals([timeout, options.signal]);
+        const ndjson = typeof body === 'string';
+        const authorization = this.options.apiKey
+            ? `ApiKey ${this.options.apiKey}`
+            : this.options.username !== undefined
+                ? `Basic ${btoa(`${this.options.username}:${this.options.password ?? ''}`)}`
+                : undefined;
+        try {
+            const response = await this.fetchImpl(`${this.base}${path}`, {
+                method,
+                headers: {
+                    ...(body === undefined ? {} : { 'content-type': ndjson ? 'application/x-ndjson' : 'application/json' }),
+                    ...(authorization ? { authorization } : {}),
+                    ...this.options.headers,
+                },
+                ...(body === undefined ? {} : { body: ndjson ? body : JSON.stringify(body) }),
+                signal: linked.signal,
+            });
+            if (response.status === 404 && options.allowNotFound)
+                return undefined;
+            const text = method === 'HEAD' ? '' : await response.text();
+            if (!response.ok) {
+                throw new ElasticsearchError(`Elasticsearch ${method} ${path} failed with ${response.status}`, response.status, text);
+            }
+            return text ? JSON.parse(text) : {};
+        }
+        finally {
+            linked.dispose();
+        }
+    }
+}