nexus-ai-pro 2.2.0 → 2.4.0

package/dist-cjs/rag/pipeline.js

@@ -0,0 +1,191 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createIngestionPipeline = createIngestionPipeline;
+exports.ingestionExecutor = ingestionExecutor;
+const ingestion_js_1 = require("./ingestion.js");
+function createIngestionPipeline(options) {
+    const batchSize = options.batchSize ?? 64;
+    if (!(Number.isInteger(batchSize) && batchSize > 0))
+        throw new RangeError('batchSize must be a positive integer');
+    const namespace = ['nexus', 'ingestion', options.name, 'documents'];
+    const keywords = options.keywords ?? [];
+    const chunkingVersion = stableJson(options.chunking ?? {});
+    const read = async (id) => (await options.manifest.get(namespace, id))?.value;
+    const removeChunks = async (ids) => {
+        if (ids.length === 0)
+            return;
+        await Promise.all([options.vectors.delete(ids), ...keywords.map((index) => index.delete(ids))]);
+    };
+    const remove = async (ids) => {
+        let chunks = 0;
+        for (const id of ids) {
+            const recorded = await read(id);
+            if (!recorded)
+                continue;
+            const chunkIds = Object.keys(recorded.chunks);
+            await removeChunks(chunkIds);
+            await options.manifest.delete(namespace, id);
+            chunks += chunkIds.length;
+        }
+        return chunks;
+    };
+    return {
+        document: read,
+        async remove(ids) {
+            await remove(ids);
+            return ids.length;
+        },
+        async run(source, runOptions = {}) {
+            const report = {
+                documents: 0,
+                unchanged: 0,
+                added: 0,
+                updated: 0,
+                deleted: 0,
+                chunksEmbedded: 0,
+                chunksKept: 0,
+                chunksDeleted: 0,
+                resumed: 0,
+            };
+            const previous = runOptions.operation?.previousHeartbeat;
+            const skip = previous?.pipeline === options.name ? previous.documents : 0;
+            const seen = new Set();
+            let batch = [];
+            const flush = async () => {
+                if (batch.length === 0)
+                    return;
+                const current = batch;
+                batch = [];
+                const versions = await Promise.all(current.map(({ document }) => hashText(stableJson([document.text, document.source ?? null, document.metadata ?? null]))));
+                const recorded = await Promise.all(current.map(({ id }) => read(id)));
+                const writes = [];
+                const stale = [];
+                const toEmbed = [];
+                for (const [index, { document, id }] of current.entries()) {
+                    const version = versions[index];
+                    const before = recorded[index];
+                    if (before &&
+                        before.version === version &&
+                        before.chunking === chunkingVersion &&
+                        before.model === options.embeddingModel) {
+                        report.unchanged += 1;
+                        continue;
+                    }
+                    if (before)
+                        report.updated += 1;
+                    else
+                        report.added += 1;
+                    const split = (0, ingestion_js_1.ingestDocuments)([{ ...document, id }], options.chunking).chunks;
+                    const hashes = await Promise.all(split.map((chunk) => hashText(stableJson([chunk.content, document.metadata ?? null]))));
+                    const used = new Map();
+                    const chunks = split.map((chunk, position) => {
+                        const base = `${id}#${hashes[position].slice(0, 12)}`;
+                        const count = (used.get(base) ?? 0) + 1;
+                        used.set(base, count);
+                        return { ...chunk, id: count === 1 ? base : `${base}-${count}` };
+                    });
+                    const reusable = before && before.model === options.embeddingModel ? before.chunks : {};
+                    for (const [position, chunk] of chunks.entries()) {
+                        if (reusable[chunk.id] === hashes[position])
+                            report.chunksKept += 1;
+                        else
+                            toEmbed.push(chunk);
+                    }
+                    const kept = new Set(chunks.map((chunk) => chunk.id));
+                    if (before)
+                        stale.push(...Object.keys(before.chunks).filter((chunkId) => !kept.has(chunkId) || !reusable[chunkId]));
+                    writes.push({ id, version, chunks, hashes });
+                }
+                if (toEmbed.length > 0) {
+                    const vectors = await options.embed(toEmbed.map((chunk) => chunk.content));
+                    if (vectors.length !== toEmbed.length) {
+                        throw new RangeError(`The embedding function returned ${vectors.length} vectors for ${toEmbed.length} texts`);
+                    }
+                    await Promise.all([
+                        options.vectors.add(toEmbed.map((chunk, index) => ({ ...chunk, embedding: vectors[index] }))),
+                        ...keywords.map((index) => index.add(toEmbed)),
+                    ]);
+                    report.chunksEmbedded += toEmbed.length;
+                }
+                const embeddedIds = new Set(toEmbed.map((chunk) => chunk.id));
+                const removable = stale.filter((chunkId) => !embeddedIds.has(chunkId));
+                await removeChunks(removable);
+                report.chunksDeleted += removable.length;
+                const updatedAt = new Date().toISOString();
+                await Promise.all(writes.map(({ id, version, chunks, hashes }) => options.manifest.put(namespace, id, {
+                    id,
+                    version,
+                    chunking: chunkingVersion,
+                    model: options.embeddingModel,
+                    chunks: Object.fromEntries(chunks.map((chunk, index) => [chunk.id, hashes[index]])),
+                    updatedAt,
+                })));
+                await runOptions.operation?.heartbeat({
+                    pipeline: options.name,
+                    documents: report.documents,
+                });
+                runOptions.onProgress?.({ ...report });
+            };
+            const sources = isSourceList(source) ? source : [source];
+            for (const loader of sources) {
+                for await (const document of loader) {
+                    runOptions.signal?.throwIfAborted();
+                    const id = document.id ?? document.source;
+                    if (!id)
+                        throw new TypeError('Durable ingestion needs a document id, or a source to use as one');
+                    seen.add(id);
+                    report.documents += 1;
+                    if (report.documents <= skip) {
+                        report.resumed += 1;
+                        continue;
+                    }
+                    batch.push({ document, id });
+                    if (batch.length >= batchSize)
+                        await flush();
+                }
+            }
+            runOptions.signal?.throwIfAborted();
+            await flush();
+            if (runOptions.deleteMissing !== false) {
+                const missing = [];
+                for (let offset = 0;; offset += 500) {
+                    const page = await options.manifest.search(namespace, { limit: 500, offset });
+                    for (const item of page)
+                        if (!seen.has(item.key))
+                            missing.push(item.key);
+                    if (page.length < 500)
+                        break;
+                }
+                for (const id of missing) {
+                    runOptions.signal?.throwIfAborted();
+                    report.chunksDeleted += await remove([id]);
+                    report.deleted += 1;
+                }
+                if (missing.length > 0)
+                    runOptions.onProgress?.({ ...report });
+            }
+            return report;
+        },
+    };
+}
+function ingestionExecutor(pipeline, source, options = {}) {
+    return (context) => pipeline.run(source(), { ...options, signal: context.signal, operation: context });
+}
+async function hashText(text) {
+    const digest = await globalThis.crypto.subtle.digest('SHA-256', new TextEncoder().encode(text));
+    let hex = '';
+    for (const byte of new Uint8Array(digest).subarray(0, 16))
+        hex += byte.toString(16).padStart(2, '0');
+    return hex;
+}
+function stableJson(value) {
+    return JSON.stringify(value, (_key, entry) => entry && typeof entry === 'object' && !Array.isArray(entry)
+        ? Object.fromEntries(Object.entries(entry).sort(([a], [b]) => a.localeCompare(b)))
+        : entry);
+}
+function isSourceList(value) {
+    if (!Array.isArray(value))
+        return false;
+    const first = value[0];
+    return first !== undefined && typeof first === 'object' && first !== null && !('text' in first);
+}

package/dist-cjs/rag/rerankers.d.ts

@@ -0,0 +1,92 @@
+import type { Reranker } from './retrievers.js';
+/** Raised when a rerank API answers with an error. */
+export declare class RerankerError extends Error {
+    /** The HTTP status the API answered with. */
+    readonly status: number;
+    /** The API's response body, as text. */
+    readonly body?: string | undefined;
+    constructor(message: string, 
+    /** The HTTP status the API answered with. */
+    status: number, 
+    /** The API's response body, as text. */
+    body?: string | undefined);
+}
+/** What every HTTP reranker takes. */
+export interface RerankerHttpOptions {
+    /** Replaces the global `fetch`, for a proxy, retries, or tests. */
+    fetch?: typeof globalThis.fetch;
+    /** Headers added to every request. */
+    headers?: Record<string, string>;
+    /** Aborts a request that takes longer, in milliseconds. Defaults to 30 seconds. */
+    timeoutMs?: number;
+    /** Characters of each chunk sent. Defaults to 4,000. */
+    maxChunkCharacters?: number;
+}
+/** Options for a hosted rerank API. */
+export interface HostedRerankerOptions extends RerankerHttpOptions {
+    /** The API key. */
+    apiKey: string;
+    /** The rerank model, such as `rerank-v3.5`. */
+    model: string;
+    /** Replaces the API's endpoint, for a regional or proxied one. */
+    url?: string;
+}
+/** Options for `httpReranker()`: any rerank API, described by its request and its response. */
+export interface HttpRerankerOptions extends RerankerHttpOptions {
+    /** The endpoint. */
+    url: string;
+    /** The request body for a query and its passages. */
+    body: (query: string, passages: string[]) => unknown;
+    /** Reads the response: each passage's index and score. Passages it leaves out score 0. */
+    scores: (response: unknown) => ReadonlyArray<{
+        index: number;
+        score: number;
+    }>;
+}
+/**
+ * A reranker over any HTTP rerank API: you describe the request body and how to read the scores,
+ * and it handles the rest — truncation, the timeout, cancellation, errors, and scores put back in
+ * the order of the chunks.
+ */
+export declare function httpReranker(options: HttpRerankerOptions): Reranker;
+/** Cohere's rerank API (`/v2/rerank`), such as `rerank-v3.5`. */
+export declare function cohereReranker(options: HostedRerankerOptions): Reranker;
+/** Voyage AI's rerank API (`/v1/rerank`), such as `rerank-2`. */
+export declare function voyageReranker(options: HostedRerankerOptions): Reranker;
+/** Jina AI's rerank API (`/v1/rerank`), such as `jina-reranker-v2-base-multilingual`. */
+export declare function jinaReranker(options: HostedRerankerOptions): Reranker;
+/** Options for `teiReranker()`. */
+export interface TeiRerankerOptions extends RerankerHttpOptions {
+    /** The server, such as `http://localhost:8080`. */
+    url: string;
+}
+/**
+ * A cross-encoder served by Hugging Face's text-embeddings-inference, on your own machine or
+ * cluster: `docker run ... --model-id BAAI/bge-reranker-base`, then `teiReranker({ url })`. The
+ * scores are the model's own, from 0 to 1.
+ */
+export declare function teiReranker(options: TeiRerankerOptions): Reranker;
+/**
+ * A cross-encoder run in process: one score per (query, passage) pair, higher meaning more relevant.
+ * A model from an ONNX runtime or a transformers library fits it in a few lines, so the package
+ * depends on neither.
+ */
+export interface CrossEncoder {
+    /** Scores each pair, in order. */
+    score(pairs: ReadonlyArray<readonly [query: string, passage: string]>): Promise<readonly number[]> | readonly number[];
+}
+/** Options for `crossEncoderReranker()`. */
+export interface CrossEncoderRerankerOptions {
+    /** Pairs scored in one call. Defaults to 32. */
+    batchSize?: number;
+    /** Characters of each chunk scored. Defaults to 2,000. */
+    maxChunkCharacters?: number;
+    /** Maps raw logits to 0–1 with a sigmoid, for a model that returns logits. Defaults to false. */
+    sigmoid?: boolean;
+}
+/**
+ * A reranker over a cross-encoder you run in process: no network call, no per-query price, and the
+ * passages never leave the machine. Pairs are scored in batches, and the run's signal is checked
+ * between batches.
+ */
+export declare function crossEncoderReranker(model: CrossEncoder, options?: CrossEncoderRerankerOptions): Reranker;

package/dist-cjs/rag/rerankers.js

@@ -0,0 +1,107 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RerankerError = void 0;
+exports.httpReranker = httpReranker;
+exports.cohereReranker = cohereReranker;
+exports.voyageReranker = voyageReranker;
+exports.jinaReranker = jinaReranker;
+exports.teiReranker = teiReranker;
+exports.crossEncoderReranker = crossEncoderReranker;
+const signals_js_1 = require("../utils/signals.js");
+class RerankerError extends Error {
+    status;
+    body;
+    constructor(message, status, body) {
+        super(message);
+        this.status = status;
+        this.body = body;
+        this.name = 'RerankerError';
+    }
+}
+exports.RerankerError = RerankerError;
+function httpReranker(options) {
+    const fetchImpl = options.fetch ?? globalThis.fetch;
+    const limit = options.maxChunkCharacters ?? 4_000;
+    return async (query, chunks, signal) => {
+        if (chunks.length === 0)
+            return [];
+        const passages = chunks.map((chunk) => chunk.content.slice(0, limit));
+        const linked = (0, signals_js_1.linkSignals)([AbortSignal.timeout(options.timeoutMs ?? 30_000), signal]);
+        try {
+            const response = await fetchImpl(options.url, {
+                method: 'POST',
+                headers: { 'content-type': 'application/json', ...options.headers },
+                body: JSON.stringify(options.body(query, passages)),
+                signal: linked.signal,
+            });
+            const text = await response.text();
+            if (!response.ok)
+                throw new RerankerError(`The rerank API answered ${response.status}`, response.status, text);
+            const scores = new Array(chunks.length).fill(0);
+            for (const { index, score } of options.scores(JSON.parse(text))) {
+                if (Number.isInteger(index) && index >= 0 && index < scores.length)
+                    scores[index] = Number(score);
+            }
+            return scores;
+        }
+        finally {
+            linked.dispose();
+        }
+    };
+}
+const bearer = (options) => ({ authorization: `Bearer ${options.apiKey}`, ...options.headers });
+const readScored = (list) => (list ?? []).map((item) => ({ index: item.index, score: item.relevance_score }));
+function cohereReranker(options) {
+    return httpReranker({
+        ...options,
+        url: options.url ?? 'https://api.cohere.com/v2/rerank',
+        headers: bearer(options),
+        body: (query, documents) => ({ model: options.model, query, documents, top_n: documents.length }),
+        scores: (response) => readScored(response.results),
+    });
+}
+function voyageReranker(options) {
+    return httpReranker({
+        ...options,
+        url: options.url ?? 'https://api.voyageai.com/v1/rerank',
+        headers: bearer(options),
+        body: (query, documents) => ({ model: options.model, query, documents, top_k: documents.length }),
+        scores: (response) => readScored(response.data),
+    });
+}
+function jinaReranker(options) {
+    return httpReranker({
+        ...options,
+        url: options.url ?? 'https://api.jina.ai/v1/rerank',
+        headers: bearer(options),
+        body: (query, documents) => ({ model: options.model, query, documents, top_n: documents.length }),
+        scores: (response) => readScored(response.results),
+    });
+}
+function teiReranker(options) {
+    return httpReranker({
+        ...options,
+        url: `${options.url.replace(/\/+$/, '')}/rerank`,
+        body: (query, texts) => ({ query, texts, raw_scores: false, truncate: true }),
+        scores: (response) => response,
+    });
+}
+function crossEncoderReranker(model, options = {}) {
+    const batchSize = Math.max(1, options.batchSize ?? 32);
+    const limit = options.maxChunkCharacters ?? 2_000;
+    return async (query, chunks, signal) => {
+        const pairs = chunks.map((chunk) => [query, chunk.content.slice(0, limit)]);
+        const scores = [];
+        for (let start = 0; start < pairs.length; start += batchSize) {
+            signal?.throwIfAborted();
+            const batch = pairs.slice(start, start + batchSize);
+            const result = await model.score(batch);
+            if (result.length !== batch.length) {
+                throw new RangeError(`The cross-encoder returned ${result.length} scores for ${batch.length} pairs`);
+            }
+            for (const raw of result)
+                scores.push(options.sigmoid ? 1 / (1 + Math.exp(-raw)) : raw);
+        }
+        return scores;
+    };
+}

package/dist-cjs/rag/retrievers.d.ts

@@ -19,6 +19,17 @@ export interface Retriever {
     /** The chunks that best answer a query, best first. */
     retrieve(query: string, options?: RetrieveOptions): Promise<VectorSearchResult[]>;
 }
+/**
+ * Keyword search that holds its own chunks: an in-memory BM25 `KeywordIndex`, Postgres full-text
+ * search, a search engine. It is a `Retriever`, so `hybridRetriever()` fuses it with vector search,
+ * and it takes writes, so ingestion keeps it in step with a vector store.
+ */
+export interface SparseRetriever extends Retriever {
+    /** Adds chunks, or replaces those whose id exists. */
+    add(chunks: readonly RagChunk[]): Promise<void> | void;
+    /** Removes chunks by id. An id that is not held is ignored. */
+    delete(ids: readonly string[]): Promise<void> | void;
+}
 /** A client that runs one completion — a `NexusAI` instance, or anything with the same `complete()`. */
 export interface RetrievalModelClient {
     /** Runs one completion and returns its text. */
@@ -47,7 +58,7 @@ export declare function tokenize(text: string): string[];
  * codes, error numbers, names — that keyword search finds; `hybridRetriever()` fuses the two.
  * Adding a chunk whose id exists replaces it, as in a vector store.
  */
-export declare class KeywordIndex implements Retriever {
+export declare class KeywordIndex implements SparseRetriever {
     private readonly chunks;
     private readonly postings;
     private totalLength;

package/dist-cjs/runtime/detect.d.ts

@@ -0,0 +1,17 @@
+/** A JavaScript runtime this package recognizes. */
+export type RuntimeName = 'node' | 'deno' | 'bun' | 'workerd' | 'edge-light' | 'browser' | 'unknown';
+/** Which runtime the code is running on. */
+export interface RuntimeInfo {
+    /**
+     * `node`, `deno`, `bun`, `workerd` for Cloudflare Workers, `edge-light` for Vercel's edge
+     * runtime, `browser` for a page or a web worker, or `unknown`.
+     */
+    name: RuntimeName;
+    /** The runtime's version, where it reports one. */
+    version?: string;
+}
+/**
+ * Which runtime the code is running on, from what each one reliably exposes. Deno and Bun are
+ * checked before Node.js, since both also present Node's `process`.
+ */
+export declare function runtimeInfo(): RuntimeInfo;

package/dist-cjs/runtime/detect.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runtimeInfo = runtimeInfo;
+function runtimeInfo() {
+    const host = globalThis;
+    if (host.Deno?.version?.deno)
+        return { name: 'deno', version: host.Deno.version.deno };
+    if (host.Bun?.version)
+        return { name: 'bun', version: host.Bun.version };
+    if (typeof host.EdgeRuntime === 'string')
+        return { name: 'edge-light' };
+    if (host.navigator?.userAgent === 'Cloudflare-Workers')
+        return { name: 'workerd' };
+    if (host.process?.versions?.node)
+        return { name: 'node', version: host.process.versions.node };
+    if (host.document !== undefined || typeof host.importScripts === 'function')
+        return { name: 'browser' };
+    return { name: 'unknown' };
+}

package/dist-cjs/runtime/index.d.ts

@@ -0,0 +1,19 @@
+/**
+ * The portable kernel: the contracts every model call and tool call is built on, for any JavaScript
+ * runtime. That means Node.js, Deno, Bun, edge runtimes such as Cloudflare Workers and Vercel's,
+ * and browsers.
+ *
+ * Nothing here, nor anything it imports, uses `fs`, `net`, `child_process`, Node's `crypto`, or
+ * Node's globals, and CI runs its tests on each runtime. The provider adapters, graphs, agents, and
+ * protocols are portable too, on their own entry points.
+ */
+export type { AssetContent, AudioContent, BinaryBuffer, CacheHint, CompletionRequest, ContentPart, ImageContent, Message, MessageRole, PromptCacheConfig, ReasoningConfig, TextContent, ToolCallResult, ToolChoice, ToolContext, ToolDefinition, ToolOutput, VideoContent, } from '../types/messages.js';
+export type { CacheOutcome, NexusResponse, NexusStream, ResponseCost, ResponseMeta, StreamChunk, TokenUsage, ToolCall, } from '../types/response.js';
+export type { ProviderCallContext } from '../types/lifecycle.js';
+export { BaseProvider, type ProviderInfo } from '../providers/base.js';
+export { categorizeProviderError, createProviderHttpError, isAbortError, isRetryableProviderError, NexusProviderError, type NexusProviderErrorCategory, type NexusProviderErrorOptions, toNexusProviderError, } from '../providers/errors.js';
+export { isToolOutput, ToolExecutor, tool, toolMessageContent, toolOutput } from '../agent/tool.js';
+export { capabilitiesOf, isSensitiveCapability, type ParsedCapability, parseCapability, } from '../agent/capabilities.js';
+export { type NegotiateOptions, type NegotiationResult, NexusCapabilityError, negotiateCompletionRequest, } from '../capabilities/negotiate.js';
+export { collectStream, createTextStream, mapStream } from '../core/streaming.js';
+export { type RuntimeInfo, type RuntimeName, runtimeInfo } from './detect.js';

package/dist-cjs/runtime/index.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runtimeInfo = exports.mapStream = exports.createTextStream = exports.collectStream = exports.negotiateCompletionRequest = exports.NexusCapabilityError = exports.parseCapability = exports.isSensitiveCapability = exports.capabilitiesOf = exports.toolOutput = exports.toolMessageContent = exports.tool = exports.ToolExecutor = exports.isToolOutput = exports.toNexusProviderError = exports.NexusProviderError = exports.isRetryableProviderError = exports.isAbortError = exports.createProviderHttpError = exports.categorizeProviderError = exports.BaseProvider = void 0;
+var base_js_1 = require("../providers/base.js");
+Object.defineProperty(exports, "BaseProvider", { enumerable: true, get: function () { return base_js_1.BaseProvider; } });
+var errors_js_1 = require("../providers/errors.js");
+Object.defineProperty(exports, "categorizeProviderError", { enumerable: true, get: function () { return errors_js_1.categorizeProviderError; } });
+Object.defineProperty(exports, "createProviderHttpError", { enumerable: true, get: function () { return errors_js_1.createProviderHttpError; } });
+Object.defineProperty(exports, "isAbortError", { enumerable: true, get: function () { return errors_js_1.isAbortError; } });
+Object.defineProperty(exports, "isRetryableProviderError", { enumerable: true, get: function () { return errors_js_1.isRetryableProviderError; } });
+Object.defineProperty(exports, "NexusProviderError", { enumerable: true, get: function () { return errors_js_1.NexusProviderError; } });
+Object.defineProperty(exports, "toNexusProviderError", { enumerable: true, get: function () { return errors_js_1.toNexusProviderError; } });
+var tool_js_1 = require("../agent/tool.js");
+Object.defineProperty(exports, "isToolOutput", { enumerable: true, get: function () { return tool_js_1.isToolOutput; } });
+Object.defineProperty(exports, "ToolExecutor", { enumerable: true, get: function () { return tool_js_1.ToolExecutor; } });
+Object.defineProperty(exports, "tool", { enumerable: true, get: function () { return tool_js_1.tool; } });
+Object.defineProperty(exports, "toolMessageContent", { enumerable: true, get: function () { return tool_js_1.toolMessageContent; } });
+Object.defineProperty(exports, "toolOutput", { enumerable: true, get: function () { return tool_js_1.toolOutput; } });
+var capabilities_js_1 = require("../agent/capabilities.js");
+Object.defineProperty(exports, "capabilitiesOf", { enumerable: true, get: function () { return capabilities_js_1.capabilitiesOf; } });
+Object.defineProperty(exports, "isSensitiveCapability", { enumerable: true, get: function () { return capabilities_js_1.isSensitiveCapability; } });
+Object.defineProperty(exports, "parseCapability", { enumerable: true, get: function () { return capabilities_js_1.parseCapability; } });
+var negotiate_js_1 = require("../capabilities/negotiate.js");
+Object.defineProperty(exports, "NexusCapabilityError", { enumerable: true, get: function () { return negotiate_js_1.NexusCapabilityError; } });
+Object.defineProperty(exports, "negotiateCompletionRequest", { enumerable: true, get: function () { return negotiate_js_1.negotiateCompletionRequest; } });
+var streaming_js_1 = require("../core/streaming.js");
+Object.defineProperty(exports, "collectStream", { enumerable: true, get: function () { return streaming_js_1.collectStream; } });
+Object.defineProperty(exports, "createTextStream", { enumerable: true, get: function () { return streaming_js_1.createTextStream; } });
+Object.defineProperty(exports, "mapStream", { enumerable: true, get: function () { return streaming_js_1.mapStream; } });
+var detect_js_1 = require("./detect.js");
+Object.defineProperty(exports, "runtimeInfo", { enumerable: true, get: function () { return detect_js_1.runtimeInfo; } });

package/dist-cjs/security/injection-detector.d.ts

@@ -7,5 +7,7 @@ export declare class InjectionDetector {
     /** Returns the request with injection markers defused. */
     neutralize(request: CompletionRequest): CompletionRequest;
     private extractText;
+    /** Defuses what `detect()` finds. Text with nothing to defuse is returned exactly as it was. */
     private escapeInstructionLikeText;
+    private escapeNormalized;
 }

package/dist-cjs/security/injection-detector.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InjectionDetector = void 0;
+const checksums_js_1 = require("../utils/checksums.js");
 const DEFAULT_PATTERNS = [
     {
         pattern: /ignore\s+(all\s+)?previous\s+instructions/i,
@@ -41,7 +42,7 @@ class InjectionDetector {
         ];
         const findings = [];
         request.messages.forEach((message, index) => {
-            for (const text of this.extractText(message)) {
+            for (const text of this.extractText(message).map(checksums_js_1.normalizeForDetection)) {
                 patterns.forEach(({ pattern, severity, label }) => {
                     const match = text.match(pattern);
                     if (!match)
@@ -75,6 +76,11 @@ class InjectionDetector {
         return message.content.filter((part) => part.type === 'text').map((part) => part.text);
     }
     escapeInstructionLikeText(text) {
+        const normalized = (0, checksums_js_1.normalizeForDetection)(text);
+        const escaped = this.escapeNormalized(normalized);
+        return escaped === normalized ? text : escaped;
+    }
+    escapeNormalized(text) {
         return text
             .replace(/ignore\s+(all\s+)?previous\s+instructions/gi, '[neutralized instruction override]')
             .replace(/disregard\s+(all\s+)?(prior|previous)\s+instructions/gi, '[neutralized instruction override]')

package/dist-cjs/security/input-guard.js

@@ -21,7 +21,7 @@ const SECRET_PATTERNS = [
 ];
 const SUSPICIOUS_URL_PATTERNS = [
     {
-        pattern: /https?:\/\/(?:localhost|127\.0\.0\.1|0\.0\.0\.0|169\.254\.169\.254)(?:[:/]|$)/gi,
+        pattern: /https?:\/\/(?:localhost|127(?:\.\d{1,3}){3}|0\.0\.0\.0|\[::1?\]|169\.254\.169\.254|metadata\.google\.internal|100\.100\.100\.200)(?:[:/]|$)/gi,
         label: 'local or metadata URL',
     },
     { pattern: /https?:\/\/[^\s]*\.(?:onion)(?:[:/]|$)/gi, label: 'onion URL' },
@@ -100,7 +100,7 @@ class InputGuard {
             let output = text;
             patterns.forEach(({ pattern, label, severity }) => {
                 output = output.replace(pattern, (value) => {
-                    findings.push({ type, severity, message: `Detected ${label}`, path, value });
+                    findings.push({ type, severity, message: `Detected ${label}`, path, value: previewSecret(value) });
                     return action === 'mask' ? '[REDACTED]' : value;
                 });
             });
@@ -130,3 +130,6 @@ class InputGuard {
     }
 }
 exports.InputGuard = InputGuard;
+function previewSecret(value) {
+    return `${value.slice(0, 4)}… (${value.length} characters)`;
+}

package/dist-cjs/security/output-guard.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OutputGuard = void 0;
 exports.redactSensitiveText = redactSensitiveText;
+const checksums_js_1 = require("../utils/checksums.js");
 const OUTPUT_SECRET_PATTERNS = [
     {
         pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
@@ -16,17 +17,18 @@ const OUTPUT_SECRET_PATTERNS = [
     { pattern: /\bAIza[0-9A-Za-z_-]{35}\b/g, label: 'google api key', severity: 'critical' },
     { pattern: /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, label: 'jwt token', severity: 'high' },
     { pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, label: 'email address', severity: 'high' },
+    { pattern: /\b(?:\d[ -]*?){13,19}\b/g, label: 'possible credit card', severity: 'high', valid: checksums_js_1.luhn },
     {
         pattern: /(?:\+?\d{1,3}[\s.-]?)?(?:\(?\d{2,4}\)?[\s.-]?)?\d{3,4}[\s.-]?\d{4}\b/g,
         label: 'phone number',
         severity: 'high',
+        valid: checksums_js_1.phoneNumber,
     },
     {
         pattern: /\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b/g,
         label: 'ip address',
         severity: 'high',
     },
-    { pattern: /\b(?:\d[ -]*?){13,19}\b/g, label: 'possible credit card', severity: 'high' },
     {
         pattern: /(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|secret|password|token)\s*[:=]\s*(?:['"][^'"\r\n]+['"]|[^\s,;]{8,})/gi,
         label: 'secret assignment',
@@ -37,8 +39,8 @@ const OUTPUT_SECRET_PATTERNS = [
 const CONNECTION_STRING_PATTERN = /\b(?:postgres|postgresql|mysql|mongodb|redis|mssql):\/\/[^\s'"<>]+/gi;
 function redactSensitiveText(value) {
     let redacted = value;
-    for (const { pattern } of OUTPUT_SECRET_PATTERNS) {
-        redacted = redacted.replace(pattern, '[REDACTED]');
+    for (const { pattern, valid } of OUTPUT_SECRET_PATTERNS) {
+        redacted = redacted.replace(pattern, (match) => (valid && !valid(match) ? match : '[REDACTED]'));
     }
     return redacted.replace(CONNECTION_STRING_PATTERN, '[REDACTED_CONNECTION_STRING]');
 }
@@ -68,9 +70,11 @@ class OutputGuard {
         }
         const shouldRedact = config.output?.piiRedaction !== false;
         if (shouldRedact) {
-            for (const { pattern, label, severity } of OUTPUT_SECRET_PATTERNS) {
+            for (const { pattern, label, severity, valid } of OUTPUT_SECRET_PATTERNS) {
                 for (const target of textTargets) {
                     target.value = target.value.replace(pattern, (value) => {
+                        if (valid && !valid(value))
+                            return value;
                         findings.push({
                             type: 'pii',
                             severity,