@promptev/context-engine 0.0.0 → 0.0.2

package/dist/storage-Dvpq2xAC.d.cts

@@ -0,0 +1,167 @@
+import { b as EmbeddingConfig } from './config-C5RZ00W6.cjs';
+import { PoolClient, Pool } from 'pg';
+
+/**
+ * Pluggable embeddings provider.
+ *
+ * openai / azure_openai / custom → OpenAI SDK (OpenAI-compatible, including TEI/vLLM).
+ * gemini / vertex_ai → lazy `@google/genai` (optional extra "gemini"). One
+ * code path: same SDK and models, differing only in how the client
+ * authenticates (API key vs ADC against a GCP project) — see
+ * `providers/google.ts`.
+ * voyage / cohere → fetch against their public REST APIs.
+ *
+ * `kind` is accepted everywhere but only voyage/cohere act on it.
+ * 30s timeout, no retries. Injectable `client` / `fetch` for tests.
+ */
+
+type EmbedKind = "document" | "query";
+type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+type EmbeddingsClient = {
+    embeddings: {
+        create(body: {
+            model: string;
+            input: string[];
+        }): Promise<{
+            data: Array<{
+                embedding: number[];
+            }>;
+            usage?: {
+                total_tokens?: number;
+            } | null;
+        }>;
+    };
+    close?: () => void | Promise<void>;
+    timeout?: number;
+    baseURL?: string;
+};
+type EmbedderOpts = {
+    client?: EmbeddingsClient | null;
+    fetch?: FetchImpl | null;
+    fetchImpl?: FetchImpl | null;
+};
+declare class Embedder {
+    cfg: EmbeddingConfig;
+    provider: EmbeddingConfig["provider"];
+    model: string;
+    dim: number | null;
+    client: EmbeddingsClient | null;
+    fetchImpl: FetchImpl | null;
+    private genaiClient;
+    constructor(cfg: EmbeddingConfig, opts?: EmbedderOpts);
+    aclose(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+    /**
+     * Embed `texts`, returning `[vectors, tokenCount]`.
+     * Auto-detects `this.dim` on first success; raises on mismatch.
+     */
+    embed(texts: string[], opts?: {
+        kind?: EmbedKind;
+    }): Promise<[number[][], number]>;
+    /** Provider dispatch. Overridable per-instance (tests stub this). */
+    rawEmbed(texts: string[], kind?: EmbedKind): Promise<[number[][], number]>;
+    private embedOpenAI;
+    private embedGemini;
+    private embedVoyage;
+    private embedCohere;
+    private requireFetch;
+}
+declare function buildEmbedder(cfg: EmbeddingConfig, opts?: EmbedderOpts): Embedder;
+
+interface ChunkRow {
+    idx: number;
+    text: string;
+    lang?: string | null;
+    embedding?: number[] | null;
+    meta?: Record<string, unknown>;
+}
+interface SearchScope {
+    sourceIds?: string[] | null;
+    /** Narrows WITHIN a source and INTERSECTS with `sourceIds` rather than
+     * widening it: a consumer that binds an agent to three of a source's
+     * documents needs a filter `sourceIds` cannot express, since every one of
+     * those documents shares the same source. */
+    documentIds?: string[] | null;
+    principals?: string[] | null;
+    limit?: number;
+}
+/** The ANN scan tuning the vector leg and the graph leg's seed query share.
+ * Named separately from `StorageBackend`, which carries it as an OPTIONAL
+ * member: a backend with no approximate index has no such knobs. */
+interface AnnTunable {
+    tuneAnnScan(client: PoolClient, limit: number, scoped: boolean): Promise<void>;
+}
+interface StorageBackend {
+    supportsFts: boolean;
+    supportsTrgm: boolean;
+    supportsAnn: boolean;
+    upsertChunks(documentId: string, sourceId: string | null, acl: string[] | null, chunks: readonly ChunkRow[]): Promise<number>;
+    deleteChunks(documentId: string): Promise<void>;
+    updateChunkEmbeddings(documentId: string, vectors: readonly number[][]): Promise<number>;
+    updateChunkAcl(documentId: string, acl: string[] | null): Promise<number>;
+    filterIds(chunkIds: readonly string[], scope: SearchScope): Promise<string[]>;
+    ftsSearch(query: string, scope: SearchScope): Promise<string[]>;
+    trgmSearch(query: string, scope: SearchScope): Promise<string[]>;
+    annSearch(vector: readonly number[], scope: SearchScope): Promise<string[]>;
+    /** Optional: only a backend with an approximate index has a scan to tune.
+     * The graph leg's seed query is the second caller — see `AnnTunable`. */
+    tuneAnnScan?: AnnTunable["tuneAnnScan"];
+}
+declare class PostgresBackend implements StorageBackend {
+    supportsFts: boolean;
+    supportsTrgm: boolean;
+    supportsAnn: boolean;
+    private readonly pool;
+    private readonly exactThresholdOverride;
+    /** Tri-state: null = not yet checked. */
+    private iterativeScan;
+    constructor(pool: Pool, exactThreshold?: number | null);
+    private get exactThreshold();
+    /**
+     * Bind values for SCOPE + the row limit.
+     * `principals` is passed through UNCHANGED: null (trusted) and [] (anonymous)
+     * mean different things.
+     */
+    private scopeParams;
+    upsertChunks(documentId: string, sourceId: string | null, acl: string[] | null, chunks: readonly ChunkRow[]): Promise<number>;
+    deleteChunks(documentId: string): Promise<void>;
+    updateChunkEmbeddings(documentId: string, vectors: readonly number[][]): Promise<number>;
+    updateChunkAcl(documentId: string, acl: string[] | null): Promise<number>;
+    filterIds(chunkIds: readonly string[], scope: SearchScope): Promise<string[]>;
+    ftsSearch(query: string, scope: SearchScope): Promise<string[]>;
+    trgmSearch(query: string, scope: SearchScope): Promise<string[]>;
+    /**
+     * Set pg_trgm's similarity threshold for this query via the `set_limit()` /
+     * `show_limit()` FUNCTIONS — never the GUC `pg_trgm.similarity_threshold`.
+     *
+     * The GUC is registered lazily, only once pg_trgm's shared library is loaded
+     * into the backend process. On a fresh pooled connection that has not yet
+     * run any pg_trgm operation, `SHOW pg_trgm.similarity_threshold` raises
+     * "unrecognized configuration parameter" — and because that error aborts the
+     * surrounding transaction, a fallback issued on the same connection then
+     * fails with "current transaction is aborted". Every connection a pool hands
+     * out is fresh, so this is the common case, not an edge case. Calling
+     * `set_limit()` loads the library and does not depend on the GUC. This
+     * mirrors the Python backend, which was hardened against exactly this.
+     *
+     * `set_limit()` is connection-scoped and survives COMMIT and the return to
+     * the pool, so the previous value is captured here and restored by the
+     * caller.
+     */
+    private applyTrgmThreshold;
+    private restoreTrgmLimit;
+    annSearch(vector: readonly number[], scope: SearchScope): Promise<string[]>;
+    private eligibleIsSmall;
+    /** Size the HNSW candidate budget, and make it iterative when filtered.
+     *
+     * Public because the graph leg's seed query (`graph/retrieval.ts`) is the
+     * same shape — an ANN walk with the scope predicate as a POST-filter — and
+     * must not carry a second copy of this. Call it inside a transaction on the
+     * client the query itself will run on: every setting here is `SET LOCAL`. */
+    tuneAnnScan(client: PoolClient, limit: number, scoped: boolean): Promise<void>;
+    private scanBudget;
+    private enableIterativeScan;
+    private supportsIterativeScan;
+}
+
+export { type AnnTunable as A, type ChunkRow as C, Embedder as E, type FetchImpl as F, PostgresBackend as P, type StorageBackend as S, type EmbedKind as a, type SearchScope as b, buildEmbedder as c };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@promptev/context-engine",
-  "version": "0.0.0",
+  "version": "0.0.2",
   "description": "Promptev Context Engine — language-agnostic ingestion + hybrid retrieval (TypeScript)",
   "license": "Apache-2.0",
   "author": "Promptev Inc",
@@ -33,39 +33,74 @@
   },
   "exports": {
     ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "require": "./dist/index.cjs"
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
     },
     "./hono": {
-      "types": "./dist/hono.d.ts",
-      "import": "./dist/hono.js",
-      "require": "./dist/hono.cjs"
+      "import": {
+        "types": "./dist/hono.d.ts",
+        "default": "./dist/hono.js"
+      },
+      "require": {
+        "types": "./dist/hono.d.cts",
+        "default": "./dist/hono.cjs"
+      }
     },
     "./express": {
-      "types": "./dist/express.d.ts",
-      "import": "./dist/express.js",
-      "require": "./dist/express.cjs"
+      "import": {
+        "types": "./dist/express.d.ts",
+        "default": "./dist/express.js"
+      },
+      "require": {
+        "types": "./dist/express.d.cts",
+        "default": "./dist/express.cjs"
+      }
     },
     "./fastify": {
-      "types": "./dist/fastify.d.ts",
-      "import": "./dist/fastify.js",
-      "require": "./dist/fastify.cjs"
+      "import": {
+        "types": "./dist/fastify.d.ts",
+        "default": "./dist/fastify.js"
+      },
+      "require": {
+        "types": "./dist/fastify.d.cts",
+        "default": "./dist/fastify.cjs"
+      }
     },
     "./mcp": {
-      "types": "./dist/mcp.d.ts",
-      "import": "./dist/mcp.js",
-      "require": "./dist/mcp.cjs"
+      "import": {
+        "types": "./dist/mcp.d.ts",
+        "default": "./dist/mcp.js"
+      },
+      "require": {
+        "types": "./dist/mcp.d.cts",
+        "default": "./dist/mcp.cjs"
+      }
     },
     "./graph": {
-      "types": "./dist/graph/index.d.ts",
-      "import": "./dist/graph/index.js",
-      "require": "./dist/graph/index.cjs"
+      "import": {
+        "types": "./dist/graph/index.d.ts",
+        "default": "./dist/graph/index.js"
+      },
+      "require": {
+        "types": "./dist/graph/index.d.cts",
+        "default": "./dist/graph/index.cjs"
+      }
     },
     "./presidio": {
-      "types": "./dist/redaction-presidio.d.ts",
-      "import": "./dist/redaction-presidio.js",
-      "require": "./dist/redaction-presidio.cjs"
+      "import": {
+        "types": "./dist/redaction-presidio.d.ts",
+        "default": "./dist/redaction-presidio.js"
+      },
+      "require": {
+        "types": "./dist/redaction-presidio.d.cts",
+        "default": "./dist/redaction-presidio.cjs"
+      }
     }
   },
   "files": [
@@ -86,7 +121,8 @@
     "typecheck": "tsc --noEmit",
     "lint": "biome check src tests",
     "lint:fix": "biome check --write src tests",
-    "context-engine": "node dist/cli.js"
+    "context-engine": "node dist/cli.js",
+    "prepublishOnly": "npm run lint && npm run typecheck && npm run build"
   },
   "dependencies": {
     "cheerio": "^1.1.0",
@@ -172,6 +208,8 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^2.1.0",
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "@napi-rs/canvas": "^1.0.6",
     "@testcontainers/postgresql": "^11.0.0",
     "@types/express": "^5.0.0",
     "@types/node": "^22.16.0",

package/src/migrations/sql/0003_tools.sql

@@ -11,6 +11,7 @@ CREATE TABLE context_engine_tools (
   requires_approval BOOLEAN NOT NULL DEFAULT false,
   approval_policy JSONB NOT NULL DEFAULT '{}'::jsonb,
   enabled BOOLEAN NOT NULL DEFAULT true,
+  meta_data JSONB NOT NULL DEFAULT '{}'::jsonb,
   created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
 );
@@ -22,6 +23,7 @@ CREATE TABLE context_engine_tool_approvals (
   tool_args_frozen JSONB NOT NULL DEFAULT '{}'::jsonb,
   source_id TEXT,
   principals JSONB,
+  approval_scope TEXT,
   status TEXT NOT NULL DEFAULT 'pending',
   approver TEXT,
   approver_meta JSONB,

package/src/migrations/sql/0004_acl_indexes.sql

@@ -1,4 +1,25 @@
 -- 0004_acl_indexes
+-- IF NOT EXISTS: this file doubles as the HEAL step runMigrate applies after
+-- every chain walk — see the comment in db.ts.
 
-CREATE INDEX ix_context_engine_chunks_acl_gin ON context_engine_chunks USING gin (acl);
-CREATE INDEX ix_context_engine_documents_acl_gin ON context_engine_documents USING gin (acl);
+CREATE INDEX IF NOT EXISTS ix_context_engine_chunks_acl_gin ON context_engine_chunks USING gin (acl);
+CREATE INDEX IF NOT EXISTS ix_context_engine_documents_acl_gin ON context_engine_documents USING gin (acl);
+
+-- tools.meta_data (2026-08-18) — engine-opaque user metadata, added after
+-- the tools chain revision shipped. Riding the heal step (this file re-runs
+-- after every chain walk) converges every existing database without
+-- re-parenting the chain; fresh databases also get it from 0003's CREATE.
+-- The Python port does the same in cli.py `_heal_schema`.
+ALTER TABLE context_engine_tools ADD COLUMN IF NOT EXISTS meta_data JSONB NOT NULL DEFAULT '{}'::jsonb;
+
+-- tool_approvals.approval_scope (2026-09-04) — the claim scope, separate from
+-- the tool-lookup source_id. The partial UNIQUE index is what makes
+-- findOrCreatePending race-safe: two identical calls in one scope can only
+-- ever leave one pending row. md5(jsonb::text) is stable because jsonb text
+-- output is canonical. The Python port does the same in cli.py _heal_schema.
+ALTER TABLE context_engine_tool_approvals ADD COLUMN IF NOT EXISTS approval_scope TEXT;
+CREATE INDEX IF NOT EXISTS ix_context_engine_tool_approvals_claim
+  ON context_engine_tool_approvals (tool_name, approval_scope, status);
+CREATE UNIQUE INDEX IF NOT EXISTS ux_context_engine_tool_approvals_pending_scope
+  ON context_engine_tool_approvals (tool_name, approval_scope, md5(tool_args_frozen::text))
+  WHERE status = 'pending' AND approval_scope IS NOT NULL;

package/src/skills/context-engine/SKILL.md

@@ -70,7 +70,11 @@ await engine.search(q, { principals: user.groups }); // scoped
   silently loses recall — queries return fewer rows than match, or none, with
   no error. The library cannot compensate.
 - **`config.enableCodeExecution` is `false` on purpose.** `engine.compute()`
-  executes generated code. Turn it on only deliberately.
+  and `computeOverFrames()` execute generated code. Turn it on only
+  deliberately. `computeOverFrames({ sheet: rows }, instruction, { config:
+  engine.config })` is the same guarded path for tables the host already holds
+  (no document to point at); redaction masks every string cell AND every column
+  header before the LLM sees a schema, and sweeps the result.
 - Filing a document under an ACL the caller does not hold returns **403**, on
   ingest and on PATCH alike.
 

package/dist/embeddings-B-jZ42mk.d.cts

@@ -1,67 +0,0 @@
-import { E as EmbeddingConfig } from './config-Bl9U789m.cjs';
-
-/**
- * Pluggable embeddings provider.
- *
- * openai / azure_openai / custom → OpenAI SDK (OpenAI-compatible, including TEI/vLLM).
- * gemini → lazy `@google/genai` (optional extra "gemini").
- * voyage / cohere → fetch against their public REST APIs.
- *
- * `kind` is accepted everywhere but only voyage/cohere act on it.
- * 30s timeout, no retries. Injectable `client` / `fetch` for tests.
- */
-
-type EmbedKind = "document" | "query";
-type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
-type EmbeddingsClient = {
-    embeddings: {
-        create(body: {
-            model: string;
-            input: string[];
-        }): Promise<{
-            data: Array<{
-                embedding: number[];
-            }>;
-            usage?: {
-                total_tokens?: number;
-            } | null;
-        }>;
-    };
-    close?: () => void | Promise<void>;
-    timeout?: number;
-    baseURL?: string;
-};
-type EmbedderOpts = {
-    client?: EmbeddingsClient | null;
-    fetch?: FetchImpl | null;
-    fetchImpl?: FetchImpl | null;
-};
-declare class Embedder {
-    cfg: EmbeddingConfig;
-    provider: EmbeddingConfig["provider"];
-    model: string;
-    dim: number | null;
-    client: EmbeddingsClient | null;
-    fetchImpl: FetchImpl | null;
-    private genaiClient;
-    constructor(cfg: EmbeddingConfig, opts?: EmbedderOpts);
-    aclose(): Promise<void>;
-    [Symbol.asyncDispose](): Promise<void>;
-    /**
-     * Embed `texts`, returning `[vectors, tokenCount]`.
-     * Auto-detects `this.dim` on first success; raises on mismatch.
-     */
-    embed(texts: string[], opts?: {
-        kind?: EmbedKind;
-    }): Promise<[number[][], number]>;
-    /** Provider dispatch. Overridable per-instance (tests stub this). */
-    rawEmbed(texts: string[], kind?: EmbedKind): Promise<[number[][], number]>;
-    private embedOpenAI;
-    private embedGemini;
-    private embedVoyage;
-    private embedCohere;
-    private requireFetch;
-}
-declare function buildEmbedder(cfg: EmbeddingConfig, opts?: EmbedderOpts): Embedder;
-
-export { Embedder as E, type FetchImpl as F, type EmbedKind as a, buildEmbedder as b };

package/dist/embeddings-DaSdAZN3.d.ts

@@ -1,67 +0,0 @@
-import { E as EmbeddingConfig } from './config-Bt9bUQqU.js';
-
-/**
- * Pluggable embeddings provider.
- *
- * openai / azure_openai / custom → OpenAI SDK (OpenAI-compatible, including TEI/vLLM).
- * gemini → lazy `@google/genai` (optional extra "gemini").
- * voyage / cohere → fetch against their public REST APIs.
- *
- * `kind` is accepted everywhere but only voyage/cohere act on it.
- * 30s timeout, no retries. Injectable `client` / `fetch` for tests.
- */
-
-type EmbedKind = "document" | "query";
-type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
-type EmbeddingsClient = {
-    embeddings: {
-        create(body: {
-            model: string;
-            input: string[];
-        }): Promise<{
-            data: Array<{
-                embedding: number[];
-            }>;
-            usage?: {
-                total_tokens?: number;
-            } | null;
-        }>;
-    };
-    close?: () => void | Promise<void>;
-    timeout?: number;
-    baseURL?: string;
-};
-type EmbedderOpts = {
-    client?: EmbeddingsClient | null;
-    fetch?: FetchImpl | null;
-    fetchImpl?: FetchImpl | null;
-};
-declare class Embedder {
-    cfg: EmbeddingConfig;
-    provider: EmbeddingConfig["provider"];
-    model: string;
-    dim: number | null;
-    client: EmbeddingsClient | null;
-    fetchImpl: FetchImpl | null;
-    private genaiClient;
-    constructor(cfg: EmbeddingConfig, opts?: EmbedderOpts);
-    aclose(): Promise<void>;
-    [Symbol.asyncDispose](): Promise<void>;
-    /**
-     * Embed `texts`, returning `[vectors, tokenCount]`.
-     * Auto-detects `this.dim` on first success; raises on mismatch.
-     */
-    embed(texts: string[], opts?: {
-        kind?: EmbedKind;
-    }): Promise<[number[][], number]>;
-    /** Provider dispatch. Overridable per-instance (tests stub this). */
-    rawEmbed(texts: string[], kind?: EmbedKind): Promise<[number[][], number]>;
-    private embedOpenAI;
-    private embedGemini;
-    private embedVoyage;
-    private embedCohere;
-    private requireFetch;
-}
-declare function buildEmbedder(cfg: EmbeddingConfig, opts?: EmbedderOpts): Embedder;
-
-export { Embedder as E, type FetchImpl as F, type EmbedKind as a, buildEmbedder as b };