@opencodehub/ingestion 0.1.0

package/dist/pipeline/phases/embedder-worker.js

@@ -0,0 +1,43 @@
+/**
+ * Piscina worker entry point for the embeddings phase.
+ *
+ * Each worker lazy-opens its own OnnxEmbedder on first task and caches it for
+ * the lifetime of the worker. Tasks carry a flat list of chunk texts; the
+ * worker returns the vectors in input order.
+ *
+ * Determinism contract: `openOnnxEmbedder()` pins intraOpNumThreads=1,
+ * interOpNumThreads=1, and graphOptimizationLevel="disabled" — the same
+ * input produces byte-identical output regardless of which worker processed
+ * it. Callers partition work deterministically (block-chunked by batch
+ * index) so repeat runs reproduce the same byte-level output.
+ */
+import { workerData } from "node:worker_threads";
+import { openOnnxEmbedder } from "@opencodehub/embedder";
+const cfg = workerData ?? { variant: "fp32" };
+let embedderPromise;
+function getEmbedder() {
+    if (embedderPromise === undefined) {
+        const opts = { variant: cfg.variant };
+        if (cfg.modelDir !== undefined)
+            opts.modelDir = cfg.modelDir;
+        embedderPromise = openOnnxEmbedder(opts);
+    }
+    return embedderPromise;
+}
+export default async function embedBatch(task) {
+    const embedder = await getEmbedder();
+    const vectors = await embedder.embedBatch(task.texts);
+    if (vectors.length === 0) {
+        return { buffer: new ArrayBuffer(0), dim: embedder.dim, count: 0 };
+    }
+    const dim = vectors[0]?.length ?? embedder.dim;
+    const out = new Float32Array(vectors.length * dim);
+    for (let i = 0; i < vectors.length; i++) {
+        const vec = vectors[i];
+        if (vec === undefined)
+            continue;
+        out.set(vec, i * dim);
+    }
+    return { buffer: out.buffer, dim, count: vectors.length };
+}
+//# sourceMappingURL=embedder-worker.js.map

package/dist/pipeline/phases/embedder-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder-worker.js","sourceRoot":"","sources":["../../../src/pipeline/phases/embedder-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAsBzD,MAAM,GAAG,GAAI,UAAuC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAE5E,IAAI,eAA8C,CAAC;AAEnD,SAAS,WAAW;IAClB,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,GAAoD,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;QACvF,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS;YAAE,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC7D,eAAe,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,UAAU,CAAC,IAAoB;IAC3D,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;IACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,MAAM,EAAE,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC;IAC/C,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;AAC5D,CAAC"}

package/dist/pipeline/phases/embeddings.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Embeddings phase — generates 768-dim vectors across one or more
+ * hierarchical tiers and materialises them into the phase output as an
+ * array of `EmbeddingRow`s the CLI upserts into DuckDB.
+ *
+ * Granularity tiers (P03):
+ *   - `"symbol"` — one vector per callable/declaration symbol. When a
+ *     `SymbolSummaryRow` exists for the node the text is fused
+ *     `signature\nsummary\nbody`; otherwise we fall back to the raw
+ *     signature/description pair.
+ *   - `"file"` — one vector per scanned file. Coarse tier used by the
+ *     `--zoom` retrieval path. Files larger than ~8192 tokens are
+ *     truncated to the first `N` chars so a single outlier never blows
+ *     up batch latency.
+ *   - `"community"` — one vector per Community node. Architectural tier
+ *     used to answer "which subsystem handles X?" queries. Text is
+ *     `inferredLabel\nkeywords…\ntop_symbols…`.
+ *
+ * Contract:
+ *   - `options.embeddings !== true` → phase is a silent no-op.
+ *   - Weights missing (EMBEDDER_NOT_SETUP) → emit a warning via the
+ *     progress callback and return zeroes. NEVER aborts the pipeline.
+ *   - Default `granularity = ["symbol"]` preserves v1.0 behaviour; callers
+ *     opt in to hierarchical tiers explicitly.
+ *
+ * Determinism:
+ *   - Rows are sorted by (granularity, node_id, chunk_index).
+ *     `embeddingsHash` hashes the canonical representation so downstream
+ *     callers can assert byte-level stability across runs. The hash is
+ *     returned in the phase output but is intentionally not folded into
+ *     graphHash.
+ */
+import type { EmbeddingGranularity, EmbeddingRow } from "@opencodehub/storage";
+import type { PipelinePhase } from "../types.js";
+export declare const EMBEDDER_PHASE_NAME: "embeddings";
+/**
+ * Options-bag extension point used by {@link runEmbeddings} to read prior
+ * `content_hash` values for the `embeddings` table. Plugged onto
+ * `ctx.options` by the orchestrator under this well-known key so the phase
+ * stays pure (no direct {@link IGraphStore} handle).
+ *
+ * When absent (or when `options.force === true`), the phase behaves as it
+ * did pre-M1-3: every eligible chunk is embedded and emitted. When present
+ * and `force !== true`, the adapter is invoked once per run; its returned
+ * map is probed per chunk so unchanged chunks skip both `embedder.embed()`
+ * and the upsert batch.
+ */
+export interface EmbeddingHashCacheAdapter {
+    /**
+     * Return every prior `content_hash` keyed by
+     * `${granularity}\0${nodeId}\0${chunkIndex}`. Empty map on a fresh
+     * database or any error the adapter wants to degrade gracefully.
+     */
+    list(): Promise<Map<string, string>>;
+}
+/**
+ * Well-known options key the orchestrator uses to attach an
+ * {@link EmbeddingHashCacheAdapter}. Kept as a `const` so callers can't
+ * typo the probe site. Matches the pattern used by `SUMMARY_CACHE_OPTIONS_KEY`
+ * in the summarize phase.
+ */
+export declare const EMBEDDING_HASH_CACHE_OPTIONS_KEY: "__embeddingHashCache";
+export interface EmbedderPhaseOptions {
+    /**
+     * Which granularity tiers to emit (P03). Defaults to `["symbol"]` so
+     * existing callers (v1.0 default) see no behavior change.
+     */
+    readonly granularity?: readonly EmbeddingGranularity[];
+}
+/** Shape of the `embeddings` phase output. */
+export interface EmbedderPhaseOutput {
+    /** Number of embeddings appended to the output `rows` array. */
+    readonly embeddingsInserted: number;
+    /** Symbols of an eligible kind that were skipped (no signature/name pair). */
+    readonly symbolsSkipped: number;
+    /** Total chunks emitted across all symbols (always >= embeddingsInserted). */
+    readonly chunksTotal: number;
+    /**
+     * Stable id tag for the embedder that produced these rows — e.g.
+     * `gte-modernbert-base/fp32`. Empty string when the phase was a
+     * no-op (flag off or weights missing).
+     */
+    readonly embeddingsModelId: string;
+    /**
+     * Content-addressable hash of the canonicalised rows. Used for acceptance
+     * and snapshot tests; NOT folded into graphHash.
+     */
+    readonly embeddingsHash: string;
+    /** Rows the CLI forwards to `store.upsertEmbeddings(...)`. */
+    readonly rows: readonly EmbeddingRow[];
+    /** `true` when the phase ran against a real embedder, `false` otherwise. */
+    readonly ranEmbedder: boolean;
+    /**
+     * Per-tier emit counts. Callers use this to surface "emitted 421
+     * symbol, 38 file, 7 community embeddings" in the analyze summary.
+     * Absent tiers are reported as `0` so consumers can iterate without
+     * null-checks.
+     */
+    readonly byGranularity: Readonly<Record<EmbeddingGranularity, number>>;
+    /**
+     * Whether the symbol tier fused `signature + summary + body` for at
+     * least one row. Diagnostic flag consumers can surface — summaries
+     * drive the biggest quality lift, so CLI output calls it out when it
+     * actually kicked in.
+     */
+    readonly summaryFused: boolean;
+    /**
+     * Chunks short-circuited by the content-hash skip. Counts
+     * chunks whose `(granularity, node_id, chunk_index)` had a prior row
+     * with identical `content_hash` in the store — so the phase neither
+     * embedded them nor emitted a row. `0` when `options.force === true`,
+     * when the hash-cache adapter is absent, or on a fresh database.
+     */
+    readonly chunksSkipped: number;
+}
+export declare const embeddingsPhase: PipelinePhase<EmbedderPhaseOutput>;
+//# sourceMappingURL=embeddings.d.ts.map

package/dist/pipeline/phases/embeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../../src/pipeline/phases/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAYH,OAAO,KAAK,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAmB,aAAa,EAAE,MAAM,aAAa,CAAC;AAiBlE,eAAO,MAAM,mBAAmB,EAAG,YAAqB,CAAC;AAEzD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,yBAAyB;IACxC;;;;OAIG;IACH,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACtC;AAED;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,EAAG,sBAA+B,CAAC;AAiFhF,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,oBAAoB,EAAE,CAAC;CACxD;AAED,8CAA8C;AAC9C,MAAM,WAAW,mBAAmB;IAClC,gEAAgE;IAChE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,8EAA8E;IAC9E,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,8EAA8E;IAC9E,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,8DAA8D;IAC9D,QAAQ,CAAC,IAAI,EAAE,SAAS,YAAY,EAAE,CAAC;IACvC,4EAA4E;IAC5E,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,CAAC;IACvE;;;;;OAKG;IACH,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAC/B;;;;;;OAMG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAioBD,eAAO,MAAM,eAAe,EAAE,aAAa,CAAC,mBAAmB,CAU9D,CAAC"}