@massa-ai/core 1.2.1

package/dist/services/embeddings/providers/local-transformers.js

@@ -0,0 +1,222 @@
+/**
+ * Local in-process embedding provider backed by transformers.js (@huggingface/transformers v3+, ONNX).
+ *
+ * Goal (roadmap A5): a fully offline, no-server, no-API-key embedding backend
+ * that composes with the existing EmbeddingProvider machinery (priority-ordered
+ * fallback chain, CachedEmbeddingProvider, rate limiter). This removes the hard
+ * runtime dependency on an external model server (Ollama) for a default install
+ * and sidesteps the documented "thinking model degrades structured output" bug.
+ *
+ * The model is loaded lazily on first use and cached by transformers.js in the
+ * local HuggingFace cache directory, so after the first run the provider needs
+ * no network access. Instantiation itself is cheap (no model I/O); the heavy
+ * ONNX load happens inside `ensureModel()` and is memoized via a single promise
+ * so concurrent embed calls share one load.
+ *
+ * The transformers.js dependency is optional at the type level: we import it
+ * dynamically (`await import("@huggingface/transformers")`) so the package only
+ * has to be present when this provider is actually selected. This keeps the dep
+ * cost off the default install path and lets tests mock the pipeline without
+ * pulling a multi-megabyte ONNX runtime.
+ *
+ * Migration note (@xenova/transformers v2 → @huggingface/transformers v3):
+ * the v3 package dropped the `quantized` pipeline option in favor of `dtype`.
+ * We pass `dtype: "q8"` to preserve the v2 `quantized: true` behavior
+ * (8-bit weights → ~25MB download, negligible retrieval-quality loss). The
+ * `pipeline`, `env`, and Tensor `.data` (typed Float32Array) surfaces are
+ * otherwise API-compatible.
+ */
+import { metrics } from "../../monitoring/metrics.js";
+import { logger } from "@massa-ai/shared";
+/**
+ * Default transformers.js model. all-MiniLM-L6-v2 is a small (~25MB ONNX),
+ * general-purpose sentence embedding model with 384 dimensions and solid
+ * semantic-search quality for its size. It is a reasonable offline default for
+ * code+text retrieval; users can override via TRANSFORMERS_EMBEDDING_MODEL.
+ *
+ * The `Xenova/...` model id remains resolvable by @huggingface/transformers v3
+ * (the weights live on the Hugging Face Hub under the Xenova org); changing it
+ * would be an out-of-scope behavioral change.
+ */
+export const DEFAULT_LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
+/** Known dimension for the default model, used when not declared in config. */
+export const DEFAULT_LOCAL_DIMENSIONS = 384;
+/**
+ * Local in-process embedding provider using transformers.js (ONNX runtime).
+ *
+ * Implements the same EmbeddingProvider interface as AISDKEmbeddingProvider so
+ * it transparently composes with CachedEmbeddingProvider and the fallback
+ * selection chain in embeddings/index.ts.
+ */
+export class LocalTransformersEmbeddingProvider {
+    id;
+    model;
+    dimensions;
+    /** Max characters to send per text (model context limit). */
+    maxChars;
+    /** Memoized model-load promise so concurrent embeds share a single load. */
+    modelPromise = null;
+    constructor(config, providerId) {
+        this.id = providerId;
+        this.model = config.model || DEFAULT_LOCAL_MODEL;
+        this.dimensions = config.dimensions || DEFAULT_LOCAL_DIMENSIONS;
+        this.maxChars = config.maxChars ?? 4000;
+    }
+    /**
+     * Lazily load the transformers.js feature-extraction pipeline.
+     *
+     * The import is dynamic so the (sizeable) ONNX runtime is only pulled in when
+     * this provider is actually selected. The load is memoized: the first caller
+     * triggers the ONNX/model load, all concurrent callers await the same promise.
+     */
+    async ensureModel() {
+        if (this.modelPromise) {
+            return this.modelPromise;
+        }
+        this.modelPromise = (async () => {
+            let transformers;
+            try {
+                // Dynamic import keeps @huggingface/transformers out of the hot path
+                // for every other provider. `import()` of an ESM package returns the
+                // namespace; pull the named exports we need.
+                const mod = (await import("@huggingface/transformers"));
+                transformers = mod;
+            }
+            catch (err) {
+                throw new Error(`[${this.id}] @huggingface/transformers is not installed. Install it in packages/core to use the local embedding provider.`);
+            }
+            // Make model-loading behavior explicit & forward-compatible. In a Node
+            // (non-browser) runtime v3 defaults allowLocalModels=true and
+            // allowRemoteModels=true, but pinning both here keeps the provider robust
+            // to env overrides and documents intent: first run downloads from the HF
+            // Hub, subsequent runs load from the local cache.
+            if (transformers.env) {
+                // allowRemoteModels defaults true; allowLocalModels defaults true on
+                // Node. Set both explicitly so a partial env override (e.g. CI setting
+                // allowRemoteModels=false) does not break the cache-fallback path.
+                transformers.env.allowLocalModels = transformers.env.allowLocalModels ?? true;
+                transformers.env.allowRemoteModels = transformers.env.allowRemoteModels ?? true;
+            }
+            logger.info(`[${this.id}] Loading local model "${this.model}" (offline ONNX)...`);
+            const extractor = await transformers.pipeline("feature-extraction", this.model, {
+                // v3 replaced the v2 `quantized: true` flag with `dtype`. "q8" is the
+                // 8-bit-quantized weight format: smaller download (~25MB for
+                // all-MiniLM-L6-v2) and faster inference, with negligible quality loss
+                // for retrieval. This preserves the v2 default behavior.
+                dtype: "q8",
+            });
+            logger.info(`[${this.id}] Local model ready (dimensions: ${this.dimensions}).`);
+            return extractor;
+        })();
+        // If the load fails, drop the memoized rejection so a later retry can
+        // attempt a fresh load instead of permanently caching the failure.
+        this.modelPromise.catch(() => {
+            this.modelPromise = null;
+        });
+        return this.modelPromise;
+    }
+    /** Truncate text to the configured per-call character limit. */
+    truncateText(text) {
+        if (text.length <= this.maxChars)
+            return text;
+        return text.substring(0, this.maxChars);
+    }
+    async embedQuery(text) {
+        const [embedding] = await this.embedBatch([text]);
+        return embedding;
+    }
+    async embedBatch(texts) {
+        if (texts.length === 0)
+            return [];
+        const startTime = Date.now();
+        const extractor = await this.ensureModel();
+        try {
+            // transformers.js returns one pooled vector per input text, in order.
+            const output = await extractor(texts.map((t) => this.truncateText(t)), { pooling: "mean", normalize: true });
+            const dims = output.dims;
+            const count = texts.length;
+            const dim = this.dimensions;
+            // transformers.js returns a typed array (Float32Array), not a regular
+            // Array — `Array.isArray()` is false for typed arrays, so read `.length`
+            // directly off the data object.
+            const dataArr = output.data;
+            const totalLen = dataArr && typeof dataArr.length === "number"
+                ? dataArr.length
+                : 0;
+            // Determine per-item length. transformers.js with pooling+normalize
+            // returns a flat Float32Array of length count*dim. Some versions expose
+            // `dims: [count, dim]`; prefer that when present.
+            const perItem = dims && dims.length >= 2 ? dims[dims.length - 1] : dim;
+            if (totalLen !== count * perItem) {
+                throw new Error(`[${this.id}] Unexpected embedding length: got ${totalLen} values for ${count} texts (expected ${count * perItem}).`);
+            }
+            const results = [];
+            for (let i = 0; i < count; i++) {
+                const slice = Array.from(output.data.slice(i * perItem, (i + 1) * perItem));
+                // Validate: no NaN/Infinity, not all-zero (degenerate).
+                if (slice.some((v) => isNaN(v) || !isFinite(v))) {
+                    throw new Error(`[${this.id}] Embedding for text ${i} contains NaN or Infinity.`);
+                }
+                results.push(slice);
+            }
+            const latency = Date.now() - startTime;
+            const tokens = Math.ceil(texts.reduce((sum, t) => sum + t.length, 0) / 4);
+            metrics.recordEmbedding({
+                provider: this.id,
+                tokens,
+                latency,
+                cached: false,
+                error: false,
+            });
+            return results;
+        }
+        catch (err) {
+            const latency = Date.now() - startTime;
+            const tokens = Math.ceil(texts.reduce((sum, t) => sum + t.length, 0) / 4);
+            metrics.recordEmbedding({
+                provider: this.id,
+                tokens,
+                latency,
+                cached: false,
+                error: true,
+            });
+            throw err;
+        }
+    }
+    /**
+     * Availability check. For the local provider this confirms the model can be
+     * loaded (which triggers the one-time ONNX/model download on first run).
+     * Cheap on subsequent calls because the load is memoized.
+     */
+    async isAvailable() {
+        try {
+            const extractor = await this.ensureModel();
+            // Smoke-test with a trivial query to confirm inference works.
+            const out = await extractor("test", { pooling: "mean", normalize: true });
+            const vec = Array.from(out.data);
+            if (!Array.isArray(vec) || vec.length !== this.dimensions) {
+                logger.error(`[${this.id}] Invalid embedding dimensions`, undefined, { expected: this.dimensions, got: vec.length });
+                return false;
+            }
+            if (vec.some((v) => typeof v !== "number" || isNaN(v))) {
+                logger.error(`[${this.id}] Invalid embedding values (not numbers)`);
+                return false;
+            }
+            return true;
+        }
+        catch (error) {
+            logger.error(`[${this.id}] Local provider unavailable`, error);
+            return false;
+        }
+    }
+    getConfig() {
+        return {
+            provider: "transformers",
+            model: this.model,
+            dimensions: this.dimensions,
+            maxChars: this.maxChars,
+            priority: 0,
+        };
+    }
+}
+//# sourceMappingURL=local-transformers.js.map

package/dist/services/embeddings/providers/local-transformers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-transformers.js","sourceRoot":"","sources":["../../../../src/services/embeddings/providers/local-transformers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE1C;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,yBAAyB,CAAC;AAE7D,+EAA+E;AAC/E,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAkC5C;;;;;;GAMG;AACH,MAAM,OAAO,kCAAkC;IAC7B,EAAE,CAAS;IACX,KAAK,CAAS;IACd,UAAU,CAAS;IAEnC,6DAA6D;IAC5C,QAAQ,CAAS;IAElC,4EAA4E;IACpE,YAAY,GAA8C,IAAI,CAAC;IAEvE,YAAY,MAA+B,EAAE,UAAkB;QAC7D,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,mBAAmB,CAAC;QACjD,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,wBAAwB,CAAC;QAChE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,YAAY,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC9B,IAAI,YAAgC,CAAC;YACrC,IAAI,CAAC;gBACH,qEAAqE;gBACrE,qEAAqE;gBACrE,6CAA6C;gBAC7C,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAY,CAAC;gBACnE,YAAY,GAAG,GAAyB,CAAC;YAC3C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CACb,IAAI,IAAI,CAAC,EAAE,gHAAgH,CAC5H,CAAC;YACJ,CAAC;YAED,uEAAuE;YACvE,8DAA8D;YAC9D,0EAA0E;YAC1E,yEAAyE;YACzE,kDAAkD;YAClD,IAAI,YAAY,CAAC,GAAG,EAAE,CAAC;gBACrB,qEAAqE;gBACrE,uEAAuE;gBACvE,mEAAmE;gBACnE,YAAY,CAAC,GAAG,CAAC,gBAAgB,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,IAAI,IAAI,CAAC;gBAC9E,YAAY,CAAC,GAAG,CAAC,iBAAiB,GAAG,YAAY,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI,CAAC;YAClF,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,0BAA0B,IAAI,CAAC,KAAK,qBAAqB,CAAC,CAAC;YAClF,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,EAAE;gBAC9E,sEAAsE;gBACtE,6DAA6D;gBAC7D,uEAAuE;gBACvE,yDAAyD;gBACzD,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,oCAAoC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;YAChF,OAAO,SAAS,CAAC;QACnB,CAAC,CAAC,EAAE,CAAC;QAEL,sEAAsE;QACtE,mEAAmE;QACnE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,gEAAgE;IACxD,YAAY,CAAC,IAAY;QAC/B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC9C,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAE3C,IAAI,CAAC;YACH,sEAAsE;YACtE,MAAM,MAAM,GAAG,MAAM,SAAS,CAC5B,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EACtC,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CACrC,CAAC;YAEF,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACzB,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC;YAC5B,sEAAsE;YACtE,yEAAyE;YACzE,gCAAgC;YAChC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;YAC5B,MAAM,QAAQ,GACZ,OAAO,IAAI,OAAQ,OAA+B,CAAC,MAAM,KAAK,QAAQ;gBACpE,CAAC,CAAE,OAA8B,CAAC,MAAM;gBACxC,CAAC,CAAC,CAAC,CAAC;YAER,oEAAoE;YACpE,wEAAwE;YACxE,kDAAkD;YAClD,MAAM,OAAO,GAAG,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;YAEvE,IAAI,QAAQ,KAAK,KAAK,GAAG,OAAO,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CACb,IAAI,IAAI,CAAC,EAAE,sCAAsC,QAAQ,eAAe,KAAK,oBAAoB,KAAK,GAAG,OAAO,IAAI,CACrH,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAe,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;gBAC5E,wDAAwD;gBACxD,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChD,MAAM,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,wBAAwB,CAAC,4BAA4B,CAAC,CAAC;gBACpF,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACvC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CACtB,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAChD,CAAC;YACF,OAAO,CAAC,eAAe,CAAC;gBACtB,QAAQ,EAAE,IAAI,CAAC,EAAE;gBACjB,MAAM;gBACN,OAAO;gBACP,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;YAEH,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACvC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1E,OAAO,CAAC,eAAe,CAAC;gBACtB,QAAQ,EAAE,IAAI,CAAC,EAAE;gBACjB,MAAM;gBACN,OAAO;gBACP,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;YACH,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,8DAA8D;YAC9D,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1E,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC1D,MAAM,CAAC,KAAK,CACV,IAAI,IAAI,CAAC,EAAE,gCAAgC,EAC3C,SAAS,EACT,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,CAC/C,CAAC;gBACF,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,0CAA0C,CAAC,CAAC;gBACpE,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,8BAA8B,EAAE,KAAc,CAAC,CAAC;YACxE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,SAAS;QACP,OAAO;YACL,QAAQ,EAAE,cAAc;YACxB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,CAAC;SACZ,CAAC;IACJ,CAAC;CACF"}

package/dist/services/embeddings/rate-limiter.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Rate Limiter for Embedding Providers
+ *
+ * Implements sliding window rate limiting for:
+ * - RPM (Requests Per Minute)
+ * - TPM (Tokens Per Minute)
+ * - RPD (Requests Per Day)
+ *
+ * Prevents hitting API rate limits by tracking requests and enforcing delays.
+ */
+interface RateLimitConfig {
+    requestsPerMinute?: number;
+    tokensPerMinute?: number;
+    requestsPerDay?: number;
+}
+export declare class EmbeddingRateLimiter {
+    private readonly requestsWindow;
+    private readonly dailyRequestsWindow;
+    private readonly config;
+    private readonly providerId;
+    constructor(providerId: string, config: RateLimitConfig);
+    /**
+     * Wait until rate limits allow the next request
+     * @param estimatedTokens Estimated tokens for this request (content.length / 4)
+     */
+    waitForCapacity(estimatedTokens?: number): Promise<void>;
+    /**
+     * Record a request for rate limiting
+     */
+    recordRequest(tokens?: number): void;
+    /**
+     * Get current rate limit status
+     */
+    getStatus(): {
+        rpm: {
+            current: number;
+            limit: number;
+            percentage: number;
+        };
+        tpm: {
+            current: number;
+            limit: number;
+            percentage: number;
+        };
+        rpd: {
+            current: number;
+            limit: number;
+            percentage: number;
+        };
+    };
+    private cleanWindow;
+    private sleep;
+}
+export {};
+//# sourceMappingURL=rate-limiter.d.ts.map

package/dist/services/embeddings/rate-limiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.d.ts","sourceRoot":"","sources":["../../../src/services/embeddings/rate-limiter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,UAAU,eAAe;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAOD,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;IACtD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAuB;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe;IAKvD;;;OAGG;IACG,eAAe,CAAC,eAAe,GAAE,MAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IA+DjE;;OAEG;IACH,aAAa,CAAC,MAAM,GAAE,MAAU,GAAG,IAAI;IAavC;;OAEG;IACH,SAAS;;;;;;;;;;;;;;;;;IAkCT,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,KAAK;CAGd"}

package/dist/services/embeddings/rate-limiter.js

@@ -0,0 +1,138 @@
+/**
+ * Rate Limiter for Embedding Providers
+ *
+ * Implements sliding window rate limiting for:
+ * - RPM (Requests Per Minute)
+ * - TPM (Tokens Per Minute)
+ * - RPD (Requests Per Day)
+ *
+ * Prevents hitting API rate limits by tracking requests and enforcing delays.
+ */
+import { logger } from "@massa-ai/shared";
+export class EmbeddingRateLimiter {
+    requestsWindow = [];
+    dailyRequestsWindow = [];
+    config;
+    providerId;
+    constructor(providerId, config) {
+        this.providerId = providerId;
+        this.config = config;
+    }
+    /**
+     * Wait until rate limits allow the next request
+     * @param estimatedTokens Estimated tokens for this request (content.length / 4)
+     */
+    async waitForCapacity(estimatedTokens = 0) {
+        const MAX_ATTEMPTS = 60;
+        for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+            const now = Date.now();
+            this.cleanWindow(this.requestsWindow, now, 60_000);
+            this.cleanWindow(this.dailyRequestsWindow, now, 86_400_000);
+            // Check RPM limit
+            if (this.config.requestsPerMinute && this.requestsWindow.length >= this.config.requestsPerMinute) {
+                const oldestRequest = this.requestsWindow[0];
+                if (oldestRequest) {
+                    const waitTime = 60_000 - (now - oldestRequest.timestamp);
+                    if (waitTime > 0) {
+                        logger.debug(`[${this.providerId}] RPM limit reached, waiting ${waitTime}ms`, {
+                            rpm: this.config.requestsPerMinute,
+                            current: this.requestsWindow.length,
+                        });
+                        await this.sleep(waitTime);
+                        continue;
+                    }
+                }
+            }
+            // Check TPM limit (skip if single request exceeds limit -- it would never pass)
+            if (this.config.tokensPerMinute && estimatedTokens > 0 && estimatedTokens <= this.config.tokensPerMinute) {
+                const tokensInLastMinute = this.requestsWindow.reduce((sum, r) => sum + r.tokens, 0);
+                if (tokensInLastMinute + estimatedTokens > this.config.tokensPerMinute) {
+                    const oldestRequest = this.requestsWindow[0];
+                    if (oldestRequest) {
+                        const waitTime = 60_000 - (now - oldestRequest.timestamp);
+                        if (waitTime > 0) {
+                            logger.debug(`[${this.providerId}] TPM limit reached, waiting ${waitTime}ms`, {
+                                tpm: this.config.tokensPerMinute,
+                                current: tokensInLastMinute,
+                                requested: estimatedTokens,
+                            });
+                            await this.sleep(waitTime);
+                            continue;
+                        }
+                    }
+                }
+            }
+            // Check RPD limit
+            if (this.config.requestsPerDay && this.dailyRequestsWindow.length >= this.config.requestsPerDay) {
+                logger.warn(`[${this.providerId}] RPD limit reached, waiting 60s`, {
+                    rpd: this.config.requestsPerDay,
+                    current: this.dailyRequestsWindow.length,
+                });
+                await this.sleep(60_000);
+                continue;
+            }
+            return; // All checks passed
+        }
+        throw new Error(`[${this.providerId}] Rate limiter: max wait attempts (${MAX_ATTEMPTS}) exceeded for ${estimatedTokens} estimated tokens`);
+    }
+    /**
+     * Record a request for rate limiting
+     */
+    recordRequest(tokens = 0) {
+        const now = Date.now();
+        const record = { timestamp: now, tokens };
+        this.requestsWindow.push(record);
+        this.dailyRequestsWindow.push(record);
+        // Periodic cleanup to prevent unbounded array growth
+        if (this.requestsWindow.length % 100 === 0) {
+            this.cleanWindow(this.requestsWindow, now, 60_000);
+            this.cleanWindow(this.dailyRequestsWindow, now, 86_400_000);
+        }
+    }
+    /**
+     * Get current rate limit status
+     */
+    getStatus() {
+        const now = Date.now();
+        this.cleanWindow(this.requestsWindow, now, 60_000);
+        this.cleanWindow(this.dailyRequestsWindow, now, 86_400_000);
+        const requestsInLastMinute = this.requestsWindow.length;
+        const tokensInLastMinute = this.requestsWindow.reduce((sum, r) => sum + r.tokens, 0);
+        const requestsToday = this.dailyRequestsWindow.length;
+        return {
+            rpm: {
+                current: requestsInLastMinute,
+                limit: this.config.requestsPerMinute || Infinity,
+                percentage: this.config.requestsPerMinute
+                    ? (requestsInLastMinute / this.config.requestsPerMinute) * 100
+                    : 0,
+            },
+            tpm: {
+                current: tokensInLastMinute,
+                limit: this.config.tokensPerMinute || Infinity,
+                percentage: this.config.tokensPerMinute
+                    ? (tokensInLastMinute / this.config.tokensPerMinute) * 100
+                    : 0,
+            },
+            rpd: {
+                current: requestsToday,
+                limit: this.config.requestsPerDay || Infinity,
+                percentage: this.config.requestsPerDay
+                    ? (requestsToday / this.config.requestsPerDay) * 100
+                    : 0,
+            },
+        };
+    }
+    cleanWindow(window, now, windowMs) {
+        const cutoff = now - windowMs;
+        let i = 0;
+        while (i < window.length && window[i].timestamp < cutoff)
+            i++;
+        if (i > 0)
+            window.splice(0, i);
+    }
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+}
+//# sourceMappingURL=rate-limiter.js.map

package/dist/services/embeddings/rate-limiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../../../src/services/embeddings/rate-limiter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAa1C,MAAM,OAAO,oBAAoB;IACd,cAAc,GAAoB,EAAE,CAAC;IACrC,mBAAmB,GAAoB,EAAE,CAAC;IAC1C,MAAM,CAAkB;IACxB,UAAU,CAAS;IAEpC,YAAY,UAAkB,EAAE,MAAuB;QACrD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,kBAA0B,CAAC;QAC/C,MAAM,YAAY,GAAG,EAAE,CAAC;QAExB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;YACxD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEvB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;YAE5D,kBAAkB;YAClB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBACjG,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;gBAC7C,IAAI,aAAa,EAAE,CAAC;oBAClB,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,GAAG,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;oBAC1D,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;wBACjB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,gCAAgC,QAAQ,IAAI,EAAE;4BAC5E,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;4BAClC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;yBACpC,CAAC,CAAC;wBACH,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;wBAC3B,SAAS;oBACX,CAAC;gBACH,CAAC;YACH,CAAC;YAED,gFAAgF;YAChF,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,eAAe,GAAG,CAAC,IAAI,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;gBACzG,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACrF,IAAI,kBAAkB,GAAG,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;oBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;oBAC7C,IAAI,aAAa,EAAE,CAAC;wBAClB,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,GAAG,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;wBAC1D,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;4BACjB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,gCAAgC,QAAQ,IAAI,EAAE;gCAC5E,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;gCAChC,OAAO,EAAE,kBAAkB;gCAC3B,SAAS,EAAE,eAAe;6BAC3B,CAAC,CAAC;4BACH,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;4BAC3B,SAAS;wBACX,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,kBAAkB;YAClB,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,mBAAmB,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;gBAChG,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,kCAAkC,EAAE;oBACjE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;oBAC/B,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM;iBACzC,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACzB,SAAS;YACX,CAAC;YAED,OAAO,CAAC,oBAAoB;QAC9B,CAAC;QAED,MAAM,IAAI,KAAK,CACb,IAAI,IAAI,CAAC,UAAU,sCAAsC,YAAY,kBAAkB,eAAe,mBAAmB,CAC1H,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,SAAiB,CAAC;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAAkB,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtC,qDAAqD;QACrD,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS;QACP,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;QAE5D,MAAM,oBAAoB,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;QACxD,MAAM,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACrF,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;QAEtD,OAAO;YACL,GAAG,EAAE;gBACH,OAAO,EAAE,oBAAoB;gBAC7B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,QAAQ;gBAChD,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;oBACvC,CAAC,CAAC,CAAC,oBAAoB,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,GAAG,GAAG;oBAC9D,CAAC,CAAC,CAAC;aACN;YACD,GAAG,EAAE;gBACH,OAAO,EAAE,kBAAkB;gBAC3B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,QAAQ;gBAC9C,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;oBACrC,CAAC,CAAC,CAAC,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,GAAG;oBAC1D,CAAC,CAAC,CAAC;aACN;YACD,GAAG,EAAE;gBACH,OAAO,EAAE,aAAa;gBACtB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,QAAQ;gBAC7C,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;oBACpC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG;oBACpD,CAAC,CAAC,CAAC;aACN;SACF,CAAC;IACJ,CAAC;IAEO,WAAW,CAAC,MAAuB,EAAE,GAAW,EAAE,QAAgB;QACxE,MAAM,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM;YAAE,CAAC,EAAE,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;CACF"}

package/dist/services/etl/graph-generation-coordinator.d.ts

@@ -0,0 +1,26 @@
+import type { ActivateGraphGenerationOutcome, GraphGenerationLease, GraphGenerationRepository } from "../../data/graph-generation/graph-generation-contract.js";
+export interface GraphInputSnapshotEntry {
+    relativePath: string;
+    contentHash: string;
+    size: number;
+}
+export declare function buildGraphInputSnapshotHash(files: readonly GraphInputSnapshotEntry[]): string;
+export interface BeginGraphBuildInput {
+    projectId: string;
+    expectedActiveGenerationId: string | null;
+    fingerprint: string;
+    inputSnapshotHash: string;
+    expectedFilesCount: number;
+}
+export declare class GraphGenerationCoordinator {
+    private readonly repository;
+    constructor(repository?: GraphGenerationRepository);
+    begin(input: BeginGraphBuildInput): Promise<GraphGenerationLease>;
+    heartbeat(lease: GraphGenerationLease): Promise<void>;
+    activate(lease: GraphGenerationLease): Promise<Extract<ActivateGraphGenerationOutcome, {
+        status: "activated";
+    }>>;
+    abort(lease: GraphGenerationLease, reason: string): Promise<void>;
+    cleanup(lease: GraphGenerationLease): Promise<void>;
+}
+//# sourceMappingURL=graph-generation-coordinator.d.ts.map

package/dist/services/etl/graph-generation-coordinator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-generation-coordinator.d.ts","sourceRoot":"","sources":["../../../src/services/etl/graph-generation-coordinator.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,8BAA8B,EAC9B,oBAAoB,EACpB,yBAAyB,EAC1B,MAAM,0DAA0D,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,SAAS,uBAAuB,EAAE,GAAG,MAAM,CAK7F;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,0BAA0B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,qBAAa,0BAA0B;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU;gBAAV,UAAU,GAAE,yBAA0D;IAE7F,KAAK,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAajE,SAAS,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKrD,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,OAAO,CAAC,8BAA8B,EAAE;QAAE,MAAM,EAAE,WAAW,CAAA;KAAE,CAAC,CAAC;IAmBhH,KAAK,CAAC,KAAK,EAAE,oBAAoB,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjE,OAAO,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;CAG1D"}

package/dist/services/etl/graph-generation-coordinator.js

@@ -0,0 +1,63 @@
+import { createHash } from "node:crypto";
+import { getGraphGenerationRepository } from "../../data/graph-generation/graph-generation-repository-factory.js";
+const GRAPH_GENERATION_LEASE_TTL_MS = 300_000;
+export function buildGraphInputSnapshotHash(files) {
+    const canonicalSnapshot = [...files]
+        .sort((left, right) => left.relativePath.localeCompare(right.relativePath))
+        .map(({ relativePath, contentHash, size }) => [relativePath.normalize("NFC"), contentHash, size]);
+    return `sha256:${createHash("sha256").update(JSON.stringify(canonicalSnapshot)).digest("hex")}`;
+}
+export class GraphGenerationCoordinator {
+    repository;
+    constructor(repository = getGraphGenerationRepository()) {
+        this.repository = repository;
+    }
+    async begin(input) {
+        const deadline = Date.now() + GRAPH_GENERATION_LEASE_TTL_MS;
+        do {
+            const outcome = await this.repository.begin({ ...input, leaseTtlMs: GRAPH_GENERATION_LEASE_TTL_MS });
+            if (outcome.status === "acquired")
+                return outcome.lease;
+            if (outcome.status === "stale_active") {
+                throw new Error(`graph_generation_stale_active:${outcome.activeGenerationId ?? "none"}`);
+            }
+            if (Date.now() >= deadline)
+                throw new Error(`graph_generation_busy:${outcome.generationId}`);
+            await new Promise((resolve) => setTimeout(resolve, 100));
+        } while (true);
+    }
+    async heartbeat(lease) {
+        const outcome = await this.repository.heartbeat(lease, GRAPH_GENERATION_LEASE_TTL_MS);
+        if (outcome.status !== "renewed")
+            throw new Error("graph_generation_lease_lost");
+    }
+    async activate(lease) {
+        const completeness = await this.repository.complete(lease);
+        if (completeness.status === "incomplete") {
+            throw new Error(`graph_generation_incomplete:${completeness.reasons.join(",")}`);
+        }
+        if (completeness.status === "lease_lost")
+            throw new Error("graph_generation_lease_lost");
+        if (completeness.status === "stale_active") {
+            throw new Error(`graph_generation_stale_active:${completeness.activeGenerationId ?? "none"}`);
+        }
+        const activation = await this.repository.activate(lease);
+        if (activation.status === "activated")
+            return activation;
+        if (activation.status === "incomplete") {
+            throw new Error(`graph_generation_incomplete:${activation.reasons.join(",")}`);
+        }
+        if (activation.status === "lease_lost")
+            throw new Error("graph_generation_lease_lost");
+        throw new Error(`graph_generation_stale_active:${activation.activeGenerationId ?? "none"}`);
+    }
+    async abort(lease, reason) {
+        const outcome = await this.repository.abort(lease, reason);
+        if (outcome.status === "lease_lost")
+            throw new Error("graph_generation_lease_lost_during_abort");
+    }
+    async cleanup(lease) {
+        await this.repository.cleanupSuperseded(lease.projectId, { retainedGenerationIds: [lease.generationId] });
+    }
+}
+//# sourceMappingURL=graph-generation-coordinator.js.map

package/dist/services/etl/graph-generation-coordinator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-generation-coordinator.js","sourceRoot":"","sources":["../../../src/services/etl/graph-generation-coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,4BAA4B,EAAE,MAAM,oEAAoE,CAAC;AAOlH,MAAM,6BAA6B,GAAG,OAAO,CAAC;AAQ9C,MAAM,UAAU,2BAA2B,CAAC,KAAyC;IACnF,MAAM,iBAAiB,GAAG,CAAC,GAAG,KAAK,CAAC;SACjC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;SAC1E,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;IACpG,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAClG,CAAC;AAUD,MAAM,OAAO,0BAA0B;IACR;IAA7B,YAA6B,aAAwC,4BAA4B,EAAE;QAAtE,eAAU,GAAV,UAAU,CAA4D;IAAG,CAAC;IAEvG,KAAK,CAAC,KAAK,CAAC,KAA2B;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,6BAA6B,CAAC;QAC5D,GAAG,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,6BAA6B,EAAE,CAAC,CAAC;YACrG,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU;gBAAE,OAAO,OAAO,CAAC,KAAK,CAAC;YACxD,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACtC,MAAM,IAAI,KAAK,CAAC,iCAAiC,OAAO,CAAC,kBAAkB,IAAI,MAAM,EAAE,CAAC,CAAC;YAC3F,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;YAC7F,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3D,CAAC,QAAQ,IAAI,EAAE;IACjB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAA2B;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,6BAA6B,CAAC,CAAC;QACtF,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACnF,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAA2B;QACxC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC3D,IAAI,YAAY,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+BAA+B,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,KAAK,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACzF,IAAI,YAAY,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,iCAAiC,YAAY,CAAC,kBAAkB,IAAI,MAAM,EAAE,CAAC,CAAC;QAChG,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzD,IAAI,UAAU,CAAC,MAAM,KAAK,WAAW;YAAE,OAAO,UAAU,CAAC;QACzD,IAAI,UAAU,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CAAC,+BAA+B,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,UAAU,CAAC,MAAM,KAAK,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACvF,MAAM,IAAI,KAAK,CAAC,iCAAiC,UAAU,CAAC,kBAAkB,IAAI,MAAM,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAA2B,EAAE,MAAc;QACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IACnG,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAA2B;QACvC,MAAM,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,qBAAqB,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;CACF"}

package/dist/services/etl/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * ETL Module Exports
+ */
+export { EtlPipeline } from "./pipeline.js";
+export type { PipelineInput } from "./pipeline.js";
+export type { EtlStageContext, EtlEvent, EtlStage, EtlResult, DiscoveredFile, ParsedFile, ResolvedFile, ResolvedImport, RawSymbol, RawImport, } from "./stage-context.js";
+export { DiscoverStage } from "./stages/discover.js";
+export { ParseStage } from "./stages/parse.js";
+export { ResolveStage } from "./stages/resolve.js";
+export { LoadStage } from "./stages/load.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/services/etl/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/etl/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,YAAY,EACV,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,cAAc,EACd,UAAU,EACV,YAAY,EACZ,cAAc,EACd,SAAS,EACT,SAAS,GACV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/services/etl/index.js

@@ -0,0 +1,9 @@
+/**
+ * ETL Module Exports
+ */
+export { EtlPipeline } from "./pipeline.js";
+export { DiscoverStage } from "./stages/discover.js";
+export { ParseStage } from "./stages/parse.js";
+export { ResolveStage } from "./stages/resolve.js";
+export { LoadStage } from "./stages/load.js";
+//# sourceMappingURL=index.js.map

package/dist/services/etl/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/services/etl/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAc5C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/services/etl/pipeline.d.ts

@@ -0,0 +1,70 @@
+/**
+ * ETL Pipeline Orchestrator
+ *
+ * Runs the 4 stages in sequence: discover → parse → resolve → load
+ *
+ * Integrates with:
+ * - IndexJobTracker (in-memory job status polling)
+ * - EventBus (real-time SSE broadcast)
+ * - SymbolRepository (centrality-based priority ordering in discover)
+ * - ContextualSearchRLM (vector clear on forceReindex)
+ *
+ * Usage:
+ *   const pipeline = EtlPipeline.getInstance();
+ *   const result = await pipeline.run({ projectId, projectPath, jobId });
+ */
+import type { EtlResult } from "./stage-context.js";
+import type { ManagedRunLease } from "../../data/managed-runs/managed-run-contract.js";
+import type { DiscoveredFile } from "./stage-context.js";
+import type { HeaderLanguageEvidence } from "../structural/language-manifest.js";
+/**
+ * Error raised by {@link EtlPipeline.runInternal} when a `managed_runs` lease
+ * for `(projectId, "indexing")` is already held by another live process
+ * (FR-09 / AC-7). IndexProjectTool maps this to a 409 response body carrying
+ * the active runId so the caller can poll get_index_status.
+ */
+export declare class EtlPipelineBusyError extends Error {
+    readonly activeRunId: string;
+    readonly leaseExpiresAt: number;
+    constructor(activeRunId: string, leaseExpiresAt: number);
+}
+export interface PipelineInput {
+    projectId: string;
+    projectPath: string;
+    jobId: string;
+    forceReindex?: boolean;
+    /** If provided, only process these relative paths (incremental mode). */
+    filesToProcess?: string[];
+    /**
+     * When true, the Discover stage does NOT exclude test/benchmark files
+     * (`.test.ts`, `__tests__/`, `*.spec.*`, etc.), so typed edges from test
+     * files are indexed. Default false (preserve search-recall hygiene —
+     * {@link loadProjectIgnore} stays unchanged for query-time callers).
+     */
+    include_tests?: boolean;
+    /**
+     * Wave 5 FR-09: managed_runs lease acquired by the caller (IndexProjectTool
+     * or auto-reindex). When provided, {@link EtlPipeline.runInternal} spawns a
+     * 30s heartbeat, aborts ETL on lease_lost, completes the lease on success,
+     * and aborts the lease on failure. The pipeline does NOT acquire the lease
+     * — the caller owns 202/409 semantics so the MCP handler can return busy
+     * synchronously without running the full ETL in the foreground.
+     */
+    managedRunLease?: ManagedRunLease;
+}
+/** Derive deterministic `.h` language evidence from the immutable discovery snapshot. */
+export declare function buildHeaderLanguageEvidence(files: readonly DiscoveredFile[]): Readonly<Record<string, HeaderLanguageEvidence>>;
+export declare class EtlPipeline {
+    private static instance;
+    private static runTails;
+    private readonly discover;
+    private readonly parse;
+    private readonly resolve;
+    private readonly load;
+    private readonly graphGenerations;
+    private constructor();
+    static getInstance(): EtlPipeline;
+    run(input: PipelineInput): Promise<EtlResult>;
+    private runInternal;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/services/etl/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/services/etl/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAcH,OAAO,KAAK,EAA6B,SAAS,EAAY,MAAM,oBAAoB,CAAC;AAWzF,OAAO,KAAK,EAEV,eAAe,EAEhB,MAAM,iDAAiD,CAAC;AAGzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAEjF;;;;;GAKG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;gBACpB,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;CAMxD;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,yEAAyE;IACzE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED,yFAAyF;AACzF,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,SAAS,cAAc,EAAE,GAC/B,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC,CAyClD;AAeD,qBAAa,WAAW;IACtB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAA4B;IACnD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAoC;IAE3D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuB;IAChD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoC;IAErE,OAAO;IAEP,MAAM,CAAC,WAAW,IAAI,WAAW;IAO3B,GAAG,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;YAkCrC,WAAW;CAmY1B"}