arceus-s 1.6.4

package/dist/core/embeddings/hf-env.d.ts

@@ -0,0 +1,135 @@
+import { CircuitBreaker } from '../../_shared/index.js';
+/** Per-attempt timeout for the full model download (5 minutes). */
+export declare const HF_DOWNLOAD_TIMEOUT_MS: number;
+/** Maximum total download attempts (1 initial + N-1 retries). */
+export declare const HF_MAX_ATTEMPTS = 3;
+/** Initial delay between retry attempts; doubles on each subsequent retry. */
+export declare const HF_BASE_DELAY_MS = 2000;
+/** Number of consecutive failures required to open the circuit. */
+export declare const CB_FAILURE_THRESHOLD = 3;
+/** How long the circuit stays open before transitioning to half-open. */
+export declare const CB_RESET_TIMEOUT_MS = 60000;
+/** Upper bound clamped on the env-override per-attempt timeout (30 minutes). */
+export declare const HF_MAX_TIMEOUT_MS: number;
+/** Upper bound clamped on the env-override attempt count. */
+export declare const HF_MAX_ATTEMPTS_CAP = 10;
+/**
+ * @internal Exported only for unit tests and the two embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
+ * public package API.
+ *
+ * Minimal subset of `@huggingface/transformers`' `env` object that arc
+ * mutates. Defining a local structural type keeps this helper free of a
+ * transitive dependency on transformers' generated `.d.ts` while still
+ * giving full type-checking on the two fields we actually touch.
+ */
+export interface HfEnvSubset {
+    cacheDir: string;
+    remoteHost: string;
+}
+/**
+ * @internal Exported only for unit tests and the two embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
+ * public package API.
+ *
+ * Apply user-controlled HuggingFace environment overrides to the
+ * `@huggingface/transformers` `env` object. Centralises the two env-var
+ * bridges so every arc embedder entry point (the analyze pipeline
+ * and the MCP server) behaves identically.
+ *
+ * - **`HF_HOME`** → `env.cacheDir` (default: `~/.cache/huggingface`).
+ *   transformers.js otherwise defaults to `./node_modules/.cache` inside
+ *   its own install dir, which is unwritable when arc is installed
+ *   globally (e.g. `/usr/lib/node_modules/`).
+ *
+ * - **`HF_ENDPOINT`** → `env.remoteHost` (#1205). transformers.js does
+ *   not read `HF_ENDPOINT` on its own — it reads `env.remoteHost` —
+ *   even though `HF_ENDPOINT` is the standard env var the upstream
+ *   `huggingface_hub` Python client and the official HF mirror docs
+ *   tell users to set. Bridging the two unblocks `--embeddings` for
+ *   users behind networks where `huggingface.co` is unreachable
+ *   (corporate proxies, the GFW, air-gapped mirrors). The trailing
+ *   slash is normalised because transformers.js builds URLs by string
+ *   concatenation and a missing slash silently falls through to its
+ *   default `huggingface.co/...` host.
+ *
+ * Mutation rather than return-and-apply because callers already hold a
+ * reference to the live `env` object imported from
+ * `@huggingface/transformers` — passing the same reference in keeps the
+ * call site a single line at each entry point.
+ */
+export declare function applyHfEnvOverrides(env: HfEnvSubset): void;
+/**
+ * @internal Exported for unit tests and the two embedder entry points.
+ *
+ * Returns true when an error message indicates a network-level fetch failure
+ * during HuggingFace model download (e.g. `TypeError: fetch failed`,
+ * `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, `ECONNRESET`).
+ *
+ * These errors are not device-specific and cannot be fixed by falling back to
+ * a different ONNX device — the caller should rethrow immediately with
+ * guidance about `HF_ENDPOINT`.
+ */
+export declare function isNetworkFetchError(message: string): boolean;
+/** @internal Used by `withHfDownloadRetry` to mark a circuit-open rejection. */
+export declare const CIRCUIT_OPEN_TAG = "hf-circuit-open";
+/**
+ * Module-level singleton shared by both embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Per-process
+ * only — not persisted across restarts. Backed by the shared
+ * `CircuitBreaker` from `arceus-shared` (same state machine, same
+ * semantics, plus the single-permit half-open gate that prevents
+ * recovery-time stampedes).
+ */
+export declare const hfDownloadCircuit: CircuitBreaker;
+/** @internal Returns true for errors that should abort without retry (circuit-open). */
+export declare function isHfCircuitOpenError(message: string): boolean;
+/**
+ * Returns true for any HuggingFace download failure that warrants showing the
+ * `HF_ENDPOINT` remediation hint: either a raw network error or a
+ * circuit-open rejection (which itself was caused by repeated network errors).
+ */
+export declare function isHfDownloadFailure(message: string): boolean;
+/** @internal Wraps `fn` in a hard time-limit. The timeout error contains
+ *  `ETIMEDOUT` so that `isNetworkFetchError` classifies it correctly.
+ */
+export declare function withDownloadTimeout<T>(fn: () => Promise<T>, timeoutMs: number): Promise<T>;
+export interface HfRetryOptions {
+    /** Maximum total attempts including the initial one (default: `HF_MAX_ATTEMPTS`). */
+    maxAttempts?: number;
+    /** Delay before the first retry; doubles on each subsequent attempt (default: `HF_BASE_DELAY_MS`). */
+    baseDelayMs?: number;
+    /** Per-attempt wall-clock timeout in ms (default: `HF_DOWNLOAD_TIMEOUT_MS`). */
+    timeoutMs?: number;
+    /**
+     * Circuit-breaker instance to use. Defaults to the module-level
+     * `hfDownloadCircuit` singleton. Pass a fresh instance in tests.
+     */
+    circuit?: CircuitBreaker;
+    /**
+     * Optional callback invoked before each retry (not the initial attempt).
+     * @param attempt - 1-based retry number
+     * @param max - total allowed attempts
+     * @param error - the error that triggered the retry
+     */
+    onRetry?: (attempt: number, max: number, error: Error) => void;
+}
+/**
+ * Retry wrapper for HuggingFace model downloads with per-attempt timeout and
+ * circuit-breaker protection.
+ *
+ * Behaviour:
+ * - If the circuit is **open**, fails immediately with a `CIRCUIT_OPEN_TAG`
+ *   message (so `isHfDownloadFailure` still returns true and the caller can
+ *   show `HF_ENDPOINT` guidance).
+ * - Each attempt is wrapped in `withDownloadTimeout`.
+ * - On a network-level error (`isNetworkFetchError`) the attempt is retried
+ *   with exponential back-off; non-network errors (e.g. ONNX device failure)
+ *   are rethrown immediately without retry.
+ * - Every network failure is recorded on the circuit breaker; a success resets
+ *   it.
+ * - After all attempts are exhausted, the last network error is rethrown
+ *   so the existing `isNetworkFetchError` / `isHfDownloadFailure` guards in
+ *   the calling code still fire.
+ */
+export declare function withHfDownloadRetry<T>(fn: () => Promise<T>, options?: HfRetryOptions): Promise<T>;

package/dist/core/embeddings/hf-env.js

@@ -0,0 +1,232 @@
+import os from 'node:os';
+import { join } from 'node:path';
+import { CircuitBreaker, withRetry } from '../../_shared/index.js';
+// ---------------------------------------------------------------------------
+// Download resilience defaults
+// ---------------------------------------------------------------------------
+/** Per-attempt timeout for the full model download (5 minutes). */
+export const HF_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1_000;
+/** Maximum total download attempts (1 initial + N-1 retries). */
+export const HF_MAX_ATTEMPTS = 3;
+/** Initial delay between retry attempts; doubles on each subsequent retry. */
+export const HF_BASE_DELAY_MS = 2_000;
+/** Number of consecutive failures required to open the circuit. */
+export const CB_FAILURE_THRESHOLD = 3;
+/** How long the circuit stays open before transitioning to half-open. */
+export const CB_RESET_TIMEOUT_MS = 60_000;
+/** Upper bound clamped on the env-override per-attempt timeout (30 minutes). */
+export const HF_MAX_TIMEOUT_MS = 30 * 60 * 1_000;
+/** Upper bound clamped on the env-override attempt count. */
+export const HF_MAX_ATTEMPTS_CAP = 10;
+/**
+ * @internal Exported only for unit tests and the two embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
+ * public package API.
+ *
+ * Apply user-controlled HuggingFace environment overrides to the
+ * `@huggingface/transformers` `env` object. Centralises the two env-var
+ * bridges so every arc embedder entry point (the analyze pipeline
+ * and the MCP server) behaves identically.
+ *
+ * - **`HF_HOME`** → `env.cacheDir` (default: `~/.cache/huggingface`).
+ *   transformers.js otherwise defaults to `./node_modules/.cache` inside
+ *   its own install dir, which is unwritable when arc is installed
+ *   globally (e.g. `/usr/lib/node_modules/`).
+ *
+ * - **`HF_ENDPOINT`** → `env.remoteHost` (#1205). transformers.js does
+ *   not read `HF_ENDPOINT` on its own — it reads `env.remoteHost` —
+ *   even though `HF_ENDPOINT` is the standard env var the upstream
+ *   `huggingface_hub` Python client and the official HF mirror docs
+ *   tell users to set. Bridging the two unblocks `--embeddings` for
+ *   users behind networks where `huggingface.co` is unreachable
+ *   (corporate proxies, the GFW, air-gapped mirrors). The trailing
+ *   slash is normalised because transformers.js builds URLs by string
+ *   concatenation and a missing slash silently falls through to its
+ *   default `huggingface.co/...` host.
+ *
+ * Mutation rather than return-and-apply because callers already hold a
+ * reference to the live `env` object imported from
+ * `@huggingface/transformers` — passing the same reference in keeps the
+ * call site a single line at each entry point.
+ */
+export function applyHfEnvOverrides(env) {
+    env.cacheDir = process.env.HF_HOME ?? join(os.homedir(), '.cache', 'huggingface');
+    // `.trim()` guards against the common copy-paste failure mode of
+    // `HF_ENDPOINT="  https://hf-mirror.com  "` (leading/trailing whitespace
+    // from shell scripts or docs) — without it, a whitespace-only value
+    // would be truthy and produce an invalid `env.remoteHost = '   /'` that
+    // silently misroutes downloads. Empty string remains falsy in JS so the
+    // truthy guard already handles the unset/empty cases.
+    const endpoint = process.env.HF_ENDPOINT?.trim();
+    if (endpoint) {
+        env.remoteHost = endpoint.endsWith('/') ? endpoint : endpoint + '/';
+    }
+}
+/**
+ * @internal Exported for unit tests and the two embedder entry points.
+ *
+ * Returns true when an error message indicates a network-level fetch failure
+ * during HuggingFace model download (e.g. `TypeError: fetch failed`,
+ * `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, `ECONNRESET`).
+ *
+ * These errors are not device-specific and cannot be fixed by falling back to
+ * a different ONNX device — the caller should rethrow immediately with
+ * guidance about `HF_ENDPOINT`.
+ */
+export function isNetworkFetchError(message) {
+    return (message.includes('fetch failed') ||
+        message.includes('ECONNREFUSED') ||
+        message.includes('ENOTFOUND') ||
+        message.includes('ETIMEDOUT') ||
+        message.includes('ECONNRESET'));
+}
+// ---------------------------------------------------------------------------
+// Circuit breaker
+// ---------------------------------------------------------------------------
+/** @internal Used by `withHfDownloadRetry` to mark a circuit-open rejection. */
+export const CIRCUIT_OPEN_TAG = 'hf-circuit-open';
+/**
+ * Module-level singleton shared by both embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Per-process
+ * only — not persisted across restarts. Backed by the shared
+ * `CircuitBreaker` from `arceus-shared` (same state machine, same
+ * semantics, plus the single-permit half-open gate that prevents
+ * recovery-time stampedes).
+ */
+export const hfDownloadCircuit = new CircuitBreaker({
+    failureThreshold: CB_FAILURE_THRESHOLD,
+    cooldownMs: CB_RESET_TIMEOUT_MS,
+    key: 'hf-download',
+});
+// ---------------------------------------------------------------------------
+// Retry + timeout wrapper
+// ---------------------------------------------------------------------------
+/** @internal Returns true for errors that should abort without retry (circuit-open). */
+export function isHfCircuitOpenError(message) {
+    return message.includes(CIRCUIT_OPEN_TAG);
+}
+/**
+ * Returns true for any HuggingFace download failure that warrants showing the
+ * `HF_ENDPOINT` remediation hint: either a raw network error or a
+ * circuit-open rejection (which itself was caused by repeated network errors).
+ */
+export function isHfDownloadFailure(message) {
+    return isNetworkFetchError(message) || isHfCircuitOpenError(message);
+}
+/** @internal Wraps `fn` in a hard time-limit. The timeout error contains
+ *  `ETIMEDOUT` so that `isNetworkFetchError` classifies it correctly.
+ */
+export function withDownloadTimeout(fn, timeoutMs) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error(`ETIMEDOUT: model download timed out after ${Math.round(timeoutMs / 1000)}s — ` +
+            `check your network speed or set HF_ENDPOINT to a faster mirror`)), timeoutMs);
+        fn().then((v) => {
+            clearTimeout(timer);
+            resolve(v);
+        }, (e) => {
+            clearTimeout(timer);
+            reject(e);
+        });
+    });
+}
+/**
+ * Retry wrapper for HuggingFace model downloads with per-attempt timeout and
+ * circuit-breaker protection.
+ *
+ * Behaviour:
+ * - If the circuit is **open**, fails immediately with a `CIRCUIT_OPEN_TAG`
+ *   message (so `isHfDownloadFailure` still returns true and the caller can
+ *   show `HF_ENDPOINT` guidance).
+ * - Each attempt is wrapped in `withDownloadTimeout`.
+ * - On a network-level error (`isNetworkFetchError`) the attempt is retried
+ *   with exponential back-off; non-network errors (e.g. ONNX device failure)
+ *   are rethrown immediately without retry.
+ * - Every network failure is recorded on the circuit breaker; a success resets
+ *   it.
+ * - After all attempts are exhausted, the last network error is rethrown
+ *   so the existing `isNetworkFetchError` / `isHfDownloadFailure` guards in
+ *   the calling code still fire.
+ */
+export async function withHfDownloadRetry(fn, options = {}) {
+    // Resolve effective values — explicit options take precedence over env vars,
+    // which take precedence over built-in defaults. This lets users lower the
+    // per-attempt timeout without rebuilding (e.g.
+    //   HF_DOWNLOAD_TIMEOUT_MS=60000 npx arc analyze --embeddings
+    // reduces the worst-case wait from 15 minutes to ~3 minutes).
+    //
+    // Upper bounds are clamped to prevent accidental runaway configuration:
+    //   - timeoutMs is capped at HF_MAX_TIMEOUT_MS (30 min)
+    //   - maxAttempts is floored (fractional values → integer) and capped at
+    //     HF_MAX_ATTEMPTS_CAP (10).  Values ≤ 0, NaN, or Infinity fall back to
+    //     the built-in defaults.
+    const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS);
+    const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS);
+    const resolvedTimeout = Number.isFinite(envTimeout) && envTimeout > 0
+        ? Math.min(envTimeout, HF_MAX_TIMEOUT_MS)
+        : HF_DOWNLOAD_TIMEOUT_MS;
+    const resolvedMaxAttempts = Number.isFinite(envMaxAttempts) && envMaxAttempts > 0
+        ? Math.min(Math.floor(envMaxAttempts), HF_MAX_ATTEMPTS_CAP)
+        : HF_MAX_ATTEMPTS;
+    const { maxAttempts = resolvedMaxAttempts, baseDelayMs = HF_BASE_DELAY_MS, timeoutMs = resolvedTimeout, circuit = hfDownloadCircuit, onRetry, } = options;
+    if (circuit.getState() === 'open') {
+        // Compute remaining cooldown without consuming a probe permit.
+        const openedAt = circuit.getOpenedAt();
+        const secsUntilReset = openedAt !== null ? Math.ceil((circuit.getCooldownMs() - (Date.now() - openedAt)) / 1000) : 0;
+        throw new Error(`${CIRCUIT_OPEN_TAG}: HuggingFace download circuit is open after repeated network failures` +
+            (secsUntilReset > 0 ? ` — will reset in ~${secsUntilReset}s` : ''));
+    }
+    // Retry budget delegated to `withRetry` from arceus-shared. The
+    // HF-specific bits — per-attempt timeout, network-vs-non-network
+    // classification, circuit-breaker recording, onRetry callback — wire
+    // through the `isRetryable` callback. `circuitTripped` is the
+    // sentinel that lets us replace the final thrown error with a
+    // CIRCUIT_OPEN_TAG message when the breaker tripped mid-loop.
+    let circuitTripped = false;
+    try {
+        return await withRetry(async () => {
+            const result = await withDownloadTimeout(fn, timeoutMs);
+            circuit.recordSuccess();
+            return result;
+        }, {
+            maxAttempts,
+            baseDelayMs,
+            // Disable the cap to match the bespoke pure-exponential
+            // progression. With the default `HF_MAX_ATTEMPTS_CAP = 10` and
+            // `baseDelayMs = 2000`, the largest possible delay is
+            // `2000 * 2^9 = ~17 minutes` — bounded enough not to need a cap.
+            capDelayMs: Number.MAX_SAFE_INTEGER,
+            isRetryable: (err, attempt) => {
+                const error = err instanceof Error ? err : new Error(String(err));
+                if (!isNetworkFetchError(error.message)) {
+                    // Non-network error (e.g. CUDA unavailable) — propagate
+                    // without retry. Use recordNeutral so the breaker's existing
+                    // failure-count progress isn't reset by a non-network failure
+                    // that says nothing about the CDN's health.
+                    circuit.recordNeutral();
+                    return { retry: false };
+                }
+                circuit.recordFailure();
+                if (circuit.getState() === 'open') {
+                    // Circuit just tripped — fail fast, no more retries.
+                    circuitTripped = true;
+                    return { retry: false };
+                }
+                // Mirror the bespoke onRetry contract: fire only when there's
+                // actually a next attempt.
+                if (attempt + 1 < maxAttempts) {
+                    onRetry?.(attempt + 1, maxAttempts, error);
+                }
+                return { retry: true };
+            },
+        });
+    }
+    catch (err) {
+        if (circuitTripped) {
+            throw new Error(`${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${CB_FAILURE_THRESHOLD} consecutive failures`);
+        }
+        // All retries exhausted — rethrow the last network error so
+        // isNetworkFetchError patterns in the calling code still match and
+        // surface HF_ENDPOINT guidance.
+        throw err;
+    }
+}

package/dist/core/embeddings/http-client.d.ts

@@ -0,0 +1,37 @@
+/**
+ * HTTP Embedding Client
+ *
+ * Shared fetch+retry logic for OpenAI-compatible /v1/embeddings endpoints.
+ * Imported by both the core embedder (batch) and MCP embedder (query).
+ *
+ * Network resilience is delegated to `resilientFetch` from
+ * `arceus-shared` — bounded retries with exponential-backoff jitter,
+ * `Retry-After` honored on 429, and an in-process circuit breaker that
+ * fails fast on a flapping endpoint. Per-attempt timeout is enforced
+ * via `AbortSignal.timeout` on the underlying fetch.
+ */
+/**
+ * Check whether HTTP embedding mode is active (env vars are set).
+ */
+export declare const isHttpMode: () => boolean;
+/**
+ * Return the configured embedding dimensions for HTTP mode, or undefined
+ * if HTTP mode is not active or no explicit dimensions are set.
+ */
+export declare const getHttpDimensions: () => number | undefined;
+/**
+ * Embed texts via the HTTP backend, splitting into batches.
+ * Reads config from env vars on every call.
+ *
+ * @param texts - Array of texts to embed
+ * @returns Array of Float32Array embedding vectors
+ */
+export declare const httpEmbed: (texts: string[]) => Promise<Float32Array[]>;
+/**
+ * Embed a single query text via the HTTP backend.
+ * Convenience for MCP search where only one vector is needed.
+ *
+ * @param text - Query text to embed
+ * @returns Embedding vector as number array
+ */
+export declare const httpEmbedQuery: (text: string) => Promise<number[]>;

package/dist/core/embeddings/http-client.js

@@ -0,0 +1,199 @@
+/**
+ * HTTP Embedding Client
+ *
+ * Shared fetch+retry logic for OpenAI-compatible /v1/embeddings endpoints.
+ * Imported by both the core embedder (batch) and MCP embedder (query).
+ *
+ * Network resilience is delegated to `resilientFetch` from
+ * `arceus-shared` — bounded retries with exponential-backoff jitter,
+ * `Retry-After` honored on 429, and an in-process circuit breaker that
+ * fails fast on a flapping endpoint. Per-attempt timeout is enforced
+ * via `AbortSignal.timeout` on the underlying fetch.
+ */
+import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from '../../_shared/index.js';
+const HTTP_TIMEOUT_MS = 30_000;
+const HTTP_MAX_RETRIES = 2;
+const HTTP_RETRY_BACKOFF_MS = 1_000;
+const HTTP_BATCH_SIZE = 64;
+const DEFAULT_DIMS = 384;
+const HTTP_BREAKER_KEY = 'embeddings-http';
+/**
+ * Build config from the current process.env snapshot.
+ * Returns null when ARC_EMBEDDING_URL + ARC_EMBEDDING_MODEL are unset.
+ * Not cached — env vars are read fresh so late configuration takes effect.
+ */
+const readConfig = () => {
+    const baseUrl = process.env.ARC_EMBEDDING_URL;
+    const model = process.env.ARC_EMBEDDING_MODEL;
+    if (!baseUrl || !model)
+        return null;
+    const rawDims = process.env.ARC_EMBEDDING_DIMS;
+    let dimensions;
+    if (rawDims !== undefined) {
+        if (!/^\d+$/.test(rawDims)) {
+            throw new Error(`ARC_EMBEDDING_DIMS must be a positive integer, got "${rawDims}"`);
+        }
+        const parsed = parseInt(rawDims, 10);
+        if (parsed <= 0) {
+            throw new Error(`ARC_EMBEDDING_DIMS must be a positive integer, got "${rawDims}"`);
+        }
+        dimensions = parsed;
+    }
+    return {
+        baseUrl: baseUrl.replace(/\/+$/, ''),
+        model,
+        apiKey: process.env.ARC_EMBEDDING_API_KEY ?? 'unused',
+        dimensions,
+    };
+};
+/**
+ * Check whether HTTP embedding mode is active (env vars are set).
+ */
+export const isHttpMode = () => readConfig() !== null;
+/**
+ * Return the configured embedding dimensions for HTTP mode, or undefined
+ * if HTTP mode is not active or no explicit dimensions are set.
+ */
+export const getHttpDimensions = () => readConfig()?.dimensions;
+/**
+ * Return a safe representation of a URL for error messages.
+ * Strips query string (may contain tokens) and userinfo.
+ */
+const safeUrl = (url) => {
+    try {
+        const u = new URL(url);
+        return `${u.protocol}//${u.host}${u.pathname}`;
+    }
+    catch {
+        return '<invalid-url>';
+    }
+};
+/**
+ * Send a single batch of texts to the embedding endpoint with retry.
+ *
+ * @param url - Full endpoint URL (e.g. https://host/v1/embeddings)
+ * @param batch - Texts to embed
+ * @param model - Model name for the request body
+ * @param apiKey - Bearer token (only used in Authorization header)
+ * @param batchIndex - Logical batch number (for error context)
+ * @param dimensions - Optional output-vector size. When provided, sent as
+ *   the `dimensions` field in the request body. Endpoints that implement
+ *   Matryoshka truncation (OpenAI text-embedding-3-*, Cohere embed-v3,
+ *   Voyage) return a truncated vector at that size; endpoints that do not
+ *   recognise the field may ignore it or return 400. Leave
+ *   `ARC_EMBEDDING_DIMS` unset for strict backends that reject
+ *   unknown fields.
+ */
+const httpEmbedBatch = async (url, batch, model, apiKey, batchIndex = 0, dimensions) => {
+    const requestBody = {
+        input: batch,
+        model,
+    };
+    if (dimensions !== undefined) {
+        requestBody.dimensions = dimensions;
+    }
+    let resp;
+    try {
+        resp = await resilientFetch(url, {
+            method: 'POST',
+            signal: AbortSignal.timeout(HTTP_TIMEOUT_MS),
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${apiKey}`,
+            },
+            body: JSON.stringify(requestBody),
+        }, {
+            breakerKey: HTTP_BREAKER_KEY,
+            retry: { maxAttempts: HTTP_MAX_RETRIES + 1, baseDelayMs: HTTP_RETRY_BACKOFF_MS },
+        });
+    }
+    catch (err) {
+        if (err instanceof CircuitOpenError) {
+            throw new Error(`Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`);
+        }
+        if (err instanceof DOMException && err.name === 'TimeoutError') {
+            throw new Error(`Embedding request timed out after ${HTTP_TIMEOUT_MS}ms (${safeUrl(url)}, batch ${batchIndex})`);
+        }
+        if (err instanceof ResilientFetchExhaustedError) {
+            throw new Error(`Embedding endpoint returned ${err.response.status} (${safeUrl(url)}, batch ${batchIndex})`);
+        }
+        const reason = err instanceof Error ? err.message : String(err);
+        throw new Error(`Embedding request failed (${safeUrl(url)}, batch ${batchIndex}): ${reason}`);
+    }
+    if (!resp.ok) {
+        // resilientFetch already retried 5xx/429; any non-OK response here is
+        // a terminal client error (4xx other than 429).
+        throw new Error(`Embedding endpoint returned ${resp.status} (${safeUrl(url)}, batch ${batchIndex})`);
+    }
+    const data = (await resp.json());
+    return data.data;
+};
+/**
+ * Embed texts via the HTTP backend, splitting into batches.
+ * Reads config from env vars on every call.
+ *
+ * @param texts - Array of texts to embed
+ * @returns Array of Float32Array embedding vectors
+ */
+export const httpEmbed = async (texts) => {
+    if (texts.length === 0)
+        return [];
+    const config = readConfig();
+    if (!config)
+        throw new Error('HTTP embedding not configured');
+    const url = `${config.baseUrl}/embeddings`;
+    const allVectors = [];
+    for (let i = 0; i < texts.length; i += HTTP_BATCH_SIZE) {
+        const batch = texts.slice(i, i + HTTP_BATCH_SIZE);
+        const batchIndex = Math.floor(i / HTTP_BATCH_SIZE);
+        const items = await httpEmbedBatch(url, batch, config.model, config.apiKey, batchIndex, config.dimensions);
+        if (items.length !== batch.length) {
+            throw new Error(`Embedding endpoint returned ${items.length} vectors for ${batch.length} texts ` +
+                `(${safeUrl(url)}, batch ${batchIndex})`);
+        }
+        for (const item of items) {
+            const vec = new Float32Array(item.embedding);
+            // Fail fast on dimension mismatch rather than inserting bad vectors
+            // into the FLOAT[N] column which would cause a cryptic Kuzu error.
+            const expected = config.dimensions ?? DEFAULT_DIMS;
+            if (vec.length !== expected) {
+                const hint = config.dimensions
+                    ? 'Update ARC_EMBEDDING_DIMS to match your model output.'
+                    : `Set ARC_EMBEDDING_DIMS=${vec.length} to match your model output.`;
+                throw new Error(`Embedding dimension mismatch: endpoint returned ${vec.length}d vector, ` +
+                    `but expected ${expected}d. ${hint}`);
+            }
+            allVectors.push(vec);
+        }
+    }
+    return allVectors;
+};
+/**
+ * Embed a single query text via the HTTP backend.
+ * Convenience for MCP search where only one vector is needed.
+ *
+ * @param text - Query text to embed
+ * @returns Embedding vector as number array
+ */
+export const httpEmbedQuery = async (text) => {
+    const config = readConfig();
+    if (!config)
+        throw new Error('HTTP embedding not configured');
+    const url = `${config.baseUrl}/embeddings`;
+    const items = await httpEmbedBatch(url, [text], config.model, config.apiKey, 0, config.dimensions);
+    if (!items.length) {
+        throw new Error(`Embedding endpoint returned empty response (${safeUrl(url)})`);
+    }
+    const embedding = items[0].embedding;
+    // Same dimension checks as httpEmbed — catch mismatches before they
+    // reach the Kuzu FLOAT[N] cast in search queries.
+    const expected = config.dimensions ?? DEFAULT_DIMS;
+    if (embedding.length !== expected) {
+        const hint = config.dimensions
+            ? 'Update ARC_EMBEDDING_DIMS to match your model output.'
+            : `Set ARC_EMBEDDING_DIMS=${embedding.length} to match your model output.`;
+        throw new Error(`Embedding dimension mismatch: endpoint returned ${embedding.length}d vector, ` +
+            `but expected ${expected}d. ${hint}`);
+    }
+    return embedding;
+};

package/dist/core/embeddings/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Embeddings Module
+ *
+ * Re-exports for the embedding pipeline system.
+ */
+export * from './types.js';
+export * from './http-client.js';
+export * from './embedder.js';
+export * from './text-generator.js';
+export * from './embedding-pipeline.js';

package/dist/core/embeddings/index.js

@@ -0,0 +1,10 @@
+/**
+ * Embeddings Module
+ *
+ * Re-exports for the embedding pipeline system.
+ */
+export * from './types.js';
+export * from './http-client.js';
+export * from './embedder.js';
+export * from './text-generator.js';
+export * from './embedding-pipeline.js';

package/dist/core/embeddings/line-index.d.ts

@@ -0,0 +1,7 @@
+export interface ResolvedLineRange {
+    startLine: number;
+    endLine: number;
+}
+export declare const buildLineIndex: (content: string) => Int32Array;
+export declare const lineFromOffset: (lineOffsets: Int32Array, charOffset: number) => number;
+export declare const resolveChunkLines: (lineOffsets: Int32Array, startOffset: number, endOffset: number, baseStartLine: number) => ResolvedLineRange;

package/dist/core/embeddings/line-index.js

@@ -0,0 +1,42 @@
+export const buildLineIndex = (content) => {
+    const offsets = [0];
+    for (let i = 0; i < content.length; i++) {
+        if (content.charCodeAt(i) === 10)
+            offsets.push(i + 1);
+    }
+    return new Int32Array(offsets);
+};
+const clampOffset = (lineOffsets, charOffset) => {
+    if (lineOffsets.length === 0)
+        return 0;
+    const maxOffset = lineOffsets[lineOffsets.length - 1];
+    if (charOffset < 0)
+        return 0;
+    if (charOffset > maxOffset)
+        return maxOffset;
+    return charOffset;
+};
+export const lineFromOffset = (lineOffsets, charOffset) => {
+    if (lineOffsets.length === 0)
+        return 0;
+    const clamped = clampOffset(lineOffsets, charOffset);
+    let lo = 0;
+    let hi = lineOffsets.length - 1;
+    while (lo < hi) {
+        const mid = (lo + hi + 1) >> 1;
+        if (lineOffsets[mid] <= clamped)
+            lo = mid;
+        else
+            hi = mid - 1;
+    }
+    return lo;
+};
+export const resolveChunkLines = (lineOffsets, startOffset, endOffset, baseStartLine) => {
+    const relativeStartLine = lineFromOffset(lineOffsets, startOffset);
+    const effectiveEndOffset = endOffset > startOffset ? endOffset - 1 : startOffset;
+    const relativeEndLine = lineFromOffset(lineOffsets, effectiveEndOffset);
+    return {
+        startLine: baseStartLine + relativeStartLine,
+        endLine: baseStartLine + relativeEndLine,
+    };
+};