@romiluz/clawmongo 0.1.0-rc.0

package/dist/retrieval/embeddings/voyage-remote-batch.js

@@ -0,0 +1,312 @@
+const VOYAGE_BATCH_ENDPOINT = "/v1/embeddings";
+const VOYAGE_BATCH_COMPLETION_WINDOW = "12h";
+const VOYAGE_BATCH_MAX_REQUESTS = 50_000;
+function normalizeBaseUrl(baseUrl) {
+    const normalized = baseUrl.trim().replace(/\/+$/, "");
+    if (!normalized) {
+        throw new Error("Voyage remote batch baseUrl is required.");
+    }
+    return normalized;
+}
+function parseRetryAfterMs(value) {
+    if (!value) {
+        return null;
+    }
+    const seconds = Number(value);
+    if (Number.isFinite(seconds) && seconds > 0) {
+        return Math.round(seconds * 1000);
+    }
+    return null;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function splitRequests(requests) {
+    if (requests.length <= VOYAGE_BATCH_MAX_REQUESTS) {
+        return [requests];
+    }
+    const groups = [];
+    for (let index = 0; index < requests.length; index += VOYAGE_BATCH_MAX_REQUESTS) {
+        groups.push(requests.slice(index, index + VOYAGE_BATCH_MAX_REQUESTS));
+    }
+    return groups;
+}
+async function requestJsonWithRetry(options) {
+    const attempts = options.maxRetries + 1;
+    for (let attempt = 0; attempt < attempts; attempt += 1) {
+        const response = await fetch(options.url, {
+            method: options.method ?? "GET",
+            headers: options.headers,
+            body: options.body
+        });
+        if ((response.status === 429 || response.status >= 500) && attempt < options.maxRetries) {
+            const retryAfterMs = parseRetryAfterMs(response.headers.get("retry-after"));
+            const waitMs = retryAfterMs ?? Math.pow(2, attempt) * 1000;
+            await sleep(waitMs);
+            continue;
+        }
+        if (!response.ok) {
+            const text = await response.text().catch(() => "");
+            throw new Error(`Voyage remote batch request failed (${response.status}): ${text}`);
+        }
+        return (await response.json());
+    }
+    throw new Error(`Voyage remote batch retry budget exhausted after ${attempts} attempts.`);
+}
+async function uploadBatchInputFile(params) {
+    const lines = params.requests.map((request) => JSON.stringify({
+        custom_id: request.customId,
+        body: { input: request.text }
+    }));
+    const jsonl = lines.join("\n");
+    const form = new FormData();
+    form.append("purpose", "batch");
+    form.append("file", new Blob([jsonl], { type: "application/jsonl" }), `clawmongo-memory-embeddings.${Date.now()}.jsonl`);
+    const response = await fetch(`${params.baseUrl}/files`, {
+        method: "POST",
+        headers: {
+            authorization: `Bearer ${params.apiKey}`
+        },
+        body: form
+    });
+    if (!response.ok) {
+        const text = await response.text().catch(() => "");
+        throw new Error(`Voyage batch file upload failed (${response.status}): ${text}`);
+    }
+    const payload = (await response.json());
+    if (!payload.id) {
+        throw new Error("Voyage batch file upload failed: missing file id.");
+    }
+    return payload.id;
+}
+async function createBatch(params) {
+    return requestJsonWithRetry({
+        url: `${params.baseUrl}/batches`,
+        method: "POST",
+        headers: {
+            authorization: `Bearer ${params.apiKey}`,
+            "content-type": "application/json"
+        },
+        body: JSON.stringify({
+            input_file_id: params.inputFileId,
+            endpoint: VOYAGE_BATCH_ENDPOINT,
+            completion_window: VOYAGE_BATCH_COMPLETION_WINDOW,
+            request_params: {
+                model: params.model,
+                input_type: "document"
+            },
+            metadata: {
+                source: "clawmongo-memory"
+            }
+        }),
+        maxRetries: params.maxRetries
+    });
+}
+async function readBatchStatus(params) {
+    return requestJsonWithRetry({
+        url: `${params.baseUrl}/batches/${params.batchId}`,
+        headers: {
+            authorization: `Bearer ${params.apiKey}`,
+            "content-type": "application/json"
+        },
+        maxRetries: 0
+    });
+}
+async function readErrorFileMessage(params) {
+    try {
+        const response = await fetch(`${params.baseUrl}/files/${params.errorFileId}/content`, {
+            headers: {
+                authorization: `Bearer ${params.apiKey}`,
+                "content-type": "application/json"
+            }
+        });
+        if (!response.ok) {
+            const text = await response.text().catch(() => "");
+            return `error file unavailable (${response.status}): ${text}`;
+        }
+        const content = await response.text();
+        if (!content.trim()) {
+            return undefined;
+        }
+        const lines = content
+            .split("\n")
+            .map((line) => line.trim())
+            .filter(Boolean)
+            .map((line) => JSON.parse(line));
+        const firstMessage = lines.find((line) => line.error?.message || line.response?.body?.error?.message);
+        return (firstMessage?.error?.message ??
+            firstMessage?.response?.body?.error?.message ??
+            undefined);
+    }
+    catch (error) {
+        return error instanceof Error ? error.message : String(error);
+    }
+}
+async function waitForBatchCompletion(params) {
+    const startedAt = Date.now();
+    let status = params.initialStatus;
+    while (true) {
+        const current = status ??
+            (await readBatchStatus({
+                baseUrl: params.baseUrl,
+                apiKey: params.apiKey,
+                batchId: params.batchId
+            }));
+        const state = current.status ?? "unknown";
+        if (state === "completed") {
+            if (!current.output_file_id) {
+                throw new Error(`Voyage batch ${params.batchId} completed without output file.`);
+            }
+            return {
+                outputFileId: current.output_file_id,
+                ...(current.error_file_id ? { errorFileId: current.error_file_id } : {})
+            };
+        }
+        if (["failed", "expired", "cancelled", "canceled"].includes(state)) {
+            const errorMessage = current.error_file_id
+                ? await readErrorFileMessage({
+                    baseUrl: params.baseUrl,
+                    apiKey: params.apiKey,
+                    errorFileId: current.error_file_id
+                })
+                : undefined;
+            throw new Error(`Voyage batch ${params.batchId} ${state}${errorMessage ? `: ${errorMessage}` : ""}`);
+        }
+        if (!params.wait) {
+            throw new Error(`Voyage batch ${params.batchId} still ${state}; wait mode is disabled.`);
+        }
+        if (Date.now() - startedAt > params.timeoutMs) {
+            throw new Error(`Voyage batch ${params.batchId} timed out after ${params.timeoutMs}ms.`);
+        }
+        await sleep(params.pollIntervalMs);
+        status = undefined;
+    }
+}
+async function readOutputEmbeddings(params) {
+    const response = await fetch(`${params.baseUrl}/files/${params.outputFileId}/content`, {
+        headers: {
+            authorization: `Bearer ${params.apiKey}`,
+            "content-type": "application/json"
+        }
+    });
+    if (!response.ok) {
+        const text = await response.text().catch(() => "");
+        throw new Error(`Voyage batch output download failed (${response.status}): ${text}`);
+    }
+    const content = await response.text();
+    const lines = content
+        .split("\n")
+        .map((line) => line.trim())
+        .filter(Boolean)
+        .map((line) => JSON.parse(line));
+    const requestedIds = new Set(params.requests.map((request) => request.customId));
+    const embeddings = new Map();
+    const errors = [];
+    for (const line of lines) {
+        const customId = line.custom_id;
+        if (!customId || !requestedIds.has(customId)) {
+            continue;
+        }
+        if (line.error?.message) {
+            errors.push(`${customId}: ${line.error.message}`);
+            continue;
+        }
+        const statusCode = line.response?.status_code ?? 0;
+        if (statusCode >= 400) {
+            const message = line.response?.body?.error?.message ??
+                "unknown voyage batch response error";
+            errors.push(`${customId}: ${message}`);
+            continue;
+        }
+        const embedding = line.response?.body?.data?.[0]?.embedding;
+        if (!Array.isArray(embedding) || embedding.length === 0) {
+            errors.push(`${customId}: empty embedding`);
+            continue;
+        }
+        const numeric = embedding.filter((value) => typeof value === "number");
+        if (numeric.length !== embedding.length) {
+            errors.push(`${customId}: non-numeric embedding values`);
+            continue;
+        }
+        embeddings.set(customId, numeric);
+    }
+    if (errors.length > 0) {
+        throw new Error(`Voyage batch output errors: ${errors.join("; ")}`);
+    }
+    const missing = params.requests.filter((request) => !embeddings.has(request.customId));
+    if (missing.length > 0) {
+        throw new Error(`Voyage batch output missing ${missing.length} embeddings.`);
+    }
+    return embeddings;
+}
+async function runWithConcurrency(tasks, concurrency) {
+    const safeConcurrency = Math.max(1, Math.floor(concurrency));
+    const results = new Array(tasks.length);
+    let nextIndex = 0;
+    async function worker() {
+        while (nextIndex < tasks.length) {
+            const current = nextIndex;
+            nextIndex += 1;
+            const task = tasks[current];
+            if (!task) {
+                continue;
+            }
+            results[current] = await task();
+        }
+    }
+    const workers = new Array(Math.min(safeConcurrency, tasks.length))
+        .fill(null)
+        .map(() => worker());
+    await Promise.all(workers);
+    return results;
+}
+export async function runVoyageRemoteEmbeddingBatches(options) {
+    if (options.requests.length === 0) {
+        return { embeddingsByCustomId: new Map(), batchIds: [] };
+    }
+    const baseUrl = normalizeBaseUrl(options.baseUrl);
+    const groups = splitRequests(options.requests);
+    const embeddingsByCustomId = new Map();
+    const batchIds = [];
+    const tasks = groups.map((group) => async () => {
+        const inputFileId = await uploadBatchInputFile({
+            baseUrl,
+            apiKey: options.apiKey,
+            requests: group
+        });
+        const created = await createBatch({
+            baseUrl,
+            apiKey: options.apiKey,
+            inputFileId,
+            model: options.model,
+            maxRetries: options.maxRetries
+        });
+        if (!created.id) {
+            throw new Error("Voyage batch creation failed: missing batch id.");
+        }
+        batchIds.push(created.id);
+        const completed = await waitForBatchCompletion({
+            baseUrl,
+            apiKey: options.apiKey,
+            batchId: created.id,
+            wait: options.wait,
+            pollIntervalMs: options.pollIntervalMs,
+            timeoutMs: options.timeoutMs,
+            initialStatus: created
+        });
+        const embeddings = await readOutputEmbeddings({
+            baseUrl,
+            apiKey: options.apiKey,
+            outputFileId: completed.outputFileId,
+            requests: group
+        });
+        for (const [customId, vector] of embeddings.entries()) {
+            embeddingsByCustomId.set(customId, vector);
+        }
+    });
+    await runWithConcurrency(tasks, options.concurrency);
+    return {
+        embeddingsByCustomId,
+        batchIds
+    };
+}

package/dist/retrieval/engine.js

@@ -0,0 +1,130 @@
+import { applyEmbeddingPolicyToScope, resolveEmbeddingPolicy } from "./embeddings/policy.js";
+function normalizeScopeForDiagnostics(scopeFilters) {
+    const normalized = {};
+    for (const [key, value] of Object.entries(scopeFilters)) {
+        if (typeof value === "string" ||
+            typeof value === "number" ||
+            typeof value === "boolean") {
+            normalized[key] = value;
+        }
+    }
+    return normalized;
+}
+function defaultStageTelemetry(stage, resultCount, scopeFilters) {
+    return {
+        stage,
+        strategy: `${stage}-default`,
+        degraded: false,
+        nativeAttempted: false,
+        nativeSucceeded: false,
+        fallbackUsed: false,
+        candidateCount: resultCount,
+        resultCount,
+        latencyMs: 0,
+        scopeFilter: normalizeScopeForDiagnostics(scopeFilters),
+        notes: ["Retriever does not expose telemetry; default stage telemetry applied."]
+    };
+}
+function buildSnippet(text, maxLength = 220) {
+    const compact = text.replace(/\s+/g, " ").trim();
+    if (compact.length <= maxLength) {
+        return compact;
+    }
+    return `${compact.slice(0, maxLength - 1)}…`;
+}
+function shapeResultEnvelope(hit) {
+    const metadata = hit.metadata ?? {};
+    const docId = typeof metadata.doc_id === "string" ? metadata.doc_id : null;
+    return {
+        id: hit.id,
+        score: Number(hit.score.toFixed(6)),
+        source: hit.source,
+        snippet: buildSnippet(hit.text),
+        citation: {
+            chunk_id: hit.id,
+            document_id: docId
+        },
+        sourceMetadata: metadata
+    };
+}
+async function readLexicalStage(retriever, query) {
+    if ("searchWithTelemetry" in retriever) {
+        const stageAware = retriever;
+        return stageAware.searchWithTelemetry(query);
+    }
+    const hits = await retriever.search(query);
+    return {
+        hits,
+        telemetry: defaultStageTelemetry("lexical", hits.length, query.scopeFilters)
+    };
+}
+async function readVectorStage(retriever, query) {
+    if ("searchWithTelemetry" in retriever) {
+        const stageAware = retriever;
+        return stageAware.searchWithTelemetry(query);
+    }
+    const hits = await retriever.search(query);
+    return {
+        hits,
+        telemetry: defaultStageTelemetry("vector", hits.length, query.scopeFilters)
+    };
+}
+export class RetrievalEngine {
+    lexical;
+    vector;
+    fusion;
+    constructor(lexical, vector, fusion) {
+        this.lexical = lexical;
+        this.vector = vector;
+        this.fusion = fusion;
+    }
+    async retrieve(query) {
+        const embeddingPolicy = resolveEmbeddingPolicy(query.scopeFilters);
+        const scopedQuery = {
+            ...query,
+            scopeFilters: applyEmbeddingPolicyToScope(query.scopeFilters, embeddingPolicy)
+        };
+        const lexicalStage = await readLexicalStage(this.lexical, scopedQuery);
+        const vectorStage = await readVectorStage(this.vector, scopedQuery);
+        const fusionResult = await this.fusion.fuse({
+            query: scopedQuery,
+            lexicalHits: lexicalStage.hits,
+            vectorHits: vectorStage.hits
+        });
+        const pipelineMode = scopedQuery.preferNativeFusion ? "hybrid" : "lexical";
+        const selectedHits = scopedQuery.preferNativeFusion
+            ? fusionResult.hits
+            : lexicalStage.hits.slice(0, scopedQuery.topK);
+        const degraded = lexicalStage.telemetry.degraded ||
+            vectorStage.telemetry.degraded ||
+            fusionResult.telemetry.degraded;
+        const results = selectedHits.map(shapeResultEnvelope);
+        return {
+            mode: pipelineMode,
+            degraded,
+            hits: selectedHits,
+            results,
+            diagnostics: {
+                scope: normalizeScopeForDiagnostics(scopedQuery.scopeFilters),
+                lexical: lexicalStage.telemetry,
+                vector: vectorStage.telemetry,
+                fusion: fusionResult.telemetry
+            },
+            debug: {
+                lexicalCount: lexicalStage.hits.length,
+                vectorCount: vectorStage.hits.length,
+                fusionStrategy: fusionResult.telemetry.strategy,
+                embeddingPolicy: {
+                    provider: embeddingPolicy.provider,
+                    model: embeddingPolicy.model,
+                    source: embeddingPolicy.source
+                },
+                modeContract: {
+                    profile: scopedQuery.profile,
+                    preferNativeFusion: scopedQuery.preferNativeFusion,
+                    pipelineMode
+                }
+            }
+        };
+    }
+}

package/dist/retrieval/fixtures.js

@@ -0,0 +1,123 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { z } from "zod";
+export const retrievalQueryClassSchema = z.enum([
+    "semantic_paraphrase",
+    "exact_token",
+    "mixed_intent",
+    "recency_scoped",
+    "channel_thread_scoped"
+]);
+const corpusChunkSchema = z.object({
+    doc_id: z.string().min(1),
+    chunk_id: z.string().min(1),
+    tenant_id: z.string().min(1),
+    workspace_id: z.string().min(1),
+    source_type: z.enum(["memory", "conversation", "operational"]),
+    channel: z.string().min(1),
+    thread_key: z.string().min(1),
+    timestamp: z.string().datetime(),
+    language: z.string().min(1),
+    embedding_provider: z.string().min(1),
+    embedding_model: z.string().min(1),
+    schema_version: z.number().int().positive(),
+    text: z.string().min(1),
+    embedding: z.array(z.number()).min(4)
+});
+const fixtureCorpusSchema = z.object({
+    dataset_id: z.string().min(1),
+    version: z.number().int().positive(),
+    generated_at: z.string().datetime(),
+    chunks: z.array(corpusChunkSchema).min(1)
+});
+const fixtureQuerySchema = z.object({
+    query_id: z.string().min(1),
+    query_class: retrievalQueryClassSchema,
+    query: z.string().min(1),
+    tenant_id: z.string().min(1),
+    workspace_id: z.string().min(1),
+    channel: z.string().min(1).optional(),
+    thread_key: z.string().min(1).optional(),
+    time_range: z
+        .object({
+        from: z.string().datetime(),
+        to: z.string().datetime()
+    })
+        .optional(),
+    expected_chunk_ids: z.array(z.string().min(1)).min(1),
+    critical_exact: z.boolean(),
+    scope_valid: z.boolean()
+});
+const fixtureQueriesSchema = z.object({
+    dataset_id: z.string().min(1),
+    version: z.number().int().positive(),
+    queries: z.array(fixtureQuerySchema).min(1)
+});
+const requiredQueryClasses = retrievalQueryClassSchema.options;
+function resolveFixturePath(candidatePath, cwd) {
+    return path.isAbsolute(candidatePath) ? candidatePath : path.resolve(cwd, candidatePath);
+}
+export function resolveFixturePaths(overrides = {}, cwd = process.cwd()) {
+    return {
+        corpusPath: resolveFixturePath(overrides.corpusPath ?? "fixtures/retrieval/corpus.v1.json", cwd),
+        queriesPath: resolveFixturePath(overrides.queriesPath ?? "fixtures/retrieval/queries.v1.json", cwd)
+    };
+}
+function countByKey(values) {
+    return values.reduce((acc, value) => {
+        acc[value] = (acc[value] ?? 0) + 1;
+        return acc;
+    }, {});
+}
+function ensureQueryTaxonomyCoverage(queries) {
+    const available = new Set(queries.queries.map((entry) => entry.query_class));
+    const missing = requiredQueryClasses.filter((queryClass) => !available.has(queryClass));
+    if (missing.length > 0) {
+        throw new Error(`Fixture queries missing required query classes: ${missing.join(", ")}`);
+    }
+}
+function ensureDatasetConsistency(corpus, queries) {
+    if (corpus.dataset_id !== queries.dataset_id) {
+        throw new Error(`Fixture dataset mismatch: corpus dataset_id='${corpus.dataset_id}' queries dataset_id='${queries.dataset_id}'.`);
+    }
+    if (corpus.version !== queries.version) {
+        throw new Error(`Fixture version mismatch: corpus version='${corpus.version}' queries version='${queries.version}'.`);
+    }
+}
+function buildFingerprint(corpusRaw, queriesRaw) {
+    return createHash("sha256").update(corpusRaw).update("\n").update(queriesRaw).digest("hex");
+}
+export async function loadRetrievalFixtureBundle(overrides = {}, cwd = process.cwd()) {
+    const { corpusPath, queriesPath } = resolveFixturePaths(overrides, cwd);
+    const [corpusRaw, queriesRaw] = await Promise.all([
+        readFile(corpusPath, "utf8"),
+        readFile(queriesPath, "utf8")
+    ]);
+    const corpus = fixtureCorpusSchema.parse(JSON.parse(corpusRaw));
+    const queries = fixtureQueriesSchema.parse(JSON.parse(queriesRaw));
+    ensureDatasetConsistency(corpus, queries);
+    ensureQueryTaxonomyCoverage(queries);
+    return {
+        corpusPath,
+        queriesPath,
+        corpus,
+        queries,
+        fingerprint: buildFingerprint(corpusRaw, queriesRaw)
+    };
+}
+export function summarizeRetrievalFixture(bundle) {
+    const sourceTypeCounts = countByKey(bundle.corpus.chunks.map((chunk) => chunk.source_type));
+    const queryClassCounts = countByKey(bundle.queries.queries.map((query) => query.query_class));
+    const tenantWorkspacePairs = new Set(bundle.corpus.chunks.map((chunk) => `${chunk.tenant_id}:${chunk.workspace_id}`)).size;
+    return {
+        datasetId: bundle.corpus.dataset_id,
+        version: bundle.corpus.version,
+        fingerprint: bundle.fingerprint,
+        chunkCount: bundle.corpus.chunks.length,
+        queryCount: bundle.queries.queries.length,
+        sourceTypeCounts,
+        queryClassCounts,
+        tenantWorkspacePairs
+    };
+}