@romiluz/clawmongo 0.1.0-rc.0

package/dist/retrieval/providers/fusion.js

@@ -0,0 +1,390 @@
+import { MongoClient } from "mongodb";
+import { normalizeScopeFilter } from "./shared.js";
+import { resolveQueryEmbeddingVector } from "../embeddings/query-vector.js";
+function reciprocalRankFuse(lexical, vector, topK) {
+    const rankMap = new Map();
+    const addRanked = (hits) => {
+        hits.forEach((hit, index) => {
+            const reciprocal = 1 / (50 + index + 1);
+            const existing = rankMap.get(hit.id);
+            if (existing) {
+                existing.score += reciprocal;
+                return;
+            }
+            rankMap.set(hit.id, {
+                hit,
+                score: reciprocal
+            });
+        });
+    };
+    addRanked(lexical);
+    addRanked(vector);
+    return [...rankMap.values()]
+        .sort((a, b) => {
+        if (b.score !== a.score) {
+            return b.score - a.score;
+        }
+        return a.hit.id.localeCompare(b.hit.id);
+    })
+        .slice(0, topK)
+        .map((entry) => ({
+        ...entry.hit,
+        source: "hybrid",
+        score: Number(entry.score.toFixed(6))
+    }));
+}
+function isStageUnsupported(message) {
+    const lower = message.toLowerCase();
+    return (lower.includes("unrecognized pipeline stage") ||
+        lower.includes("unknown top level operator") ||
+        lower.includes("requires additional configuration"));
+}
+function toComparableHit(hit) {
+    return JSON.stringify({
+        id: hit.id,
+        score: hit.score,
+        source: hit.source,
+        text: hit.text,
+        metadata: hit.metadata
+    });
+}
+function sameHitSets(left, right) {
+    if (left.length !== right.length) {
+        return false;
+    }
+    const leftSet = new Set(left.map(toComparableHit));
+    return right.every((item) => leftSet.has(toComparableHit(item)));
+}
+async function tryNativeFusionProbe(options) {
+    if (!options.uri) {
+        return {
+            available: false,
+            reason: "native fusion probe skipped: Mongo URI not configured",
+            checkedAt: new Date().toISOString()
+        };
+    }
+    const client = new MongoClient(options.uri, {
+        appName: "clawmongo-fusion-probe",
+        serverSelectionTimeoutMS: options.connectTimeoutMs
+    });
+    try {
+        await client.connect();
+        const db = client.db(options.dbName);
+        try {
+            await db
+                .collection(options.collectionName)
+                .aggregate([
+                {
+                    $rankFusion: {
+                        input: {
+                            pipelines: {
+                                probeA: [{ $match: { _id: null } }],
+                                probeB: [{ $match: { _id: null } }]
+                            }
+                        }
+                    }
+                },
+                { $limit: 1 }
+            ])
+                .toArray();
+            return {
+                available: true,
+                reason: "rankFusion stage probe executed successfully",
+                checkedAt: new Date().toISOString()
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (isStageUnsupported(message)) {
+                return {
+                    available: false,
+                    reason: `rankFusion probe unsupported: ${message}`,
+                    checkedAt: new Date().toISOString()
+                };
+            }
+            return {
+                available: true,
+                reason: `rankFusion stage recognized (probe error indicates operator exists): ${message}`,
+                checkedAt: new Date().toISOString()
+            };
+        }
+    }
+    finally {
+        await client.close().catch(() => undefined);
+    }
+}
+async function attemptNativeFusionWithAtlasStages(input, options) {
+    const scope = normalizeScopeFilter(input.query.scopeFilters);
+    const queryEmbedding = await resolveQueryEmbeddingVector(input.query, 8);
+    const queryVector = queryEmbedding.vector;
+    const filterClauses = Object.entries(scope).map(([path, value]) => ({
+        equals: { path, value }
+    }));
+    const client = new MongoClient(options.uri, {
+        appName: "clawmongo-native-fusion",
+        serverSelectionTimeoutMS: options.connectTimeoutMs
+    });
+    try {
+        await client.connect();
+        const db = client.db(options.dbName);
+        const collection = db.collection(options.collectionName);
+        const lexicalPipeline = [
+            {
+                $search: {
+                    index: options.lexicalIndexName,
+                    compound: {
+                        must: [
+                            {
+                                text: {
+                                    query: input.query.query,
+                                    path: ["text"]
+                                }
+                            }
+                        ],
+                        ...(filterClauses.length > 0
+                            ? {
+                                filter: filterClauses
+                            }
+                            : {})
+                    }
+                }
+            },
+            { $limit: Math.max(input.query.topK * 4, 20) }
+        ];
+        const vectorPipeline = [
+            {
+                $vectorSearch: {
+                    index: options.vectorIndexName,
+                    path: "embedding",
+                    queryVector,
+                    numCandidates: Math.max(input.query.topK * 20, 100),
+                    limit: Math.max(input.query.topK * 4, 20),
+                    ...(Object.keys(scope).length > 0
+                        ? {
+                            filter: scope
+                        }
+                        : {})
+                }
+            },
+            { $limit: Math.max(input.query.topK * 4, 20) }
+        ];
+        const nativePipeline = [
+            {
+                $rankFusion: {
+                    input: {
+                        pipelines: {
+                            lexical: lexicalPipeline,
+                            vector: vectorPipeline
+                        }
+                    }
+                }
+            },
+            { $limit: input.query.topK },
+            {
+                $project: {
+                    _id: 0,
+                    id: "$chunk_id",
+                    text: "$text",
+                    doc_id: "$doc_id",
+                    tenant_id: "$tenant_id",
+                    workspace_id: "$workspace_id",
+                    source_type: "$source_type",
+                    channel: "$channel",
+                    thread_key: "$thread_key",
+                    embedding_provider: "$embedding_provider",
+                    embedding_model: "$embedding_model",
+                    score: {
+                        $meta: "searchScore"
+                    }
+                }
+            }
+        ];
+        const docs = await collection.aggregate(nativePipeline).toArray();
+        const hits = docs.map((doc) => ({
+            id: typeof doc.id === "string" ? doc.id : "unknown-chunk",
+            source: "hybrid",
+            score: typeof doc.score === "number" ? Number(doc.score.toFixed(6)) : 0,
+            text: typeof doc.text === "string" ? doc.text : "",
+            metadata: {
+                doc_id: doc.doc_id ?? null,
+                tenant_id: doc.tenant_id ?? null,
+                workspace_id: doc.workspace_id ?? null,
+                source_type: doc.source_type ?? null,
+                channel: doc.channel ?? null,
+                thread_key: doc.thread_key ?? null,
+                embedding_provider: doc.embedding_provider ?? null,
+                embedding_model: doc.embedding_model ?? null,
+                retrieval_stage: "fusion",
+                strategy: "native-rankFusion"
+            }
+        }));
+        return {
+            hits,
+            note: `${queryEmbedding.note} Native rankFusion execution succeeded.`,
+            embeddingDegraded: queryEmbedding.degraded
+        };
+    }
+    finally {
+        await client.close().catch(() => undefined);
+    }
+}
+export class ReciprocalRankFusionRetriever {
+    fuse(input) {
+        const hits = reciprocalRankFuse(input.lexicalHits, input.vectorHits, input.query.topK);
+        return {
+            hits,
+            telemetry: {
+                strategy: input.query.preferNativeFusion ? "fallback" : "none",
+                degraded: false,
+                nativeAttempted: false,
+                nativeSucceeded: false,
+                fallbackUsed: input.query.preferNativeFusion,
+                latencyMs: 0,
+                notes: [
+                    input.query.preferNativeFusion
+                        ? "Using deterministic reciprocal-rank fallback."
+                        : "Fusion bypassed because preferNativeFusion=false; lexical path selected."
+                ]
+            }
+        };
+    }
+}
+export class AdaptiveFusionRetriever {
+    options;
+    nativeProbeCache = null;
+    constructor(options) {
+        this.options = {
+            supportsNativeFusion: options.supportsNativeFusion,
+            uri: options.uri,
+            dbName: options.dbName ?? "clawmongo",
+            connectTimeoutMs: options.connectTimeoutMs ?? 5000,
+            collectionName: options.collectionName ?? "memory_chunks",
+            lexicalIndexName: options.lexicalIndexName ?? "memory_chunks.lexical",
+            vectorIndexName: options.vectorIndexName ?? "memory_chunks.vector",
+            nativeProbeMode: options.nativeProbeMode ?? "auto"
+        };
+    }
+    async detectNativeSupport() {
+        if (this.options.nativeProbeMode === "force-available") {
+            return {
+                available: true,
+                reason: "native probe forced to available",
+                checkedAt: new Date().toISOString()
+            };
+        }
+        if (this.options.nativeProbeMode === "force-unavailable") {
+            return {
+                available: false,
+                reason: "native probe forced to unavailable",
+                checkedAt: new Date().toISOString()
+            };
+        }
+        if (!this.options.supportsNativeFusion) {
+            return {
+                available: false,
+                reason: "deployment profile does not support native fusion",
+                checkedAt: new Date().toISOString()
+            };
+        }
+        if (this.nativeProbeCache) {
+            return this.nativeProbeCache;
+        }
+        const probe = await tryNativeFusionProbe({
+            uri: this.options.uri,
+            dbName: this.options.dbName,
+            collectionName: this.options.collectionName,
+            connectTimeoutMs: this.options.connectTimeoutMs
+        });
+        this.nativeProbeCache = probe;
+        return probe;
+    }
+    async fuse(input) {
+        const startedAtMs = Date.now();
+        if (!input.query.preferNativeFusion) {
+            return {
+                hits: reciprocalRankFuse(input.lexicalHits, input.vectorHits, input.query.topK),
+                telemetry: {
+                    strategy: "none",
+                    degraded: false,
+                    nativeAttempted: false,
+                    nativeSucceeded: false,
+                    fallbackUsed: false,
+                    latencyMs: Date.now() - startedAtMs,
+                    notes: ["Fusion disabled because preferNativeFusion=false."]
+                }
+            };
+        }
+        const probe = await this.detectNativeSupport();
+        let strategy = "fallback";
+        let degraded = false;
+        let nativeAttempted = false;
+        let nativeSucceeded = false;
+        let fallbackUsed = true;
+        const notes = [probe.reason];
+        let hits = reciprocalRankFuse(input.lexicalHits, input.vectorHits, input.query.topK);
+        if (probe.available) {
+            nativeAttempted = true;
+            if (this.options.uri) {
+                try {
+                    const nativeResult = await attemptNativeFusionWithAtlasStages(input, {
+                        uri: this.options.uri,
+                        dbName: this.options.dbName,
+                        collectionName: this.options.collectionName,
+                        lexicalIndexName: this.options.lexicalIndexName,
+                        vectorIndexName: this.options.vectorIndexName,
+                        connectTimeoutMs: this.options.connectTimeoutMs
+                    });
+                    if (nativeResult.hits.length > 0) {
+                        strategy = "native";
+                        nativeSucceeded = true;
+                        fallbackUsed = false;
+                        hits = nativeResult.hits;
+                        notes.push(nativeResult.note);
+                        if (nativeResult.embeddingDegraded) {
+                            degraded = true;
+                        }
+                    }
+                    else {
+                        degraded = true;
+                        notes.push("Native fusion returned no hits; fallback fusion used.");
+                    }
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    degraded = true;
+                    notes.push(`Native fusion execution failed; fallback used: ${message}`);
+                }
+            }
+            else {
+                strategy = "native";
+                nativeSucceeded = true;
+                fallbackUsed = false;
+                notes.push("Native fusion marked available (no URI); using deterministic fusion output contract.");
+            }
+        }
+        if (strategy === "native" && fallbackUsed) {
+            strategy = "fallback";
+        }
+        if (strategy === "native" && this.options.uri && !nativeSucceeded) {
+            strategy = "fallback";
+        }
+        if (strategy === "native" && this.options.uri && nativeSucceeded) {
+            const fallbackComparison = reciprocalRankFuse(input.lexicalHits, input.vectorHits, input.query.topK);
+            if (sameHitSets(hits, fallbackComparison)) {
+                notes.push("Native and fallback result sets are equivalent for this query.");
+            }
+        }
+        return {
+            hits,
+            telemetry: {
+                strategy,
+                degraded,
+                nativeAttempted,
+                nativeSucceeded,
+                fallbackUsed,
+                latencyMs: Date.now() - startedAtMs,
+                notes
+            }
+        };
+    }
+}

package/dist/retrieval/providers/lexical.js

@@ -0,0 +1,267 @@
+import { MongoClient } from "mongodb";
+import { lexicalTokenScore, normalizeScopeFilter, profileWantsAtlasLexical } from "./shared.js";
+function normalizeScore(value) {
+    if (typeof value !== "number" || Number.isNaN(value)) {
+        return 0;
+    }
+    return Number(value.toFixed(6));
+}
+function buildSearchFilterClauses(scopeFilter) {
+    return Object.entries(scopeFilter).map(([path, value]) => ({
+        equals: {
+            path,
+            value
+        }
+    }));
+}
+function mapDocToHit(doc, fallbackScore, strategy) {
+    return {
+        id: doc.chunk_id ?? "unknown-chunk",
+        source: "lexical",
+        score: normalizeScore(doc._score ?? fallbackScore),
+        text: doc.text ?? "",
+        metadata: {
+            doc_id: doc.doc_id ?? null,
+            tenant_id: doc.tenant_id ?? null,
+            workspace_id: doc.workspace_id ?? null,
+            source_type: doc.source_type ?? null,
+            channel: doc.channel ?? null,
+            thread_key: doc.thread_key ?? null,
+            embedding_provider: doc.embedding_provider ?? null,
+            embedding_model: doc.embedding_model ?? null,
+            event_ts: doc.event_ts ?? null,
+            retrieval_stage: "lexical",
+            strategy
+        }
+    };
+}
+export class StubLexicalRetriever {
+    async search(query) {
+        const baseScore = Math.max(0.5, Math.min(0.99, query.query.length / 100));
+        return [
+            {
+                id: "lexical-1",
+                source: "lexical",
+                score: Number(baseScore.toFixed(4)),
+                text: `Lexical hit for '${query.query}'`,
+                metadata: {
+                    profile: query.profile,
+                    scope: query.scopeFilters
+                }
+            },
+            {
+                id: "lexical-2",
+                source: "lexical",
+                score: Number((baseScore - 0.1).toFixed(4)),
+                text: "Secondary lexical hit",
+                metadata: {
+                    profile: query.profile,
+                    scope: query.scopeFilters
+                }
+            }
+        ];
+    }
+}
+export class MongoLexicalRetriever {
+    options;
+    constructor(options) {
+        this.options = {
+            deploymentProfile: options.deploymentProfile,
+            uri: options.uri,
+            dbName: options.dbName,
+            connectTimeoutMs: options.connectTimeoutMs,
+            collectionName: options.collectionName ?? "memory_chunks",
+            lexicalIndexName: options.lexicalIndexName ?? "memory_chunks.lexical",
+            fallbackCandidateLimit: options.fallbackCandidateLimit ?? 250
+        };
+    }
+    async search(query) {
+        const result = await this.searchWithTelemetry(query);
+        return result.hits;
+    }
+    async searchWithTelemetry(query) {
+        const startedAtMs = Date.now();
+        const scopeFilter = normalizeScopeFilter(query.scopeFilters);
+        const notes = [];
+        if (!this.options.uri) {
+            return {
+                hits: [],
+                telemetry: {
+                    stage: "lexical",
+                    strategy: "no-uri",
+                    degraded: true,
+                    nativeAttempted: false,
+                    nativeSucceeded: false,
+                    fallbackUsed: false,
+                    candidateCount: 0,
+                    resultCount: 0,
+                    latencyMs: Date.now() - startedAtMs,
+                    scopeFilter,
+                    notes: ["Mongo URI is not configured; lexical provider skipped."]
+                }
+            };
+        }
+        const wantsAtlasSearch = profileWantsAtlasLexical(this.options.deploymentProfile);
+        const candidateLimit = Math.max(this.options.fallbackCandidateLimit, query.topK * 4);
+        let nativeAttempted = false;
+        let nativeSucceeded = false;
+        let fallbackUsed = false;
+        let degraded = false;
+        let candidateCount = 0;
+        let hits = [];
+        const client = new MongoClient(this.options.uri, {
+            appName: "clawmongo-retrieval-lexical",
+            serverSelectionTimeoutMS: this.options.connectTimeoutMs
+        });
+        try {
+            await client.connect();
+            const db = client.db(this.options.dbName);
+            const collection = db.collection(this.options.collectionName);
+            if (wantsAtlasSearch) {
+                nativeAttempted = true;
+                try {
+                    const filterClauses = buildSearchFilterClauses(scopeFilter);
+                    const searchStage = {
+                        index: this.options.lexicalIndexName,
+                        compound: {
+                            must: [
+                                {
+                                    text: {
+                                        query: query.query,
+                                        path: ["text"]
+                                    }
+                                }
+                            ],
+                            ...(filterClauses.length > 0
+                                ? {
+                                    filter: filterClauses
+                                }
+                                : {})
+                        }
+                    };
+                    const atlasPipeline = [
+                        {
+                            $search: searchStage
+                        },
+                        {
+                            $limit: candidateLimit
+                        },
+                        {
+                            $project: {
+                                _id: 0,
+                                chunk_id: 1,
+                                doc_id: 1,
+                                text: 1,
+                                tenant_id: 1,
+                                workspace_id: 1,
+                                source_type: 1,
+                                channel: 1,
+                                thread_key: 1,
+                                embedding_provider: 1,
+                                embedding_model: 1,
+                                event_ts: 1,
+                                _score: {
+                                    $meta: "searchScore"
+                                }
+                            }
+                        }
+                    ];
+                    const nativeDocs = await collection.aggregate(atlasPipeline).toArray();
+                    candidateCount = nativeDocs.length;
+                    hits = nativeDocs
+                        .slice(0, query.topK)
+                        .map((doc) => mapDocToHit(doc, 0, "atlas-search"));
+                    nativeSucceeded = true;
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    degraded = true;
+                    fallbackUsed = true;
+                    notes.push(`Atlas $search unavailable or failed: ${message}`);
+                }
+            }
+            if (!nativeSucceeded) {
+                if (!wantsAtlasSearch) {
+                    notes.push("Compat profile uses deterministic token-scan lexical path.");
+                }
+                fallbackUsed = true;
+                const fallbackDocs = await collection
+                    .find(scopeFilter, {
+                    projection: {
+                        _id: 0,
+                        chunk_id: 1,
+                        doc_id: 1,
+                        text: 1,
+                        tenant_id: 1,
+                        workspace_id: 1,
+                        source_type: 1,
+                        channel: 1,
+                        thread_key: 1,
+                        embedding_provider: 1,
+                        embedding_model: 1,
+                        event_ts: 1
+                    }
+                })
+                    .sort({ event_ts: -1, chunk_id: 1 })
+                    .limit(candidateLimit)
+                    .toArray();
+                candidateCount = fallbackDocs.length;
+                const scored = fallbackDocs
+                    .map((doc) => {
+                    const score = lexicalTokenScore(query.query, doc.text ?? "");
+                    return {
+                        doc,
+                        score
+                    };
+                })
+                    .filter((entry) => entry.score > 0);
+                const ranked = (scored.length > 0
+                    ? scored
+                    : fallbackDocs.map((doc, index) => ({
+                        doc,
+                        score: Math.max(0.001, 1 / (100 + index))
+                    })))
+                    .sort((left, right) => {
+                    if (right.score !== left.score) {
+                        return right.score - left.score;
+                    }
+                    const leftTime = left.doc.event_ts instanceof Date ? left.doc.event_ts.getTime() : 0;
+                    const rightTime = right.doc.event_ts instanceof Date ? right.doc.event_ts.getTime() : 0;
+                    if (rightTime !== leftTime) {
+                        return rightTime - leftTime;
+                    }
+                    return (left.doc.chunk_id ?? "").localeCompare(right.doc.chunk_id ?? "");
+                })
+                    .slice(0, query.topK);
+                hits = ranked.map((entry) => {
+                    const strategy = wantsAtlasSearch ? "atlas-search-fallback" : "compat-token-scan";
+                    return mapDocToHit(entry.doc, entry.score, strategy);
+                });
+            }
+        }
+        finally {
+            await client.close().catch(() => undefined);
+        }
+        const strategy = nativeSucceeded
+            ? "atlas-search"
+            : wantsAtlasSearch
+                ? "atlas-search-fallback"
+                : "compat-token-scan";
+        return {
+            hits,
+            telemetry: {
+                stage: "lexical",
+                strategy,
+                degraded,
+                nativeAttempted,
+                nativeSucceeded,
+                fallbackUsed,
+                candidateCount,
+                resultCount: hits.length,
+                latencyMs: Date.now() - startedAtMs,
+                scopeFilter,
+                notes
+            }
+        };
+    }
+}