claude-flow 3.7.0-alpha.51 → 3.7.0-alpha.53

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-flow",
-  "version": "3.7.0-alpha.51",
+  "version": "3.7.0-alpha.53",
   "description": "Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration",
   "main": "dist/index.js",
   "type": "module",

package/v3/@claude-flow/cli/dist/src/mcp-tools/embeddings-tools.js

@@ -1280,6 +1280,282 @@ export const embeddingsTools = [
         },
     },
     // ============================================================
+    // ADR-121 Phase 12 — HyDE embedding-level fusion (alpha.53 CLI)
+    // ============================================================
+    //
+    // HyDE (Gao, Ma, Lin, Callan 2022 — "Precise Zero-Shot Dense
+    // Retrieval without Relevance Labels"): question embeddings live in
+    // "question space" while documents embed into "answer space", so
+    // cosine search systematically underweights relevant docs. The
+    // standard fix: have an LLM generate N hypothetical answers, embed
+    // each, AVERAGE the embeddings into a single query vector, search
+    // once with that.
+    //
+    // Distinct from `embeddings_search_text_ensemble` (Phase 11):
+    //   - HyDE fuses at the EMBEDDING level (1 search after average).
+    //     Cheaper, finds the centroid hit, interpolates between
+    //     hypothetical answers.
+    //   - RRF fuses at the RANK level (N searches then merge ranks).
+    //     More expensive, preserves intent boundaries between variants.
+    //
+    // Both are useful. Production systems often combine them — HyDE
+    // inside one ranked list, RRF across multiple lists.
+    {
+        name: 'embeddings_search_text_hyde',
+        description: "Embed N hypothetical-answer texts, AVERAGE their embeddings into a single query vector, and search a named AnnRouter handle once. Implements the HyDE recipe (Gao et al. 2022) for zero-shot dense retrieval — the LLM-generated hypothetical answers live in the same answer-space as the corpus, so the averaged vector lands near the true relevant docs. Distinct from embeddings_search_text_ensemble (which fuses at the rank level via RRF — more expensive, preserves intent boundaries); HyDE fuses at the embedding level (cheaper, one search, finds the centroid hit). Optional `weights` per text (e.g. weight the user's original question 0.5× and LLM answers 1.0× each — the paper's recommended recipe). Returns hits + averaged-vector metadata (unitNorm assertion + contributing-text count) for transparency.",
+        category: 'embeddings',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                texts: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Array of hypothetical-answer texts (LLM-generated) to fuse. The caller is responsible for generating these — typically via a few-shot prompt to an LLM asking it to answer the user question.',
+                },
+                name: { type: 'string', description: 'AnnRouter handle name (set by embeddings_ann_router_build).' },
+                k: { type: 'number', description: 'Number of nearest neighbors to return.' },
+                weights: {
+                    type: 'array',
+                    items: { type: 'number' },
+                    description: 'Per-text weights for the embedding average. Length must equal texts.length. Default uniform.',
+                },
+            },
+            required: ['texts', 'name', 'k'],
+        },
+        handler: async (input) => {
+            const config = loadConfig();
+            if (!config) {
+                return { success: false, error: 'Embeddings not initialized. Run embeddings_init first.' };
+            }
+            const texts = input.texts;
+            const name = input.name;
+            const k = input.k;
+            const weights = Array.isArray(input.weights) ? input.weights : undefined;
+            if (!Array.isArray(texts) || texts.length === 0) {
+                return { success: false, error: 'texts must be a non-empty array' };
+            }
+            if (!Number.isInteger(k) || k < 1) {
+                return { success: false, error: 'k must be a positive integer' };
+            }
+            for (let i = 0; i < texts.length; i++) {
+                if (typeof texts[i] !== 'string') {
+                    return { success: false, error: `texts[${i}] is not a string` };
+                }
+            }
+            if (weights && weights.length !== texts.length) {
+                return {
+                    success: false,
+                    error: `weights.length (${weights.length}) must match texts.length (${texts.length})`,
+                };
+            }
+            if (weights) {
+                for (let i = 0; i < weights.length; i++) {
+                    if (weights[i] < 0) {
+                        return { success: false, error: `weights[${i}] is negative` };
+                    }
+                }
+            }
+            const { getAnnRouterRegistry } = await import('../memory/ann-router-registry.js');
+            const registry = getAnnRouterRegistry();
+            // Stage 1 — embed each hypothetical text in parallel.
+            const embedT0 = Date.now();
+            let embeddings;
+            try {
+                embeddings = await Promise.all(texts.map(t => generateRealEmbedding(t, config.dimension)));
+            }
+            catch (err) {
+                return { success: false, name, error: `batch embed failed: ${err instanceof Error ? err.message : String(err)}` };
+            }
+            const embeddingMs = Date.now() - embedT0;
+            // Stage 2 — average the embeddings (HyDE fusion).
+            const { averageEmbeddings, isUnitNorm } = await import('@claude-flow/embeddings/embedding-fusion');
+            const fuseT0 = Date.now();
+            const avgVec = averageEmbeddings(embeddings, {
+                weights,
+                normalizeInputs: true,
+                normalizeOutput: true,
+            });
+            const fuseMs = Date.now() - fuseT0;
+            const fusedUnit = isUnitNorm(avgVec);
+            // Stage 3 — single search with the averaged vector.
+            const searchT0 = Date.now();
+            let hits;
+            try {
+                hits = await registry.search(name, avgVec, k);
+            }
+            catch (err) {
+                return {
+                    success: false,
+                    name,
+                    error: err instanceof Error ? err.message : String(err),
+                    latency: { embeddingMs, fuseMs, searchMs: 0 },
+                };
+            }
+            const searchMs = Date.now() - searchT0;
+            return {
+                success: true,
+                name,
+                k,
+                hits,
+                hyde: {
+                    textsFused: texts.length,
+                    weights: weights ?? null,
+                    averagedVectorUnitNorm: fusedUnit,
+                    dimension: avgVec.length,
+                },
+                latency: {
+                    embeddingMs,
+                    fuseMs,
+                    searchMs,
+                    totalMs: embeddingMs + fuseMs + searchMs,
+                },
+            };
+        },
+    },
+    // ============================================================
+    // ADR-121 Phase 11 — RRF ensemble retrieval (alpha.52 CLI)
+    // ============================================================
+    //
+    // Question-reformulation pipelines produce N parallel result lists.
+    // Reciprocal Rank Fusion (Cormack-Clarke-Büttcher 2009) fuses them
+    // into a single ranking without needing score comparability across
+    // lists. Composes `embeddings_search_text_batch` (N parallel
+    // searches) with `reciprocalRankFusion` (rank-level merge).
+    //
+    // Standard production ensemble-RAG pattern. Pairs naturally with
+    // LLM query rewriting upstream (expand "how does auth work?" into
+    // {"how does authentication work?", "what's the login flow?",
+    //  "describe the OAuth2 handshake"} → batch search each →
+    // RRF-fuse) — recovers more relevant docs than a single search.
+    {
+        name: 'embeddings_search_text_ensemble',
+        description: "Embed N text query variants, search a named AnnRouter handle for each in parallel, then RRF-fuse (Reciprocal Rank Fusion, Cormack-Clarke-Büttcher 2009) the N hit-lists into a single merged top-k ranking. Standard production shape for question-reformulation RAG: agents expand one user question into N variants, retrieve top-k for each, get a single fused list back — items appearing high in MORE variants outrank items appearing high in only one. Returns fused hits with per-list ranks for transparency + aggregate latency. λ-equivalent here is `kRrf` (default 60 per SIGIR 2009). Per-list `listWeights` available for biased ensemble (e.g. weight the original-query list 2× over reformulations). For non-fused multi-query results use embeddings_search_text_batch; for single-query diverse retrieval use embeddings_search_text_diverse.",
+        category: 'embeddings',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                texts: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Array of query text variants. Order preserved in per-list ranks.',
+                },
+                name: { type: 'string', description: 'AnnRouter handle name (set by embeddings_ann_router_build).' },
+                k: { type: 'number', description: 'Number of fused results to return.' },
+                perQueryK: {
+                    type: 'number',
+                    description: 'Top-k per query before fusion. Default 2*k. Larger = wider candidate pool, more compute.',
+                },
+                kRrf: {
+                    type: 'number',
+                    description: 'RRF smoothing constant. Default 60 (SIGIR 2009). Smaller = top-rank dominance.',
+                },
+                listWeights: {
+                    type: 'array',
+                    items: { type: 'number' },
+                    description: 'Per-query weights. Length must equal texts.length. Default = 1 each.',
+                },
+            },
+            required: ['texts', 'name', 'k'],
+        },
+        handler: async (input) => {
+            const config = loadConfig();
+            if (!config) {
+                return { success: false, error: 'Embeddings not initialized. Run embeddings_init first.' };
+            }
+            const texts = input.texts;
+            const name = input.name;
+            const k = input.k;
+            const perQueryK = typeof input.perQueryK === 'number' && input.perQueryK >= 1
+                ? input.perQueryK
+                : Math.max(k * 2, k);
+            const kRrf = typeof input.kRrf === 'number' && input.kRrf > 0 ? input.kRrf : 60;
+            const listWeights = Array.isArray(input.listWeights) ? input.listWeights : undefined;
+            if (!Array.isArray(texts) || texts.length === 0) {
+                return { success: false, error: 'texts must be a non-empty array' };
+            }
+            if (!Number.isInteger(k) || k < 1) {
+                return { success: false, error: 'k must be a positive integer' };
+            }
+            for (let i = 0; i < texts.length; i++) {
+                if (typeof texts[i] !== 'string') {
+                    return { success: false, error: `texts[${i}] is not a string` };
+                }
+            }
+            if (listWeights && listWeights.length !== texts.length) {
+                return {
+                    success: false,
+                    error: `listWeights.length (${listWeights.length}) must match texts.length (${texts.length})`,
+                };
+            }
+            const { getAnnRouterRegistry } = await import('../memory/ann-router-registry.js');
+            const registry = getAnnRouterRegistry();
+            // Stage 1 — embed all queries in parallel.
+            const embedT0 = Date.now();
+            let embeddings;
+            try {
+                embeddings = await Promise.all(texts.map(t => generateRealEmbedding(t, config.dimension)));
+            }
+            catch (err) {
+                return { success: false, name, error: `batch embed failed: ${err instanceof Error ? err.message : String(err)}` };
+            }
+            const embeddingMs = Date.now() - embedT0;
+            // Stage 2 — search each variant in parallel. Per-query errors
+            // become empty result lists (the variant just doesn't contribute
+            // to the fusion) rather than aborting the ensemble.
+            const searchT0 = Date.now();
+            const perQueryResults = await Promise.all(embeddings.map(async (emb, i) => {
+                try {
+                    const hits = await registry.search(name, new Float32Array(emb), perQueryK);
+                    return { index: i, text: texts[i], hits, success: true };
+                }
+                catch (err) {
+                    return {
+                        index: i,
+                        text: texts[i],
+                        hits: [],
+                        success: false,
+                        error: err instanceof Error ? err.message : String(err),
+                    };
+                }
+            }));
+            const searchMs = Date.now() - searchT0;
+            // Stage 3 — RRF fusion across the per-query lists.
+            const { reciprocalRankFusion } = await import('@claude-flow/embeddings/rrf');
+            const fuseT0 = Date.now();
+            const lists = perQueryResults.map(r => r.hits.map(h => ({ id: h.id, payload: { score: h.score } })));
+            const fused = reciprocalRankFusion(lists, { k, kRrf, listWeights });
+            const fuseMs = Date.now() - fuseT0;
+            const successCount = perQueryResults.filter(r => r.success).length;
+            return {
+                success: successCount === perQueryResults.length,
+                name,
+                k,
+                queryCount: texts.length,
+                perQueryK,
+                kRrf,
+                listWeights: listWeights ?? null,
+                successCount,
+                failureCount: perQueryResults.length - successCount,
+                hits: fused,
+                perQuery: perQueryResults.map(r => ({
+                    index: r.index,
+                    text: r.text,
+                    success: r.success,
+                    hitCount: r.hits.length,
+                    error: r.success ? undefined : r.error,
+                })),
+                latency: {
+                    embeddingMs,
+                    searchMs,
+                    fuseMs,
+                    totalMs: embeddingMs + searchMs + fuseMs,
+                    avgPerQueryMs: Math.round(((embeddingMs + searchMs) / texts.length) * 100) / 100,
+                },
+                embeddingDimension: embeddings[0]?.length,
+            };
+        },
+    },
+    // ============================================================
     // ADR-121 Phase 10 — MMR diversity rerank (alpha.51 CLI)
     // ============================================================
     //

package/v3/@claude-flow/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@claude-flow/cli",
-  "version": "3.7.0-alpha.51",
+  "version": "3.7.0-alpha.53",
   "type": "module",
   "description": "Ruflo CLI - Enterprise AI agent orchestration with 60+ specialized agents, swarm coordination, MCP server, self-learning hooks, and vector memory for Claude Code",
   "main": "dist/src/index.js",
@@ -104,14 +104,14 @@
     "semver": "^7.6.0",
     "yaml": "^2.8.0",
     "@claude-flow/memory": "^3.0.0-alpha.17",
-    "@claude-flow/embeddings": "^3.0.0-alpha.31",
+    "@claude-flow/embeddings": "^3.0.0-alpha.33",
     "@claude-flow/security": "^3.0.0-alpha.8",
     "@claude-flow/swarm": "^3.0.0-alpha.8"
   },
   "optionalDependencies": {
     "@claude-flow/aidefence": "^3.0.2",
     "@claude-flow/codex": "^3.0.0-alpha.8",
-    "@claude-flow/embeddings": "^3.0.0-alpha.31",
+    "@claude-flow/embeddings": "^3.0.0-alpha.33",
     "@claude-flow/guidance": "^3.0.0-alpha.1",
     "@claude-flow/memory": "^3.0.0-alpha.17",
     "@claude-flow/plugin-gastown-bridge": "^0.1.3",