claude-flow 3.58.0 → 3.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/.claude/proven-config.json +1 -1
  2. package/README.md +2 -0
  3. package/node_modules/@claude-flow/codex/dist/cli.js +0 -0
  4. package/node_modules/@claude-flow/plugin-agent-federation/dist/bin.js +0 -0
  5. package/package.json +2 -1
  6. package/v3/@claude-flow/cli/README.md +62 -174
  7. package/v3/@claude-flow/cli/bin/cli.js +19 -1
  8. package/v3/@claude-flow/cli/catalog-manifest.json +5 -5
  9. package/v3/@claude-flow/cli/dist/src/commands/harness.d.ts +40 -0
  10. package/v3/@claude-flow/cli/dist/src/commands/harness.js +142 -0
  11. package/v3/@claude-flow/cli/dist/src/commands/index.js +1 -0
  12. package/v3/@claude-flow/cli/dist/src/commands/init.js +116 -0
  13. package/v3/@claude-flow/cli/dist/src/commands/memory.js +6 -1
  14. package/v3/@claude-flow/cli/dist/src/commands/status.js +2 -2
  15. package/v3/@claude-flow/cli/dist/src/index.d.ts +3 -0
  16. package/v3/@claude-flow/cli/dist/src/index.js +12 -2
  17. package/v3/@claude-flow/cli/dist/src/init/grok-generator.d.ts +96 -0
  18. package/v3/@claude-flow/cli/dist/src/init/grok-generator.js +371 -0
  19. package/v3/@claude-flow/cli/dist/src/init/grok-templates.d.ts +11 -0
  20. package/v3/@claude-flow/cli/dist/src/init/grok-templates.js +28 -0
  21. package/v3/@claude-flow/cli/dist/src/init/index.d.ts +2 -0
  22. package/v3/@claude-flow/cli/dist/src/init/index.js +2 -0
  23. package/v3/@claude-flow/cli/dist/src/mcp-client.js +4 -0
  24. package/v3/@claude-flow/cli/dist/src/mcp-tools/agent-execute-core.js +59 -12
  25. package/v3/@claude-flow/cli/dist/src/mcp-tools/capability-brain.d.ts +1 -1
  26. package/v3/@claude-flow/cli/dist/src/mcp-tools/capability-brain.js +5 -3
  27. package/v3/@claude-flow/cli/dist/src/mcp-tools/embeddings-tools.js +11 -4
  28. package/v3/@claude-flow/cli/dist/src/mcp-tools/hooks-tools.js +13 -3
  29. package/v3/@claude-flow/cli/dist/src/mcp-tools/index.d.ts +1 -0
  30. package/v3/@claude-flow/cli/dist/src/mcp-tools/index.js +2 -0
  31. package/v3/@claude-flow/cli/dist/src/mcp-tools/memory-tools.js +42 -4
  32. package/v3/@claude-flow/cli/dist/src/mcp-tools/task-tools.js +7 -3
  33. package/v3/@claude-flow/cli/dist/src/mcp-tools/team-tools.d.ts +15 -0
  34. package/v3/@claude-flow/cli/dist/src/mcp-tools/team-tools.js +244 -0
  35. package/v3/@claude-flow/cli/dist/src/memory/memory-bridge.d.ts +19 -0
  36. package/v3/@claude-flow/cli/dist/src/memory/memory-bridge.js +163 -3
  37. package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.d.ts +4 -0
  38. package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.js +70 -37
  39. package/v3/@claude-flow/cli/dist/src/memory/rabitq-index.d.ts +19 -0
  40. package/v3/@claude-flow/cli/dist/src/memory/rabitq-index.js +117 -1
  41. package/v3/@claude-flow/cli/dist/src/missions/schemas.d.ts +113 -113
  42. package/v3/@claude-flow/cli/dist/src/mods/command-registry/schema.d.ts +193 -193
  43. package/v3/@claude-flow/cli/dist/src/parser.js +3 -2
  44. package/v3/@claude-flow/cli/dist/src/services/live-harness.d.ts +111 -0
  45. package/v3/@claude-flow/cli/dist/src/services/live-harness.js +427 -0
  46. package/v3/@claude-flow/cli/package.json +4 -3
  47. package/v3/@claude-flow/cli/templates/grok/agents/ruflo-architect.md +35 -0
  48. package/v3/@claude-flow/cli/templates/grok/agents/ruflo-coder.md +34 -0
  49. package/v3/@claude-flow/cli/templates/grok/agents/ruflo-reviewer.md +30 -0
  50. package/v3/@claude-flow/cli/templates/grok/agents/ruflo-tester.md +26 -0
  51. package/v3/@claude-flow/cli/templates/grok/config.toml +63 -0
  52. package/v3/@claude-flow/cli/templates/grok/docs/README.md +186 -0
  53. package/v3/@claude-flow/cli/templates/grok/hooks/subagent-stop-team.json +15 -0
  54. package/v3/@claude-flow/cli/templates/grok/rules/ruflo-grok.md +132 -0
  55. package/v3/@claude-flow/cli/templates/grok/scripts/grok-subagent-stop-hook.mjs +90 -0
  56. package/v3/@claude-flow/cli/templates/grok/scripts/grok-team-bus.mjs +525 -0
  57. package/v3/@claude-flow/cli/templates/grok/scripts/grok-team-store.mjs +287 -0
  58. package/v3/@claude-flow/cli/templates/grok/scripts/host-statusline.mjs +281 -0
  59. package/v3/@claude-flow/cli/templates/grok/skills/agent-teams-grok/SKILL.md +98 -0
@@ -1438,7 +1438,7 @@ export async function bridgeListEntries(options) {
1438
1438
  if (!ctx)
1439
1439
  return null;
1440
1440
  try {
1441
- const { namespace, limit = 20, offset = 0, provenanceFilter } = options;
1441
+ const { namespace, limit = 20, offset = 0, provenanceFilter, tags: requiredTags } = options;
1442
1442
  if (provenanceFilter?.some(p => !isValidProvenanceType(p)))
1443
1443
  return null;
1444
1444
  const filters = [];
@@ -1451,6 +1451,20 @@ export async function bridgeListEntries(options) {
1451
1451
  filters.push(`provenance_type IN (${provenanceFilter.map(() => '?').join(',')})`);
1452
1452
  filterParams.push(...provenanceFilter);
1453
1453
  }
1454
+ // Tags are stored as JSON-array TEXT (e.g. `["a","b"]`). Match each
1455
+ // required tag as an exact element via SQLite's json_each so the filter
1456
+ // runs in SQL instead of pulling the whole table into memory to filter
1457
+ // there. A LIKE-based substring match is ASCII case-insensitive (so
1458
+ // `prod` would match `PROD`) and matches raw JSON bytes rather than
1459
+ // parsed array elements (so a tag literally containing `"prod` could
1460
+ // false-match) — json_each compares real, decoded array values.
1461
+ const tagFilter = Array.isArray(requiredTags)
1462
+ ? requiredTags.filter((t) => typeof t === 'string' && t.length > 0)
1463
+ : [];
1464
+ for (const tag of tagFilter) {
1465
+ filters.push(`EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)`);
1466
+ filterParams.push(tag);
1467
+ }
1454
1468
  const extraFilter = filters.length > 0 ? `AND ${filters.join(' AND ')}` : '';
1455
1469
  // #2120 — `status IS NULL` accepted alongside `'active'`. Old
1456
1470
  // databases imported by the auto-memory bridge (before the status
@@ -1461,6 +1475,13 @@ export async function bridgeListEntries(options) {
1461
1475
  // "legacy-active" — the safe default for any entry that predates the
1462
1476
  // status column.
1463
1477
  const statusFilter = liveMemoryRowSql({ includeExpired: options.includeExpired });
1478
+ // `|| 20` would treat an explicit `limit: 0` as falsy and silently
1479
+ // substitute the default (returning 20 rows instead of the requested
1480
+ // zero) — only fall back to the default when parsing genuinely failed.
1481
+ const parsedLimit = parseInt(String(limit), 10);
1482
+ const safeLimit = Number.isNaN(parsedLimit) ? 20 : parsedLimit;
1483
+ const parsedOffset = parseInt(String(offset), 10);
1484
+ const safeOffset = Number.isNaN(parsedOffset) ? 0 : parsedOffset;
1464
1485
  // Count
1465
1486
  let total = 0;
1466
1487
  try {
@@ -1474,15 +1495,26 @@ export async function bridgeListEntries(options) {
1474
1495
  // List
1475
1496
  const entries = [];
1476
1497
  try {
1498
+ // Include tags so export/list can round-trip them.
1477
1499
  const stmt = ctx.db.prepare(`
1478
- SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type, expires_at
1500
+ SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type, expires_at, tags
1479
1501
  FROM memory_entries
1480
1502
  WHERE ${statusFilter} ${extraFilter}
1481
1503
  ORDER BY updated_at DESC
1482
1504
  LIMIT ? OFFSET ?
1483
1505
  `);
1484
- const rows = stmt.all(...filterParams, options.includeEmbedding ? Math.min(limit, MAX_LIST_EMBEDDINGS) : limit, offset);
1506
+ const rows = stmt.all(...filterParams, options.includeEmbedding ? Math.min(safeLimit, MAX_LIST_EMBEDDINGS) : safeLimit, safeOffset);
1485
1507
  for (const row of rows) {
1508
+ let tags = [];
1509
+ if (row.tags) {
1510
+ try {
1511
+ const parsed = typeof row.tags === 'string' ? JSON.parse(row.tags) : row.tags;
1512
+ if (Array.isArray(parsed)) {
1513
+ tags = parsed.filter((t) => typeof t === 'string');
1514
+ }
1515
+ }
1516
+ catch { /* invalid tags JSON */ }
1517
+ }
1486
1518
  const entry = {
1487
1519
  // #2073: don't truncate id when content is requested — callers
1488
1520
  // (notably memory_export) need the full id to round-trip via import.
@@ -1496,6 +1528,7 @@ export async function bridgeListEntries(options) {
1496
1528
  expiresAt: row.expires_at ?? null,
1497
1529
  hasEmbedding: !!(row.embedding && String(row.embedding).length > 10),
1498
1530
  provenanceType: row.provenance_type || 'unknown',
1531
+ tags,
1499
1532
  };
1500
1533
  if (options.includeContent) {
1501
1534
  entry.content = row.content || '';
@@ -1861,10 +1894,91 @@ export async function bridgeGetVectorSearchStatus(dbPath) {
1861
1894
  return null;
1862
1895
  }
1863
1896
  }
1897
+ /**
1898
+ * #4203: RaBitQ pre-filter for the bridge's vector search.
1899
+ *
1900
+ * The index yields candidate ids from a 32×-compressed Hamming scan; each is
1901
+ * reranked here with the same exact cosine similarity the exhaustive scan uses,
1902
+ * so scores keep their meaning and only the scanned set shrinks (the candidate
1903
+ * window `k * 2` is then widened 3× by `searchRabitq` itself).
1904
+ *
1905
+ * Returns null whenever the index cannot answer this query — WASM missing, too
1906
+ * few vectors, or an index built for another store/dimension — so the caller
1907
+ * runs its unchanged brute-force scan. Keyword/BM25 recall is unaffected: this
1908
+ * only ever replaces the cosine scan.
1909
+ */
1910
+ async function rabitqSearchAndRerank(db, queryEmbedding, options) {
1911
+ try {
1912
+ const { ensureRabitqIndex, searchRabitq } = await import('./rabitq-index.js');
1913
+ const ready = await ensureRabitqIndex({ dbPath: options.dbPath, dimensions: queryEmbedding.length });
1914
+ if (!ready)
1915
+ return null;
1916
+ const candidates = await searchRabitq(queryEmbedding, {
1917
+ k: options.k * 2,
1918
+ namespace: options.namespace,
1919
+ });
1920
+ if (!candidates || candidates.length === 0)
1921
+ return null;
1922
+ // Fetch exactly the candidates (id is the primary key) instead of the
1923
+ // 10k-row full scan; chunked to stay well inside SQLite's parameter limit.
1924
+ const rowsById = new Map();
1925
+ const ids = candidates.map(c => c.id);
1926
+ for (let i = 0; i < ids.length; i += 500) {
1927
+ const chunk = ids.slice(i, i + 500);
1928
+ const placeholders = chunk.map(() => '?').join(',');
1929
+ const rows = db.prepare(`
1930
+ SELECT id, key, namespace, content, embedding
1931
+ FROM memory_entries
1932
+ WHERE id IN (${placeholders}) AND ${liveMemoryRowSql()} AND embedding IS NOT NULL
1933
+ `).all(...chunk);
1934
+ for (const row of rows) {
1935
+ let embedding;
1936
+ try {
1937
+ embedding = JSON.parse(row.embedding);
1938
+ }
1939
+ catch {
1940
+ continue;
1941
+ }
1942
+ if (!Array.isArray(embedding) || embedding.length !== queryEmbedding.length)
1943
+ continue;
1944
+ rowsById.set(String(row.id), {
1945
+ key: row.key || String(row.id),
1946
+ namespace: row.namespace || 'default',
1947
+ content: row.content || '',
1948
+ score: cosineSim(queryEmbedding, embedding),
1949
+ });
1950
+ }
1951
+ }
1952
+ const results = [];
1953
+ for (const candidate of candidates) {
1954
+ const row = rowsById.get(candidate.id);
1955
+ if (!row || row.score < options.threshold)
1956
+ continue;
1957
+ results.push({
1958
+ id: candidate.id.substring(0, 12),
1959
+ key: row.key || candidate.id.substring(0, 15),
1960
+ content: row.content.substring(0, 60) + (row.content.length > 60 ? '...' : ''),
1961
+ score: row.score,
1962
+ namespace: row.namespace,
1963
+ });
1964
+ }
1965
+ if (results.length === 0)
1966
+ return null;
1967
+ results.sort((a, b) => b.score - a.score);
1968
+ return results.slice(0, options.k);
1969
+ }
1970
+ catch {
1971
+ return null;
1972
+ }
1973
+ }
1864
1974
  /**
1865
1975
  * Search AgentDB v3's embedder + SQLite entries via a full-table scan and
1866
1976
  * brute-force cosine similarity. NOT HNSW-accelerated — see #2922. Returns
1867
1977
  * null if unavailable.
1978
+ *
1979
+ * #4203: when a RaBitQ index is available the Hamming-scan candidate set is
1980
+ * reranked exactly instead of scanning the whole table; the exhaustive loop
1981
+ * below stays the fallback, so results and scores keep their meaning.
1868
1982
  */
1869
1983
  export async function bridgeSearchBruteForceCosine(queryEmbedding, options, dbPath) {
1870
1984
  if (!operationContext.getStore()?.active)
@@ -1881,6 +1995,17 @@ export async function bridgeSearchBruteForceCosine(queryEmbedding, options, dbPa
1881
1995
  const nsFilter = options?.namespace && options.namespace !== 'all'
1882
1996
  ? `AND namespace = ?`
1883
1997
  : '';
1998
+ // #4203: try the RaBitQ pre-filter before the exhaustive scan below. A null
1999
+ // result (no index / unavailable / nothing above threshold) falls through
2000
+ // to the unchanged brute-force path.
2001
+ const accelerated = await rabitqSearchAndRerank(ctx.db, queryEmbedding, {
2002
+ k,
2003
+ threshold,
2004
+ namespace: options?.namespace,
2005
+ dbPath,
2006
+ });
2007
+ if (accelerated)
2008
+ return accelerated;
1884
2009
  let rows;
1885
2010
  try {
1886
2011
  // #2982/#2976: same unordered pre-rank truncation as bridgeSearchEntries
@@ -3361,6 +3486,41 @@ export async function bridgeGetAllEmbeddings(options) {
3361
3486
  return null;
3362
3487
  }
3363
3488
  }
3489
+ /**
3490
+ * Count the embeddings `bridgeGetAllEmbeddings()` would export (#4203).
3491
+ * RaBitQ's auto-build uses this as its cheap "has the corpus drifted?" probe —
3492
+ * a COUNT instead of re-reading every embedding blob. With `dimensions` it
3493
+ * counts only rows build() would keep, so the drift ratio compares like with
3494
+ * like. Returns null if the bridge is unavailable.
3495
+ */
3496
+ export async function bridgeCountEmbeddings(options) {
3497
+ if (!operationContext.getStore()?.active)
3498
+ return withBridgeOperation(() => bridgeCountEmbeddings(options));
3499
+ const registry = await getRegistry(options?.dbPath);
3500
+ if (!registry)
3501
+ return null;
3502
+ const ctx = getDb(registry);
3503
+ if (!ctx)
3504
+ return null;
3505
+ const dims = options?.dimensions;
3506
+ try {
3507
+ if (dims != null) {
3508
+ try {
3509
+ const row = ctx.db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE ${liveMemoryRowSql()} AND embedding IS NOT NULL AND json_array_length(embedding) = ?`).get(dims);
3510
+ return row?.cnt ?? 0;
3511
+ }
3512
+ catch {
3513
+ // JSON1 unavailable in this SQLite build — fall through to the
3514
+ // unfiltered count rather than losing the probe entirely.
3515
+ }
3516
+ }
3517
+ const row = ctx.db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE ${liveMemoryRowSql()} AND embedding IS NOT NULL`).get();
3518
+ return row?.cnt ?? 0;
3519
+ }
3520
+ catch {
3521
+ return null;
3522
+ }
3523
+ }
3364
3524
  // ===== Utility =====
3365
3525
  function cosineSim(a, b) {
3366
3526
  if (!a || !b || a.length === 0 || b.length === 0)
@@ -513,6 +513,8 @@ export declare function listEntries(options: {
513
513
  includeEmbedding?: boolean;
514
514
  /** #4056: also return rows whose TTL has passed (with `expiresAt`). For cleanup only; normal reads hide them. */
515
515
  includeExpired?: boolean;
516
+ /** Require entries to include all of these tags (AND). */
517
+ tags?: string[];
516
518
  }): Promise<{
517
519
  success: boolean;
518
520
  entries: {
@@ -531,6 +533,8 @@ export declare function listEntries(options: {
531
533
  provenanceType?: string;
532
534
  /** ADR-472: present when `includeEmbedding: true` and the row has a valid stored vector. */
533
535
  embeddingQ8?: EmbeddingQ8;
536
+ /** Tag set stored with the entry (empty array when none). */
537
+ tags?: string[];
534
538
  }[];
535
539
  total: number;
536
540
  error?: string;
@@ -2808,7 +2808,11 @@ export async function searchEntries(options) {
2808
2808
  const queryEmbedding = queryEmb.embedding;
2809
2809
  // Try RaBitQ pre-filter first (32× compressed Hamming scan)
2810
2810
  try {
2811
- const { searchRabitq } = await import('./rabitq-index.js');
2811
+ const { ensureRabitqIndex, searchRabitq } = await import('./rabitq-index.js');
2812
+ // #4203: build/rebuild on demand — without this the index was only ever
2813
+ // populated by the explicit `embeddings_rabitq_build` tool, so this
2814
+ // pre-filter always missed.
2815
+ await ensureRabitqIndex({ dbPath, dimensions: queryEmbedding.length });
2812
2816
  const rabitqCandidates = await searchRabitq(queryEmbedding, { k: limit * 2, namespace: effectiveNamespace });
2813
2817
  if (rabitqCandidates && rabitqCandidates.length > 0) {
2814
2818
  // Rerank candidates with exact cosine similarity from SQLite
@@ -3043,7 +3047,7 @@ export async function listEntries(options) {
3043
3047
  return bridgeResult;
3044
3048
  }
3045
3049
  // Fallback: raw sql.js
3046
- const { namespace, limit = 20, offset = 0, dbPath: customPath, provenanceFilter } = options;
3050
+ const { namespace, limit = 20, offset = 0, dbPath: customPath, provenanceFilter, tags: requiredTags, } = options;
3047
3051
  const swarmDir = getMemoryRoot();
3048
3052
  const dbPath = customPath || path.join(swarmDir, 'memory.db');
3049
3053
  try {
@@ -3065,18 +3069,41 @@ export async function listEntries(options) {
3065
3069
  // #2120 — accept `status IS NULL` alongside `'active'`. Old DBs
3066
3070
  // that predate the status column may have NULL after migration.
3067
3071
  // See memory-bridge.ts:bridgeListEntries for full context.
3068
- // Get total count
3069
3072
  const whereClauses = [liveMemoryRowSql({ includeExpired: options.includeExpired })];
3070
3073
  const whereParams = [];
3071
3074
  if (namespace) {
3072
3075
  whereClauses.push('namespace = ?');
3073
3076
  whereParams.push(namespace);
3074
3077
  }
3078
+ // ADR-323: restrict to the requested provenance types. This must be
3079
+ // ANDed alongside any tags filter below, never replaced by it — dropping
3080
+ // this clause lets unverified user_claim rows leak into results that
3081
+ // are supposed to be tool_result-only.
3075
3082
  if (provenanceFilter?.length) {
3076
3083
  whereClauses.push(`provenance_type IN (${provenanceFilter.map(() => '?').join(',')})`);
3077
3084
  whereParams.push(...provenanceFilter);
3078
3085
  }
3086
+ // Tags are stored as JSON-array TEXT (e.g. `["a","b"]`). Match each
3087
+ // required tag as an exact element via SQLite's json_each so a tag of
3088
+ // `prod` can't be satisfied by a substring like `"prod` or by a
3089
+ // differently-cased value like `PROD` (LIKE is ASCII case-insensitive
3090
+ // and matches raw JSON-array bytes, not parsed array elements).
3091
+ const tagFilter = Array.isArray(requiredTags)
3092
+ ? requiredTags.filter((t) => typeof t === 'string' && t.length > 0)
3093
+ : [];
3094
+ for (const tag of tagFilter) {
3095
+ whereClauses.push(`EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)`);
3096
+ whereParams.push(tag);
3097
+ }
3079
3098
  const whereSql = whereClauses.join(' AND ');
3099
+ // `|| 100` would treat an explicit `limit: 0` as falsy and silently
3100
+ // substitute the default (returning rows instead of the requested
3101
+ // zero) — only fall back to the default when parsing genuinely failed.
3102
+ const parsedLimit = parseInt(String(limit), 10);
3103
+ const safeLimit = Number.isNaN(parsedLimit) ? 100 : parsedLimit;
3104
+ const parsedOffset = parseInt(String(offset), 10);
3105
+ const safeOffset = Number.isNaN(parsedOffset) ? 0 : parsedOffset;
3106
+ // Get total count
3080
3107
  const countStmt = db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE ${whereSql}`);
3081
3108
  if (whereParams.length > 0)
3082
3109
  countStmt.bind(whereParams);
@@ -3085,50 +3112,56 @@ export async function listEntries(options) {
3085
3112
  countRows.push(countStmt.get());
3086
3113
  }
3087
3114
  countStmt.free();
3088
- const countResult = countRows.length > 0 ? [{ values: countRows }] : [];
3089
- const total = countResult[0]?.values?.[0]?.[0] || 0;
3090
- // Get entries
3091
- const safeLimit = Math.min(parseInt(String(limit), 10) || 100, options.includeEmbedding ? MAX_LIST_EMBEDDINGS : Number.MAX_SAFE_INTEGER);
3092
- const safeOffset = parseInt(String(offset), 10) || 0;
3093
- // #2120 — same NULL-as-active acceptance as the count above.
3094
- const listStmt = db.prepare(`SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type, expires_at
3115
+ const total = countRows[0]?.[0] || 0;
3116
+ // Get entries. ADR-472: embedding rows are capped at MAX_LIST_EMBEDDINGS.
3117
+ const listLimit = options.includeEmbedding ? Math.min(safeLimit, MAX_LIST_EMBEDDINGS) : safeLimit;
3118
+ // Include tags so export/list can round-trip them.
3119
+ const listStmt = db.prepare(`SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type, expires_at, tags
3095
3120
  FROM memory_entries WHERE ${whereSql}
3096
3121
  ORDER BY updated_at DESC LIMIT ? OFFSET ?`);
3097
- listStmt.bind([...whereParams, safeLimit, safeOffset]);
3122
+ listStmt.bind([...whereParams, listLimit, safeOffset]);
3098
3123
  const listRows = [];
3099
3124
  while (listStmt.step()) {
3100
3125
  listRows.push(listStmt.get());
3101
3126
  }
3102
3127
  listStmt.free();
3103
- const result = listRows.length > 0 ? [{ values: listRows }] : [];
3104
3128
  const entries = [];
3105
- if (result[0]?.values) {
3106
- for (const row of result[0].values) {
3107
- const [id, key, ns, content, embedding, accessCount, createdAt, updatedAt, provenanceTypeVal, expiresAtVal] = row;
3108
- const entry = {
3109
- // #2073: don't truncate id when content is requested — callers
3110
- // (notably memory_export) need the full id to round-trip via import.
3111
- id: options.includeContent ? String(id) : String(id).substring(0, 20),
3112
- key: key || String(id).substring(0, 15),
3113
- namespace: ns || 'default',
3114
- size: (content || '').length,
3115
- accessCount: accessCount || 0,
3116
- createdAt: createdAt || new Date().toISOString(),
3117
- updatedAt: updatedAt || new Date().toISOString(),
3118
- expiresAt: expiresAtVal ?? null,
3119
- hasEmbedding: !!embedding && embedding.length > 10,
3120
- provenanceType: provenanceTypeVal || 'unknown'
3121
- };
3122
- if (options.includeContent) {
3123
- entry.content = content || '';
3124
- }
3125
- if (options.includeEmbedding) {
3126
- const q8 = encodeEmbeddingQ8(embedding);
3127
- if (q8)
3128
- entry.embeddingQ8 = q8;
3129
+ for (const row of listRows) {
3130
+ const [id, key, ns, content, embedding, accessCount, createdAt, updatedAt, provenanceTypeVal, expiresAtVal, tagsJson] = row;
3131
+ let tags = [];
3132
+ if (tagsJson) {
3133
+ try {
3134
+ const parsed = JSON.parse(tagsJson);
3135
+ if (Array.isArray(parsed)) {
3136
+ tags = parsed.filter((t) => typeof t === 'string');
3137
+ }
3129
3138
  }
3130
- entries.push(entry);
3139
+ catch { /* invalid tags JSON */ }
3140
+ }
3141
+ const entry = {
3142
+ // #2073: don't truncate id when content is requested — callers
3143
+ // (notably memory_export) need the full id to round-trip via import.
3144
+ id: options.includeContent ? String(id) : String(id).substring(0, 20),
3145
+ key: key || String(id).substring(0, 15),
3146
+ namespace: ns || 'default',
3147
+ size: (content || '').length,
3148
+ accessCount: accessCount || 0,
3149
+ createdAt: createdAt || new Date().toISOString(),
3150
+ updatedAt: updatedAt || new Date().toISOString(),
3151
+ expiresAt: expiresAtVal ?? null,
3152
+ hasEmbedding: !!embedding && embedding.length > 10,
3153
+ provenanceType: provenanceTypeVal || 'unknown',
3154
+ tags,
3155
+ };
3156
+ if (options.includeContent) {
3157
+ entry.content = content || '';
3158
+ }
3159
+ if (options.includeEmbedding) {
3160
+ const q8 = encodeEmbeddingQ8(embedding);
3161
+ if (q8)
3162
+ entry.embeddingQ8 = q8;
3131
3163
  }
3164
+ entries.push(entry);
3132
3165
  }
3133
3166
  db.close();
3134
3167
  return { success: true, entries, total };
@@ -9,6 +9,11 @@
9
9
  * 1. build() — bulk-load all embeddings from SQLite into the WASM index
10
10
  * 2. search() — fast Hamming scan → candidate ids → caller reranks
11
11
  * 3. rebuild() — called when entry count drifts >20% from last build
12
+ *
13
+ * #4203: step 3 is driven by `ensureRabitqIndex()`, which the search paths
14
+ * call before `searchRabitq()`. Previously the only caller of the build was
15
+ * the explicit `embeddings_rabitq_build` tool, so the index was effectively
16
+ * never populated and every `searchRabitq()` returned null.
12
17
  */
13
18
  /**
14
19
  * Build or rebuild the RaBitQ index from SQLite embeddings.
@@ -46,6 +51,20 @@ export declare function searchRabitq(queryEmbedding: number[], options?: {
46
51
  * Check if the RaBitQ index needs rebuilding.
47
52
  */
48
53
  export declare function shouldRebuildRabitq(currentEntryCount: number): Promise<boolean>;
54
+ /**
55
+ * Auto-build / auto-rebuild entry point (#4203).
56
+ *
57
+ * Search paths call this before `searchRabitq()`: it lets `shouldRebuildRabitq()`
58
+ * decide whether the index is missing or has drifted past the 20% threshold and
59
+ * builds it when so. Never throws — a missing WASM module, an unreadable store
60
+ * or an index built for another store/dimension just leaves the caller's
61
+ * existing non-RaBitQ path untouched. Returns whether an index usable for these
62
+ * options exists afterwards.
63
+ */
64
+ export declare function ensureRabitqIndex(options?: {
65
+ dbPath?: string;
66
+ dimensions?: number;
67
+ }): Promise<boolean>;
49
68
  /**
50
69
  * Get RaBitQ index status.
51
70
  */
@@ -9,14 +9,27 @@
9
9
  * 1. build() — bulk-load all embeddings from SQLite into the WASM index
10
10
  * 2. search() — fast Hamming scan → candidate ids → caller reranks
11
11
  * 3. rebuild() — called when entry count drifts >20% from last build
12
+ *
13
+ * #4203: step 3 is driven by `ensureRabitqIndex()`, which the search paths
14
+ * call before `searchRabitq()`. Previously the only caller of the build was
15
+ * the explicit `embeddings_rabitq_build` tool, so the index was effectively
16
+ * never populated and every `searchRabitq()` returned null.
12
17
  */
13
18
  import * as fs from 'fs';
14
19
  import * as path from 'path';
15
20
  const RABITQ_SEED = 42n;
16
21
  const RABITQ_RERANK_FACTOR = 20;
17
22
  const REBUILD_DRIFT_THRESHOLD = 0.2; // rebuild when count drifts >20%
23
+ /** #4203: minimum gap between auto-build attempts, so a store that is still
24
+ * too small cannot turn every search into a build attempt. */
25
+ const AUTO_BUILD_RETRY_MS = 60_000;
26
+ const RABITQ_UNAVAILABLE_ERROR = '@ruvector/rabitq-wasm not available';
18
27
  let rabitqState = null;
19
28
  let rabitqInitializing = false;
29
+ let lastAutoBuildAttempt = 0;
30
+ /** Set once the WASM module is known to be missing for this process: retrying
31
+ * the same failing dynamic import on every search buys nothing. */
32
+ let autoBuildDisabled = false;
20
33
  async function loadRabitqModule() {
21
34
  try {
22
35
  const mod = await import('@ruvector/rabitq-wasm');
@@ -50,7 +63,7 @@ export async function buildRabitqIndex(options) {
50
63
  const mod = await loadRabitqModule();
51
64
  if (!mod) {
52
65
  rabitqInitializing = false;
53
- return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: '@ruvector/rabitq-wasm not available' };
66
+ return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: RABITQ_UNAVAILABLE_ERROR };
54
67
  }
55
68
  const dimensions = options?.dimensions ?? 384;
56
69
  const swarmDir = path.resolve(process.cwd(), '.swarm');
@@ -126,6 +139,7 @@ export async function buildRabitqIndex(options) {
126
139
  dimensions,
127
140
  builtAt: Date.now(),
128
141
  vectorCount: entries.length,
142
+ sourcePath: options?.dbPath ? path.resolve(options.dbPath) : undefined,
129
143
  };
130
144
  // Persist metadata for fast reload hint
131
145
  try {
@@ -226,6 +240,108 @@ export async function shouldRebuildRabitq(currentEntryCount) {
226
240
  const drift = Math.abs(currentEntryCount - rabitqState.vectorCount) / Math.max(rabitqState.vectorCount, 1);
227
241
  return drift > REBUILD_DRIFT_THRESHOLD;
228
242
  }
243
+ /**
244
+ * Whether the in-memory index can answer a query for this store/dimension.
245
+ * `undefined` dbPath means "the default store", matching build's own default.
246
+ */
247
+ function isRabitqUsable(options) {
248
+ if (!rabitqState)
249
+ return false;
250
+ if (options?.dimensions != null && rabitqState.dimensions !== options.dimensions)
251
+ return false;
252
+ const sourcePath = options?.dbPath ? path.resolve(options.dbPath) : undefined;
253
+ return rabitqState.sourcePath === sourcePath;
254
+ }
255
+ /**
256
+ * Count the rows `buildRabitqIndex()` would load, using the cheapest source
257
+ * available: the bridge's COUNT (WAL-visible, same rows build prefers) with a
258
+ * sql.js COUNT of the same file as fallback. Returns 0 when neither is
259
+ * readable, which simply leaves the caller on its non-RaBitQ path.
260
+ */
261
+ async function countIndexableEntries(options) {
262
+ const dimensions = options?.dimensions ?? 384;
263
+ try {
264
+ const { bridgeCountEmbeddings } = await import('./memory-bridge.js');
265
+ const bridgeCount = await bridgeCountEmbeddings({ dimensions, dbPath: options?.dbPath });
266
+ if (bridgeCount !== null)
267
+ return bridgeCount;
268
+ }
269
+ catch { /* bridge unavailable, fall through */ }
270
+ const dbPath = options?.dbPath
271
+ ? path.resolve(options.dbPath)
272
+ : path.join(path.resolve(process.cwd(), '.swarm'), 'memory.db');
273
+ if (!fs.existsSync(dbPath))
274
+ return 0;
275
+ let db = null;
276
+ try {
277
+ const initSqlJs = (await import('sql.js')).default;
278
+ const SQL = await initSqlJs();
279
+ db = new SQL.Database(fs.readFileSync(dbPath));
280
+ const count = (dimFilter) => db.exec(`
281
+ SELECT COUNT(*) FROM memory_entries
282
+ WHERE status = 'active' AND embedding IS NOT NULL${dimFilter}
283
+ `);
284
+ let result;
285
+ try {
286
+ // Same rows build() keeps: only embeddings of the requested width.
287
+ result = count(` AND json_array_length(embedding) = ${dimensions}`);
288
+ }
289
+ catch {
290
+ result = count('');
291
+ }
292
+ return Number(result[0]?.values?.[0]?.[0] ?? 0);
293
+ }
294
+ catch {
295
+ return 0;
296
+ }
297
+ finally {
298
+ try {
299
+ db?.close();
300
+ }
301
+ catch { /* ignore */ }
302
+ }
303
+ }
304
+ /**
305
+ * Auto-build / auto-rebuild entry point (#4203).
306
+ *
307
+ * Search paths call this before `searchRabitq()`: it lets `shouldRebuildRabitq()`
308
+ * decide whether the index is missing or has drifted past the 20% threshold and
309
+ * builds it when so. Never throws — a missing WASM module, an unreadable store
310
+ * or an index built for another store/dimension just leaves the caller's
311
+ * existing non-RaBitQ path untouched. Returns whether an index usable for these
312
+ * options exists afterwards.
313
+ */
314
+ export async function ensureRabitqIndex(options) {
315
+ if (autoBuildDisabled || rabitqInitializing)
316
+ return isRabitqUsable(options);
317
+ const dimensions = options?.dimensions ?? 384;
318
+ const now = Date.now();
319
+ if (now - lastAutoBuildAttempt < AUTO_BUILD_RETRY_MS)
320
+ return isRabitqUsable(options);
321
+ lastAutoBuildAttempt = now;
322
+ let currentEntryCount = 0;
323
+ try {
324
+ currentEntryCount = await countIndexableEntries({ dbPath: options?.dbPath, dimensions });
325
+ }
326
+ catch {
327
+ currentEntryCount = 0;
328
+ }
329
+ if (isRabitqUsable(options)) {
330
+ // Same store and width: rebuild only once the corpus has drifted >20%.
331
+ if (!(await shouldRebuildRabitq(currentEntryCount)))
332
+ return true;
333
+ }
334
+ else if (!rabitqState) {
335
+ // Nothing built yet: honour the "enough vectors to be worth it" floor.
336
+ if (!(await shouldRebuildRabitq(currentEntryCount)))
337
+ return false;
338
+ }
339
+ // Either the index is stale for this store/width, or the entry count drifted.
340
+ const result = await buildRabitqIndex({ dbPath: options?.dbPath, dimensions });
341
+ if (!result.success && result.error === RABITQ_UNAVAILABLE_ERROR)
342
+ autoBuildDisabled = true;
343
+ return isRabitqUsable(options);
344
+ }
229
345
  /**
230
346
  * Get RaBitQ index status.
231
347
  */