claude-flow 3.58.0 → 3.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/proven-config.json +1 -1
- package/README.md +2 -0
- package/node_modules/@claude-flow/codex/dist/cli.js +0 -0
- package/node_modules/@claude-flow/plugin-agent-federation/dist/bin.js +0 -0
- package/package.json +2 -1
- package/v3/@claude-flow/cli/README.md +62 -174
- package/v3/@claude-flow/cli/bin/cli.js +19 -1
- package/v3/@claude-flow/cli/catalog-manifest.json +5 -5
- package/v3/@claude-flow/cli/dist/src/commands/harness.d.ts +40 -0
- package/v3/@claude-flow/cli/dist/src/commands/harness.js +142 -0
- package/v3/@claude-flow/cli/dist/src/commands/index.js +1 -0
- package/v3/@claude-flow/cli/dist/src/commands/init.js +116 -0
- package/v3/@claude-flow/cli/dist/src/commands/memory.js +6 -1
- package/v3/@claude-flow/cli/dist/src/commands/status.js +2 -2
- package/v3/@claude-flow/cli/dist/src/index.d.ts +3 -0
- package/v3/@claude-flow/cli/dist/src/index.js +12 -2
- package/v3/@claude-flow/cli/dist/src/init/grok-generator.d.ts +96 -0
- package/v3/@claude-flow/cli/dist/src/init/grok-generator.js +371 -0
- package/v3/@claude-flow/cli/dist/src/init/grok-templates.d.ts +11 -0
- package/v3/@claude-flow/cli/dist/src/init/grok-templates.js +28 -0
- package/v3/@claude-flow/cli/dist/src/init/index.d.ts +2 -0
- package/v3/@claude-flow/cli/dist/src/init/index.js +2 -0
- package/v3/@claude-flow/cli/dist/src/mcp-client.js +4 -0
- package/v3/@claude-flow/cli/dist/src/mcp-tools/agent-execute-core.js +59 -12
- package/v3/@claude-flow/cli/dist/src/mcp-tools/capability-brain.d.ts +1 -1
- package/v3/@claude-flow/cli/dist/src/mcp-tools/capability-brain.js +5 -3
- package/v3/@claude-flow/cli/dist/src/mcp-tools/embeddings-tools.js +11 -4
- package/v3/@claude-flow/cli/dist/src/mcp-tools/hooks-tools.js +13 -3
- package/v3/@claude-flow/cli/dist/src/mcp-tools/index.d.ts +1 -0
- package/v3/@claude-flow/cli/dist/src/mcp-tools/index.js +2 -0
- package/v3/@claude-flow/cli/dist/src/mcp-tools/memory-tools.js +42 -4
- package/v3/@claude-flow/cli/dist/src/mcp-tools/task-tools.js +7 -3
- package/v3/@claude-flow/cli/dist/src/mcp-tools/team-tools.d.ts +15 -0
- package/v3/@claude-flow/cli/dist/src/mcp-tools/team-tools.js +244 -0
- package/v3/@claude-flow/cli/dist/src/memory/memory-bridge.d.ts +19 -0
- package/v3/@claude-flow/cli/dist/src/memory/memory-bridge.js +163 -3
- package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.d.ts +4 -0
- package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.js +70 -37
- package/v3/@claude-flow/cli/dist/src/memory/rabitq-index.d.ts +19 -0
- package/v3/@claude-flow/cli/dist/src/memory/rabitq-index.js +117 -1
- package/v3/@claude-flow/cli/dist/src/missions/schemas.d.ts +113 -113
- package/v3/@claude-flow/cli/dist/src/mods/command-registry/schema.d.ts +193 -193
- package/v3/@claude-flow/cli/dist/src/parser.js +3 -2
- package/v3/@claude-flow/cli/dist/src/services/live-harness.d.ts +111 -0
- package/v3/@claude-flow/cli/dist/src/services/live-harness.js +427 -0
- package/v3/@claude-flow/cli/package.json +4 -3
- package/v3/@claude-flow/cli/templates/grok/agents/ruflo-architect.md +35 -0
- package/v3/@claude-flow/cli/templates/grok/agents/ruflo-coder.md +34 -0
- package/v3/@claude-flow/cli/templates/grok/agents/ruflo-reviewer.md +30 -0
- package/v3/@claude-flow/cli/templates/grok/agents/ruflo-tester.md +26 -0
- package/v3/@claude-flow/cli/templates/grok/config.toml +63 -0
- package/v3/@claude-flow/cli/templates/grok/docs/README.md +186 -0
- package/v3/@claude-flow/cli/templates/grok/hooks/subagent-stop-team.json +15 -0
- package/v3/@claude-flow/cli/templates/grok/rules/ruflo-grok.md +132 -0
- package/v3/@claude-flow/cli/templates/grok/scripts/grok-subagent-stop-hook.mjs +90 -0
- package/v3/@claude-flow/cli/templates/grok/scripts/grok-team-bus.mjs +525 -0
- package/v3/@claude-flow/cli/templates/grok/scripts/grok-team-store.mjs +287 -0
- package/v3/@claude-flow/cli/templates/grok/scripts/host-statusline.mjs +281 -0
- package/v3/@claude-flow/cli/templates/grok/skills/agent-teams-grok/SKILL.md +98 -0
|
@@ -1438,7 +1438,7 @@ export async function bridgeListEntries(options) {
|
|
|
1438
1438
|
if (!ctx)
|
|
1439
1439
|
return null;
|
|
1440
1440
|
try {
|
|
1441
|
-
const { namespace, limit = 20, offset = 0, provenanceFilter } = options;
|
|
1441
|
+
const { namespace, limit = 20, offset = 0, provenanceFilter, tags: requiredTags } = options;
|
|
1442
1442
|
if (provenanceFilter?.some(p => !isValidProvenanceType(p)))
|
|
1443
1443
|
return null;
|
|
1444
1444
|
const filters = [];
|
|
@@ -1451,6 +1451,20 @@ export async function bridgeListEntries(options) {
|
|
|
1451
1451
|
filters.push(`provenance_type IN (${provenanceFilter.map(() => '?').join(',')})`);
|
|
1452
1452
|
filterParams.push(...provenanceFilter);
|
|
1453
1453
|
}
|
|
1454
|
+
// Tags are stored as JSON-array TEXT (e.g. `["a","b"]`). Match each
|
|
1455
|
+
// required tag as an exact element via SQLite's json_each so the filter
|
|
1456
|
+
// runs in SQL instead of pulling the whole table into memory to filter
|
|
1457
|
+
// there. A LIKE-based substring match is ASCII case-insensitive (so
|
|
1458
|
+
// `prod` would match `PROD`) and matches raw JSON bytes rather than
|
|
1459
|
+
// parsed array elements (so a tag literally containing `"prod` could
|
|
1460
|
+
// false-match) — json_each compares real, decoded array values.
|
|
1461
|
+
const tagFilter = Array.isArray(requiredTags)
|
|
1462
|
+
? requiredTags.filter((t) => typeof t === 'string' && t.length > 0)
|
|
1463
|
+
: [];
|
|
1464
|
+
for (const tag of tagFilter) {
|
|
1465
|
+
filters.push(`EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)`);
|
|
1466
|
+
filterParams.push(tag);
|
|
1467
|
+
}
|
|
1454
1468
|
const extraFilter = filters.length > 0 ? `AND ${filters.join(' AND ')}` : '';
|
|
1455
1469
|
// #2120 — `status IS NULL` accepted alongside `'active'`. Old
|
|
1456
1470
|
// databases imported by the auto-memory bridge (before the status
|
|
@@ -1461,6 +1475,13 @@ export async function bridgeListEntries(options) {
|
|
|
1461
1475
|
// "legacy-active" — the safe default for any entry that predates the
|
|
1462
1476
|
// status column.
|
|
1463
1477
|
const statusFilter = liveMemoryRowSql({ includeExpired: options.includeExpired });
|
|
1478
|
+
// `|| 20` would treat an explicit `limit: 0` as falsy and silently
|
|
1479
|
+
// substitute the default (returning 20 rows instead of the requested
|
|
1480
|
+
// zero) — only fall back to the default when parsing genuinely failed.
|
|
1481
|
+
const parsedLimit = parseInt(String(limit), 10);
|
|
1482
|
+
const safeLimit = Number.isNaN(parsedLimit) ? 20 : parsedLimit;
|
|
1483
|
+
const parsedOffset = parseInt(String(offset), 10);
|
|
1484
|
+
const safeOffset = Number.isNaN(parsedOffset) ? 0 : parsedOffset;
|
|
1464
1485
|
// Count
|
|
1465
1486
|
let total = 0;
|
|
1466
1487
|
try {
|
|
@@ -1474,15 +1495,26 @@ export async function bridgeListEntries(options) {
|
|
|
1474
1495
|
// List
|
|
1475
1496
|
const entries = [];
|
|
1476
1497
|
try {
|
|
1498
|
+
// Include tags so export/list can round-trip them.
|
|
1477
1499
|
const stmt = ctx.db.prepare(`
|
|
1478
|
-
SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type, expires_at
|
|
1500
|
+
SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type, expires_at, tags
|
|
1479
1501
|
FROM memory_entries
|
|
1480
1502
|
WHERE ${statusFilter} ${extraFilter}
|
|
1481
1503
|
ORDER BY updated_at DESC
|
|
1482
1504
|
LIMIT ? OFFSET ?
|
|
1483
1505
|
`);
|
|
1484
|
-
const rows = stmt.all(...filterParams, options.includeEmbedding ? Math.min(
|
|
1506
|
+
const rows = stmt.all(...filterParams, options.includeEmbedding ? Math.min(safeLimit, MAX_LIST_EMBEDDINGS) : safeLimit, safeOffset);
|
|
1485
1507
|
for (const row of rows) {
|
|
1508
|
+
let tags = [];
|
|
1509
|
+
if (row.tags) {
|
|
1510
|
+
try {
|
|
1511
|
+
const parsed = typeof row.tags === 'string' ? JSON.parse(row.tags) : row.tags;
|
|
1512
|
+
if (Array.isArray(parsed)) {
|
|
1513
|
+
tags = parsed.filter((t) => typeof t === 'string');
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
catch { /* invalid tags JSON */ }
|
|
1517
|
+
}
|
|
1486
1518
|
const entry = {
|
|
1487
1519
|
// #2073: don't truncate id when content is requested — callers
|
|
1488
1520
|
// (notably memory_export) need the full id to round-trip via import.
|
|
@@ -1496,6 +1528,7 @@ export async function bridgeListEntries(options) {
|
|
|
1496
1528
|
expiresAt: row.expires_at ?? null,
|
|
1497
1529
|
hasEmbedding: !!(row.embedding && String(row.embedding).length > 10),
|
|
1498
1530
|
provenanceType: row.provenance_type || 'unknown',
|
|
1531
|
+
tags,
|
|
1499
1532
|
};
|
|
1500
1533
|
if (options.includeContent) {
|
|
1501
1534
|
entry.content = row.content || '';
|
|
@@ -1861,10 +1894,91 @@ export async function bridgeGetVectorSearchStatus(dbPath) {
|
|
|
1861
1894
|
return null;
|
|
1862
1895
|
}
|
|
1863
1896
|
}
|
|
1897
|
+
/**
|
|
1898
|
+
* #4203: RaBitQ pre-filter for the bridge's vector search.
|
|
1899
|
+
*
|
|
1900
|
+
* The index yields candidate ids from a 32×-compressed Hamming scan; each is
|
|
1901
|
+
* reranked here with the same exact cosine similarity the exhaustive scan uses,
|
|
1902
|
+
* so scores keep their meaning and only the scanned set shrinks (the candidate
|
|
1903
|
+
* window `k * 2` is then widened 3× by `searchRabitq` itself).
|
|
1904
|
+
*
|
|
1905
|
+
* Returns null whenever the index cannot answer this query — WASM missing, too
|
|
1906
|
+
* few vectors, or an index built for another store/dimension — so the caller
|
|
1907
|
+
* runs its unchanged brute-force scan. Keyword/BM25 recall is unaffected: this
|
|
1908
|
+
* only ever replaces the cosine scan.
|
|
1909
|
+
*/
|
|
1910
|
+
async function rabitqSearchAndRerank(db, queryEmbedding, options) {
|
|
1911
|
+
try {
|
|
1912
|
+
const { ensureRabitqIndex, searchRabitq } = await import('./rabitq-index.js');
|
|
1913
|
+
const ready = await ensureRabitqIndex({ dbPath: options.dbPath, dimensions: queryEmbedding.length });
|
|
1914
|
+
if (!ready)
|
|
1915
|
+
return null;
|
|
1916
|
+
const candidates = await searchRabitq(queryEmbedding, {
|
|
1917
|
+
k: options.k * 2,
|
|
1918
|
+
namespace: options.namespace,
|
|
1919
|
+
});
|
|
1920
|
+
if (!candidates || candidates.length === 0)
|
|
1921
|
+
return null;
|
|
1922
|
+
// Fetch exactly the candidates (id is the primary key) instead of the
|
|
1923
|
+
// 10k-row full scan; chunked to stay well inside SQLite's parameter limit.
|
|
1924
|
+
const rowsById = new Map();
|
|
1925
|
+
const ids = candidates.map(c => c.id);
|
|
1926
|
+
for (let i = 0; i < ids.length; i += 500) {
|
|
1927
|
+
const chunk = ids.slice(i, i + 500);
|
|
1928
|
+
const placeholders = chunk.map(() => '?').join(',');
|
|
1929
|
+
const rows = db.prepare(`
|
|
1930
|
+
SELECT id, key, namespace, content, embedding
|
|
1931
|
+
FROM memory_entries
|
|
1932
|
+
WHERE id IN (${placeholders}) AND ${liveMemoryRowSql()} AND embedding IS NOT NULL
|
|
1933
|
+
`).all(...chunk);
|
|
1934
|
+
for (const row of rows) {
|
|
1935
|
+
let embedding;
|
|
1936
|
+
try {
|
|
1937
|
+
embedding = JSON.parse(row.embedding);
|
|
1938
|
+
}
|
|
1939
|
+
catch {
|
|
1940
|
+
continue;
|
|
1941
|
+
}
|
|
1942
|
+
if (!Array.isArray(embedding) || embedding.length !== queryEmbedding.length)
|
|
1943
|
+
continue;
|
|
1944
|
+
rowsById.set(String(row.id), {
|
|
1945
|
+
key: row.key || String(row.id),
|
|
1946
|
+
namespace: row.namespace || 'default',
|
|
1947
|
+
content: row.content || '',
|
|
1948
|
+
score: cosineSim(queryEmbedding, embedding),
|
|
1949
|
+
});
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
const results = [];
|
|
1953
|
+
for (const candidate of candidates) {
|
|
1954
|
+
const row = rowsById.get(candidate.id);
|
|
1955
|
+
if (!row || row.score < options.threshold)
|
|
1956
|
+
continue;
|
|
1957
|
+
results.push({
|
|
1958
|
+
id: candidate.id.substring(0, 12),
|
|
1959
|
+
key: row.key || candidate.id.substring(0, 15),
|
|
1960
|
+
content: row.content.substring(0, 60) + (row.content.length > 60 ? '...' : ''),
|
|
1961
|
+
score: row.score,
|
|
1962
|
+
namespace: row.namespace,
|
|
1963
|
+
});
|
|
1964
|
+
}
|
|
1965
|
+
if (results.length === 0)
|
|
1966
|
+
return null;
|
|
1967
|
+
results.sort((a, b) => b.score - a.score);
|
|
1968
|
+
return results.slice(0, options.k);
|
|
1969
|
+
}
|
|
1970
|
+
catch {
|
|
1971
|
+
return null;
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1864
1974
|
/**
|
|
1865
1975
|
* Search AgentDB v3's embedder + SQLite entries via a full-table scan and
|
|
1866
1976
|
* brute-force cosine similarity. NOT HNSW-accelerated — see #2922. Returns
|
|
1867
1977
|
* null if unavailable.
|
|
1978
|
+
*
|
|
1979
|
+
* #4203: when a RaBitQ index is available the Hamming-scan candidate set is
|
|
1980
|
+
* reranked exactly instead of scanning the whole table; the exhaustive loop
|
|
1981
|
+
* below stays the fallback, so results and scores keep their meaning.
|
|
1868
1982
|
*/
|
|
1869
1983
|
export async function bridgeSearchBruteForceCosine(queryEmbedding, options, dbPath) {
|
|
1870
1984
|
if (!operationContext.getStore()?.active)
|
|
@@ -1881,6 +1995,17 @@ export async function bridgeSearchBruteForceCosine(queryEmbedding, options, dbPa
|
|
|
1881
1995
|
const nsFilter = options?.namespace && options.namespace !== 'all'
|
|
1882
1996
|
? `AND namespace = ?`
|
|
1883
1997
|
: '';
|
|
1998
|
+
// #4203: try the RaBitQ pre-filter before the exhaustive scan below. A null
|
|
1999
|
+
// result (no index / unavailable / nothing above threshold) falls through
|
|
2000
|
+
// to the unchanged brute-force path.
|
|
2001
|
+
const accelerated = await rabitqSearchAndRerank(ctx.db, queryEmbedding, {
|
|
2002
|
+
k,
|
|
2003
|
+
threshold,
|
|
2004
|
+
namespace: options?.namespace,
|
|
2005
|
+
dbPath,
|
|
2006
|
+
});
|
|
2007
|
+
if (accelerated)
|
|
2008
|
+
return accelerated;
|
|
1884
2009
|
let rows;
|
|
1885
2010
|
try {
|
|
1886
2011
|
// #2982/#2976: same unordered pre-rank truncation as bridgeSearchEntries
|
|
@@ -3361,6 +3486,41 @@ export async function bridgeGetAllEmbeddings(options) {
|
|
|
3361
3486
|
return null;
|
|
3362
3487
|
}
|
|
3363
3488
|
}
|
|
3489
|
+
/**
|
|
3490
|
+
* Count the embeddings `bridgeGetAllEmbeddings()` would export (#4203).
|
|
3491
|
+
* RaBitQ's auto-build uses this as its cheap "has the corpus drifted?" probe —
|
|
3492
|
+
* a COUNT instead of re-reading every embedding blob. With `dimensions` it
|
|
3493
|
+
* counts only rows build() would keep, so the drift ratio compares like with
|
|
3494
|
+
* like. Returns null if the bridge is unavailable.
|
|
3495
|
+
*/
|
|
3496
|
+
export async function bridgeCountEmbeddings(options) {
|
|
3497
|
+
if (!operationContext.getStore()?.active)
|
|
3498
|
+
return withBridgeOperation(() => bridgeCountEmbeddings(options));
|
|
3499
|
+
const registry = await getRegistry(options?.dbPath);
|
|
3500
|
+
if (!registry)
|
|
3501
|
+
return null;
|
|
3502
|
+
const ctx = getDb(registry);
|
|
3503
|
+
if (!ctx)
|
|
3504
|
+
return null;
|
|
3505
|
+
const dims = options?.dimensions;
|
|
3506
|
+
try {
|
|
3507
|
+
if (dims != null) {
|
|
3508
|
+
try {
|
|
3509
|
+
const row = ctx.db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE ${liveMemoryRowSql()} AND embedding IS NOT NULL AND json_array_length(embedding) = ?`).get(dims);
|
|
3510
|
+
return row?.cnt ?? 0;
|
|
3511
|
+
}
|
|
3512
|
+
catch {
|
|
3513
|
+
// JSON1 unavailable in this SQLite build — fall through to the
|
|
3514
|
+
// unfiltered count rather than losing the probe entirely.
|
|
3515
|
+
}
|
|
3516
|
+
}
|
|
3517
|
+
const row = ctx.db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE ${liveMemoryRowSql()} AND embedding IS NOT NULL`).get();
|
|
3518
|
+
return row?.cnt ?? 0;
|
|
3519
|
+
}
|
|
3520
|
+
catch {
|
|
3521
|
+
return null;
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3364
3524
|
// ===== Utility =====
|
|
3365
3525
|
function cosineSim(a, b) {
|
|
3366
3526
|
if (!a || !b || a.length === 0 || b.length === 0)
|
|
@@ -513,6 +513,8 @@ export declare function listEntries(options: {
|
|
|
513
513
|
includeEmbedding?: boolean;
|
|
514
514
|
/** #4056: also return rows whose TTL has passed (with `expiresAt`). For cleanup only; normal reads hide them. */
|
|
515
515
|
includeExpired?: boolean;
|
|
516
|
+
/** Require entries to include all of these tags (AND). */
|
|
517
|
+
tags?: string[];
|
|
516
518
|
}): Promise<{
|
|
517
519
|
success: boolean;
|
|
518
520
|
entries: {
|
|
@@ -531,6 +533,8 @@ export declare function listEntries(options: {
|
|
|
531
533
|
provenanceType?: string;
|
|
532
534
|
/** ADR-472: present when `includeEmbedding: true` and the row has a valid stored vector. */
|
|
533
535
|
embeddingQ8?: EmbeddingQ8;
|
|
536
|
+
/** Tag set stored with the entry (empty array when none). */
|
|
537
|
+
tags?: string[];
|
|
534
538
|
}[];
|
|
535
539
|
total: number;
|
|
536
540
|
error?: string;
|
|
@@ -2808,7 +2808,11 @@ export async function searchEntries(options) {
|
|
|
2808
2808
|
const queryEmbedding = queryEmb.embedding;
|
|
2809
2809
|
// Try RaBitQ pre-filter first (32× compressed Hamming scan)
|
|
2810
2810
|
try {
|
|
2811
|
-
const { searchRabitq } = await import('./rabitq-index.js');
|
|
2811
|
+
const { ensureRabitqIndex, searchRabitq } = await import('./rabitq-index.js');
|
|
2812
|
+
// #4203: build/rebuild on demand — without this the index was only ever
|
|
2813
|
+
// populated by the explicit `embeddings_rabitq_build` tool, so this
|
|
2814
|
+
// pre-filter always missed.
|
|
2815
|
+
await ensureRabitqIndex({ dbPath, dimensions: queryEmbedding.length });
|
|
2812
2816
|
const rabitqCandidates = await searchRabitq(queryEmbedding, { k: limit * 2, namespace: effectiveNamespace });
|
|
2813
2817
|
if (rabitqCandidates && rabitqCandidates.length > 0) {
|
|
2814
2818
|
// Rerank candidates with exact cosine similarity from SQLite
|
|
@@ -3043,7 +3047,7 @@ export async function listEntries(options) {
|
|
|
3043
3047
|
return bridgeResult;
|
|
3044
3048
|
}
|
|
3045
3049
|
// Fallback: raw sql.js
|
|
3046
|
-
const { namespace, limit = 20, offset = 0, dbPath: customPath, provenanceFilter } = options;
|
|
3050
|
+
const { namespace, limit = 20, offset = 0, dbPath: customPath, provenanceFilter, tags: requiredTags, } = options;
|
|
3047
3051
|
const swarmDir = getMemoryRoot();
|
|
3048
3052
|
const dbPath = customPath || path.join(swarmDir, 'memory.db');
|
|
3049
3053
|
try {
|
|
@@ -3065,18 +3069,41 @@ export async function listEntries(options) {
|
|
|
3065
3069
|
// #2120 — accept `status IS NULL` alongside `'active'`. Old DBs
|
|
3066
3070
|
// that predate the status column may have NULL after migration.
|
|
3067
3071
|
// See memory-bridge.ts:bridgeListEntries for full context.
|
|
3068
|
-
// Get total count
|
|
3069
3072
|
const whereClauses = [liveMemoryRowSql({ includeExpired: options.includeExpired })];
|
|
3070
3073
|
const whereParams = [];
|
|
3071
3074
|
if (namespace) {
|
|
3072
3075
|
whereClauses.push('namespace = ?');
|
|
3073
3076
|
whereParams.push(namespace);
|
|
3074
3077
|
}
|
|
3078
|
+
// ADR-323: restrict to the requested provenance types. This must be
|
|
3079
|
+
// ANDed alongside any tags filter below, never replaced by it — dropping
|
|
3080
|
+
// this clause lets unverified user_claim rows leak into results that
|
|
3081
|
+
// are supposed to be tool_result-only.
|
|
3075
3082
|
if (provenanceFilter?.length) {
|
|
3076
3083
|
whereClauses.push(`provenance_type IN (${provenanceFilter.map(() => '?').join(',')})`);
|
|
3077
3084
|
whereParams.push(...provenanceFilter);
|
|
3078
3085
|
}
|
|
3086
|
+
// Tags are stored as JSON-array TEXT (e.g. `["a","b"]`). Match each
|
|
3087
|
+
// required tag as an exact element via SQLite's json_each so a tag of
|
|
3088
|
+
// `prod` can't be satisfied by a substring like `"prod` or by a
|
|
3089
|
+
// differently-cased value like `PROD` (LIKE is ASCII case-insensitive
|
|
3090
|
+
// and matches raw JSON-array bytes, not parsed array elements).
|
|
3091
|
+
const tagFilter = Array.isArray(requiredTags)
|
|
3092
|
+
? requiredTags.filter((t) => typeof t === 'string' && t.length > 0)
|
|
3093
|
+
: [];
|
|
3094
|
+
for (const tag of tagFilter) {
|
|
3095
|
+
whereClauses.push(`EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)`);
|
|
3096
|
+
whereParams.push(tag);
|
|
3097
|
+
}
|
|
3079
3098
|
const whereSql = whereClauses.join(' AND ');
|
|
3099
|
+
// `|| 100` would treat an explicit `limit: 0` as falsy and silently
|
|
3100
|
+
// substitute the default (returning rows instead of the requested
|
|
3101
|
+
// zero) — only fall back to the default when parsing genuinely failed.
|
|
3102
|
+
const parsedLimit = parseInt(String(limit), 10);
|
|
3103
|
+
const safeLimit = Number.isNaN(parsedLimit) ? 100 : parsedLimit;
|
|
3104
|
+
const parsedOffset = parseInt(String(offset), 10);
|
|
3105
|
+
const safeOffset = Number.isNaN(parsedOffset) ? 0 : parsedOffset;
|
|
3106
|
+
// Get total count
|
|
3080
3107
|
const countStmt = db.prepare(`SELECT COUNT(*) as cnt FROM memory_entries WHERE ${whereSql}`);
|
|
3081
3108
|
if (whereParams.length > 0)
|
|
3082
3109
|
countStmt.bind(whereParams);
|
|
@@ -3085,50 +3112,56 @@ export async function listEntries(options) {
|
|
|
3085
3112
|
countRows.push(countStmt.get());
|
|
3086
3113
|
}
|
|
3087
3114
|
countStmt.free();
|
|
3088
|
-
const
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
const
|
|
3093
|
-
// #2120 — same NULL-as-active acceptance as the count above.
|
|
3094
|
-
const listStmt = db.prepare(`SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type, expires_at
|
|
3115
|
+
const total = countRows[0]?.[0] || 0;
|
|
3116
|
+
// Get entries. ADR-472: embedding rows are capped at MAX_LIST_EMBEDDINGS.
|
|
3117
|
+
const listLimit = options.includeEmbedding ? Math.min(safeLimit, MAX_LIST_EMBEDDINGS) : safeLimit;
|
|
3118
|
+
// Include tags so export/list can round-trip them.
|
|
3119
|
+
const listStmt = db.prepare(`SELECT id, key, namespace, content, embedding, access_count, created_at, updated_at, provenance_type, expires_at, tags
|
|
3095
3120
|
FROM memory_entries WHERE ${whereSql}
|
|
3096
3121
|
ORDER BY updated_at DESC LIMIT ? OFFSET ?`);
|
|
3097
|
-
listStmt.bind([...whereParams,
|
|
3122
|
+
listStmt.bind([...whereParams, listLimit, safeOffset]);
|
|
3098
3123
|
const listRows = [];
|
|
3099
3124
|
while (listStmt.step()) {
|
|
3100
3125
|
listRows.push(listStmt.get());
|
|
3101
3126
|
}
|
|
3102
3127
|
listStmt.free();
|
|
3103
|
-
const result = listRows.length > 0 ? [{ values: listRows }] : [];
|
|
3104
3128
|
const entries = [];
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
size: (content || '').length,
|
|
3115
|
-
accessCount: accessCount || 0,
|
|
3116
|
-
createdAt: createdAt || new Date().toISOString(),
|
|
3117
|
-
updatedAt: updatedAt || new Date().toISOString(),
|
|
3118
|
-
expiresAt: expiresAtVal ?? null,
|
|
3119
|
-
hasEmbedding: !!embedding && embedding.length > 10,
|
|
3120
|
-
provenanceType: provenanceTypeVal || 'unknown'
|
|
3121
|
-
};
|
|
3122
|
-
if (options.includeContent) {
|
|
3123
|
-
entry.content = content || '';
|
|
3124
|
-
}
|
|
3125
|
-
if (options.includeEmbedding) {
|
|
3126
|
-
const q8 = encodeEmbeddingQ8(embedding);
|
|
3127
|
-
if (q8)
|
|
3128
|
-
entry.embeddingQ8 = q8;
|
|
3129
|
+
for (const row of listRows) {
|
|
3130
|
+
const [id, key, ns, content, embedding, accessCount, createdAt, updatedAt, provenanceTypeVal, expiresAtVal, tagsJson] = row;
|
|
3131
|
+
let tags = [];
|
|
3132
|
+
if (tagsJson) {
|
|
3133
|
+
try {
|
|
3134
|
+
const parsed = JSON.parse(tagsJson);
|
|
3135
|
+
if (Array.isArray(parsed)) {
|
|
3136
|
+
tags = parsed.filter((t) => typeof t === 'string');
|
|
3137
|
+
}
|
|
3129
3138
|
}
|
|
3130
|
-
|
|
3139
|
+
catch { /* invalid tags JSON */ }
|
|
3140
|
+
}
|
|
3141
|
+
const entry = {
|
|
3142
|
+
// #2073: don't truncate id when content is requested — callers
|
|
3143
|
+
// (notably memory_export) need the full id to round-trip via import.
|
|
3144
|
+
id: options.includeContent ? String(id) : String(id).substring(0, 20),
|
|
3145
|
+
key: key || String(id).substring(0, 15),
|
|
3146
|
+
namespace: ns || 'default',
|
|
3147
|
+
size: (content || '').length,
|
|
3148
|
+
accessCount: accessCount || 0,
|
|
3149
|
+
createdAt: createdAt || new Date().toISOString(),
|
|
3150
|
+
updatedAt: updatedAt || new Date().toISOString(),
|
|
3151
|
+
expiresAt: expiresAtVal ?? null,
|
|
3152
|
+
hasEmbedding: !!embedding && embedding.length > 10,
|
|
3153
|
+
provenanceType: provenanceTypeVal || 'unknown',
|
|
3154
|
+
tags,
|
|
3155
|
+
};
|
|
3156
|
+
if (options.includeContent) {
|
|
3157
|
+
entry.content = content || '';
|
|
3158
|
+
}
|
|
3159
|
+
if (options.includeEmbedding) {
|
|
3160
|
+
const q8 = encodeEmbeddingQ8(embedding);
|
|
3161
|
+
if (q8)
|
|
3162
|
+
entry.embeddingQ8 = q8;
|
|
3131
3163
|
}
|
|
3164
|
+
entries.push(entry);
|
|
3132
3165
|
}
|
|
3133
3166
|
db.close();
|
|
3134
3167
|
return { success: true, entries, total };
|
|
@@ -9,6 +9,11 @@
|
|
|
9
9
|
* 1. build() — bulk-load all embeddings from SQLite into the WASM index
|
|
10
10
|
* 2. search() — fast Hamming scan → candidate ids → caller reranks
|
|
11
11
|
* 3. rebuild() — called when entry count drifts >20% from last build
|
|
12
|
+
*
|
|
13
|
+
* #4203: step 3 is driven by `ensureRabitqIndex()`, which the search paths
|
|
14
|
+
* call before `searchRabitq()`. Previously the only caller of the build was
|
|
15
|
+
* the explicit `embeddings_rabitq_build` tool, so the index was effectively
|
|
16
|
+
* never populated and every `searchRabitq()` returned null.
|
|
12
17
|
*/
|
|
13
18
|
/**
|
|
14
19
|
* Build or rebuild the RaBitQ index from SQLite embeddings.
|
|
@@ -46,6 +51,20 @@ export declare function searchRabitq(queryEmbedding: number[], options?: {
|
|
|
46
51
|
* Check if the RaBitQ index needs rebuilding.
|
|
47
52
|
*/
|
|
48
53
|
export declare function shouldRebuildRabitq(currentEntryCount: number): Promise<boolean>;
|
|
54
|
+
/**
|
|
55
|
+
* Auto-build / auto-rebuild entry point (#4203).
|
|
56
|
+
*
|
|
57
|
+
* Search paths call this before `searchRabitq()`: it lets `shouldRebuildRabitq()`
|
|
58
|
+
* decide whether the index is missing or has drifted past the 20% threshold and
|
|
59
|
+
* builds it when so. Never throws — a missing WASM module, an unreadable store
|
|
60
|
+
* or an index built for another store/dimension just leaves the caller's
|
|
61
|
+
* existing non-RaBitQ path untouched. Returns whether an index usable for these
|
|
62
|
+
* options exists afterwards.
|
|
63
|
+
*/
|
|
64
|
+
export declare function ensureRabitqIndex(options?: {
|
|
65
|
+
dbPath?: string;
|
|
66
|
+
dimensions?: number;
|
|
67
|
+
}): Promise<boolean>;
|
|
49
68
|
/**
|
|
50
69
|
* Get RaBitQ index status.
|
|
51
70
|
*/
|
|
@@ -9,14 +9,27 @@
|
|
|
9
9
|
* 1. build() — bulk-load all embeddings from SQLite into the WASM index
|
|
10
10
|
* 2. search() — fast Hamming scan → candidate ids → caller reranks
|
|
11
11
|
* 3. rebuild() — called when entry count drifts >20% from last build
|
|
12
|
+
*
|
|
13
|
+
* #4203: step 3 is driven by `ensureRabitqIndex()`, which the search paths
|
|
14
|
+
* call before `searchRabitq()`. Previously the only caller of the build was
|
|
15
|
+
* the explicit `embeddings_rabitq_build` tool, so the index was effectively
|
|
16
|
+
* never populated and every `searchRabitq()` returned null.
|
|
12
17
|
*/
|
|
13
18
|
import * as fs from 'fs';
|
|
14
19
|
import * as path from 'path';
|
|
15
20
|
const RABITQ_SEED = 42n;
|
|
16
21
|
const RABITQ_RERANK_FACTOR = 20;
|
|
17
22
|
const REBUILD_DRIFT_THRESHOLD = 0.2; // rebuild when count drifts >20%
|
|
23
|
+
/** #4203: minimum gap between auto-build attempts, so a store that is still
|
|
24
|
+
* too small cannot turn every search into a build attempt. */
|
|
25
|
+
const AUTO_BUILD_RETRY_MS = 60_000;
|
|
26
|
+
const RABITQ_UNAVAILABLE_ERROR = '@ruvector/rabitq-wasm not available';
|
|
18
27
|
let rabitqState = null;
|
|
19
28
|
let rabitqInitializing = false;
|
|
29
|
+
let lastAutoBuildAttempt = 0;
|
|
30
|
+
/** Set once the WASM module is known to be missing for this process: retrying
|
|
31
|
+
* the same failing dynamic import on every search buys nothing. */
|
|
32
|
+
let autoBuildDisabled = false;
|
|
20
33
|
async function loadRabitqModule() {
|
|
21
34
|
try {
|
|
22
35
|
const mod = await import('@ruvector/rabitq-wasm');
|
|
@@ -50,7 +63,7 @@ export async function buildRabitqIndex(options) {
|
|
|
50
63
|
const mod = await loadRabitqModule();
|
|
51
64
|
if (!mod) {
|
|
52
65
|
rabitqInitializing = false;
|
|
53
|
-
return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error:
|
|
66
|
+
return { success: false, vectorCount: 0, dimensions: 0, compressionRatio: 0, buildTimeMs: 0, error: RABITQ_UNAVAILABLE_ERROR };
|
|
54
67
|
}
|
|
55
68
|
const dimensions = options?.dimensions ?? 384;
|
|
56
69
|
const swarmDir = path.resolve(process.cwd(), '.swarm');
|
|
@@ -126,6 +139,7 @@ export async function buildRabitqIndex(options) {
|
|
|
126
139
|
dimensions,
|
|
127
140
|
builtAt: Date.now(),
|
|
128
141
|
vectorCount: entries.length,
|
|
142
|
+
sourcePath: options?.dbPath ? path.resolve(options.dbPath) : undefined,
|
|
129
143
|
};
|
|
130
144
|
// Persist metadata for fast reload hint
|
|
131
145
|
try {
|
|
@@ -226,6 +240,108 @@ export async function shouldRebuildRabitq(currentEntryCount) {
|
|
|
226
240
|
const drift = Math.abs(currentEntryCount - rabitqState.vectorCount) / Math.max(rabitqState.vectorCount, 1);
|
|
227
241
|
return drift > REBUILD_DRIFT_THRESHOLD;
|
|
228
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* Whether the in-memory index can answer a query for this store/dimension.
|
|
245
|
+
* `undefined` dbPath means "the default store", matching build's own default.
|
|
246
|
+
*/
|
|
247
|
+
function isRabitqUsable(options) {
|
|
248
|
+
if (!rabitqState)
|
|
249
|
+
return false;
|
|
250
|
+
if (options?.dimensions != null && rabitqState.dimensions !== options.dimensions)
|
|
251
|
+
return false;
|
|
252
|
+
const sourcePath = options?.dbPath ? path.resolve(options.dbPath) : undefined;
|
|
253
|
+
return rabitqState.sourcePath === sourcePath;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Count the rows `buildRabitqIndex()` would load, using the cheapest source
|
|
257
|
+
* available: the bridge's COUNT (WAL-visible, same rows build prefers) with a
|
|
258
|
+
* sql.js COUNT of the same file as fallback. Returns 0 when neither is
|
|
259
|
+
* readable, which simply leaves the caller on its non-RaBitQ path.
|
|
260
|
+
*/
|
|
261
|
+
async function countIndexableEntries(options) {
|
|
262
|
+
const dimensions = options?.dimensions ?? 384;
|
|
263
|
+
try {
|
|
264
|
+
const { bridgeCountEmbeddings } = await import('./memory-bridge.js');
|
|
265
|
+
const bridgeCount = await bridgeCountEmbeddings({ dimensions, dbPath: options?.dbPath });
|
|
266
|
+
if (bridgeCount !== null)
|
|
267
|
+
return bridgeCount;
|
|
268
|
+
}
|
|
269
|
+
catch { /* bridge unavailable, fall through */ }
|
|
270
|
+
const dbPath = options?.dbPath
|
|
271
|
+
? path.resolve(options.dbPath)
|
|
272
|
+
: path.join(path.resolve(process.cwd(), '.swarm'), 'memory.db');
|
|
273
|
+
if (!fs.existsSync(dbPath))
|
|
274
|
+
return 0;
|
|
275
|
+
let db = null;
|
|
276
|
+
try {
|
|
277
|
+
const initSqlJs = (await import('sql.js')).default;
|
|
278
|
+
const SQL = await initSqlJs();
|
|
279
|
+
db = new SQL.Database(fs.readFileSync(dbPath));
|
|
280
|
+
const count = (dimFilter) => db.exec(`
|
|
281
|
+
SELECT COUNT(*) FROM memory_entries
|
|
282
|
+
WHERE status = 'active' AND embedding IS NOT NULL${dimFilter}
|
|
283
|
+
`);
|
|
284
|
+
let result;
|
|
285
|
+
try {
|
|
286
|
+
// Same rows build() keeps: only embeddings of the requested width.
|
|
287
|
+
result = count(` AND json_array_length(embedding) = ${dimensions}`);
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
result = count('');
|
|
291
|
+
}
|
|
292
|
+
return Number(result[0]?.values?.[0]?.[0] ?? 0);
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
return 0;
|
|
296
|
+
}
|
|
297
|
+
finally {
|
|
298
|
+
try {
|
|
299
|
+
db?.close();
|
|
300
|
+
}
|
|
301
|
+
catch { /* ignore */ }
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* Auto-build / auto-rebuild entry point (#4203).
|
|
306
|
+
*
|
|
307
|
+
* Search paths call this before `searchRabitq()`: it lets `shouldRebuildRabitq()`
|
|
308
|
+
* decide whether the index is missing or has drifted past the 20% threshold and
|
|
309
|
+
* builds it when so. Never throws — a missing WASM module, an unreadable store
|
|
310
|
+
* or an index built for another store/dimension just leaves the caller's
|
|
311
|
+
* existing non-RaBitQ path untouched. Returns whether an index usable for these
|
|
312
|
+
* options exists afterwards.
|
|
313
|
+
*/
|
|
314
|
+
export async function ensureRabitqIndex(options) {
|
|
315
|
+
if (autoBuildDisabled || rabitqInitializing)
|
|
316
|
+
return isRabitqUsable(options);
|
|
317
|
+
const dimensions = options?.dimensions ?? 384;
|
|
318
|
+
const now = Date.now();
|
|
319
|
+
if (now - lastAutoBuildAttempt < AUTO_BUILD_RETRY_MS)
|
|
320
|
+
return isRabitqUsable(options);
|
|
321
|
+
lastAutoBuildAttempt = now;
|
|
322
|
+
let currentEntryCount = 0;
|
|
323
|
+
try {
|
|
324
|
+
currentEntryCount = await countIndexableEntries({ dbPath: options?.dbPath, dimensions });
|
|
325
|
+
}
|
|
326
|
+
catch {
|
|
327
|
+
currentEntryCount = 0;
|
|
328
|
+
}
|
|
329
|
+
if (isRabitqUsable(options)) {
|
|
330
|
+
// Same store and width: rebuild only once the corpus has drifted >20%.
|
|
331
|
+
if (!(await shouldRebuildRabitq(currentEntryCount)))
|
|
332
|
+
return true;
|
|
333
|
+
}
|
|
334
|
+
else if (!rabitqState) {
|
|
335
|
+
// Nothing built yet: honour the "enough vectors to be worth it" floor.
|
|
336
|
+
if (!(await shouldRebuildRabitq(currentEntryCount)))
|
|
337
|
+
return false;
|
|
338
|
+
}
|
|
339
|
+
// Either the index is stale for this store/width, or the entry count drifted.
|
|
340
|
+
const result = await buildRabitqIndex({ dbPath: options?.dbPath, dimensions });
|
|
341
|
+
if (!result.success && result.error === RABITQ_UNAVAILABLE_ERROR)
|
|
342
|
+
autoBuildDisabled = true;
|
|
343
|
+
return isRabitqUsable(options);
|
|
344
|
+
}
|
|
229
345
|
/**
|
|
230
346
|
* Get RaBitQ index status.
|
|
231
347
|
*/
|