claude-memory-layer 1.0.11 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +2389 -286
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1017 -132
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1347 -202
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1339 -194
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1343 -198
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1351 -206
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1347 -202
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +1436 -211
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +1445 -220
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1345 -199
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +69 -2
- package/dist/ui/index.html +8 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +2 -1
- package/scripts/build.ts +6 -0
- package/src/cli/index.ts +281 -2
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +350 -1
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/types.ts +28 -0
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +3 -1
- package/src/server/api/stats.ts +46 -1
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +373 -68
- package/src/ui/app.js +69 -2
- package/src/ui/index.html +8 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
package/dist/core/index.js
CHANGED
|
@@ -487,6 +487,15 @@ var ConsolidatedMemorySchema = z.object({
|
|
|
487
487
|
accessedAt: z.date().optional(),
|
|
488
488
|
accessCount: z.number().default(0)
|
|
489
489
|
});
|
|
490
|
+
var ConsolidationRuleSchema = z.object({
|
|
491
|
+
ruleId: z.string(),
|
|
492
|
+
rule: z.string(),
|
|
493
|
+
topics: z.array(z.string()),
|
|
494
|
+
sourceMemoryIds: z.array(z.string()),
|
|
495
|
+
sourceEvents: z.array(z.string()),
|
|
496
|
+
confidence: z.number(),
|
|
497
|
+
createdAt: z.date()
|
|
498
|
+
});
|
|
490
499
|
var TransitionTypeSchema = z.enum(["seamless", "topic_shift", "break"]);
|
|
491
500
|
var ContinuityLogSchema = z.object({
|
|
492
501
|
logId: z.string(),
|
|
@@ -639,11 +648,11 @@ function toDate(value) {
|
|
|
639
648
|
return new Date(value);
|
|
640
649
|
return new Date(String(value));
|
|
641
650
|
}
|
|
642
|
-
function createDatabase(
|
|
651
|
+
function createDatabase(path2, options) {
|
|
643
652
|
if (options?.readOnly) {
|
|
644
|
-
return new duckdb.Database(
|
|
653
|
+
return new duckdb.Database(path2, { access_mode: "READ_ONLY" });
|
|
645
654
|
}
|
|
646
|
-
return new duckdb.Database(
|
|
655
|
+
return new duckdb.Database(path2);
|
|
647
656
|
}
|
|
648
657
|
function dbRun(db, sql, params = []) {
|
|
649
658
|
return new Promise((resolve, reject) => {
|
|
@@ -907,6 +916,17 @@ var EventStore = class {
|
|
|
907
916
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
908
917
|
)
|
|
909
918
|
`);
|
|
919
|
+
await dbRun(this.db, `
|
|
920
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
921
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
922
|
+
rule TEXT NOT NULL,
|
|
923
|
+
topics JSON,
|
|
924
|
+
source_memory_ids JSON,
|
|
925
|
+
source_events JSON,
|
|
926
|
+
confidence FLOAT DEFAULT 0.5,
|
|
927
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
928
|
+
)
|
|
929
|
+
`);
|
|
910
930
|
await dbRun(this.db, `
|
|
911
931
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
912
932
|
key VARCHAR PRIMARY KEY,
|
|
@@ -926,6 +946,7 @@ var EventStore = class {
|
|
|
926
946
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
927
947
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
928
948
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
949
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
929
950
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
930
951
|
this.initialized = true;
|
|
931
952
|
}
|
|
@@ -1312,12 +1333,12 @@ var EventStore = class {
|
|
|
1312
1333
|
import Database from "better-sqlite3";
|
|
1313
1334
|
import * as fs from "fs";
|
|
1314
1335
|
import * as nodePath from "path";
|
|
1315
|
-
function createSQLiteDatabase(
|
|
1316
|
-
const dir = nodePath.dirname(
|
|
1336
|
+
function createSQLiteDatabase(path2, options) {
|
|
1337
|
+
const dir = nodePath.dirname(path2);
|
|
1317
1338
|
if (!fs.existsSync(dir)) {
|
|
1318
1339
|
fs.mkdirSync(dir, { recursive: true });
|
|
1319
1340
|
}
|
|
1320
|
-
const db = new Database(
|
|
1341
|
+
const db = new Database(path2, {
|
|
1321
1342
|
readonly: options?.readonly ?? false
|
|
1322
1343
|
});
|
|
1323
1344
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -1363,6 +1384,66 @@ function toSQLiteTimestamp(date) {
|
|
|
1363
1384
|
|
|
1364
1385
|
// src/core/sqlite-event-store.ts
|
|
1365
1386
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
1387
|
+
|
|
1388
|
+
// src/core/markdown-mirror.ts
|
|
1389
|
+
import * as fs2 from "fs/promises";
|
|
1390
|
+
import * as path from "path";
|
|
1391
|
+
var DEFAULT_NAMESPACE = "default";
|
|
1392
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
1393
|
+
function sanitizeSegment(input, fallback) {
|
|
1394
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
1395
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
1396
|
+
if (!safe || safe === "." || safe === "..")
|
|
1397
|
+
return fallback;
|
|
1398
|
+
return safe;
|
|
1399
|
+
}
|
|
1400
|
+
function getCategorySegments(metadata, eventType) {
|
|
1401
|
+
const raw = metadata?.categoryPath;
|
|
1402
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
1403
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
1404
|
+
}
|
|
1405
|
+
const single = metadata?.category;
|
|
1406
|
+
if (typeof single === "string" && single.trim()) {
|
|
1407
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
1408
|
+
}
|
|
1409
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
1410
|
+
}
|
|
1411
|
+
function buildMirrorPath(rootDir, event) {
|
|
1412
|
+
const metadata = event.metadata;
|
|
1413
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
1414
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
1415
|
+
const d = event.timestamp;
|
|
1416
|
+
const yyyy = d.getFullYear();
|
|
1417
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
1418
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
1419
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
1420
|
+
}
|
|
1421
|
+
function formatMirrorEntry(event) {
|
|
1422
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
1423
|
+
return [
|
|
1424
|
+
"",
|
|
1425
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
1426
|
+
` id: ${event.id}`,
|
|
1427
|
+
` type: ${event.eventType}`,
|
|
1428
|
+
` session: ${event.sessionId}`,
|
|
1429
|
+
` category: ${category}`,
|
|
1430
|
+
" content: |",
|
|
1431
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
1432
|
+
].join("\n") + "\n";
|
|
1433
|
+
}
|
|
1434
|
+
var MarkdownMirror = class {
|
|
1435
|
+
constructor(rootDir) {
|
|
1436
|
+
this.rootDir = rootDir;
|
|
1437
|
+
}
|
|
1438
|
+
async append(event) {
|
|
1439
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
1440
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
1441
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
1442
|
+
return outPath;
|
|
1443
|
+
}
|
|
1444
|
+
};
|
|
1445
|
+
|
|
1446
|
+
// src/core/sqlite-event-store.ts
|
|
1366
1447
|
var SQLiteEventStore = class {
|
|
1367
1448
|
constructor(dbPath, options) {
|
|
1368
1449
|
this.dbPath = dbPath;
|
|
@@ -1371,10 +1452,12 @@ var SQLiteEventStore = class {
|
|
|
1371
1452
|
readonly: this.readOnly,
|
|
1372
1453
|
walMode: !this.readOnly
|
|
1373
1454
|
});
|
|
1455
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
1374
1456
|
}
|
|
1375
1457
|
db;
|
|
1376
1458
|
initialized = false;
|
|
1377
1459
|
readOnly;
|
|
1460
|
+
markdownMirror;
|
|
1378
1461
|
/**
|
|
1379
1462
|
* Initialize database schema
|
|
1380
1463
|
*/
|
|
@@ -1581,6 +1664,17 @@ var SQLiteEventStore = class {
|
|
|
1581
1664
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1582
1665
|
);
|
|
1583
1666
|
|
|
1667
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1668
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1669
|
+
rule_id TEXT PRIMARY KEY,
|
|
1670
|
+
rule TEXT NOT NULL,
|
|
1671
|
+
topics TEXT,
|
|
1672
|
+
source_memory_ids TEXT,
|
|
1673
|
+
source_events TEXT,
|
|
1674
|
+
confidence REAL DEFAULT 0.5,
|
|
1675
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1676
|
+
);
|
|
1677
|
+
|
|
1584
1678
|
-- Endless Mode Config table
|
|
1585
1679
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1586
1680
|
key TEXT PRIMARY KEY,
|
|
@@ -1605,6 +1699,24 @@ var SQLiteEventStore = class {
|
|
|
1605
1699
|
measured_at TEXT
|
|
1606
1700
|
);
|
|
1607
1701
|
|
|
1702
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1703
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1704
|
+
trace_id TEXT PRIMARY KEY,
|
|
1705
|
+
session_id TEXT,
|
|
1706
|
+
project_hash TEXT,
|
|
1707
|
+
query_text TEXT NOT NULL,
|
|
1708
|
+
strategy TEXT,
|
|
1709
|
+
candidate_event_ids TEXT,
|
|
1710
|
+
selected_event_ids TEXT,
|
|
1711
|
+
candidate_details_json TEXT,
|
|
1712
|
+
selected_details_json TEXT,
|
|
1713
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1714
|
+
selected_count INTEGER DEFAULT 0,
|
|
1715
|
+
confidence TEXT,
|
|
1716
|
+
fallback_trace TEXT,
|
|
1717
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1718
|
+
);
|
|
1719
|
+
|
|
1608
1720
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1609
1721
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1610
1722
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1629,10 +1741,14 @@ var SQLiteEventStore = class {
|
|
|
1629
1741
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1630
1742
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1631
1743
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1744
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1632
1745
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1633
1746
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1634
1747
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1635
1748
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1749
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1750
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1751
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1636
1752
|
|
|
1637
1753
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1638
1754
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1656,6 +1772,14 @@ var SQLiteEventStore = class {
|
|
|
1656
1772
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1657
1773
|
END;
|
|
1658
1774
|
`);
|
|
1775
|
+
try {
|
|
1776
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1777
|
+
} catch {
|
|
1778
|
+
}
|
|
1779
|
+
try {
|
|
1780
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1781
|
+
} catch {
|
|
1782
|
+
}
|
|
1659
1783
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1660
1784
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1661
1785
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1755,6 +1879,21 @@ var SQLiteEventStore = class {
|
|
|
1755
1879
|
insertLevel.run(id);
|
|
1756
1880
|
});
|
|
1757
1881
|
transaction();
|
|
1882
|
+
if (this.markdownMirror) {
|
|
1883
|
+
const event = {
|
|
1884
|
+
id,
|
|
1885
|
+
eventType: input.eventType,
|
|
1886
|
+
sessionId: input.sessionId,
|
|
1887
|
+
timestamp: input.timestamp,
|
|
1888
|
+
content: input.content,
|
|
1889
|
+
canonicalKey,
|
|
1890
|
+
dedupeKey,
|
|
1891
|
+
metadata
|
|
1892
|
+
};
|
|
1893
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1894
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1895
|
+
});
|
|
1896
|
+
}
|
|
1758
1897
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1759
1898
|
} catch (error) {
|
|
1760
1899
|
return {
|
|
@@ -1813,6 +1952,92 @@ var SQLiteEventStore = class {
|
|
|
1813
1952
|
);
|
|
1814
1953
|
return rows.map(this.rowToEvent);
|
|
1815
1954
|
}
|
|
1955
|
+
/**
|
|
1956
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1957
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1958
|
+
*/
|
|
1959
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1960
|
+
await this.initialize();
|
|
1961
|
+
const rows = sqliteAll(
|
|
1962
|
+
this.db,
|
|
1963
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1964
|
+
[lastRowid, limit]
|
|
1965
|
+
);
|
|
1966
|
+
return rows.map((row) => ({
|
|
1967
|
+
rowid: row._rowid,
|
|
1968
|
+
event: this.rowToEvent(row)
|
|
1969
|
+
}));
|
|
1970
|
+
}
|
|
1971
|
+
/**
|
|
1972
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1973
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1974
|
+
*
|
|
1975
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1976
|
+
*/
|
|
1977
|
+
async importEvents(events) {
|
|
1978
|
+
if (events.length === 0)
|
|
1979
|
+
return { inserted: 0, skipped: 0 };
|
|
1980
|
+
if (this.readOnly)
|
|
1981
|
+
return { inserted: 0, skipped: events.length };
|
|
1982
|
+
await this.initialize();
|
|
1983
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1984
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1985
|
+
const insertEvent = this.db.prepare(`
|
|
1986
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1987
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1988
|
+
`);
|
|
1989
|
+
const insertDedup = this.db.prepare(`
|
|
1990
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1991
|
+
`);
|
|
1992
|
+
const insertLevel = this.db.prepare(`
|
|
1993
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1994
|
+
`);
|
|
1995
|
+
let inserted = 0;
|
|
1996
|
+
let skipped = 0;
|
|
1997
|
+
const insertedEvents = [];
|
|
1998
|
+
const tx = this.db.transaction((batch) => {
|
|
1999
|
+
for (const ev of batch) {
|
|
2000
|
+
const existingById = getById.get(ev.id);
|
|
2001
|
+
if (existingById) {
|
|
2002
|
+
skipped++;
|
|
2003
|
+
continue;
|
|
2004
|
+
}
|
|
2005
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
2006
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
2007
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
2008
|
+
if (existingByDedupe) {
|
|
2009
|
+
skipped++;
|
|
2010
|
+
continue;
|
|
2011
|
+
}
|
|
2012
|
+
const metadata = ev.metadata || {};
|
|
2013
|
+
const turnId = metadata.turnId;
|
|
2014
|
+
insertEvent.run(
|
|
2015
|
+
ev.id,
|
|
2016
|
+
ev.eventType,
|
|
2017
|
+
ev.sessionId,
|
|
2018
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
2019
|
+
ev.content,
|
|
2020
|
+
canonicalKey,
|
|
2021
|
+
dedupeKey,
|
|
2022
|
+
JSON.stringify(metadata),
|
|
2023
|
+
turnId ?? null
|
|
2024
|
+
);
|
|
2025
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
2026
|
+
insertLevel.run(ev.id);
|
|
2027
|
+
inserted++;
|
|
2028
|
+
insertedEvents.push(ev);
|
|
2029
|
+
}
|
|
2030
|
+
});
|
|
2031
|
+
tx(events);
|
|
2032
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
2033
|
+
for (const ev of insertedEvents) {
|
|
2034
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
2035
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
2036
|
+
});
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
return { inserted, skipped };
|
|
2040
|
+
}
|
|
1816
2041
|
/**
|
|
1817
2042
|
* Create or update session
|
|
1818
2043
|
*/
|
|
@@ -1975,6 +2200,35 @@ var SQLiteEventStore = class {
|
|
|
1975
2200
|
[error, ...ids]
|
|
1976
2201
|
);
|
|
1977
2202
|
}
|
|
2203
|
+
/**
|
|
2204
|
+
* Get embedding/vector outbox health statistics
|
|
2205
|
+
*/
|
|
2206
|
+
async getOutboxStats() {
|
|
2207
|
+
await this.initialize();
|
|
2208
|
+
const embeddingRows = sqliteAll(
|
|
2209
|
+
this.db,
|
|
2210
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
2211
|
+
);
|
|
2212
|
+
const vectorRows = sqliteAll(
|
|
2213
|
+
this.db,
|
|
2214
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
2215
|
+
);
|
|
2216
|
+
const fromRows = (rows) => {
|
|
2217
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
2218
|
+
for (const row of rows) {
|
|
2219
|
+
const key = row.status;
|
|
2220
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
2221
|
+
out[key] += row.count;
|
|
2222
|
+
}
|
|
2223
|
+
out.total += row.count;
|
|
2224
|
+
}
|
|
2225
|
+
return out;
|
|
2226
|
+
};
|
|
2227
|
+
return {
|
|
2228
|
+
embedding: fromRows(embeddingRows),
|
|
2229
|
+
vector: fromRows(vectorRows)
|
|
2230
|
+
};
|
|
2231
|
+
}
|
|
1978
2232
|
/**
|
|
1979
2233
|
* Update memory level
|
|
1980
2234
|
*/
|
|
@@ -2333,6 +2587,79 @@ var SQLiteEventStore = class {
|
|
|
2333
2587
|
getDatabase() {
|
|
2334
2588
|
return this.db;
|
|
2335
2589
|
}
|
|
2590
|
+
async recordRetrievalTrace(input) {
|
|
2591
|
+
await this.initialize();
|
|
2592
|
+
const traceId = randomUUID2();
|
|
2593
|
+
sqliteRun(
|
|
2594
|
+
this.db,
|
|
2595
|
+
`INSERT INTO retrieval_traces (
|
|
2596
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2597
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2598
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2599
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2600
|
+
[
|
|
2601
|
+
traceId,
|
|
2602
|
+
input.sessionId || null,
|
|
2603
|
+
input.projectHash || null,
|
|
2604
|
+
input.queryText,
|
|
2605
|
+
input.strategy || null,
|
|
2606
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2607
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2608
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2609
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2610
|
+
(input.candidateEventIds || []).length,
|
|
2611
|
+
(input.selectedEventIds || []).length,
|
|
2612
|
+
input.confidence || null,
|
|
2613
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2614
|
+
]
|
|
2615
|
+
);
|
|
2616
|
+
}
|
|
2617
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2618
|
+
await this.initialize();
|
|
2619
|
+
const rows = sqliteAll(
|
|
2620
|
+
this.db,
|
|
2621
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2622
|
+
[limit]
|
|
2623
|
+
);
|
|
2624
|
+
return rows.map((row) => ({
|
|
2625
|
+
traceId: row.trace_id,
|
|
2626
|
+
sessionId: row.session_id || void 0,
|
|
2627
|
+
projectHash: row.project_hash || void 0,
|
|
2628
|
+
queryText: row.query_text,
|
|
2629
|
+
strategy: row.strategy || void 0,
|
|
2630
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2631
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2632
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2633
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2634
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2635
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2636
|
+
confidence: row.confidence || void 0,
|
|
2637
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2638
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2639
|
+
}));
|
|
2640
|
+
}
|
|
2641
|
+
async getRetrievalTraceStats() {
|
|
2642
|
+
await this.initialize();
|
|
2643
|
+
const row = sqliteGet(
|
|
2644
|
+
this.db,
|
|
2645
|
+
`SELECT
|
|
2646
|
+
COUNT(*) as total_queries,
|
|
2647
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2648
|
+
AVG(selected_count) as avg_selected_count,
|
|
2649
|
+
CASE
|
|
2650
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2651
|
+
ELSE 0
|
|
2652
|
+
END as selection_rate
|
|
2653
|
+
FROM retrieval_traces`,
|
|
2654
|
+
[]
|
|
2655
|
+
);
|
|
2656
|
+
return {
|
|
2657
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2658
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2659
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2660
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2661
|
+
};
|
|
2662
|
+
}
|
|
2336
2663
|
/**
|
|
2337
2664
|
* Close database connection
|
|
2338
2665
|
*/
|
|
@@ -2666,8 +2993,282 @@ var SyncWorker = class {
|
|
|
2666
2993
|
}
|
|
2667
2994
|
};
|
|
2668
2995
|
|
|
2669
|
-
// src/core/
|
|
2996
|
+
// src/core/mongo-sync-worker.ts
|
|
2670
2997
|
import { randomUUID as randomUUID3 } from "crypto";
|
|
2998
|
+
import * as os from "os";
|
|
2999
|
+
import { MongoClient } from "mongodb";
|
|
3000
|
+
function redactMongoUri(uri) {
|
|
3001
|
+
const schemeIdx = uri.indexOf("://");
|
|
3002
|
+
if (schemeIdx === -1)
|
|
3003
|
+
return uri;
|
|
3004
|
+
const atIdx = uri.indexOf("@", schemeIdx + 3);
|
|
3005
|
+
if (atIdx === -1)
|
|
3006
|
+
return uri;
|
|
3007
|
+
const creds = uri.slice(schemeIdx + 3, atIdx);
|
|
3008
|
+
const colonIdx = creds.indexOf(":");
|
|
3009
|
+
if (colonIdx === -1)
|
|
3010
|
+
return uri;
|
|
3011
|
+
const prefix = uri.slice(0, schemeIdx + 3 + colonIdx + 1);
|
|
3012
|
+
const suffix = uri.slice(atIdx);
|
|
3013
|
+
return `${prefix}***${suffix}`;
|
|
3014
|
+
}
|
|
3015
|
+
function parseIntOrZero(value) {
|
|
3016
|
+
if (!value)
|
|
3017
|
+
return 0;
|
|
3018
|
+
const n = parseInt(value, 10);
|
|
3019
|
+
return Number.isFinite(n) ? n : 0;
|
|
3020
|
+
}
|
|
3021
|
+
var MongoSyncWorker = class {
|
|
3022
|
+
constructor(sqliteStore, config) {
|
|
3023
|
+
this.sqliteStore = sqliteStore;
|
|
3024
|
+
this.config = {
|
|
3025
|
+
uri: config.uri,
|
|
3026
|
+
dbName: config.dbName,
|
|
3027
|
+
projectKey: config.projectKey,
|
|
3028
|
+
direction: config.direction ?? "both",
|
|
3029
|
+
intervalMs: config.intervalMs ?? 3e4,
|
|
3030
|
+
batchSize: config.batchSize ?? 500,
|
|
3031
|
+
instanceId: config.instanceId ?? randomUUID3()
|
|
3032
|
+
};
|
|
3033
|
+
}
|
|
3034
|
+
config;
|
|
3035
|
+
intervalHandle = null;
|
|
3036
|
+
running = false;
|
|
3037
|
+
client = null;
|
|
3038
|
+
db = null;
|
|
3039
|
+
counters = null;
|
|
3040
|
+
events = null;
|
|
3041
|
+
indexesEnsured = false;
|
|
3042
|
+
stats = {
|
|
3043
|
+
lastSyncAt: null,
|
|
3044
|
+
pushedEvents: 0,
|
|
3045
|
+
pulledEvents: 0,
|
|
3046
|
+
errors: 0,
|
|
3047
|
+
status: "idle"
|
|
3048
|
+
};
|
|
3049
|
+
start() {
|
|
3050
|
+
if (this.running)
|
|
3051
|
+
return;
|
|
3052
|
+
this.running = true;
|
|
3053
|
+
this.stats.status = "idle";
|
|
3054
|
+
this.syncNow().catch((err) => {
|
|
3055
|
+
console.error("[MongoSyncWorker] Initial sync failed:", err);
|
|
3056
|
+
});
|
|
3057
|
+
this.intervalHandle = setInterval(() => {
|
|
3058
|
+
this.syncNow().catch((err) => {
|
|
3059
|
+
console.error("[MongoSyncWorker] Periodic sync failed:", err);
|
|
3060
|
+
});
|
|
3061
|
+
}, this.config.intervalMs);
|
|
3062
|
+
}
|
|
3063
|
+
stop() {
|
|
3064
|
+
this.running = false;
|
|
3065
|
+
this.stats.status = "stopped";
|
|
3066
|
+
if (this.intervalHandle) {
|
|
3067
|
+
clearInterval(this.intervalHandle);
|
|
3068
|
+
this.intervalHandle = null;
|
|
3069
|
+
}
|
|
3070
|
+
}
|
|
3071
|
+
async shutdown() {
|
|
3072
|
+
this.stop();
|
|
3073
|
+
await this.disconnect();
|
|
3074
|
+
}
|
|
3075
|
+
getStats() {
|
|
3076
|
+
return { ...this.stats };
|
|
3077
|
+
}
|
|
3078
|
+
isRunning() {
|
|
3079
|
+
return this.running;
|
|
3080
|
+
}
|
|
3081
|
+
async syncNow() {
|
|
3082
|
+
if (this.stats.status === "syncing")
|
|
3083
|
+
return { pushed: 0, pulled: 0 };
|
|
3084
|
+
this.stats.status = "syncing";
|
|
3085
|
+
let pushed = 0;
|
|
3086
|
+
let pulled = 0;
|
|
3087
|
+
try {
|
|
3088
|
+
await this.sqliteStore.initialize();
|
|
3089
|
+
await this.ensureConnected();
|
|
3090
|
+
await this.ensureIndexes();
|
|
3091
|
+
if (this.config.direction === "push" || this.config.direction === "both") {
|
|
3092
|
+
pushed = await this.pushEvents();
|
|
3093
|
+
this.stats.pushedEvents += pushed;
|
|
3094
|
+
}
|
|
3095
|
+
if (this.config.direction === "pull" || this.config.direction === "both") {
|
|
3096
|
+
pulled = await this.pullEvents();
|
|
3097
|
+
this.stats.pulledEvents += pulled;
|
|
3098
|
+
}
|
|
3099
|
+
this.stats.lastSyncAt = /* @__PURE__ */ new Date();
|
|
3100
|
+
this.stats.status = "idle";
|
|
3101
|
+
return { pushed, pulled };
|
|
3102
|
+
} catch (error) {
|
|
3103
|
+
this.stats.errors++;
|
|
3104
|
+
this.stats.status = "error";
|
|
3105
|
+
throw error;
|
|
3106
|
+
}
|
|
3107
|
+
}
|
|
3108
|
+
async ensureConnected() {
|
|
3109
|
+
if (this.client && this.db && this.counters && this.events)
|
|
3110
|
+
return;
|
|
3111
|
+
try {
|
|
3112
|
+
this.client = new MongoClient(this.config.uri, {
|
|
3113
|
+
appName: "claude-memory-layer",
|
|
3114
|
+
serverSelectionTimeoutMS: 5e3
|
|
3115
|
+
});
|
|
3116
|
+
await this.client.connect();
|
|
3117
|
+
this.db = this.client.db(this.config.dbName);
|
|
3118
|
+
this.counters = this.db.collection("cml_counters");
|
|
3119
|
+
this.events = this.db.collection("cml_events");
|
|
3120
|
+
} catch (err) {
|
|
3121
|
+
const safeUri = redactMongoUri(this.config.uri);
|
|
3122
|
+
throw new Error(`MongoDB connection failed (${safeUri}, db=${this.config.dbName}): ${String(err)}`);
|
|
3123
|
+
}
|
|
3124
|
+
}
|
|
3125
|
+
async disconnect() {
|
|
3126
|
+
try {
|
|
3127
|
+
await this.client?.close();
|
|
3128
|
+
} finally {
|
|
3129
|
+
this.client = null;
|
|
3130
|
+
this.db = null;
|
|
3131
|
+
this.counters = null;
|
|
3132
|
+
this.events = null;
|
|
3133
|
+
this.indexesEnsured = false;
|
|
3134
|
+
}
|
|
3135
|
+
}
|
|
3136
|
+
async ensureIndexes() {
|
|
3137
|
+
if (this.indexesEnsured)
|
|
3138
|
+
return;
|
|
3139
|
+
if (!this.events || !this.counters)
|
|
3140
|
+
throw new Error("Mongo not connected");
|
|
3141
|
+
try {
|
|
3142
|
+
await this.events.createIndex({ projectKey: 1, seq: 1 }, { unique: true });
|
|
3143
|
+
await this.events.createIndex({ projectKey: 1, eventId: 1 }, { unique: true });
|
|
3144
|
+
await this.events.createIndex({ projectKey: 1, dedupeKey: 1 });
|
|
3145
|
+
} catch (err) {
|
|
3146
|
+
console.warn("[MongoSyncWorker] Failed to ensure indexes (continuing):", err);
|
|
3147
|
+
}
|
|
3148
|
+
this.indexesEnsured = true;
|
|
3149
|
+
}
|
|
3150
|
+
counterKey(kind) {
|
|
3151
|
+
return `${kind}:${this.config.projectKey}`;
|
|
3152
|
+
}
|
|
3153
|
+
async allocateSeqRange(kind, count) {
|
|
3154
|
+
if (!this.counters)
|
|
3155
|
+
throw new Error("Mongo not connected");
|
|
3156
|
+
if (count <= 0)
|
|
3157
|
+
return 1;
|
|
3158
|
+
const key = this.counterKey(kind);
|
|
3159
|
+
const doc = await this.counters.findOneAndUpdate(
|
|
3160
|
+
{ _id: key },
|
|
3161
|
+
{ $inc: { seq: count } },
|
|
3162
|
+
{ upsert: true, returnDocument: "after" }
|
|
3163
|
+
);
|
|
3164
|
+
const endSeq = doc?.seq;
|
|
3165
|
+
if (typeof endSeq !== "number") {
|
|
3166
|
+
throw new Error(`Failed to allocate seq range for ${key}`);
|
|
3167
|
+
}
|
|
3168
|
+
return endSeq - count + 1;
|
|
3169
|
+
}
|
|
3170
|
+
pushTargetName() {
|
|
3171
|
+
return `mongo_push_events_rowid:${this.config.projectKey}`;
|
|
3172
|
+
}
|
|
3173
|
+
pullTargetName() {
|
|
3174
|
+
return `mongo_pull_events_seq:${this.config.projectKey}`;
|
|
3175
|
+
}
|
|
3176
|
+
async pushEvents() {
|
|
3177
|
+
if (!this.events)
|
|
3178
|
+
throw new Error("Mongo not connected");
|
|
3179
|
+
const position = await this.sqliteStore.getSyncPosition(this.pushTargetName());
|
|
3180
|
+
let lastRowid = parseIntOrZero(position.lastEventId);
|
|
3181
|
+
let pushed = 0;
|
|
3182
|
+
while (true) {
|
|
3183
|
+
const batch = await this.sqliteStore.getEventsSinceRowid(lastRowid, this.config.batchSize);
|
|
3184
|
+
if (batch.length === 0)
|
|
3185
|
+
break;
|
|
3186
|
+
const startSeq = await this.allocateSeqRange("events", batch.length);
|
|
3187
|
+
const now = /* @__PURE__ */ new Date();
|
|
3188
|
+
const hostname2 = os.hostname();
|
|
3189
|
+
const ops = batch.map((item, idx) => {
|
|
3190
|
+
const event = item.event;
|
|
3191
|
+
const seq = startSeq + idx;
|
|
3192
|
+
const docId = `${this.config.projectKey}:${event.id}`;
|
|
3193
|
+
return {
|
|
3194
|
+
updateOne: {
|
|
3195
|
+
filter: { _id: docId },
|
|
3196
|
+
update: {
|
|
3197
|
+
$setOnInsert: {
|
|
3198
|
+
_id: docId,
|
|
3199
|
+
projectKey: this.config.projectKey,
|
|
3200
|
+
seq,
|
|
3201
|
+
eventId: event.id,
|
|
3202
|
+
eventType: event.eventType,
|
|
3203
|
+
sessionId: event.sessionId,
|
|
3204
|
+
timestamp: event.timestamp,
|
|
3205
|
+
content: event.content,
|
|
3206
|
+
canonicalKey: event.canonicalKey,
|
|
3207
|
+
dedupeKey: event.dedupeKey,
|
|
3208
|
+
metadata: event.metadata ?? null,
|
|
3209
|
+
insertedAt: now,
|
|
3210
|
+
updatedAt: now,
|
|
3211
|
+
source: { hostname: hostname2, instanceId: this.config.instanceId }
|
|
3212
|
+
}
|
|
3213
|
+
},
|
|
3214
|
+
upsert: true
|
|
3215
|
+
}
|
|
3216
|
+
};
|
|
3217
|
+
});
|
|
3218
|
+
await this.events.bulkWrite(ops, { ordered: false });
|
|
3219
|
+
const last = batch[batch.length - 1];
|
|
3220
|
+
lastRowid = last.rowid;
|
|
3221
|
+
await this.sqliteStore.updateSyncPosition(
|
|
3222
|
+
this.pushTargetName(),
|
|
3223
|
+
String(lastRowid),
|
|
3224
|
+
last.event.timestamp.toISOString()
|
|
3225
|
+
);
|
|
3226
|
+
pushed += batch.length;
|
|
3227
|
+
if (batch.length < this.config.batchSize)
|
|
3228
|
+
break;
|
|
3229
|
+
}
|
|
3230
|
+
return pushed;
|
|
3231
|
+
}
|
|
3232
|
+
async pullEvents() {
|
|
3233
|
+
if (!this.events)
|
|
3234
|
+
throw new Error("Mongo not connected");
|
|
3235
|
+
const position = await this.sqliteStore.getSyncPosition(this.pullTargetName());
|
|
3236
|
+
let lastSeq = parseIntOrZero(position.lastEventId);
|
|
3237
|
+
let pulled = 0;
|
|
3238
|
+
while (true) {
|
|
3239
|
+
const docs = await this.events.find(
|
|
3240
|
+
{ projectKey: this.config.projectKey, seq: { $gt: lastSeq } },
|
|
3241
|
+
{ sort: { seq: 1 }, limit: this.config.batchSize }
|
|
3242
|
+
).toArray();
|
|
3243
|
+
if (docs.length === 0)
|
|
3244
|
+
break;
|
|
3245
|
+
const events = docs.map((d) => ({
|
|
3246
|
+
id: d.eventId,
|
|
3247
|
+
eventType: d.eventType,
|
|
3248
|
+
sessionId: d.sessionId,
|
|
3249
|
+
timestamp: d.timestamp instanceof Date ? d.timestamp : new Date(d.timestamp),
|
|
3250
|
+
content: d.content,
|
|
3251
|
+
canonicalKey: d.canonicalKey,
|
|
3252
|
+
dedupeKey: d.dedupeKey,
|
|
3253
|
+
metadata: d.metadata ?? void 0
|
|
3254
|
+
}));
|
|
3255
|
+
const result = await this.sqliteStore.importEvents(events);
|
|
3256
|
+
pulled += result.inserted;
|
|
3257
|
+
lastSeq = docs[docs.length - 1].seq;
|
|
3258
|
+
await this.sqliteStore.updateSyncPosition(
|
|
3259
|
+
this.pullTargetName(),
|
|
3260
|
+
String(lastSeq),
|
|
3261
|
+
(/* @__PURE__ */ new Date()).toISOString()
|
|
3262
|
+
);
|
|
3263
|
+
if (docs.length < this.config.batchSize)
|
|
3264
|
+
break;
|
|
3265
|
+
}
|
|
3266
|
+
return pulled;
|
|
3267
|
+
}
|
|
3268
|
+
};
|
|
3269
|
+
|
|
3270
|
+
// src/core/entity-repo.ts
|
|
3271
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
2671
3272
|
var EntityRepo = class {
|
|
2672
3273
|
constructor(db) {
|
|
2673
3274
|
this.db = db;
|
|
@@ -2676,7 +3277,7 @@ var EntityRepo = class {
|
|
|
2676
3277
|
* Create a new entity
|
|
2677
3278
|
*/
|
|
2678
3279
|
async create(input) {
|
|
2679
|
-
const entityId =
|
|
3280
|
+
const entityId = randomUUID4();
|
|
2680
3281
|
const canonicalKey = makeEntityCanonicalKey(input.entityType, input.title, {
|
|
2681
3282
|
project: input.project
|
|
2682
3283
|
});
|
|
@@ -2930,7 +3531,7 @@ var EntityRepo = class {
|
|
|
2930
3531
|
};
|
|
2931
3532
|
|
|
2932
3533
|
// src/core/edge-repo.ts
|
|
2933
|
-
import { randomUUID as
|
|
3534
|
+
import { randomUUID as randomUUID5 } from "crypto";
|
|
2934
3535
|
var EdgeRepo = class {
|
|
2935
3536
|
constructor(db) {
|
|
2936
3537
|
this.db = db;
|
|
@@ -2939,7 +3540,7 @@ var EdgeRepo = class {
|
|
|
2939
3540
|
* Create a new edge (idempotent - ignores duplicates)
|
|
2940
3541
|
*/
|
|
2941
3542
|
async create(input) {
|
|
2942
|
-
const edgeId =
|
|
3543
|
+
const edgeId = randomUUID5();
|
|
2943
3544
|
const now = /* @__PURE__ */ new Date();
|
|
2944
3545
|
await dbRun(
|
|
2945
3546
|
this.db,
|
|
@@ -3430,7 +4031,7 @@ function getDefaultEmbedder() {
|
|
|
3430
4031
|
}
|
|
3431
4032
|
|
|
3432
4033
|
// src/core/vector-outbox.ts
|
|
3433
|
-
import { randomUUID as
|
|
4034
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
3434
4035
|
var DEFAULT_CONFIG2 = {
|
|
3435
4036
|
embeddingVersion: "v1",
|
|
3436
4037
|
maxRetries: 3,
|
|
@@ -3449,7 +4050,7 @@ var VectorOutbox = class {
|
|
|
3449
4050
|
*/
|
|
3450
4051
|
async enqueue(itemKind, itemId, embeddingVersion) {
|
|
3451
4052
|
const version = embeddingVersion ?? this.config.embeddingVersion;
|
|
3452
|
-
const jobId =
|
|
4053
|
+
const jobId = randomUUID6();
|
|
3453
4054
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3454
4055
|
await dbRun(
|
|
3455
4056
|
this.db,
|
|
@@ -4610,7 +5211,20 @@ var DEFAULT_OPTIONS2 = {
|
|
|
4610
5211
|
topK: 5,
|
|
4611
5212
|
minScore: 0.7,
|
|
4612
5213
|
maxTokens: 2e3,
|
|
4613
|
-
includeSessionContext: true
|
|
5214
|
+
includeSessionContext: true,
|
|
5215
|
+
strategy: "auto",
|
|
5216
|
+
rerankWithKeyword: true,
|
|
5217
|
+
decayPolicy: {
|
|
5218
|
+
enabled: true,
|
|
5219
|
+
windowDays: 30,
|
|
5220
|
+
maxPenalty: 0.15
|
|
5221
|
+
},
|
|
5222
|
+
graphHop: {
|
|
5223
|
+
enabled: true,
|
|
5224
|
+
maxHops: 1,
|
|
5225
|
+
hopPenalty: 0.08
|
|
5226
|
+
},
|
|
5227
|
+
projectScopeMode: "global"
|
|
4614
5228
|
};
|
|
4615
5229
|
var Retriever = class {
|
|
4616
5230
|
eventStore;
|
|
@@ -4620,6 +5234,7 @@ var Retriever = class {
|
|
|
4620
5234
|
sharedStore;
|
|
4621
5235
|
sharedVectorStore;
|
|
4622
5236
|
graduation;
|
|
5237
|
+
queryRewriter;
|
|
4623
5238
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
4624
5239
|
this.eventStore = eventStore;
|
|
4625
5240
|
this.vectorStore = vectorStore;
|
|
@@ -4628,47 +5243,105 @@ var Retriever = class {
|
|
|
4628
5243
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
4629
5244
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
4630
5245
|
}
|
|
4631
|
-
/**
|
|
4632
|
-
* Set graduation pipeline for access tracking
|
|
4633
|
-
*/
|
|
4634
5246
|
setGraduationPipeline(graduation) {
|
|
4635
5247
|
this.graduation = graduation;
|
|
4636
5248
|
}
|
|
4637
|
-
/**
|
|
4638
|
-
* Set shared stores after construction
|
|
4639
|
-
*/
|
|
4640
5249
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
4641
5250
|
this.sharedStore = sharedStore;
|
|
4642
5251
|
this.sharedVectorStore = sharedVectorStore;
|
|
4643
5252
|
}
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
5253
|
+
setQueryRewriter(rewriter) {
|
|
5254
|
+
this.queryRewriter = rewriter;
|
|
5255
|
+
}
|
|
4647
5256
|
async retrieve(query, options = {}) {
|
|
4648
5257
|
const opts = { ...DEFAULT_OPTIONS2, ...options };
|
|
4649
|
-
const
|
|
4650
|
-
const
|
|
4651
|
-
|
|
4652
|
-
|
|
5258
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
5259
|
+
const fallbackTrace = [];
|
|
5260
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
5261
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
5262
|
+
let current = await this.runStage(query, {
|
|
5263
|
+
strategy: primaryStrategy,
|
|
5264
|
+
topK: opts.topK,
|
|
4653
5265
|
minScore: opts.minScore,
|
|
4654
|
-
sessionId:
|
|
5266
|
+
sessionId: sessionFilter,
|
|
5267
|
+
scope: opts.scope,
|
|
5268
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
5269
|
+
rerankWeights: opts.rerankWeights,
|
|
5270
|
+
decayPolicy: opts.decayPolicy,
|
|
5271
|
+
intentRewrite: opts.intentRewrite === true,
|
|
5272
|
+
graphHop: opts.graphHop,
|
|
5273
|
+
projectScopeMode: opts.projectScopeMode,
|
|
5274
|
+
projectHash: opts.projectHash,
|
|
5275
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
4655
5276
|
});
|
|
4656
|
-
|
|
4657
|
-
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
|
|
5277
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
5278
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
5279
|
+
current = await this.runStage(query, {
|
|
5280
|
+
strategy: "deep",
|
|
5281
|
+
topK: opts.topK,
|
|
5282
|
+
minScore: opts.minScore,
|
|
5283
|
+
sessionId: sessionFilter,
|
|
5284
|
+
scope: opts.scope,
|
|
5285
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
5286
|
+
rerankWeights: opts.rerankWeights,
|
|
5287
|
+
decayPolicy: opts.decayPolicy,
|
|
5288
|
+
graphHop: opts.graphHop,
|
|
5289
|
+
projectScopeMode: opts.projectScopeMode,
|
|
5290
|
+
projectHash: opts.projectHash,
|
|
5291
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
5292
|
+
});
|
|
5293
|
+
fallbackTrace.push("fallback:deep");
|
|
5294
|
+
}
|
|
5295
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
5296
|
+
current = await this.runStage(query, {
|
|
5297
|
+
strategy: "deep",
|
|
5298
|
+
topK: opts.topK,
|
|
5299
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
5300
|
+
sessionId: void 0,
|
|
5301
|
+
scope: void 0,
|
|
5302
|
+
rerankWithKeyword: true,
|
|
5303
|
+
rerankWeights: opts.rerankWeights,
|
|
5304
|
+
decayPolicy: opts.decayPolicy,
|
|
5305
|
+
graphHop: opts.graphHop,
|
|
5306
|
+
projectScopeMode: opts.projectScopeMode,
|
|
5307
|
+
projectHash: opts.projectHash,
|
|
5308
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
5309
|
+
});
|
|
5310
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
5311
|
+
}
|
|
5312
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
5313
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
5314
|
+
current = {
|
|
5315
|
+
results: summary,
|
|
5316
|
+
candidateResults: summary,
|
|
5317
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
5318
|
+
};
|
|
5319
|
+
fallbackTrace.push("fallback:summary");
|
|
5320
|
+
}
|
|
5321
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
4661
5322
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
4662
5323
|
return {
|
|
4663
5324
|
memories,
|
|
4664
|
-
matchResult,
|
|
5325
|
+
matchResult: current.matchResult,
|
|
4665
5326
|
totalTokens: this.estimateTokens(context),
|
|
4666
|
-
context
|
|
5327
|
+
context,
|
|
5328
|
+
fallbackTrace,
|
|
5329
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
5330
|
+
eventId: r.eventId,
|
|
5331
|
+
score: r.score,
|
|
5332
|
+
semanticScore: r.semanticScore,
|
|
5333
|
+
lexicalScore: r.lexicalScore,
|
|
5334
|
+
recencyScore: r.recencyScore
|
|
5335
|
+
})),
|
|
5336
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
5337
|
+
eventId: r.eventId,
|
|
5338
|
+
score: r.score,
|
|
5339
|
+
semanticScore: r.semanticScore,
|
|
5340
|
+
lexicalScore: r.lexicalScore,
|
|
5341
|
+
recencyScore: r.recencyScore
|
|
5342
|
+
}))
|
|
4667
5343
|
};
|
|
4668
5344
|
}
|
|
4669
|
-
/**
|
|
4670
|
-
* Retrieve with unified search (project + shared)
|
|
4671
|
-
*/
|
|
4672
5345
|
async retrieveUnified(query, options = {}) {
|
|
4673
5346
|
const projectResult = await this.retrieve(query, options);
|
|
4674
5347
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -4676,22 +5349,19 @@ var Retriever = class {
|
|
|
4676
5349
|
}
|
|
4677
5350
|
try {
|
|
4678
5351
|
const queryEmbedding = await this.embedder.embed(query);
|
|
4679
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
excludeProjectHash: options.projectHash
|
|
4685
|
-
}
|
|
4686
|
-
);
|
|
5352
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
5353
|
+
limit: options.topK || 5,
|
|
5354
|
+
minScore: options.minScore || 0.7,
|
|
5355
|
+
excludeProjectHash: options.projectHash
|
|
5356
|
+
});
|
|
4687
5357
|
const sharedMemories = [];
|
|
4688
5358
|
for (const result of sharedVectorResults) {
|
|
4689
5359
|
const entry = await this.sharedStore.get(result.entryId);
|
|
4690
|
-
if (entry)
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
5360
|
+
if (!entry)
|
|
5361
|
+
continue;
|
|
5362
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
5363
|
+
sharedMemories.push(entry);
|
|
5364
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
4695
5365
|
}
|
|
4696
5366
|
}
|
|
4697
5367
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -4706,50 +5376,243 @@ var Retriever = class {
|
|
|
4706
5376
|
return projectResult;
|
|
4707
5377
|
}
|
|
4708
5378
|
}
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
5379
|
+
async runStage(query, input) {
|
|
5380
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
5381
|
+
strategy: input.strategy,
|
|
5382
|
+
topK: input.topK,
|
|
5383
|
+
minScore: input.minScore,
|
|
5384
|
+
sessionId: input.sessionId
|
|
5385
|
+
});
|
|
5386
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
5387
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
5388
|
+
if (rewritten && rewritten !== query) {
|
|
5389
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
5390
|
+
strategy: "deep",
|
|
5391
|
+
topK: input.topK,
|
|
5392
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
5393
|
+
sessionId: input.sessionId
|
|
5394
|
+
});
|
|
5395
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
5396
|
+
}
|
|
5397
|
+
}
|
|
5398
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
5399
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
5400
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
5401
|
+
limit: input.topK * 4
|
|
5402
|
+
});
|
|
5403
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
5404
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
5405
|
+
scope: input.scope,
|
|
5406
|
+
projectScopeMode: input.projectScopeMode,
|
|
5407
|
+
projectHash: input.projectHash,
|
|
5408
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
5409
|
+
});
|
|
5410
|
+
const top = filtered.slice(0, input.topK);
|
|
5411
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
5412
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
5413
|
+
}
|
|
5414
|
+
mergeResults(primary, secondary, limit) {
|
|
5415
|
+
const byId = /* @__PURE__ */ new Map();
|
|
5416
|
+
for (const row of primary)
|
|
5417
|
+
byId.set(row.eventId, row);
|
|
5418
|
+
for (const row of secondary) {
|
|
5419
|
+
const prev = byId.get(row.eventId);
|
|
5420
|
+
if (!prev || row.score > prev.score) {
|
|
5421
|
+
byId.set(row.eventId, row);
|
|
5422
|
+
}
|
|
5423
|
+
}
|
|
5424
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
5425
|
+
}
|
|
5426
|
+
async expandGraphHops(seeds, opts) {
|
|
5427
|
+
const byId = /* @__PURE__ */ new Map();
|
|
5428
|
+
for (const s of seeds)
|
|
5429
|
+
byId.set(s.eventId, s);
|
|
5430
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
5431
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
5432
|
+
const next = [];
|
|
5433
|
+
for (const f of frontier) {
|
|
5434
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
5435
|
+
if (!ev)
|
|
5436
|
+
continue;
|
|
5437
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
5438
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
5439
|
+
for (const rid of relatedIds) {
|
|
5440
|
+
if (byId.has(rid))
|
|
5441
|
+
continue;
|
|
5442
|
+
const target = await this.eventStore.getEvent(rid);
|
|
5443
|
+
if (!target)
|
|
5444
|
+
continue;
|
|
5445
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
5446
|
+
const row = {
|
|
5447
|
+
id: `hop-${hop}-${rid}`,
|
|
5448
|
+
eventId: target.id,
|
|
5449
|
+
content: target.content,
|
|
5450
|
+
score,
|
|
5451
|
+
sessionId: target.sessionId,
|
|
5452
|
+
eventType: target.eventType,
|
|
5453
|
+
timestamp: target.timestamp.toISOString()
|
|
5454
|
+
};
|
|
5455
|
+
byId.set(row.eventId, row);
|
|
5456
|
+
next.push({ row, hop });
|
|
5457
|
+
if (byId.size >= opts.limit)
|
|
5458
|
+
break;
|
|
4730
5459
|
}
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
`;
|
|
5460
|
+
if (byId.size >= opts.limit)
|
|
5461
|
+
break;
|
|
4734
5462
|
}
|
|
5463
|
+
frontier = next;
|
|
5464
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
5465
|
+
break;
|
|
4735
5466
|
}
|
|
4736
|
-
return
|
|
5467
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
5468
|
+
}
|
|
5469
|
+
shouldFallback(matchResult, results) {
|
|
5470
|
+
if (results.length === 0)
|
|
5471
|
+
return true;
|
|
5472
|
+
if (matchResult.confidence === "none")
|
|
5473
|
+
return true;
|
|
5474
|
+
return false;
|
|
5475
|
+
}
|
|
5476
|
+
async buildSummaryFallback(query, topK) {
|
|
5477
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
5478
|
+
const q = this.tokenize(query);
|
|
5479
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
5480
|
+
id: `summary-${row.e.id}`,
|
|
5481
|
+
eventId: row.e.id,
|
|
5482
|
+
content: row.e.content,
|
|
5483
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
5484
|
+
sessionId: row.e.sessionId,
|
|
5485
|
+
eventType: row.e.eventType,
|
|
5486
|
+
timestamp: row.e.timestamp.toISOString()
|
|
5487
|
+
}));
|
|
5488
|
+
return ranked;
|
|
5489
|
+
}
|
|
5490
|
+
async searchByStrategy(query, input) {
|
|
5491
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
5492
|
+
if (strategy === "fast") {
|
|
5493
|
+
const keyword = await this.searchByKeyword(query, {
|
|
5494
|
+
limit: Math.max(5, input.topK * 3),
|
|
5495
|
+
sessionId: input.sessionId
|
|
5496
|
+
});
|
|
5497
|
+
return keyword;
|
|
5498
|
+
}
|
|
5499
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
5500
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
5501
|
+
limit: Math.max(5, input.topK * 3),
|
|
5502
|
+
minScore: input.minScore,
|
|
5503
|
+
sessionId: input.sessionId
|
|
5504
|
+
});
|
|
5505
|
+
}
|
|
5506
|
+
async searchByKeyword(query, input) {
|
|
5507
|
+
if (this.eventStore.keywordSearch) {
|
|
5508
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
5509
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
5510
|
+
return filtered2.map((row, idx) => ({
|
|
5511
|
+
id: `kw-${row.event.id}`,
|
|
5512
|
+
eventId: row.event.id,
|
|
5513
|
+
content: row.event.content,
|
|
5514
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
5515
|
+
sessionId: row.event.sessionId,
|
|
5516
|
+
eventType: row.event.eventType,
|
|
5517
|
+
timestamp: row.event.timestamp.toISOString()
|
|
5518
|
+
}));
|
|
5519
|
+
}
|
|
5520
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
5521
|
+
const tokens = this.tokenize(query);
|
|
5522
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
5523
|
+
return filtered.map((row, idx) => ({
|
|
5524
|
+
id: `kw-fallback-${row.e.id}`,
|
|
5525
|
+
eventId: row.e.id,
|
|
5526
|
+
content: row.e.content,
|
|
5527
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
5528
|
+
sessionId: row.e.sessionId,
|
|
5529
|
+
eventType: row.e.eventType,
|
|
5530
|
+
timestamp: row.e.timestamp.toISOString()
|
|
5531
|
+
}));
|
|
5532
|
+
}
|
|
5533
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
5534
|
+
const q = this.tokenize(query);
|
|
5535
|
+
const now = Date.now();
|
|
5536
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
5537
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
5538
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
5539
|
+
const total = sw + lw + rw || 1;
|
|
5540
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
5541
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
5542
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
5543
|
+
return [...results].map((r) => {
|
|
5544
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
5545
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
5546
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
5547
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
5548
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
5549
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
5550
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
5551
|
+
}
|
|
5552
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
5553
|
+
}).sort((a, b) => b.score - a.score);
|
|
5554
|
+
}
|
|
5555
|
+
async applyScopeFilters(results, options) {
|
|
5556
|
+
const scope = options?.scope;
|
|
5557
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
5558
|
+
const allowedProjectHashes = new Set(
|
|
5559
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
5560
|
+
(value) => typeof value === "string" && value.length > 0
|
|
5561
|
+
)
|
|
5562
|
+
);
|
|
5563
|
+
if (!scope && projectScopeMode === "global")
|
|
5564
|
+
return results;
|
|
5565
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
5566
|
+
const filtered = [];
|
|
5567
|
+
for (const result of results) {
|
|
5568
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
5569
|
+
continue;
|
|
5570
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
5571
|
+
continue;
|
|
5572
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
5573
|
+
continue;
|
|
5574
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
5575
|
+
if (!event)
|
|
5576
|
+
continue;
|
|
5577
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
5578
|
+
continue;
|
|
5579
|
+
if (normalizedIncludes.length > 0) {
|
|
5580
|
+
const lc = event.content.toLowerCase();
|
|
5581
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
5582
|
+
continue;
|
|
5583
|
+
}
|
|
5584
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
5585
|
+
continue;
|
|
5586
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
5587
|
+
filtered.push({ result, projectHash });
|
|
5588
|
+
}
|
|
5589
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
5590
|
+
return filtered.map((x) => x.result);
|
|
5591
|
+
}
|
|
5592
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
5593
|
+
if (projectScopeMode === "strict") {
|
|
5594
|
+
return projectMatched.map((x) => x.result);
|
|
5595
|
+
}
|
|
5596
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
5597
|
+
}
|
|
5598
|
+
extractProjectHash(metadata) {
|
|
5599
|
+
if (!metadata || typeof metadata !== "object")
|
|
5600
|
+
return void 0;
|
|
5601
|
+
const scope = metadata.scope;
|
|
5602
|
+
if (!scope || typeof scope !== "object")
|
|
5603
|
+
return void 0;
|
|
5604
|
+
const project = scope.project;
|
|
5605
|
+
if (!project || typeof project !== "object")
|
|
5606
|
+
return void 0;
|
|
5607
|
+
const hash = project.hash;
|
|
5608
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
4737
5609
|
}
|
|
4738
|
-
/**
|
|
4739
|
-
* Retrieve memories from a specific session
|
|
4740
|
-
*/
|
|
4741
5610
|
async retrieveFromSession(sessionId) {
|
|
4742
5611
|
return this.eventStore.getSessionEvents(sessionId);
|
|
4743
5612
|
}
|
|
4744
|
-
/**
|
|
4745
|
-
* Get recent memories across all sessions
|
|
4746
|
-
*/
|
|
4747
5613
|
async retrieveRecent(limit = 100) {
|
|
4748
5614
|
return this.eventStore.getRecentEvents(limit);
|
|
4749
5615
|
}
|
|
4750
|
-
/**
|
|
4751
|
-
* Enrich search results with full event data
|
|
4752
|
-
*/
|
|
4753
5616
|
async enrichResults(results, options) {
|
|
4754
5617
|
const memories = [];
|
|
4755
5618
|
for (const result of results) {
|
|
@@ -4757,27 +5620,16 @@ var Retriever = class {
|
|
|
4757
5620
|
if (!event)
|
|
4758
5621
|
continue;
|
|
4759
5622
|
if (this.graduation) {
|
|
4760
|
-
this.graduation.recordAccess(
|
|
4761
|
-
event.id,
|
|
4762
|
-
options.sessionId || "unknown",
|
|
4763
|
-
result.score
|
|
4764
|
-
);
|
|
5623
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
4765
5624
|
}
|
|
4766
5625
|
let sessionContext;
|
|
4767
5626
|
if (options.includeSessionContext) {
|
|
4768
5627
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
4769
5628
|
}
|
|
4770
|
-
memories.push({
|
|
4771
|
-
event,
|
|
4772
|
-
score: result.score,
|
|
4773
|
-
sessionContext
|
|
4774
|
-
});
|
|
5629
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
4775
5630
|
}
|
|
4776
5631
|
return memories;
|
|
4777
5632
|
}
|
|
4778
|
-
/**
|
|
4779
|
-
* Get surrounding context from the same session
|
|
4780
|
-
*/
|
|
4781
5633
|
async getSessionContext(sessionId, eventId) {
|
|
4782
5634
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
4783
5635
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -4790,55 +5642,86 @@ var Retriever = class {
|
|
|
4790
5642
|
return void 0;
|
|
4791
5643
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
4792
5644
|
}
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
|
|
5645
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
5646
|
+
let context = projectResult.context;
|
|
5647
|
+
if (sharedMemories.length === 0)
|
|
5648
|
+
return context;
|
|
5649
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
5650
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
5651
|
+
context += `### ${memory.title}
|
|
5652
|
+
`;
|
|
5653
|
+
if (memory.symptoms.length > 0)
|
|
5654
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
5655
|
+
`;
|
|
5656
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
5657
|
+
`;
|
|
5658
|
+
context += `**Solution:** ${memory.solution}
|
|
5659
|
+
`;
|
|
5660
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
5661
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
5662
|
+
`;
|
|
5663
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
5664
|
+
|
|
5665
|
+
`;
|
|
5666
|
+
}
|
|
5667
|
+
return context;
|
|
5668
|
+
}
|
|
4796
5669
|
buildContext(memories, maxTokens) {
|
|
4797
5670
|
const parts = [];
|
|
4798
5671
|
let currentTokens = 0;
|
|
4799
5672
|
for (const memory of memories) {
|
|
4800
5673
|
const memoryText = this.formatMemory(memory);
|
|
4801
5674
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
4802
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
5675
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
4803
5676
|
break;
|
|
4804
|
-
}
|
|
4805
5677
|
parts.push(memoryText);
|
|
4806
5678
|
currentTokens += memoryTokens;
|
|
4807
5679
|
}
|
|
4808
|
-
if (parts.length === 0)
|
|
5680
|
+
if (parts.length === 0)
|
|
4809
5681
|
return "";
|
|
4810
|
-
}
|
|
4811
5682
|
return `## Relevant Memories
|
|
4812
5683
|
|
|
4813
5684
|
${parts.join("\n\n---\n\n")}`;
|
|
4814
5685
|
}
|
|
4815
|
-
/**
|
|
4816
|
-
* Format a single memory for context
|
|
4817
|
-
*/
|
|
4818
5686
|
formatMemory(memory) {
|
|
4819
5687
|
const { event, score, sessionContext } = memory;
|
|
4820
5688
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
4821
5689
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
4822
5690
|
${event.content}`;
|
|
4823
|
-
if (sessionContext)
|
|
5691
|
+
if (sessionContext)
|
|
4824
5692
|
text += `
|
|
4825
5693
|
|
|
4826
5694
|
_Context:_ ${sessionContext}`;
|
|
4827
|
-
}
|
|
4828
5695
|
return text;
|
|
4829
5696
|
}
|
|
4830
|
-
|
|
4831
|
-
|
|
4832
|
-
|
|
5697
|
+
matchesMetadataScope(metadata, expected) {
|
|
5698
|
+
if (!metadata)
|
|
5699
|
+
return false;
|
|
5700
|
+
return Object.entries(expected).every(([path2, value]) => {
|
|
5701
|
+
const actual = path2.split(".").reduce((acc, key) => {
|
|
5702
|
+
if (typeof acc !== "object" || acc === null)
|
|
5703
|
+
return void 0;
|
|
5704
|
+
return acc[key];
|
|
5705
|
+
}, metadata);
|
|
5706
|
+
return actual === value;
|
|
5707
|
+
});
|
|
5708
|
+
}
|
|
5709
|
+
tokenize(text) {
|
|
5710
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
5711
|
+
}
|
|
5712
|
+
keywordOverlap(a, b) {
|
|
5713
|
+
if (a.length === 0 || b.length === 0)
|
|
5714
|
+
return 0;
|
|
5715
|
+
const bs = new Set(b);
|
|
5716
|
+
let hit = 0;
|
|
5717
|
+
for (const t of a)
|
|
5718
|
+
if (bs.has(t))
|
|
5719
|
+
hit += 1;
|
|
5720
|
+
return hit / a.length;
|
|
5721
|
+
}
|
|
4833
5722
|
estimateTokens(text) {
|
|
4834
5723
|
return Math.ceil(text.length / 4);
|
|
4835
5724
|
}
|
|
4836
|
-
/**
|
|
4837
|
-
* Get event age in days (for recency scoring)
|
|
4838
|
-
*/
|
|
4839
|
-
getEventAgeDays(eventId) {
|
|
4840
|
-
return 0;
|
|
4841
|
-
}
|
|
4842
5725
|
};
|
|
4843
5726
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
4844
5727
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -5403,7 +6286,7 @@ var TaskMatcher = class {
|
|
|
5403
6286
|
};
|
|
5404
6287
|
|
|
5405
6288
|
// src/core/task/blocker-resolver.ts
|
|
5406
|
-
import { randomUUID as
|
|
6289
|
+
import { randomUUID as randomUUID7 } from "crypto";
|
|
5407
6290
|
var URL_PATTERN = /^https?:\/\/.+/;
|
|
5408
6291
|
var JIRA_PATTERN = /^[A-Z]+-\d+$/;
|
|
5409
6292
|
var GITHUB_ISSUE_PATTERN = /^[^\/]+\/[^#]+#\d+$/;
|
|
@@ -5540,7 +6423,7 @@ var BlockerResolver = class {
|
|
|
5540
6423
|
* Declare a new condition entity
|
|
5541
6424
|
*/
|
|
5542
6425
|
async declareCondition(text, canonicalKey, candidates) {
|
|
5543
|
-
const entityId =
|
|
6426
|
+
const entityId = randomUUID7();
|
|
5544
6427
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5545
6428
|
const currentJson = {
|
|
5546
6429
|
text,
|
|
@@ -5583,7 +6466,7 @@ var BlockerResolver = class {
|
|
|
5583
6466
|
* Declare a new artifact entity
|
|
5584
6467
|
*/
|
|
5585
6468
|
async declareArtifact(identifier, canonicalKey) {
|
|
5586
|
-
const entityId =
|
|
6469
|
+
const entityId = randomUUID7();
|
|
5587
6470
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5588
6471
|
let artifactType = "generic";
|
|
5589
6472
|
if (URL_PATTERN.test(identifier)) {
|
|
@@ -5648,7 +6531,7 @@ var BlockerResolver = class {
|
|
|
5648
6531
|
};
|
|
5649
6532
|
|
|
5650
6533
|
// src/core/task/task-resolver.ts
|
|
5651
|
-
import { randomUUID as
|
|
6534
|
+
import { randomUUID as randomUUID8 } from "crypto";
|
|
5652
6535
|
var VALID_TRANSITIONS = {
|
|
5653
6536
|
pending: ["in_progress", "cancelled"],
|
|
5654
6537
|
in_progress: ["blocked", "done", "cancelled"],
|
|
@@ -5723,7 +6606,7 @@ var TaskResolver = class {
|
|
|
5723
6606
|
isNew: false
|
|
5724
6607
|
};
|
|
5725
6608
|
}
|
|
5726
|
-
const taskId =
|
|
6609
|
+
const taskId = randomUUID8();
|
|
5727
6610
|
const canonicalKey = makeEntityCanonicalKey("task", extracted.title, {
|
|
5728
6611
|
project: extracted.project
|
|
5729
6612
|
});
|
|
@@ -5876,7 +6759,7 @@ var TaskResolver = class {
|
|
|
5876
6759
|
* Emit task event to events table
|
|
5877
6760
|
*/
|
|
5878
6761
|
async emitTaskEvent(eventType, payload) {
|
|
5879
|
-
const eventId =
|
|
6762
|
+
const eventId = randomUUID8();
|
|
5880
6763
|
const now = /* @__PURE__ */ new Date();
|
|
5881
6764
|
const dedupeKey = makeTaskEventDedupeKey(
|
|
5882
6765
|
eventType,
|
|
@@ -5934,14 +6817,14 @@ var TaskResolver = class {
|
|
|
5934
6817
|
`INSERT INTO edges (edge_id, src_type, src_id, rel_type, dst_type, dst_id, meta_json)
|
|
5935
6818
|
VALUES (?, 'entity', ?, 'resolves_to', 'entity', ?, ?)
|
|
5936
6819
|
ON CONFLICT DO NOTHING`,
|
|
5937
|
-
[
|
|
6820
|
+
[randomUUID8(), conditionId, taskId, JSON.stringify({ resolved_at: (/* @__PURE__ */ new Date()).toISOString() })]
|
|
5938
6821
|
);
|
|
5939
6822
|
return eventId;
|
|
5940
6823
|
}
|
|
5941
6824
|
};
|
|
5942
6825
|
|
|
5943
6826
|
// src/core/task/task-projector.ts
|
|
5944
|
-
import { randomUUID as
|
|
6827
|
+
import { randomUUID as randomUUID9 } from "crypto";
|
|
5945
6828
|
var PROJECTOR_NAME = "task_projector";
|
|
5946
6829
|
var TASK_EVENT_TYPES = [
|
|
5947
6830
|
"task_created",
|
|
@@ -6137,7 +7020,7 @@ var TaskProjector = class {
|
|
|
6137
7020
|
* Create blocker edge
|
|
6138
7021
|
*/
|
|
6139
7022
|
async createBlockerEdge(taskId, blocker, relType) {
|
|
6140
|
-
const edgeId =
|
|
7023
|
+
const edgeId = randomUUID9();
|
|
6141
7024
|
await dbRun(
|
|
6142
7025
|
this.db,
|
|
6143
7026
|
`INSERT INTO edges (edge_id, src_type, src_id, rel_type, dst_type, dst_id, meta_json, created_at)
|
|
@@ -6175,7 +7058,7 @@ var TaskProjector = class {
|
|
|
6175
7058
|
* Enqueue entity for vectorization
|
|
6176
7059
|
*/
|
|
6177
7060
|
async enqueueForVectorization(itemId, itemKind) {
|
|
6178
|
-
const jobId =
|
|
7061
|
+
const jobId = randomUUID9();
|
|
6179
7062
|
const embeddingVersion = "v1";
|
|
6180
7063
|
await dbRun(
|
|
6181
7064
|
this.db,
|
|
@@ -6295,7 +7178,7 @@ function createSharedEventStore(dbPath) {
|
|
|
6295
7178
|
}
|
|
6296
7179
|
|
|
6297
7180
|
// src/core/shared-store.ts
|
|
6298
|
-
import { randomUUID as
|
|
7181
|
+
import { randomUUID as randomUUID10 } from "crypto";
|
|
6299
7182
|
var SharedStore = class {
|
|
6300
7183
|
constructor(sharedEventStore) {
|
|
6301
7184
|
this.sharedEventStore = sharedEventStore;
|
|
@@ -6307,7 +7190,7 @@ var SharedStore = class {
|
|
|
6307
7190
|
* Promote a verified troubleshooting entry to shared storage
|
|
6308
7191
|
*/
|
|
6309
7192
|
async promoteEntry(input) {
|
|
6310
|
-
const entryId =
|
|
7193
|
+
const entryId = randomUUID10();
|
|
6311
7194
|
await dbRun(
|
|
6312
7195
|
this.db,
|
|
6313
7196
|
`INSERT INTO shared_troubleshooting (
|
|
@@ -6672,7 +7555,7 @@ function createSharedVectorStore(dbPath) {
|
|
|
6672
7555
|
}
|
|
6673
7556
|
|
|
6674
7557
|
// src/core/shared-promoter.ts
|
|
6675
|
-
import { randomUUID as
|
|
7558
|
+
import { randomUUID as randomUUID11 } from "crypto";
|
|
6676
7559
|
var SharedPromoter = class {
|
|
6677
7560
|
constructor(sharedStore, sharedVectorStore, embedder, config) {
|
|
6678
7561
|
this.sharedStore = sharedStore;
|
|
@@ -6740,7 +7623,7 @@ var SharedPromoter = class {
|
|
|
6740
7623
|
const embeddingContent = this.createEmbeddingContent(input);
|
|
6741
7624
|
const embedding = await this.embedder.embed(embeddingContent);
|
|
6742
7625
|
await this.sharedVectorStore.upsert({
|
|
6743
|
-
id:
|
|
7626
|
+
id: randomUUID11(),
|
|
6744
7627
|
entryId,
|
|
6745
7628
|
entryType: "troubleshooting",
|
|
6746
7629
|
content: embeddingContent,
|
|
@@ -6865,6 +7748,7 @@ export {
|
|
|
6865
7748
|
CitationUsageSchema,
|
|
6866
7749
|
ConfigSchema,
|
|
6867
7750
|
ConsolidatedMemorySchema,
|
|
7751
|
+
ConsolidationRuleSchema,
|
|
6868
7752
|
ContinuityLogSchema,
|
|
6869
7753
|
DefaultContentProvider,
|
|
6870
7754
|
EdgeRepo,
|
|
@@ -6901,6 +7785,7 @@ export {
|
|
|
6901
7785
|
MemoryLevelSchema,
|
|
6902
7786
|
MemoryMatchSchema,
|
|
6903
7787
|
MemoryModeSchema,
|
|
7788
|
+
MongoSyncWorker,
|
|
6904
7789
|
NodeTypeSchema,
|
|
6905
7790
|
OutboxItemKindSchema,
|
|
6906
7791
|
OutboxJobSchema,
|