claude-memory-layer 1.0.11 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +2389 -286
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1017 -132
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1347 -202
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1339 -194
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1343 -198
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1351 -206
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1347 -202
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +1436 -211
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +1445 -220
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1345 -199
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +69 -2
- package/dist/ui/index.html +8 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +2 -1
- package/scripts/build.ts +6 -0
- package/scripts/bump-patch-version.sh +18 -0
- package/src/cli/index.ts +281 -2
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +350 -1
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/types.ts +28 -0
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +3 -1
- package/src/server/api/stats.ts +46 -1
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +373 -68
- package/src/services/session-history-importer.ts +53 -25
- package/src/ui/app.js +69 -2
- package/src/ui/index.html +8 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
|
@@ -6,9 +6,9 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
6
6
|
const __dirname = dirname(__filename);
|
|
7
7
|
|
|
8
8
|
// src/services/memory-service.ts
|
|
9
|
-
import * as
|
|
9
|
+
import * as path3 from "path";
|
|
10
10
|
import * as os from "os";
|
|
11
|
-
import * as
|
|
11
|
+
import * as fs4 from "fs";
|
|
12
12
|
import * as crypto2 from "crypto";
|
|
13
13
|
|
|
14
14
|
// src/core/event-store.ts
|
|
@@ -66,11 +66,11 @@ function toDate(value) {
|
|
|
66
66
|
return new Date(value);
|
|
67
67
|
return new Date(String(value));
|
|
68
68
|
}
|
|
69
|
-
function createDatabase(
|
|
69
|
+
function createDatabase(path4, options) {
|
|
70
70
|
if (options?.readOnly) {
|
|
71
|
-
return new duckdb.Database(
|
|
71
|
+
return new duckdb.Database(path4, { access_mode: "READ_ONLY" });
|
|
72
72
|
}
|
|
73
|
-
return new duckdb.Database(
|
|
73
|
+
return new duckdb.Database(path4);
|
|
74
74
|
}
|
|
75
75
|
function dbRun(db, sql, params = []) {
|
|
76
76
|
return new Promise((resolve2, reject) => {
|
|
@@ -334,6 +334,17 @@ var EventStore = class {
|
|
|
334
334
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
335
335
|
)
|
|
336
336
|
`);
|
|
337
|
+
await dbRun(this.db, `
|
|
338
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
339
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
340
|
+
rule TEXT NOT NULL,
|
|
341
|
+
topics JSON,
|
|
342
|
+
source_memory_ids JSON,
|
|
343
|
+
source_events JSON,
|
|
344
|
+
confidence FLOAT DEFAULT 0.5,
|
|
345
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
346
|
+
)
|
|
347
|
+
`);
|
|
337
348
|
await dbRun(this.db, `
|
|
338
349
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
339
350
|
key VARCHAR PRIMARY KEY,
|
|
@@ -353,6 +364,7 @@ var EventStore = class {
|
|
|
353
364
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
354
365
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
355
366
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
367
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
356
368
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
357
369
|
this.initialized = true;
|
|
358
370
|
}
|
|
@@ -742,12 +754,12 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
742
754
|
import Database from "better-sqlite3";
|
|
743
755
|
import * as fs from "fs";
|
|
744
756
|
import * as nodePath from "path";
|
|
745
|
-
function createSQLiteDatabase(
|
|
746
|
-
const dir = nodePath.dirname(
|
|
757
|
+
function createSQLiteDatabase(path4, options) {
|
|
758
|
+
const dir = nodePath.dirname(path4);
|
|
747
759
|
if (!fs.existsSync(dir)) {
|
|
748
760
|
fs.mkdirSync(dir, { recursive: true });
|
|
749
761
|
}
|
|
750
|
-
const db = new Database(
|
|
762
|
+
const db = new Database(path4, {
|
|
751
763
|
readonly: options?.readonly ?? false
|
|
752
764
|
});
|
|
753
765
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -788,6 +800,64 @@ function toSQLiteTimestamp(date) {
|
|
|
788
800
|
return date.toISOString();
|
|
789
801
|
}
|
|
790
802
|
|
|
803
|
+
// src/core/markdown-mirror.ts
|
|
804
|
+
import * as fs2 from "fs/promises";
|
|
805
|
+
import * as path from "path";
|
|
806
|
+
var DEFAULT_NAMESPACE = "default";
|
|
807
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
808
|
+
function sanitizeSegment(input, fallback) {
|
|
809
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
810
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
811
|
+
if (!safe || safe === "." || safe === "..")
|
|
812
|
+
return fallback;
|
|
813
|
+
return safe;
|
|
814
|
+
}
|
|
815
|
+
function getCategorySegments(metadata, eventType) {
|
|
816
|
+
const raw = metadata?.categoryPath;
|
|
817
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
818
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
819
|
+
}
|
|
820
|
+
const single = metadata?.category;
|
|
821
|
+
if (typeof single === "string" && single.trim()) {
|
|
822
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
823
|
+
}
|
|
824
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
825
|
+
}
|
|
826
|
+
function buildMirrorPath(rootDir, event) {
|
|
827
|
+
const metadata = event.metadata;
|
|
828
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
829
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
830
|
+
const d = event.timestamp;
|
|
831
|
+
const yyyy = d.getFullYear();
|
|
832
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
833
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
834
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
835
|
+
}
|
|
836
|
+
function formatMirrorEntry(event) {
|
|
837
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
838
|
+
return [
|
|
839
|
+
"",
|
|
840
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
841
|
+
` id: ${event.id}`,
|
|
842
|
+
` type: ${event.eventType}`,
|
|
843
|
+
` session: ${event.sessionId}`,
|
|
844
|
+
` category: ${category}`,
|
|
845
|
+
" content: |",
|
|
846
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
847
|
+
].join("\n") + "\n";
|
|
848
|
+
}
|
|
849
|
+
var MarkdownMirror = class {
|
|
850
|
+
constructor(rootDir) {
|
|
851
|
+
this.rootDir = rootDir;
|
|
852
|
+
}
|
|
853
|
+
async append(event) {
|
|
854
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
855
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
856
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
857
|
+
return outPath;
|
|
858
|
+
}
|
|
859
|
+
};
|
|
860
|
+
|
|
791
861
|
// src/core/sqlite-event-store.ts
|
|
792
862
|
var SQLiteEventStore = class {
|
|
793
863
|
constructor(dbPath, options) {
|
|
@@ -797,10 +867,12 @@ var SQLiteEventStore = class {
|
|
|
797
867
|
readonly: this.readOnly,
|
|
798
868
|
walMode: !this.readOnly
|
|
799
869
|
});
|
|
870
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
800
871
|
}
|
|
801
872
|
db;
|
|
802
873
|
initialized = false;
|
|
803
874
|
readOnly;
|
|
875
|
+
markdownMirror;
|
|
804
876
|
/**
|
|
805
877
|
* Initialize database schema
|
|
806
878
|
*/
|
|
@@ -1007,6 +1079,17 @@ var SQLiteEventStore = class {
|
|
|
1007
1079
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1008
1080
|
);
|
|
1009
1081
|
|
|
1082
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1083
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1084
|
+
rule_id TEXT PRIMARY KEY,
|
|
1085
|
+
rule TEXT NOT NULL,
|
|
1086
|
+
topics TEXT,
|
|
1087
|
+
source_memory_ids TEXT,
|
|
1088
|
+
source_events TEXT,
|
|
1089
|
+
confidence REAL DEFAULT 0.5,
|
|
1090
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1091
|
+
);
|
|
1092
|
+
|
|
1010
1093
|
-- Endless Mode Config table
|
|
1011
1094
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1012
1095
|
key TEXT PRIMARY KEY,
|
|
@@ -1031,6 +1114,24 @@ var SQLiteEventStore = class {
|
|
|
1031
1114
|
measured_at TEXT
|
|
1032
1115
|
);
|
|
1033
1116
|
|
|
1117
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1118
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1119
|
+
trace_id TEXT PRIMARY KEY,
|
|
1120
|
+
session_id TEXT,
|
|
1121
|
+
project_hash TEXT,
|
|
1122
|
+
query_text TEXT NOT NULL,
|
|
1123
|
+
strategy TEXT,
|
|
1124
|
+
candidate_event_ids TEXT,
|
|
1125
|
+
selected_event_ids TEXT,
|
|
1126
|
+
candidate_details_json TEXT,
|
|
1127
|
+
selected_details_json TEXT,
|
|
1128
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1129
|
+
selected_count INTEGER DEFAULT 0,
|
|
1130
|
+
confidence TEXT,
|
|
1131
|
+
fallback_trace TEXT,
|
|
1132
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1133
|
+
);
|
|
1134
|
+
|
|
1034
1135
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1035
1136
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1036
1137
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1055,10 +1156,14 @@ var SQLiteEventStore = class {
|
|
|
1055
1156
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1056
1157
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1057
1158
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1159
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1058
1160
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1059
1161
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1060
1162
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1061
1163
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1164
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1165
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1166
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1062
1167
|
|
|
1063
1168
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1064
1169
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1082,6 +1187,14 @@ var SQLiteEventStore = class {
|
|
|
1082
1187
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1083
1188
|
END;
|
|
1084
1189
|
`);
|
|
1190
|
+
try {
|
|
1191
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1192
|
+
} catch {
|
|
1193
|
+
}
|
|
1194
|
+
try {
|
|
1195
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1196
|
+
} catch {
|
|
1197
|
+
}
|
|
1085
1198
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1086
1199
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1087
1200
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1181,6 +1294,21 @@ var SQLiteEventStore = class {
|
|
|
1181
1294
|
insertLevel.run(id);
|
|
1182
1295
|
});
|
|
1183
1296
|
transaction();
|
|
1297
|
+
if (this.markdownMirror) {
|
|
1298
|
+
const event = {
|
|
1299
|
+
id,
|
|
1300
|
+
eventType: input.eventType,
|
|
1301
|
+
sessionId: input.sessionId,
|
|
1302
|
+
timestamp: input.timestamp,
|
|
1303
|
+
content: input.content,
|
|
1304
|
+
canonicalKey,
|
|
1305
|
+
dedupeKey,
|
|
1306
|
+
metadata
|
|
1307
|
+
};
|
|
1308
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1309
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1310
|
+
});
|
|
1311
|
+
}
|
|
1184
1312
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1185
1313
|
} catch (error) {
|
|
1186
1314
|
return {
|
|
@@ -1239,6 +1367,92 @@ var SQLiteEventStore = class {
|
|
|
1239
1367
|
);
|
|
1240
1368
|
return rows.map(this.rowToEvent);
|
|
1241
1369
|
}
|
|
1370
|
+
/**
|
|
1371
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1372
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1373
|
+
*/
|
|
1374
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1375
|
+
await this.initialize();
|
|
1376
|
+
const rows = sqliteAll(
|
|
1377
|
+
this.db,
|
|
1378
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1379
|
+
[lastRowid, limit]
|
|
1380
|
+
);
|
|
1381
|
+
return rows.map((row) => ({
|
|
1382
|
+
rowid: row._rowid,
|
|
1383
|
+
event: this.rowToEvent(row)
|
|
1384
|
+
}));
|
|
1385
|
+
}
|
|
1386
|
+
/**
|
|
1387
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1388
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1389
|
+
*
|
|
1390
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1391
|
+
*/
|
|
1392
|
+
async importEvents(events) {
|
|
1393
|
+
if (events.length === 0)
|
|
1394
|
+
return { inserted: 0, skipped: 0 };
|
|
1395
|
+
if (this.readOnly)
|
|
1396
|
+
return { inserted: 0, skipped: events.length };
|
|
1397
|
+
await this.initialize();
|
|
1398
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1399
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1400
|
+
const insertEvent = this.db.prepare(`
|
|
1401
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1402
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1403
|
+
`);
|
|
1404
|
+
const insertDedup = this.db.prepare(`
|
|
1405
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1406
|
+
`);
|
|
1407
|
+
const insertLevel = this.db.prepare(`
|
|
1408
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1409
|
+
`);
|
|
1410
|
+
let inserted = 0;
|
|
1411
|
+
let skipped = 0;
|
|
1412
|
+
const insertedEvents = [];
|
|
1413
|
+
const tx = this.db.transaction((batch) => {
|
|
1414
|
+
for (const ev of batch) {
|
|
1415
|
+
const existingById = getById.get(ev.id);
|
|
1416
|
+
if (existingById) {
|
|
1417
|
+
skipped++;
|
|
1418
|
+
continue;
|
|
1419
|
+
}
|
|
1420
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1421
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1422
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1423
|
+
if (existingByDedupe) {
|
|
1424
|
+
skipped++;
|
|
1425
|
+
continue;
|
|
1426
|
+
}
|
|
1427
|
+
const metadata = ev.metadata || {};
|
|
1428
|
+
const turnId = metadata.turnId;
|
|
1429
|
+
insertEvent.run(
|
|
1430
|
+
ev.id,
|
|
1431
|
+
ev.eventType,
|
|
1432
|
+
ev.sessionId,
|
|
1433
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1434
|
+
ev.content,
|
|
1435
|
+
canonicalKey,
|
|
1436
|
+
dedupeKey,
|
|
1437
|
+
JSON.stringify(metadata),
|
|
1438
|
+
turnId ?? null
|
|
1439
|
+
);
|
|
1440
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1441
|
+
insertLevel.run(ev.id);
|
|
1442
|
+
inserted++;
|
|
1443
|
+
insertedEvents.push(ev);
|
|
1444
|
+
}
|
|
1445
|
+
});
|
|
1446
|
+
tx(events);
|
|
1447
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1448
|
+
for (const ev of insertedEvents) {
|
|
1449
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1450
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
return { inserted, skipped };
|
|
1455
|
+
}
|
|
1242
1456
|
/**
|
|
1243
1457
|
* Create or update session
|
|
1244
1458
|
*/
|
|
@@ -1401,6 +1615,35 @@ var SQLiteEventStore = class {
|
|
|
1401
1615
|
[error, ...ids]
|
|
1402
1616
|
);
|
|
1403
1617
|
}
|
|
1618
|
+
/**
|
|
1619
|
+
* Get embedding/vector outbox health statistics
|
|
1620
|
+
*/
|
|
1621
|
+
async getOutboxStats() {
|
|
1622
|
+
await this.initialize();
|
|
1623
|
+
const embeddingRows = sqliteAll(
|
|
1624
|
+
this.db,
|
|
1625
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1626
|
+
);
|
|
1627
|
+
const vectorRows = sqliteAll(
|
|
1628
|
+
this.db,
|
|
1629
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1630
|
+
);
|
|
1631
|
+
const fromRows = (rows) => {
|
|
1632
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1633
|
+
for (const row of rows) {
|
|
1634
|
+
const key = row.status;
|
|
1635
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1636
|
+
out[key] += row.count;
|
|
1637
|
+
}
|
|
1638
|
+
out.total += row.count;
|
|
1639
|
+
}
|
|
1640
|
+
return out;
|
|
1641
|
+
};
|
|
1642
|
+
return {
|
|
1643
|
+
embedding: fromRows(embeddingRows),
|
|
1644
|
+
vector: fromRows(vectorRows)
|
|
1645
|
+
};
|
|
1646
|
+
}
|
|
1404
1647
|
/**
|
|
1405
1648
|
* Update memory level
|
|
1406
1649
|
*/
|
|
@@ -1759,6 +2002,79 @@ var SQLiteEventStore = class {
|
|
|
1759
2002
|
getDatabase() {
|
|
1760
2003
|
return this.db;
|
|
1761
2004
|
}
|
|
2005
|
+
async recordRetrievalTrace(input) {
|
|
2006
|
+
await this.initialize();
|
|
2007
|
+
const traceId = randomUUID2();
|
|
2008
|
+
sqliteRun(
|
|
2009
|
+
this.db,
|
|
2010
|
+
`INSERT INTO retrieval_traces (
|
|
2011
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2012
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2013
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2014
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2015
|
+
[
|
|
2016
|
+
traceId,
|
|
2017
|
+
input.sessionId || null,
|
|
2018
|
+
input.projectHash || null,
|
|
2019
|
+
input.queryText,
|
|
2020
|
+
input.strategy || null,
|
|
2021
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2022
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2023
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2024
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2025
|
+
(input.candidateEventIds || []).length,
|
|
2026
|
+
(input.selectedEventIds || []).length,
|
|
2027
|
+
input.confidence || null,
|
|
2028
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2029
|
+
]
|
|
2030
|
+
);
|
|
2031
|
+
}
|
|
2032
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2033
|
+
await this.initialize();
|
|
2034
|
+
const rows = sqliteAll(
|
|
2035
|
+
this.db,
|
|
2036
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2037
|
+
[limit]
|
|
2038
|
+
);
|
|
2039
|
+
return rows.map((row) => ({
|
|
2040
|
+
traceId: row.trace_id,
|
|
2041
|
+
sessionId: row.session_id || void 0,
|
|
2042
|
+
projectHash: row.project_hash || void 0,
|
|
2043
|
+
queryText: row.query_text,
|
|
2044
|
+
strategy: row.strategy || void 0,
|
|
2045
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2046
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2047
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2048
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2049
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2050
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2051
|
+
confidence: row.confidence || void 0,
|
|
2052
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2053
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2054
|
+
}));
|
|
2055
|
+
}
|
|
2056
|
+
async getRetrievalTraceStats() {
|
|
2057
|
+
await this.initialize();
|
|
2058
|
+
const row = sqliteGet(
|
|
2059
|
+
this.db,
|
|
2060
|
+
`SELECT
|
|
2061
|
+
COUNT(*) as total_queries,
|
|
2062
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2063
|
+
AVG(selected_count) as avg_selected_count,
|
|
2064
|
+
CASE
|
|
2065
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2066
|
+
ELSE 0
|
|
2067
|
+
END as selection_rate
|
|
2068
|
+
FROM retrieval_traces`,
|
|
2069
|
+
[]
|
|
2070
|
+
);
|
|
2071
|
+
return {
|
|
2072
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2073
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2074
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2075
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2076
|
+
};
|
|
2077
|
+
}
|
|
1762
2078
|
/**
|
|
1763
2079
|
* Close database connection
|
|
1764
2080
|
*/
|
|
@@ -2620,7 +2936,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2620
2936
|
topK: 5,
|
|
2621
2937
|
minScore: 0.7,
|
|
2622
2938
|
maxTokens: 2e3,
|
|
2623
|
-
includeSessionContext: true
|
|
2939
|
+
includeSessionContext: true,
|
|
2940
|
+
strategy: "auto",
|
|
2941
|
+
rerankWithKeyword: true,
|
|
2942
|
+
decayPolicy: {
|
|
2943
|
+
enabled: true,
|
|
2944
|
+
windowDays: 30,
|
|
2945
|
+
maxPenalty: 0.15
|
|
2946
|
+
},
|
|
2947
|
+
graphHop: {
|
|
2948
|
+
enabled: true,
|
|
2949
|
+
maxHops: 1,
|
|
2950
|
+
hopPenalty: 0.08
|
|
2951
|
+
},
|
|
2952
|
+
projectScopeMode: "global"
|
|
2624
2953
|
};
|
|
2625
2954
|
var Retriever = class {
|
|
2626
2955
|
eventStore;
|
|
@@ -2630,6 +2959,7 @@ var Retriever = class {
|
|
|
2630
2959
|
sharedStore;
|
|
2631
2960
|
sharedVectorStore;
|
|
2632
2961
|
graduation;
|
|
2962
|
+
queryRewriter;
|
|
2633
2963
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2634
2964
|
this.eventStore = eventStore;
|
|
2635
2965
|
this.vectorStore = vectorStore;
|
|
@@ -2638,47 +2968,105 @@ var Retriever = class {
|
|
|
2638
2968
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2639
2969
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2640
2970
|
}
|
|
2641
|
-
/**
|
|
2642
|
-
* Set graduation pipeline for access tracking
|
|
2643
|
-
*/
|
|
2644
2971
|
setGraduationPipeline(graduation) {
|
|
2645
2972
|
this.graduation = graduation;
|
|
2646
2973
|
}
|
|
2647
|
-
/**
|
|
2648
|
-
* Set shared stores after construction
|
|
2649
|
-
*/
|
|
2650
2974
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2651
2975
|
this.sharedStore = sharedStore;
|
|
2652
2976
|
this.sharedVectorStore = sharedVectorStore;
|
|
2653
2977
|
}
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2978
|
+
setQueryRewriter(rewriter) {
|
|
2979
|
+
this.queryRewriter = rewriter;
|
|
2980
|
+
}
|
|
2657
2981
|
async retrieve(query, options = {}) {
|
|
2658
2982
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2659
|
-
const
|
|
2660
|
-
const
|
|
2661
|
-
|
|
2662
|
-
|
|
2983
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
2984
|
+
const fallbackTrace = [];
|
|
2985
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
2986
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
2987
|
+
let current = await this.runStage(query, {
|
|
2988
|
+
strategy: primaryStrategy,
|
|
2989
|
+
topK: opts.topK,
|
|
2663
2990
|
minScore: opts.minScore,
|
|
2664
|
-
sessionId:
|
|
2991
|
+
sessionId: sessionFilter,
|
|
2992
|
+
scope: opts.scope,
|
|
2993
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
2994
|
+
rerankWeights: opts.rerankWeights,
|
|
2995
|
+
decayPolicy: opts.decayPolicy,
|
|
2996
|
+
intentRewrite: opts.intentRewrite === true,
|
|
2997
|
+
graphHop: opts.graphHop,
|
|
2998
|
+
projectScopeMode: opts.projectScopeMode,
|
|
2999
|
+
projectHash: opts.projectHash,
|
|
3000
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2665
3001
|
});
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
3002
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3003
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3004
|
+
current = await this.runStage(query, {
|
|
3005
|
+
strategy: "deep",
|
|
3006
|
+
topK: opts.topK,
|
|
3007
|
+
minScore: opts.minScore,
|
|
3008
|
+
sessionId: sessionFilter,
|
|
3009
|
+
scope: opts.scope,
|
|
3010
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3011
|
+
rerankWeights: opts.rerankWeights,
|
|
3012
|
+
decayPolicy: opts.decayPolicy,
|
|
3013
|
+
graphHop: opts.graphHop,
|
|
3014
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3015
|
+
projectHash: opts.projectHash,
|
|
3016
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3017
|
+
});
|
|
3018
|
+
fallbackTrace.push("fallback:deep");
|
|
3019
|
+
}
|
|
3020
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3021
|
+
current = await this.runStage(query, {
|
|
3022
|
+
strategy: "deep",
|
|
3023
|
+
topK: opts.topK,
|
|
3024
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3025
|
+
sessionId: void 0,
|
|
3026
|
+
scope: void 0,
|
|
3027
|
+
rerankWithKeyword: true,
|
|
3028
|
+
rerankWeights: opts.rerankWeights,
|
|
3029
|
+
decayPolicy: opts.decayPolicy,
|
|
3030
|
+
graphHop: opts.graphHop,
|
|
3031
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3032
|
+
projectHash: opts.projectHash,
|
|
3033
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3034
|
+
});
|
|
3035
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3036
|
+
}
|
|
3037
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3038
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3039
|
+
current = {
|
|
3040
|
+
results: summary,
|
|
3041
|
+
candidateResults: summary,
|
|
3042
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3043
|
+
};
|
|
3044
|
+
fallbackTrace.push("fallback:summary");
|
|
3045
|
+
}
|
|
3046
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2671
3047
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2672
3048
|
return {
|
|
2673
3049
|
memories,
|
|
2674
|
-
matchResult,
|
|
3050
|
+
matchResult: current.matchResult,
|
|
2675
3051
|
totalTokens: this.estimateTokens(context),
|
|
2676
|
-
context
|
|
3052
|
+
context,
|
|
3053
|
+
fallbackTrace,
|
|
3054
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3055
|
+
eventId: r.eventId,
|
|
3056
|
+
score: r.score,
|
|
3057
|
+
semanticScore: r.semanticScore,
|
|
3058
|
+
lexicalScore: r.lexicalScore,
|
|
3059
|
+
recencyScore: r.recencyScore
|
|
3060
|
+
})),
|
|
3061
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3062
|
+
eventId: r.eventId,
|
|
3063
|
+
score: r.score,
|
|
3064
|
+
semanticScore: r.semanticScore,
|
|
3065
|
+
lexicalScore: r.lexicalScore,
|
|
3066
|
+
recencyScore: r.recencyScore
|
|
3067
|
+
}))
|
|
2677
3068
|
};
|
|
2678
3069
|
}
|
|
2679
|
-
/**
|
|
2680
|
-
* Retrieve with unified search (project + shared)
|
|
2681
|
-
*/
|
|
2682
3070
|
async retrieveUnified(query, options = {}) {
|
|
2683
3071
|
const projectResult = await this.retrieve(query, options);
|
|
2684
3072
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2686,22 +3074,19 @@ var Retriever = class {
|
|
|
2686
3074
|
}
|
|
2687
3075
|
try {
|
|
2688
3076
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2689
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
excludeProjectHash: options.projectHash
|
|
2695
|
-
}
|
|
2696
|
-
);
|
|
3077
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3078
|
+
limit: options.topK || 5,
|
|
3079
|
+
minScore: options.minScore || 0.7,
|
|
3080
|
+
excludeProjectHash: options.projectHash
|
|
3081
|
+
});
|
|
2697
3082
|
const sharedMemories = [];
|
|
2698
3083
|
for (const result of sharedVectorResults) {
|
|
2699
3084
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2700
|
-
if (entry)
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
3085
|
+
if (!entry)
|
|
3086
|
+
continue;
|
|
3087
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3088
|
+
sharedMemories.push(entry);
|
|
3089
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2705
3090
|
}
|
|
2706
3091
|
}
|
|
2707
3092
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2716,50 +3101,243 @@ var Retriever = class {
|
|
|
2716
3101
|
return projectResult;
|
|
2717
3102
|
}
|
|
2718
3103
|
}
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
3104
|
+
async runStage(query, input) {
|
|
3105
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3106
|
+
strategy: input.strategy,
|
|
3107
|
+
topK: input.topK,
|
|
3108
|
+
minScore: input.minScore,
|
|
3109
|
+
sessionId: input.sessionId
|
|
3110
|
+
});
|
|
3111
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3112
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3113
|
+
if (rewritten && rewritten !== query) {
|
|
3114
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3115
|
+
strategy: "deep",
|
|
3116
|
+
topK: input.topK,
|
|
3117
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3118
|
+
sessionId: input.sessionId
|
|
3119
|
+
});
|
|
3120
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3121
|
+
}
|
|
3122
|
+
}
|
|
3123
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3124
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3125
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3126
|
+
limit: input.topK * 4
|
|
3127
|
+
});
|
|
3128
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3129
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3130
|
+
scope: input.scope,
|
|
3131
|
+
projectScopeMode: input.projectScopeMode,
|
|
3132
|
+
projectHash: input.projectHash,
|
|
3133
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3134
|
+
});
|
|
3135
|
+
const top = filtered.slice(0, input.topK);
|
|
3136
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3137
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3138
|
+
}
|
|
3139
|
+
mergeResults(primary, secondary, limit) {
|
|
3140
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3141
|
+
for (const row of primary)
|
|
3142
|
+
byId.set(row.eventId, row);
|
|
3143
|
+
for (const row of secondary) {
|
|
3144
|
+
const prev = byId.get(row.eventId);
|
|
3145
|
+
if (!prev || row.score > prev.score) {
|
|
3146
|
+
byId.set(row.eventId, row);
|
|
3147
|
+
}
|
|
3148
|
+
}
|
|
3149
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3150
|
+
}
|
|
3151
|
+
async expandGraphHops(seeds, opts) {
|
|
3152
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3153
|
+
for (const s of seeds)
|
|
3154
|
+
byId.set(s.eventId, s);
|
|
3155
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3156
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3157
|
+
const next = [];
|
|
3158
|
+
for (const f of frontier) {
|
|
3159
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3160
|
+
if (!ev)
|
|
3161
|
+
continue;
|
|
3162
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3163
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3164
|
+
for (const rid of relatedIds) {
|
|
3165
|
+
if (byId.has(rid))
|
|
3166
|
+
continue;
|
|
3167
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3168
|
+
if (!target)
|
|
3169
|
+
continue;
|
|
3170
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3171
|
+
const row = {
|
|
3172
|
+
id: `hop-${hop}-${rid}`,
|
|
3173
|
+
eventId: target.id,
|
|
3174
|
+
content: target.content,
|
|
3175
|
+
score,
|
|
3176
|
+
sessionId: target.sessionId,
|
|
3177
|
+
eventType: target.eventType,
|
|
3178
|
+
timestamp: target.timestamp.toISOString()
|
|
3179
|
+
};
|
|
3180
|
+
byId.set(row.eventId, row);
|
|
3181
|
+
next.push({ row, hop });
|
|
3182
|
+
if (byId.size >= opts.limit)
|
|
3183
|
+
break;
|
|
2740
3184
|
}
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
`;
|
|
3185
|
+
if (byId.size >= opts.limit)
|
|
3186
|
+
break;
|
|
2744
3187
|
}
|
|
3188
|
+
frontier = next;
|
|
3189
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3190
|
+
break;
|
|
2745
3191
|
}
|
|
2746
|
-
return
|
|
3192
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3193
|
+
}
|
|
3194
|
+
shouldFallback(matchResult, results) {
|
|
3195
|
+
if (results.length === 0)
|
|
3196
|
+
return true;
|
|
3197
|
+
if (matchResult.confidence === "none")
|
|
3198
|
+
return true;
|
|
3199
|
+
return false;
|
|
3200
|
+
}
|
|
3201
|
+
async buildSummaryFallback(query, topK) {
|
|
3202
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3203
|
+
const q = this.tokenize(query);
|
|
3204
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3205
|
+
id: `summary-${row.e.id}`,
|
|
3206
|
+
eventId: row.e.id,
|
|
3207
|
+
content: row.e.content,
|
|
3208
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3209
|
+
sessionId: row.e.sessionId,
|
|
3210
|
+
eventType: row.e.eventType,
|
|
3211
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3212
|
+
}));
|
|
3213
|
+
return ranked;
|
|
3214
|
+
}
|
|
3215
|
+
async searchByStrategy(query, input) {
|
|
3216
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3217
|
+
if (strategy === "fast") {
|
|
3218
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3219
|
+
limit: Math.max(5, input.topK * 3),
|
|
3220
|
+
sessionId: input.sessionId
|
|
3221
|
+
});
|
|
3222
|
+
return keyword;
|
|
3223
|
+
}
|
|
3224
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3225
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3226
|
+
limit: Math.max(5, input.topK * 3),
|
|
3227
|
+
minScore: input.minScore,
|
|
3228
|
+
sessionId: input.sessionId
|
|
3229
|
+
});
|
|
3230
|
+
}
|
|
3231
|
+
async searchByKeyword(query, input) {
|
|
3232
|
+
if (this.eventStore.keywordSearch) {
|
|
3233
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3234
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3235
|
+
return filtered2.map((row, idx) => ({
|
|
3236
|
+
id: `kw-${row.event.id}`,
|
|
3237
|
+
eventId: row.event.id,
|
|
3238
|
+
content: row.event.content,
|
|
3239
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3240
|
+
sessionId: row.event.sessionId,
|
|
3241
|
+
eventType: row.event.eventType,
|
|
3242
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3243
|
+
}));
|
|
3244
|
+
}
|
|
3245
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3246
|
+
const tokens = this.tokenize(query);
|
|
3247
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3248
|
+
return filtered.map((row, idx) => ({
|
|
3249
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3250
|
+
eventId: row.e.id,
|
|
3251
|
+
content: row.e.content,
|
|
3252
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3253
|
+
sessionId: row.e.sessionId,
|
|
3254
|
+
eventType: row.e.eventType,
|
|
3255
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3256
|
+
}));
|
|
3257
|
+
}
|
|
3258
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3259
|
+
const q = this.tokenize(query);
|
|
3260
|
+
const now = Date.now();
|
|
3261
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3262
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3263
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3264
|
+
const total = sw + lw + rw || 1;
|
|
3265
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3266
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3267
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3268
|
+
return [...results].map((r) => {
|
|
3269
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3270
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3271
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3272
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3273
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3274
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3275
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3276
|
+
}
|
|
3277
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3278
|
+
}).sort((a, b) => b.score - a.score);
|
|
3279
|
+
}
|
|
3280
|
+
async applyScopeFilters(results, options) {
|
|
3281
|
+
const scope = options?.scope;
|
|
3282
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3283
|
+
const allowedProjectHashes = new Set(
|
|
3284
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3285
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3286
|
+
)
|
|
3287
|
+
);
|
|
3288
|
+
if (!scope && projectScopeMode === "global")
|
|
3289
|
+
return results;
|
|
3290
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3291
|
+
const filtered = [];
|
|
3292
|
+
for (const result of results) {
|
|
3293
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3294
|
+
continue;
|
|
3295
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3296
|
+
continue;
|
|
3297
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3298
|
+
continue;
|
|
3299
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3300
|
+
if (!event)
|
|
3301
|
+
continue;
|
|
3302
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3303
|
+
continue;
|
|
3304
|
+
if (normalizedIncludes.length > 0) {
|
|
3305
|
+
const lc = event.content.toLowerCase();
|
|
3306
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3307
|
+
continue;
|
|
3308
|
+
}
|
|
3309
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3310
|
+
continue;
|
|
3311
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3312
|
+
filtered.push({ result, projectHash });
|
|
3313
|
+
}
|
|
3314
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3315
|
+
return filtered.map((x) => x.result);
|
|
3316
|
+
}
|
|
3317
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3318
|
+
if (projectScopeMode === "strict") {
|
|
3319
|
+
return projectMatched.map((x) => x.result);
|
|
3320
|
+
}
|
|
3321
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3322
|
+
}
|
|
3323
|
+
extractProjectHash(metadata) {
|
|
3324
|
+
if (!metadata || typeof metadata !== "object")
|
|
3325
|
+
return void 0;
|
|
3326
|
+
const scope = metadata.scope;
|
|
3327
|
+
if (!scope || typeof scope !== "object")
|
|
3328
|
+
return void 0;
|
|
3329
|
+
const project = scope.project;
|
|
3330
|
+
if (!project || typeof project !== "object")
|
|
3331
|
+
return void 0;
|
|
3332
|
+
const hash = project.hash;
|
|
3333
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2747
3334
|
}
|
|
2748
|
-
/**
|
|
2749
|
-
* Retrieve memories from a specific session
|
|
2750
|
-
*/
|
|
2751
3335
|
async retrieveFromSession(sessionId) {
|
|
2752
3336
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2753
3337
|
}
|
|
2754
|
-
/**
|
|
2755
|
-
* Get recent memories across all sessions
|
|
2756
|
-
*/
|
|
2757
3338
|
async retrieveRecent(limit = 100) {
|
|
2758
3339
|
return this.eventStore.getRecentEvents(limit);
|
|
2759
3340
|
}
|
|
2760
|
-
/**
|
|
2761
|
-
* Enrich search results with full event data
|
|
2762
|
-
*/
|
|
2763
3341
|
async enrichResults(results, options) {
|
|
2764
3342
|
const memories = [];
|
|
2765
3343
|
for (const result of results) {
|
|
@@ -2767,27 +3345,16 @@ var Retriever = class {
|
|
|
2767
3345
|
if (!event)
|
|
2768
3346
|
continue;
|
|
2769
3347
|
if (this.graduation) {
|
|
2770
|
-
this.graduation.recordAccess(
|
|
2771
|
-
event.id,
|
|
2772
|
-
options.sessionId || "unknown",
|
|
2773
|
-
result.score
|
|
2774
|
-
);
|
|
3348
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2775
3349
|
}
|
|
2776
3350
|
let sessionContext;
|
|
2777
3351
|
if (options.includeSessionContext) {
|
|
2778
3352
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2779
3353
|
}
|
|
2780
|
-
memories.push({
|
|
2781
|
-
event,
|
|
2782
|
-
score: result.score,
|
|
2783
|
-
sessionContext
|
|
2784
|
-
});
|
|
3354
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2785
3355
|
}
|
|
2786
3356
|
return memories;
|
|
2787
3357
|
}
|
|
2788
|
-
/**
|
|
2789
|
-
* Get surrounding context from the same session
|
|
2790
|
-
*/
|
|
2791
3358
|
async getSessionContext(sessionId, eventId) {
|
|
2792
3359
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2793
3360
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2800,55 +3367,86 @@ var Retriever = class {
|
|
|
2800
3367
|
return void 0;
|
|
2801
3368
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2802
3369
|
}
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
3370
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3371
|
+
let context = projectResult.context;
|
|
3372
|
+
if (sharedMemories.length === 0)
|
|
3373
|
+
return context;
|
|
3374
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3375
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3376
|
+
context += `### ${memory.title}
|
|
3377
|
+
`;
|
|
3378
|
+
if (memory.symptoms.length > 0)
|
|
3379
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3380
|
+
`;
|
|
3381
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3382
|
+
`;
|
|
3383
|
+
context += `**Solution:** ${memory.solution}
|
|
3384
|
+
`;
|
|
3385
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3386
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3387
|
+
`;
|
|
3388
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3389
|
+
|
|
3390
|
+
`;
|
|
3391
|
+
}
|
|
3392
|
+
return context;
|
|
3393
|
+
}
|
|
2806
3394
|
buildContext(memories, maxTokens) {
|
|
2807
3395
|
const parts = [];
|
|
2808
3396
|
let currentTokens = 0;
|
|
2809
3397
|
for (const memory of memories) {
|
|
2810
3398
|
const memoryText = this.formatMemory(memory);
|
|
2811
3399
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2812
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3400
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2813
3401
|
break;
|
|
2814
|
-
}
|
|
2815
3402
|
parts.push(memoryText);
|
|
2816
3403
|
currentTokens += memoryTokens;
|
|
2817
3404
|
}
|
|
2818
|
-
if (parts.length === 0)
|
|
3405
|
+
if (parts.length === 0)
|
|
2819
3406
|
return "";
|
|
2820
|
-
}
|
|
2821
3407
|
return `## Relevant Memories
|
|
2822
3408
|
|
|
2823
3409
|
${parts.join("\n\n---\n\n")}`;
|
|
2824
3410
|
}
|
|
2825
|
-
/**
|
|
2826
|
-
* Format a single memory for context
|
|
2827
|
-
*/
|
|
2828
3411
|
formatMemory(memory) {
|
|
2829
3412
|
const { event, score, sessionContext } = memory;
|
|
2830
3413
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2831
3414
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2832
3415
|
${event.content}`;
|
|
2833
|
-
if (sessionContext)
|
|
3416
|
+
if (sessionContext)
|
|
2834
3417
|
text += `
|
|
2835
3418
|
|
|
2836
3419
|
_Context:_ ${sessionContext}`;
|
|
2837
|
-
}
|
|
2838
3420
|
return text;
|
|
2839
3421
|
}
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
3422
|
+
matchesMetadataScope(metadata, expected) {
|
|
3423
|
+
if (!metadata)
|
|
3424
|
+
return false;
|
|
3425
|
+
return Object.entries(expected).every(([path4, value]) => {
|
|
3426
|
+
const actual = path4.split(".").reduce((acc, key) => {
|
|
3427
|
+
if (typeof acc !== "object" || acc === null)
|
|
3428
|
+
return void 0;
|
|
3429
|
+
return acc[key];
|
|
3430
|
+
}, metadata);
|
|
3431
|
+
return actual === value;
|
|
3432
|
+
});
|
|
3433
|
+
}
|
|
3434
|
+
tokenize(text) {
|
|
3435
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3436
|
+
}
|
|
3437
|
+
keywordOverlap(a, b) {
|
|
3438
|
+
if (a.length === 0 || b.length === 0)
|
|
3439
|
+
return 0;
|
|
3440
|
+
const bs = new Set(b);
|
|
3441
|
+
let hit = 0;
|
|
3442
|
+
for (const t of a)
|
|
3443
|
+
if (bs.has(t))
|
|
3444
|
+
hit += 1;
|
|
3445
|
+
return hit / a.length;
|
|
3446
|
+
}
|
|
2843
3447
|
estimateTokens(text) {
|
|
2844
3448
|
return Math.ceil(text.length / 4);
|
|
2845
3449
|
}
|
|
2846
|
-
/**
|
|
2847
|
-
* Get event age in days (for recency scoring)
|
|
2848
|
-
*/
|
|
2849
|
-
getEventAgeDays(eventId) {
|
|
2850
|
-
return 0;
|
|
2851
|
-
}
|
|
2852
3450
|
};
|
|
2853
3451
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2854
3452
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -4138,6 +4736,59 @@ var ConsolidatedStore = class {
|
|
|
4138
4736
|
[memoryId]
|
|
4139
4737
|
);
|
|
4140
4738
|
}
|
|
4739
|
+
/**
|
|
4740
|
+
* Create a long-term rule promoted from stable summaries
|
|
4741
|
+
*/
|
|
4742
|
+
async createRule(input) {
|
|
4743
|
+
const ruleId = randomUUID6();
|
|
4744
|
+
await dbRun(
|
|
4745
|
+
this.db,
|
|
4746
|
+
`INSERT INTO consolidated_rules
|
|
4747
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4748
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4749
|
+
[
|
|
4750
|
+
ruleId,
|
|
4751
|
+
input.rule,
|
|
4752
|
+
JSON.stringify(input.topics),
|
|
4753
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4754
|
+
JSON.stringify(input.sourceEvents),
|
|
4755
|
+
input.confidence
|
|
4756
|
+
]
|
|
4757
|
+
);
|
|
4758
|
+
return ruleId;
|
|
4759
|
+
}
|
|
4760
|
+
async getRules(options) {
|
|
4761
|
+
const limit = options?.limit || 100;
|
|
4762
|
+
const rows = await dbAll(
|
|
4763
|
+
this.db,
|
|
4764
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4765
|
+
[limit]
|
|
4766
|
+
);
|
|
4767
|
+
return rows.map((row) => ({
|
|
4768
|
+
ruleId: row.rule_id,
|
|
4769
|
+
rule: row.rule,
|
|
4770
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4771
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4772
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4773
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4774
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4775
|
+
}));
|
|
4776
|
+
}
|
|
4777
|
+
async countRules() {
|
|
4778
|
+
const result = await dbAll(
|
|
4779
|
+
this.db,
|
|
4780
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4781
|
+
);
|
|
4782
|
+
return result[0]?.count || 0;
|
|
4783
|
+
}
|
|
4784
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4785
|
+
const rows = await dbAll(
|
|
4786
|
+
this.db,
|
|
4787
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4788
|
+
[`%"${memoryId}"%`]
|
|
4789
|
+
);
|
|
4790
|
+
return (rows[0]?.count || 0) > 0;
|
|
4791
|
+
}
|
|
4141
4792
|
/**
|
|
4142
4793
|
* Get count of consolidated memories
|
|
4143
4794
|
*/
|
|
@@ -4287,7 +4938,14 @@ var ConsolidationWorker = class {
|
|
|
4287
4938
|
* Force a consolidation run (manual trigger)
|
|
4288
4939
|
*/
|
|
4289
4940
|
async forceRun() {
|
|
4290
|
-
|
|
4941
|
+
const out = await this.consolidateWithReport();
|
|
4942
|
+
return out.consolidatedCount;
|
|
4943
|
+
}
|
|
4944
|
+
/**
|
|
4945
|
+
* Force a consolidation run and return metrics report
|
|
4946
|
+
*/
|
|
4947
|
+
async forceRunWithReport() {
|
|
4948
|
+
return this.consolidateWithReport();
|
|
4291
4949
|
}
|
|
4292
4950
|
/**
|
|
4293
4951
|
* Schedule the next consolidation check
|
|
@@ -4327,12 +4985,21 @@ var ConsolidationWorker = class {
|
|
|
4327
4985
|
* Perform consolidation
|
|
4328
4986
|
*/
|
|
4329
4987
|
async consolidate() {
|
|
4988
|
+
const out = await this.consolidateWithReport();
|
|
4989
|
+
return out.consolidatedCount;
|
|
4990
|
+
}
|
|
4991
|
+
async consolidateWithReport() {
|
|
4330
4992
|
const workingSet = await this.workingSetStore.get();
|
|
4331
4993
|
if (workingSet.recentEvents.length < 3) {
|
|
4332
|
-
return
|
|
4994
|
+
return {
|
|
4995
|
+
consolidatedCount: 0,
|
|
4996
|
+
promotedRuleCount: 0,
|
|
4997
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
4998
|
+
};
|
|
4333
4999
|
}
|
|
4334
5000
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
4335
5001
|
let consolidatedCount = 0;
|
|
5002
|
+
const createdMemoryIds = [];
|
|
4336
5003
|
for (const group of groups) {
|
|
4337
5004
|
if (group.events.length < 3)
|
|
4338
5005
|
continue;
|
|
@@ -4341,14 +5008,16 @@ var ConsolidationWorker = class {
|
|
|
4341
5008
|
if (alreadyConsolidated)
|
|
4342
5009
|
continue;
|
|
4343
5010
|
const summary = await this.summarize(group);
|
|
4344
|
-
await this.consolidatedStore.create({
|
|
5011
|
+
const memoryId = await this.consolidatedStore.create({
|
|
4345
5012
|
summary,
|
|
4346
5013
|
topics: group.topics,
|
|
4347
5014
|
sourceEvents: eventIds,
|
|
4348
5015
|
confidence: this.calculateConfidence(group)
|
|
4349
5016
|
});
|
|
5017
|
+
createdMemoryIds.push(memoryId);
|
|
4350
5018
|
consolidatedCount++;
|
|
4351
5019
|
}
|
|
5020
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
4352
5021
|
if (consolidatedCount > 0) {
|
|
4353
5022
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
4354
5023
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4362,7 +5031,61 @@ var ConsolidationWorker = class {
|
|
|
4362
5031
|
await this.workingSetStore.prune(oldEventIds);
|
|
4363
5032
|
}
|
|
4364
5033
|
}
|
|
4365
|
-
|
|
5034
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5035
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5036
|
+
}
|
|
5037
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5038
|
+
let promoted = 0;
|
|
5039
|
+
for (const memoryId of memoryIds) {
|
|
5040
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5041
|
+
if (!memory)
|
|
5042
|
+
continue;
|
|
5043
|
+
if (memory.confidence < 0.55)
|
|
5044
|
+
continue;
|
|
5045
|
+
if (memory.sourceEvents.length < 4)
|
|
5046
|
+
continue;
|
|
5047
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5048
|
+
if (exists)
|
|
5049
|
+
continue;
|
|
5050
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5051
|
+
if (!rule)
|
|
5052
|
+
continue;
|
|
5053
|
+
await this.consolidatedStore.createRule({
|
|
5054
|
+
rule,
|
|
5055
|
+
topics: memory.topics,
|
|
5056
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5057
|
+
sourceEvents: memory.sourceEvents,
|
|
5058
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5059
|
+
});
|
|
5060
|
+
promoted++;
|
|
5061
|
+
}
|
|
5062
|
+
return promoted;
|
|
5063
|
+
}
|
|
5064
|
+
buildRuleFromSummary(summary, topics) {
|
|
5065
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5066
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5067
|
+
const seed = bullet || lines[0];
|
|
5068
|
+
if (!seed || seed.length < 8)
|
|
5069
|
+
return null;
|
|
5070
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5071
|
+
return `${topicPrefix}${seed}`;
|
|
5072
|
+
}
|
|
5073
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5074
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5075
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5076
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5077
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5078
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5079
|
+
return {
|
|
5080
|
+
beforeTokenEstimate,
|
|
5081
|
+
afterTokenEstimate,
|
|
5082
|
+
reductionRatio,
|
|
5083
|
+
qualityGuardPassed,
|
|
5084
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5085
|
+
};
|
|
5086
|
+
}
|
|
5087
|
+
estimateTokens(text) {
|
|
5088
|
+
return Math.ceil((text || "").length / 4);
|
|
4366
5089
|
}
|
|
4367
5090
|
/**
|
|
4368
5091
|
* Check if consolidation should run
|
|
@@ -4922,13 +5645,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
4922
5645
|
);
|
|
4923
5646
|
}
|
|
4924
5647
|
|
|
5648
|
+
// src/core/md-mirror.ts
|
|
5649
|
+
import * as fs3 from "node:fs";
|
|
5650
|
+
import * as path2 from "node:path";
|
|
5651
|
+
function sanitizeSegment2(input, fallback) {
|
|
5652
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5653
|
+
return v || fallback;
|
|
5654
|
+
}
|
|
5655
|
+
function getAtPath(obj, dotted) {
|
|
5656
|
+
if (!obj)
|
|
5657
|
+
return void 0;
|
|
5658
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5659
|
+
if (!acc || typeof acc !== "object")
|
|
5660
|
+
return void 0;
|
|
5661
|
+
return acc[key];
|
|
5662
|
+
}, obj);
|
|
5663
|
+
}
|
|
5664
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5665
|
+
const meta = event.metadata;
|
|
5666
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5667
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5668
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5669
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5670
|
+
const d = event.timestamp;
|
|
5671
|
+
const yyyy = d.getFullYear();
|
|
5672
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5673
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5674
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5675
|
+
}
|
|
5676
|
+
var MarkdownMirror2 = class {
|
|
5677
|
+
constructor(rootDir) {
|
|
5678
|
+
this.rootDir = rootDir;
|
|
5679
|
+
}
|
|
5680
|
+
async append(event, eventId) {
|
|
5681
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5682
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5683
|
+
const lines = [
|
|
5684
|
+
"",
|
|
5685
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5686
|
+
`- type: ${event.eventType}`,
|
|
5687
|
+
`- session: ${event.sessionId}`,
|
|
5688
|
+
event.content
|
|
5689
|
+
];
|
|
5690
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5691
|
+
await this.refreshIndex();
|
|
5692
|
+
}
|
|
5693
|
+
async refreshIndex() {
|
|
5694
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5695
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5696
|
+
const files = [];
|
|
5697
|
+
await this.walk(memoryRoot, files);
|
|
5698
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5699
|
+
const index = [
|
|
5700
|
+
"# Memory Index",
|
|
5701
|
+
"",
|
|
5702
|
+
"Generated automatically by MarkdownMirror.",
|
|
5703
|
+
"",
|
|
5704
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5705
|
+
""
|
|
5706
|
+
].join("\n");
|
|
5707
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5708
|
+
}
|
|
5709
|
+
async walk(dir, out) {
|
|
5710
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5711
|
+
for (const e of entries) {
|
|
5712
|
+
const full = path2.join(dir, e.name);
|
|
5713
|
+
if (e.isDirectory()) {
|
|
5714
|
+
await this.walk(full, out);
|
|
5715
|
+
} else {
|
|
5716
|
+
out.push(full);
|
|
5717
|
+
}
|
|
5718
|
+
}
|
|
5719
|
+
}
|
|
5720
|
+
};
|
|
5721
|
+
|
|
5722
|
+
// src/core/ingest-interceptor.ts
|
|
5723
|
+
var IngestInterceptorRegistry = class {
|
|
5724
|
+
before = [];
|
|
5725
|
+
after = [];
|
|
5726
|
+
onError = [];
|
|
5727
|
+
registerBefore(interceptor) {
|
|
5728
|
+
this.before.push(interceptor);
|
|
5729
|
+
return () => {
|
|
5730
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5731
|
+
};
|
|
5732
|
+
}
|
|
5733
|
+
registerAfter(interceptor) {
|
|
5734
|
+
this.after.push(interceptor);
|
|
5735
|
+
return () => {
|
|
5736
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5737
|
+
};
|
|
5738
|
+
}
|
|
5739
|
+
registerOnError(interceptor) {
|
|
5740
|
+
this.onError.push(interceptor);
|
|
5741
|
+
return () => {
|
|
5742
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5743
|
+
};
|
|
5744
|
+
}
|
|
5745
|
+
async run(stage, context) {
|
|
5746
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5747
|
+
for (const interceptor of interceptors) {
|
|
5748
|
+
await interceptor({ ...context, stage });
|
|
5749
|
+
}
|
|
5750
|
+
}
|
|
5751
|
+
};
|
|
5752
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5753
|
+
if (!base && !patch)
|
|
5754
|
+
return void 0;
|
|
5755
|
+
if (!base)
|
|
5756
|
+
return patch;
|
|
5757
|
+
if (!patch)
|
|
5758
|
+
return base;
|
|
5759
|
+
const result = { ...base };
|
|
5760
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5761
|
+
const current = result[key];
|
|
5762
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5763
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5764
|
+
current,
|
|
5765
|
+
value
|
|
5766
|
+
);
|
|
5767
|
+
} else {
|
|
5768
|
+
result[key] = value;
|
|
5769
|
+
}
|
|
5770
|
+
}
|
|
5771
|
+
return result;
|
|
5772
|
+
}
|
|
5773
|
+
|
|
5774
|
+
// src/core/tag-taxonomy.ts
|
|
5775
|
+
var TAG_NAMESPACES = {
|
|
5776
|
+
SYSTEM: "sys:",
|
|
5777
|
+
QUALITY: "q:",
|
|
5778
|
+
PROJECT: "proj:",
|
|
5779
|
+
TOPIC: "topic:",
|
|
5780
|
+
TEMPORAL: "t:",
|
|
5781
|
+
USER: "user:",
|
|
5782
|
+
AGENT: "agent:"
|
|
5783
|
+
};
|
|
5784
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5785
|
+
function parseTag(tag) {
|
|
5786
|
+
const value = (tag || "").trim();
|
|
5787
|
+
const idx = value.indexOf(":");
|
|
5788
|
+
if (idx <= 0)
|
|
5789
|
+
return { value };
|
|
5790
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5791
|
+
const tagValue = value.slice(idx + 1);
|
|
5792
|
+
if (!tagValue)
|
|
5793
|
+
return { value };
|
|
5794
|
+
return { namespace, value: tagValue };
|
|
5795
|
+
}
|
|
5796
|
+
function validateTag(tag) {
|
|
5797
|
+
const normalized = (tag || "").trim();
|
|
5798
|
+
if (!normalized)
|
|
5799
|
+
return false;
|
|
5800
|
+
const { namespace } = parseTag(normalized);
|
|
5801
|
+
if (!namespace)
|
|
5802
|
+
return true;
|
|
5803
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5804
|
+
}
|
|
5805
|
+
function normalizeTags(tags) {
|
|
5806
|
+
if (!Array.isArray(tags))
|
|
5807
|
+
return [];
|
|
5808
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5809
|
+
for (const item of tags) {
|
|
5810
|
+
if (typeof item !== "string")
|
|
5811
|
+
continue;
|
|
5812
|
+
const normalized = item.trim();
|
|
5813
|
+
if (!validateTag(normalized))
|
|
5814
|
+
continue;
|
|
5815
|
+
dedup.add(normalized);
|
|
5816
|
+
}
|
|
5817
|
+
return [...dedup];
|
|
5818
|
+
}
|
|
5819
|
+
|
|
4925
5820
|
// src/services/memory-service.ts
|
|
4926
5821
|
function normalizePath(projectPath) {
|
|
4927
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5822
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
4928
5823
|
try {
|
|
4929
|
-
return
|
|
5824
|
+
return fs4.realpathSync(expanded);
|
|
4930
5825
|
} catch {
|
|
4931
|
-
return
|
|
5826
|
+
return path3.resolve(expanded);
|
|
4932
5827
|
}
|
|
4933
5828
|
}
|
|
4934
5829
|
function hashProjectPath(projectPath) {
|
|
@@ -4937,14 +5832,14 @@ function hashProjectPath(projectPath) {
|
|
|
4937
5832
|
}
|
|
4938
5833
|
function getProjectStoragePath(projectPath) {
|
|
4939
5834
|
const hash = hashProjectPath(projectPath);
|
|
4940
|
-
return
|
|
5835
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
4941
5836
|
}
|
|
4942
|
-
var REGISTRY_PATH =
|
|
4943
|
-
var SHARED_STORAGE_PATH =
|
|
5837
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5838
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
4944
5839
|
function loadSessionRegistry() {
|
|
4945
5840
|
try {
|
|
4946
|
-
if (
|
|
4947
|
-
const data =
|
|
5841
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5842
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
4948
5843
|
return JSON.parse(data);
|
|
4949
5844
|
}
|
|
4950
5845
|
} catch (error) {
|
|
@@ -4953,13 +5848,13 @@ function loadSessionRegistry() {
|
|
|
4953
5848
|
return { version: 1, sessions: {} };
|
|
4954
5849
|
}
|
|
4955
5850
|
function saveSessionRegistry(registry) {
|
|
4956
|
-
const dir =
|
|
4957
|
-
if (!
|
|
4958
|
-
|
|
5851
|
+
const dir = path3.dirname(REGISTRY_PATH);
|
|
5852
|
+
if (!fs4.existsSync(dir)) {
|
|
5853
|
+
fs4.mkdirSync(dir, { recursive: true });
|
|
4959
5854
|
}
|
|
4960
5855
|
const tempPath = REGISTRY_PATH + ".tmp";
|
|
4961
|
-
|
|
4962
|
-
|
|
5856
|
+
fs4.writeFileSync(tempPath, JSON.stringify(registry, null, 2));
|
|
5857
|
+
fs4.renameSync(tempPath, REGISTRY_PATH);
|
|
4963
5858
|
}
|
|
4964
5859
|
function registerSession(sessionId, projectPath) {
|
|
4965
5860
|
const registry = loadSessionRegistry();
|
|
@@ -4995,6 +5890,7 @@ var MemoryService = class {
|
|
|
4995
5890
|
vectorWorker = null;
|
|
4996
5891
|
graduationWorker = null;
|
|
4997
5892
|
initialized = false;
|
|
5893
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
4998
5894
|
// Endless Mode components
|
|
4999
5895
|
workingSetStore = null;
|
|
5000
5896
|
consolidatedStore = null;
|
|
@@ -5008,20 +5904,27 @@ var MemoryService = class {
|
|
|
5008
5904
|
sharedPromoter = null;
|
|
5009
5905
|
sharedStoreConfig = null;
|
|
5010
5906
|
projectHash = null;
|
|
5907
|
+
projectPath = null;
|
|
5011
5908
|
readOnly;
|
|
5012
5909
|
lightweightMode;
|
|
5910
|
+
mdMirror;
|
|
5013
5911
|
constructor(config) {
|
|
5014
5912
|
const storagePath = this.expandPath(config.storagePath);
|
|
5015
5913
|
this.readOnly = config.readOnly ?? false;
|
|
5016
5914
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
5017
|
-
|
|
5018
|
-
|
|
5915
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
5916
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
5917
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
5019
5918
|
}
|
|
5020
5919
|
this.projectHash = config.projectHash || null;
|
|
5920
|
+
this.projectPath = config.projectPath || null;
|
|
5021
5921
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
5022
5922
|
this.sqliteStore = new SQLiteEventStore(
|
|
5023
|
-
|
|
5024
|
-
{
|
|
5923
|
+
path3.join(storagePath, "events.sqlite"),
|
|
5924
|
+
{
|
|
5925
|
+
readonly: this.readOnly,
|
|
5926
|
+
markdownMirrorRoot: storagePath
|
|
5927
|
+
}
|
|
5025
5928
|
);
|
|
5026
5929
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
5027
5930
|
if (!analyticsEnabled) {
|
|
@@ -5029,7 +5932,7 @@ var MemoryService = class {
|
|
|
5029
5932
|
} else if (this.readOnly) {
|
|
5030
5933
|
try {
|
|
5031
5934
|
this.analyticsStore = new EventStore(
|
|
5032
|
-
|
|
5935
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5033
5936
|
{ readOnly: true }
|
|
5034
5937
|
);
|
|
5035
5938
|
} catch {
|
|
@@ -5037,11 +5940,11 @@ var MemoryService = class {
|
|
|
5037
5940
|
}
|
|
5038
5941
|
} else {
|
|
5039
5942
|
this.analyticsStore = new EventStore(
|
|
5040
|
-
|
|
5943
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5041
5944
|
{ readOnly: false }
|
|
5042
5945
|
);
|
|
5043
5946
|
}
|
|
5044
|
-
this.vectorStore = new VectorStore(
|
|
5947
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
5045
5948
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
5046
5949
|
this.matcher = getDefaultMatcher();
|
|
5047
5950
|
this.retriever = createRetriever(
|
|
@@ -5051,6 +5954,7 @@ var MemoryService = class {
|
|
|
5051
5954
|
this.embedder,
|
|
5052
5955
|
this.matcher
|
|
5053
5956
|
);
|
|
5957
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
5054
5958
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
5055
5959
|
}
|
|
5056
5960
|
/**
|
|
@@ -5110,16 +6014,16 @@ var MemoryService = class {
|
|
|
5110
6014
|
*/
|
|
5111
6015
|
async initializeSharedStore() {
|
|
5112
6016
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
5113
|
-
if (!
|
|
5114
|
-
|
|
6017
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
6018
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
5115
6019
|
}
|
|
5116
6020
|
this.sharedEventStore = createSharedEventStore(
|
|
5117
|
-
|
|
6021
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
5118
6022
|
);
|
|
5119
6023
|
await this.sharedEventStore.initialize();
|
|
5120
6024
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
5121
6025
|
this.sharedVectorStore = createSharedVectorStore(
|
|
5122
|
-
|
|
6026
|
+
path3.join(sharedPath, "vectors")
|
|
5123
6027
|
);
|
|
5124
6028
|
await this.sharedVectorStore.initialize();
|
|
5125
6029
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -5130,6 +6034,86 @@ var MemoryService = class {
|
|
|
5130
6034
|
);
|
|
5131
6035
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
5132
6036
|
}
|
|
6037
|
+
registerIngestBefore(interceptor) {
|
|
6038
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6039
|
+
}
|
|
6040
|
+
registerIngestAfter(interceptor) {
|
|
6041
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6042
|
+
}
|
|
6043
|
+
registerIngestOnError(interceptor) {
|
|
6044
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6045
|
+
}
|
|
6046
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6047
|
+
const normalizedInput = {
|
|
6048
|
+
...input,
|
|
6049
|
+
metadata: mergeHierarchicalMetadata(
|
|
6050
|
+
{
|
|
6051
|
+
ingest: {
|
|
6052
|
+
operation,
|
|
6053
|
+
pipeline: "default",
|
|
6054
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6055
|
+
},
|
|
6056
|
+
...this.projectHash ? {
|
|
6057
|
+
scope: {
|
|
6058
|
+
project: {
|
|
6059
|
+
hash: this.projectHash,
|
|
6060
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6061
|
+
}
|
|
6062
|
+
},
|
|
6063
|
+
tags: [`proj:${this.projectHash}`]
|
|
6064
|
+
} : {}
|
|
6065
|
+
},
|
|
6066
|
+
input.metadata
|
|
6067
|
+
)
|
|
6068
|
+
};
|
|
6069
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6070
|
+
const meta = normalizedInput.metadata;
|
|
6071
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6072
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6073
|
+
if (!currentTags.includes(projectTag)) {
|
|
6074
|
+
meta.tags = [...currentTags, projectTag];
|
|
6075
|
+
}
|
|
6076
|
+
}
|
|
6077
|
+
if (normalizedInput.metadata) {
|
|
6078
|
+
const meta = normalizedInput.metadata;
|
|
6079
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6080
|
+
if (normalizedTags.length > 0) {
|
|
6081
|
+
meta.tags = normalizedTags;
|
|
6082
|
+
}
|
|
6083
|
+
}
|
|
6084
|
+
await this.ingestInterceptors.run("before", {
|
|
6085
|
+
operation,
|
|
6086
|
+
sessionId: normalizedInput.sessionId,
|
|
6087
|
+
event: normalizedInput
|
|
6088
|
+
});
|
|
6089
|
+
try {
|
|
6090
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6091
|
+
if (result.success && !result.isDuplicate) {
|
|
6092
|
+
if (onSuccess) {
|
|
6093
|
+
await onSuccess(result.eventId);
|
|
6094
|
+
}
|
|
6095
|
+
try {
|
|
6096
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6097
|
+
} catch {
|
|
6098
|
+
}
|
|
6099
|
+
}
|
|
6100
|
+
await this.ingestInterceptors.run("after", {
|
|
6101
|
+
operation,
|
|
6102
|
+
sessionId: normalizedInput.sessionId,
|
|
6103
|
+
event: normalizedInput
|
|
6104
|
+
});
|
|
6105
|
+
return result;
|
|
6106
|
+
} catch (error) {
|
|
6107
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6108
|
+
await this.ingestInterceptors.run("error", {
|
|
6109
|
+
operation,
|
|
6110
|
+
sessionId: normalizedInput.sessionId,
|
|
6111
|
+
event: normalizedInput,
|
|
6112
|
+
error: normalizedError
|
|
6113
|
+
});
|
|
6114
|
+
throw error;
|
|
6115
|
+
}
|
|
6116
|
+
}
|
|
5133
6117
|
/**
|
|
5134
6118
|
* Start a new session
|
|
5135
6119
|
*/
|
|
@@ -5157,50 +6141,57 @@ var MemoryService = class {
|
|
|
5157
6141
|
*/
|
|
5158
6142
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
5159
6143
|
await this.initialize();
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
|
|
5169
|
-
|
|
5170
|
-
|
|
6144
|
+
return this.ingestWithInterceptors(
|
|
6145
|
+
"user_prompt",
|
|
6146
|
+
{
|
|
6147
|
+
eventType: "user_prompt",
|
|
6148
|
+
sessionId,
|
|
6149
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6150
|
+
content,
|
|
6151
|
+
metadata
|
|
6152
|
+
},
|
|
6153
|
+
async (eventId) => {
|
|
6154
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6155
|
+
}
|
|
6156
|
+
);
|
|
5171
6157
|
}
|
|
5172
6158
|
/**
|
|
5173
6159
|
* Store an agent response
|
|
5174
6160
|
*/
|
|
5175
6161
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
5176
6162
|
await this.initialize();
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
6163
|
+
return this.ingestWithInterceptors(
|
|
6164
|
+
"agent_response",
|
|
6165
|
+
{
|
|
6166
|
+
eventType: "agent_response",
|
|
6167
|
+
sessionId,
|
|
6168
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6169
|
+
content,
|
|
6170
|
+
metadata
|
|
6171
|
+
},
|
|
6172
|
+
async (eventId) => {
|
|
6173
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6174
|
+
}
|
|
6175
|
+
);
|
|
5188
6176
|
}
|
|
5189
6177
|
/**
|
|
5190
6178
|
* Store a session summary
|
|
5191
6179
|
*/
|
|
5192
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6180
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
5193
6181
|
await this.initialize();
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
6182
|
+
return this.ingestWithInterceptors(
|
|
6183
|
+
"session_summary",
|
|
6184
|
+
{
|
|
6185
|
+
eventType: "session_summary",
|
|
6186
|
+
sessionId,
|
|
6187
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6188
|
+
content: summary,
|
|
6189
|
+
metadata
|
|
6190
|
+
},
|
|
6191
|
+
async (eventId) => {
|
|
6192
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6193
|
+
}
|
|
6194
|
+
);
|
|
5204
6195
|
}
|
|
5205
6196
|
/**
|
|
5206
6197
|
* Store a tool observation
|
|
@@ -5209,40 +6200,181 @@ var MemoryService = class {
|
|
|
5209
6200
|
await this.initialize();
|
|
5210
6201
|
const content = JSON.stringify(payload);
|
|
5211
6202
|
const turnId = payload.metadata?.turnId;
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
6203
|
+
return this.ingestWithInterceptors(
|
|
6204
|
+
"tool_observation",
|
|
6205
|
+
{
|
|
6206
|
+
eventType: "tool_observation",
|
|
6207
|
+
sessionId,
|
|
6208
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6209
|
+
content,
|
|
6210
|
+
metadata: {
|
|
6211
|
+
toolName: payload.toolName,
|
|
6212
|
+
success: payload.success,
|
|
6213
|
+
...turnId ? { turnId } : {}
|
|
6214
|
+
}
|
|
6215
|
+
},
|
|
6216
|
+
async (eventId) => {
|
|
6217
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6218
|
+
payload.toolName,
|
|
6219
|
+
payload.metadata || {},
|
|
6220
|
+
payload.success
|
|
6221
|
+
);
|
|
6222
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
5221
6223
|
}
|
|
5222
|
-
|
|
5223
|
-
if (result.success && !result.isDuplicate) {
|
|
5224
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
5225
|
-
payload.toolName,
|
|
5226
|
-
payload.metadata || {},
|
|
5227
|
-
payload.success
|
|
5228
|
-
);
|
|
5229
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
5230
|
-
}
|
|
5231
|
-
return result;
|
|
6224
|
+
);
|
|
5232
6225
|
}
|
|
5233
6226
|
/**
|
|
5234
6227
|
* Retrieve relevant memories for a query
|
|
5235
6228
|
*/
|
|
5236
6229
|
async retrieveMemories(query, options) {
|
|
5237
6230
|
await this.initialize();
|
|
6231
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6232
|
+
let result;
|
|
5238
6233
|
if (options?.includeShared && this.sharedStore) {
|
|
5239
|
-
|
|
6234
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
5240
6235
|
...options,
|
|
6236
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6237
|
+
rerankWeights,
|
|
5241
6238
|
includeShared: true,
|
|
5242
|
-
projectHash: this.projectHash || void 0
|
|
6239
|
+
projectHash: this.projectHash || void 0,
|
|
6240
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6241
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6242
|
+
});
|
|
6243
|
+
} else {
|
|
6244
|
+
result = await this.retriever.retrieve(query, {
|
|
6245
|
+
...options,
|
|
6246
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6247
|
+
rerankWeights,
|
|
6248
|
+
projectHash: this.projectHash || void 0,
|
|
6249
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6250
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6251
|
+
});
|
|
6252
|
+
}
|
|
6253
|
+
try {
|
|
6254
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6255
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6256
|
+
eventId: d.eventId,
|
|
6257
|
+
score: d.score,
|
|
6258
|
+
semanticScore: d.semanticScore,
|
|
6259
|
+
lexicalScore: d.lexicalScore,
|
|
6260
|
+
recencyScore: d.recencyScore
|
|
6261
|
+
}));
|
|
6262
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6263
|
+
eventId: d.eventId,
|
|
6264
|
+
score: d.score,
|
|
6265
|
+
semanticScore: d.semanticScore,
|
|
6266
|
+
lexicalScore: d.lexicalScore,
|
|
6267
|
+
recencyScore: d.recencyScore
|
|
6268
|
+
}));
|
|
6269
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6270
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6271
|
+
sessionId: options?.sessionId,
|
|
6272
|
+
projectHash: this.projectHash || void 0,
|
|
6273
|
+
queryText: query,
|
|
6274
|
+
strategy: options?.strategy || "auto",
|
|
6275
|
+
candidateEventIds,
|
|
6276
|
+
selectedEventIds,
|
|
6277
|
+
candidateDetails,
|
|
6278
|
+
selectedDetails,
|
|
6279
|
+
confidence: result.matchResult.confidence,
|
|
6280
|
+
fallbackTrace: result.fallbackTrace || []
|
|
6281
|
+
});
|
|
6282
|
+
} catch {
|
|
6283
|
+
}
|
|
6284
|
+
return result;
|
|
6285
|
+
}
|
|
6286
|
+
getConfiguredRerankWeights() {
|
|
6287
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6288
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6289
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6290
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6291
|
+
if (!allFinite)
|
|
6292
|
+
return void 0;
|
|
6293
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6294
|
+
const total = semantic + lexical + recency;
|
|
6295
|
+
if (!nonNegative || total <= 0)
|
|
6296
|
+
return void 0;
|
|
6297
|
+
return {
|
|
6298
|
+
semantic: semantic / total,
|
|
6299
|
+
lexical: lexical / total,
|
|
6300
|
+
recency: recency / total
|
|
6301
|
+
};
|
|
6302
|
+
}
|
|
6303
|
+
async getRerankWeights(adaptive) {
|
|
6304
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6305
|
+
if (configured)
|
|
6306
|
+
return configured;
|
|
6307
|
+
if (adaptive)
|
|
6308
|
+
return this.getAdaptiveRerankWeights();
|
|
6309
|
+
return void 0;
|
|
6310
|
+
}
|
|
6311
|
+
async rewriteQueryIntent(query) {
|
|
6312
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6313
|
+
return null;
|
|
6314
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6315
|
+
if (!apiUrl)
|
|
6316
|
+
return null;
|
|
6317
|
+
const controller = new AbortController();
|
|
6318
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6319
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6320
|
+
try {
|
|
6321
|
+
const prompt = [
|
|
6322
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6323
|
+
"Return plain text only, one line, no markdown.",
|
|
6324
|
+
`Query: ${query}`
|
|
6325
|
+
].join("\n");
|
|
6326
|
+
const res = await fetch(apiUrl, {
|
|
6327
|
+
method: "POST",
|
|
6328
|
+
headers: {
|
|
6329
|
+
"Content-Type": "application/json",
|
|
6330
|
+
Accept: "*/*",
|
|
6331
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6332
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6333
|
+
},
|
|
6334
|
+
body: JSON.stringify({
|
|
6335
|
+
question: prompt,
|
|
6336
|
+
company_name: null,
|
|
6337
|
+
conversation_id: null
|
|
6338
|
+
}),
|
|
6339
|
+
signal: controller.signal
|
|
5243
6340
|
});
|
|
6341
|
+
const text = (await res.text()).trim();
|
|
6342
|
+
if (!text)
|
|
6343
|
+
return null;
|
|
6344
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6345
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6346
|
+
return null;
|
|
6347
|
+
return oneLine;
|
|
6348
|
+
} catch {
|
|
6349
|
+
return null;
|
|
6350
|
+
} finally {
|
|
6351
|
+
clearTimeout(timeout);
|
|
6352
|
+
}
|
|
6353
|
+
}
|
|
6354
|
+
async getAdaptiveRerankWeights() {
|
|
6355
|
+
try {
|
|
6356
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6357
|
+
if (s.totalEvaluated < 20)
|
|
6358
|
+
return void 0;
|
|
6359
|
+
let semantic = 0.7;
|
|
6360
|
+
let lexical = 0.2;
|
|
6361
|
+
let recency = 0.1;
|
|
6362
|
+
if (s.avgScore < 0.45) {
|
|
6363
|
+
semantic -= 0.1;
|
|
6364
|
+
lexical += 0.1;
|
|
6365
|
+
} else if (s.avgScore > 0.75) {
|
|
6366
|
+
semantic += 0.05;
|
|
6367
|
+
lexical -= 0.05;
|
|
6368
|
+
}
|
|
6369
|
+
if (s.unhelpful > s.helpful) {
|
|
6370
|
+
recency += 0.05;
|
|
6371
|
+
semantic -= 0.03;
|
|
6372
|
+
lexical -= 0.02;
|
|
6373
|
+
}
|
|
6374
|
+
return { semantic, lexical, recency };
|
|
6375
|
+
} catch {
|
|
6376
|
+
return void 0;
|
|
5244
6377
|
}
|
|
5245
|
-
return this.retriever.retrieve(query, options);
|
|
5246
6378
|
}
|
|
5247
6379
|
/**
|
|
5248
6380
|
* Fast keyword search using SQLite FTS5
|
|
@@ -5284,6 +6416,18 @@ var MemoryService = class {
|
|
|
5284
6416
|
/**
|
|
5285
6417
|
* Get memory statistics
|
|
5286
6418
|
*/
|
|
6419
|
+
async getOutboxStats() {
|
|
6420
|
+
await this.initialize();
|
|
6421
|
+
return this.sqliteStore.getOutboxStats();
|
|
6422
|
+
}
|
|
6423
|
+
async getRetrievalTraceStats() {
|
|
6424
|
+
await this.initialize();
|
|
6425
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6426
|
+
}
|
|
6427
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6428
|
+
await this.initialize();
|
|
6429
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6430
|
+
}
|
|
5287
6431
|
async getStats() {
|
|
5288
6432
|
await this.initialize();
|
|
5289
6433
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5774,7 +6918,7 @@ var MemoryService = class {
|
|
|
5774
6918
|
*/
|
|
5775
6919
|
expandPath(p) {
|
|
5776
6920
|
if (p.startsWith("~")) {
|
|
5777
|
-
return
|
|
6921
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5778
6922
|
}
|
|
5779
6923
|
return p;
|
|
5780
6924
|
}
|
|
@@ -5810,6 +6954,7 @@ function getMemoryServiceForProject(projectPath, sharedStoreConfig) {
|
|
|
5810
6954
|
serviceCache.set(hash, new MemoryService({
|
|
5811
6955
|
storagePath,
|
|
5812
6956
|
projectHash: hash,
|
|
6957
|
+
projectPath,
|
|
5813
6958
|
// Override shared store config - hooks don't need DuckDB
|
|
5814
6959
|
sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
|
|
5815
6960
|
analyticsEnabled: false
|
|
@@ -5829,10 +6974,11 @@ function getLightweightMemoryService(sessionId) {
|
|
|
5829
6974
|
const projectInfo = getSessionProject(sessionId);
|
|
5830
6975
|
const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : "lightweight_global";
|
|
5831
6976
|
if (!serviceCache.has(key)) {
|
|
5832
|
-
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) :
|
|
6977
|
+
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) : path3.join(os.homedir(), ".claude-code", "memory");
|
|
5833
6978
|
serviceCache.set(key, new MemoryService({
|
|
5834
6979
|
storagePath,
|
|
5835
6980
|
projectHash: projectInfo?.projectHash,
|
|
6981
|
+
projectPath: projectInfo?.projectPath,
|
|
5836
6982
|
lightweightMode: true,
|
|
5837
6983
|
// Skip embedder/vector/workers
|
|
5838
6984
|
analyticsEnabled: false,
|