claude-memory-layer 1.0.11 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +2389 -286
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1017 -132
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1347 -202
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1339 -194
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1343 -198
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1351 -206
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1347 -202
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +1436 -211
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +1445 -220
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1345 -199
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +69 -2
- package/dist/ui/index.html +8 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +2 -1
- package/scripts/build.ts +6 -0
- package/src/cli/index.ts +281 -2
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +350 -1
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/types.ts +28 -0
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +3 -1
- package/src/server/api/stats.ts +46 -1
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +373 -68
- package/src/ui/app.js +69 -2
- package/src/ui/index.html +8 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
|
@@ -10,9 +10,9 @@ const __dirname = dirname(__filename);
|
|
|
10
10
|
import { randomUUID as randomUUID9 } from "crypto";
|
|
11
11
|
|
|
12
12
|
// src/services/memory-service.ts
|
|
13
|
-
import * as
|
|
13
|
+
import * as path3 from "path";
|
|
14
14
|
import * as os from "os";
|
|
15
|
-
import * as
|
|
15
|
+
import * as fs4 from "fs";
|
|
16
16
|
import * as crypto2 from "crypto";
|
|
17
17
|
|
|
18
18
|
// src/core/event-store.ts
|
|
@@ -70,11 +70,11 @@ function toDate(value) {
|
|
|
70
70
|
return new Date(value);
|
|
71
71
|
return new Date(String(value));
|
|
72
72
|
}
|
|
73
|
-
function createDatabase(
|
|
73
|
+
function createDatabase(path5, options) {
|
|
74
74
|
if (options?.readOnly) {
|
|
75
|
-
return new duckdb.Database(
|
|
75
|
+
return new duckdb.Database(path5, { access_mode: "READ_ONLY" });
|
|
76
76
|
}
|
|
77
|
-
return new duckdb.Database(
|
|
77
|
+
return new duckdb.Database(path5);
|
|
78
78
|
}
|
|
79
79
|
function dbRun(db, sql, params = []) {
|
|
80
80
|
return new Promise((resolve2, reject) => {
|
|
@@ -338,6 +338,17 @@ var EventStore = class {
|
|
|
338
338
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
339
339
|
)
|
|
340
340
|
`);
|
|
341
|
+
await dbRun(this.db, `
|
|
342
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
343
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
344
|
+
rule TEXT NOT NULL,
|
|
345
|
+
topics JSON,
|
|
346
|
+
source_memory_ids JSON,
|
|
347
|
+
source_events JSON,
|
|
348
|
+
confidence FLOAT DEFAULT 0.5,
|
|
349
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
350
|
+
)
|
|
351
|
+
`);
|
|
341
352
|
await dbRun(this.db, `
|
|
342
353
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
343
354
|
key VARCHAR PRIMARY KEY,
|
|
@@ -357,6 +368,7 @@ var EventStore = class {
|
|
|
357
368
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
358
369
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
359
370
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
371
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
360
372
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
361
373
|
this.initialized = true;
|
|
362
374
|
}
|
|
@@ -746,12 +758,12 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
746
758
|
import Database from "better-sqlite3";
|
|
747
759
|
import * as fs from "fs";
|
|
748
760
|
import * as nodePath from "path";
|
|
749
|
-
function createSQLiteDatabase(
|
|
750
|
-
const dir = nodePath.dirname(
|
|
761
|
+
function createSQLiteDatabase(path5, options) {
|
|
762
|
+
const dir = nodePath.dirname(path5);
|
|
751
763
|
if (!fs.existsSync(dir)) {
|
|
752
764
|
fs.mkdirSync(dir, { recursive: true });
|
|
753
765
|
}
|
|
754
|
-
const db = new Database(
|
|
766
|
+
const db = new Database(path5, {
|
|
755
767
|
readonly: options?.readonly ?? false
|
|
756
768
|
});
|
|
757
769
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -792,6 +804,64 @@ function toSQLiteTimestamp(date) {
|
|
|
792
804
|
return date.toISOString();
|
|
793
805
|
}
|
|
794
806
|
|
|
807
|
+
// src/core/markdown-mirror.ts
|
|
808
|
+
import * as fs2 from "fs/promises";
|
|
809
|
+
import * as path from "path";
|
|
810
|
+
var DEFAULT_NAMESPACE = "default";
|
|
811
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
812
|
+
function sanitizeSegment(input, fallback) {
|
|
813
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
814
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
815
|
+
if (!safe || safe === "." || safe === "..")
|
|
816
|
+
return fallback;
|
|
817
|
+
return safe;
|
|
818
|
+
}
|
|
819
|
+
function getCategorySegments(metadata, eventType) {
|
|
820
|
+
const raw = metadata?.categoryPath;
|
|
821
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
822
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
823
|
+
}
|
|
824
|
+
const single = metadata?.category;
|
|
825
|
+
if (typeof single === "string" && single.trim()) {
|
|
826
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
827
|
+
}
|
|
828
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
829
|
+
}
|
|
830
|
+
function buildMirrorPath(rootDir, event) {
|
|
831
|
+
const metadata = event.metadata;
|
|
832
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
833
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
834
|
+
const d = event.timestamp;
|
|
835
|
+
const yyyy = d.getFullYear();
|
|
836
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
837
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
838
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
839
|
+
}
|
|
840
|
+
function formatMirrorEntry(event) {
|
|
841
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
842
|
+
return [
|
|
843
|
+
"",
|
|
844
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
845
|
+
` id: ${event.id}`,
|
|
846
|
+
` type: ${event.eventType}`,
|
|
847
|
+
` session: ${event.sessionId}`,
|
|
848
|
+
` category: ${category}`,
|
|
849
|
+
" content: |",
|
|
850
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
851
|
+
].join("\n") + "\n";
|
|
852
|
+
}
|
|
853
|
+
var MarkdownMirror = class {
|
|
854
|
+
constructor(rootDir) {
|
|
855
|
+
this.rootDir = rootDir;
|
|
856
|
+
}
|
|
857
|
+
async append(event) {
|
|
858
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
859
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
860
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
861
|
+
return outPath;
|
|
862
|
+
}
|
|
863
|
+
};
|
|
864
|
+
|
|
795
865
|
// src/core/sqlite-event-store.ts
|
|
796
866
|
var SQLiteEventStore = class {
|
|
797
867
|
constructor(dbPath, options) {
|
|
@@ -801,10 +871,12 @@ var SQLiteEventStore = class {
|
|
|
801
871
|
readonly: this.readOnly,
|
|
802
872
|
walMode: !this.readOnly
|
|
803
873
|
});
|
|
874
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
804
875
|
}
|
|
805
876
|
db;
|
|
806
877
|
initialized = false;
|
|
807
878
|
readOnly;
|
|
879
|
+
markdownMirror;
|
|
808
880
|
/**
|
|
809
881
|
* Initialize database schema
|
|
810
882
|
*/
|
|
@@ -1011,6 +1083,17 @@ var SQLiteEventStore = class {
|
|
|
1011
1083
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1012
1084
|
);
|
|
1013
1085
|
|
|
1086
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1087
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1088
|
+
rule_id TEXT PRIMARY KEY,
|
|
1089
|
+
rule TEXT NOT NULL,
|
|
1090
|
+
topics TEXT,
|
|
1091
|
+
source_memory_ids TEXT,
|
|
1092
|
+
source_events TEXT,
|
|
1093
|
+
confidence REAL DEFAULT 0.5,
|
|
1094
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1095
|
+
);
|
|
1096
|
+
|
|
1014
1097
|
-- Endless Mode Config table
|
|
1015
1098
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1016
1099
|
key TEXT PRIMARY KEY,
|
|
@@ -1035,6 +1118,24 @@ var SQLiteEventStore = class {
|
|
|
1035
1118
|
measured_at TEXT
|
|
1036
1119
|
);
|
|
1037
1120
|
|
|
1121
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1122
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1123
|
+
trace_id TEXT PRIMARY KEY,
|
|
1124
|
+
session_id TEXT,
|
|
1125
|
+
project_hash TEXT,
|
|
1126
|
+
query_text TEXT NOT NULL,
|
|
1127
|
+
strategy TEXT,
|
|
1128
|
+
candidate_event_ids TEXT,
|
|
1129
|
+
selected_event_ids TEXT,
|
|
1130
|
+
candidate_details_json TEXT,
|
|
1131
|
+
selected_details_json TEXT,
|
|
1132
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1133
|
+
selected_count INTEGER DEFAULT 0,
|
|
1134
|
+
confidence TEXT,
|
|
1135
|
+
fallback_trace TEXT,
|
|
1136
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1137
|
+
);
|
|
1138
|
+
|
|
1038
1139
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1039
1140
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1040
1141
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1059,10 +1160,14 @@ var SQLiteEventStore = class {
|
|
|
1059
1160
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1060
1161
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1061
1162
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1163
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1062
1164
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1063
1165
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1064
1166
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1065
1167
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1168
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1169
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1170
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1066
1171
|
|
|
1067
1172
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1068
1173
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1086,6 +1191,14 @@ var SQLiteEventStore = class {
|
|
|
1086
1191
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1087
1192
|
END;
|
|
1088
1193
|
`);
|
|
1194
|
+
try {
|
|
1195
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1196
|
+
} catch {
|
|
1197
|
+
}
|
|
1198
|
+
try {
|
|
1199
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1200
|
+
} catch {
|
|
1201
|
+
}
|
|
1089
1202
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1090
1203
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1091
1204
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1185,6 +1298,21 @@ var SQLiteEventStore = class {
|
|
|
1185
1298
|
insertLevel.run(id);
|
|
1186
1299
|
});
|
|
1187
1300
|
transaction();
|
|
1301
|
+
if (this.markdownMirror) {
|
|
1302
|
+
const event = {
|
|
1303
|
+
id,
|
|
1304
|
+
eventType: input.eventType,
|
|
1305
|
+
sessionId: input.sessionId,
|
|
1306
|
+
timestamp: input.timestamp,
|
|
1307
|
+
content: input.content,
|
|
1308
|
+
canonicalKey,
|
|
1309
|
+
dedupeKey,
|
|
1310
|
+
metadata
|
|
1311
|
+
};
|
|
1312
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1313
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1314
|
+
});
|
|
1315
|
+
}
|
|
1188
1316
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1189
1317
|
} catch (error) {
|
|
1190
1318
|
return {
|
|
@@ -1243,6 +1371,92 @@ var SQLiteEventStore = class {
|
|
|
1243
1371
|
);
|
|
1244
1372
|
return rows.map(this.rowToEvent);
|
|
1245
1373
|
}
|
|
1374
|
+
/**
|
|
1375
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1376
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1377
|
+
*/
|
|
1378
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1379
|
+
await this.initialize();
|
|
1380
|
+
const rows = sqliteAll(
|
|
1381
|
+
this.db,
|
|
1382
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1383
|
+
[lastRowid, limit]
|
|
1384
|
+
);
|
|
1385
|
+
return rows.map((row) => ({
|
|
1386
|
+
rowid: row._rowid,
|
|
1387
|
+
event: this.rowToEvent(row)
|
|
1388
|
+
}));
|
|
1389
|
+
}
|
|
1390
|
+
/**
|
|
1391
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1392
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1393
|
+
*
|
|
1394
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1395
|
+
*/
|
|
1396
|
+
async importEvents(events) {
|
|
1397
|
+
if (events.length === 0)
|
|
1398
|
+
return { inserted: 0, skipped: 0 };
|
|
1399
|
+
if (this.readOnly)
|
|
1400
|
+
return { inserted: 0, skipped: events.length };
|
|
1401
|
+
await this.initialize();
|
|
1402
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1403
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1404
|
+
const insertEvent = this.db.prepare(`
|
|
1405
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1406
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1407
|
+
`);
|
|
1408
|
+
const insertDedup = this.db.prepare(`
|
|
1409
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1410
|
+
`);
|
|
1411
|
+
const insertLevel = this.db.prepare(`
|
|
1412
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1413
|
+
`);
|
|
1414
|
+
let inserted = 0;
|
|
1415
|
+
let skipped = 0;
|
|
1416
|
+
const insertedEvents = [];
|
|
1417
|
+
const tx = this.db.transaction((batch) => {
|
|
1418
|
+
for (const ev of batch) {
|
|
1419
|
+
const existingById = getById.get(ev.id);
|
|
1420
|
+
if (existingById) {
|
|
1421
|
+
skipped++;
|
|
1422
|
+
continue;
|
|
1423
|
+
}
|
|
1424
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1425
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1426
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1427
|
+
if (existingByDedupe) {
|
|
1428
|
+
skipped++;
|
|
1429
|
+
continue;
|
|
1430
|
+
}
|
|
1431
|
+
const metadata = ev.metadata || {};
|
|
1432
|
+
const turnId = metadata.turnId;
|
|
1433
|
+
insertEvent.run(
|
|
1434
|
+
ev.id,
|
|
1435
|
+
ev.eventType,
|
|
1436
|
+
ev.sessionId,
|
|
1437
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1438
|
+
ev.content,
|
|
1439
|
+
canonicalKey,
|
|
1440
|
+
dedupeKey,
|
|
1441
|
+
JSON.stringify(metadata),
|
|
1442
|
+
turnId ?? null
|
|
1443
|
+
);
|
|
1444
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1445
|
+
insertLevel.run(ev.id);
|
|
1446
|
+
inserted++;
|
|
1447
|
+
insertedEvents.push(ev);
|
|
1448
|
+
}
|
|
1449
|
+
});
|
|
1450
|
+
tx(events);
|
|
1451
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1452
|
+
for (const ev of insertedEvents) {
|
|
1453
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1454
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1455
|
+
});
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
return { inserted, skipped };
|
|
1459
|
+
}
|
|
1246
1460
|
/**
|
|
1247
1461
|
* Create or update session
|
|
1248
1462
|
*/
|
|
@@ -1405,6 +1619,35 @@ var SQLiteEventStore = class {
|
|
|
1405
1619
|
[error, ...ids]
|
|
1406
1620
|
);
|
|
1407
1621
|
}
|
|
1622
|
+
/**
|
|
1623
|
+
* Get embedding/vector outbox health statistics
|
|
1624
|
+
*/
|
|
1625
|
+
async getOutboxStats() {
|
|
1626
|
+
await this.initialize();
|
|
1627
|
+
const embeddingRows = sqliteAll(
|
|
1628
|
+
this.db,
|
|
1629
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1630
|
+
);
|
|
1631
|
+
const vectorRows = sqliteAll(
|
|
1632
|
+
this.db,
|
|
1633
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1634
|
+
);
|
|
1635
|
+
const fromRows = (rows) => {
|
|
1636
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1637
|
+
for (const row of rows) {
|
|
1638
|
+
const key = row.status;
|
|
1639
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1640
|
+
out[key] += row.count;
|
|
1641
|
+
}
|
|
1642
|
+
out.total += row.count;
|
|
1643
|
+
}
|
|
1644
|
+
return out;
|
|
1645
|
+
};
|
|
1646
|
+
return {
|
|
1647
|
+
embedding: fromRows(embeddingRows),
|
|
1648
|
+
vector: fromRows(vectorRows)
|
|
1649
|
+
};
|
|
1650
|
+
}
|
|
1408
1651
|
/**
|
|
1409
1652
|
* Update memory level
|
|
1410
1653
|
*/
|
|
@@ -1763,6 +2006,79 @@ var SQLiteEventStore = class {
|
|
|
1763
2006
|
getDatabase() {
|
|
1764
2007
|
return this.db;
|
|
1765
2008
|
}
|
|
2009
|
+
async recordRetrievalTrace(input) {
|
|
2010
|
+
await this.initialize();
|
|
2011
|
+
const traceId = randomUUID2();
|
|
2012
|
+
sqliteRun(
|
|
2013
|
+
this.db,
|
|
2014
|
+
`INSERT INTO retrieval_traces (
|
|
2015
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2016
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2017
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2018
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2019
|
+
[
|
|
2020
|
+
traceId,
|
|
2021
|
+
input.sessionId || null,
|
|
2022
|
+
input.projectHash || null,
|
|
2023
|
+
input.queryText,
|
|
2024
|
+
input.strategy || null,
|
|
2025
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2026
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2027
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2028
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2029
|
+
(input.candidateEventIds || []).length,
|
|
2030
|
+
(input.selectedEventIds || []).length,
|
|
2031
|
+
input.confidence || null,
|
|
2032
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2033
|
+
]
|
|
2034
|
+
);
|
|
2035
|
+
}
|
|
2036
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2037
|
+
await this.initialize();
|
|
2038
|
+
const rows = sqliteAll(
|
|
2039
|
+
this.db,
|
|
2040
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2041
|
+
[limit]
|
|
2042
|
+
);
|
|
2043
|
+
return rows.map((row) => ({
|
|
2044
|
+
traceId: row.trace_id,
|
|
2045
|
+
sessionId: row.session_id || void 0,
|
|
2046
|
+
projectHash: row.project_hash || void 0,
|
|
2047
|
+
queryText: row.query_text,
|
|
2048
|
+
strategy: row.strategy || void 0,
|
|
2049
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2050
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2051
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2052
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2053
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2054
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2055
|
+
confidence: row.confidence || void 0,
|
|
2056
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2057
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2058
|
+
}));
|
|
2059
|
+
}
|
|
2060
|
+
async getRetrievalTraceStats() {
|
|
2061
|
+
await this.initialize();
|
|
2062
|
+
const row = sqliteGet(
|
|
2063
|
+
this.db,
|
|
2064
|
+
`SELECT
|
|
2065
|
+
COUNT(*) as total_queries,
|
|
2066
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2067
|
+
AVG(selected_count) as avg_selected_count,
|
|
2068
|
+
CASE
|
|
2069
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2070
|
+
ELSE 0
|
|
2071
|
+
END as selection_rate
|
|
2072
|
+
FROM retrieval_traces`,
|
|
2073
|
+
[]
|
|
2074
|
+
);
|
|
2075
|
+
return {
|
|
2076
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2077
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2078
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2079
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2080
|
+
};
|
|
2081
|
+
}
|
|
1766
2082
|
/**
|
|
1767
2083
|
* Close database connection
|
|
1768
2084
|
*/
|
|
@@ -2624,7 +2940,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2624
2940
|
topK: 5,
|
|
2625
2941
|
minScore: 0.7,
|
|
2626
2942
|
maxTokens: 2e3,
|
|
2627
|
-
includeSessionContext: true
|
|
2943
|
+
includeSessionContext: true,
|
|
2944
|
+
strategy: "auto",
|
|
2945
|
+
rerankWithKeyword: true,
|
|
2946
|
+
decayPolicy: {
|
|
2947
|
+
enabled: true,
|
|
2948
|
+
windowDays: 30,
|
|
2949
|
+
maxPenalty: 0.15
|
|
2950
|
+
},
|
|
2951
|
+
graphHop: {
|
|
2952
|
+
enabled: true,
|
|
2953
|
+
maxHops: 1,
|
|
2954
|
+
hopPenalty: 0.08
|
|
2955
|
+
},
|
|
2956
|
+
projectScopeMode: "global"
|
|
2628
2957
|
};
|
|
2629
2958
|
var Retriever = class {
|
|
2630
2959
|
eventStore;
|
|
@@ -2634,6 +2963,7 @@ var Retriever = class {
|
|
|
2634
2963
|
sharedStore;
|
|
2635
2964
|
sharedVectorStore;
|
|
2636
2965
|
graduation;
|
|
2966
|
+
queryRewriter;
|
|
2637
2967
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2638
2968
|
this.eventStore = eventStore;
|
|
2639
2969
|
this.vectorStore = vectorStore;
|
|
@@ -2642,47 +2972,105 @@ var Retriever = class {
|
|
|
2642
2972
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2643
2973
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2644
2974
|
}
|
|
2645
|
-
/**
|
|
2646
|
-
* Set graduation pipeline for access tracking
|
|
2647
|
-
*/
|
|
2648
2975
|
setGraduationPipeline(graduation) {
|
|
2649
2976
|
this.graduation = graduation;
|
|
2650
2977
|
}
|
|
2651
|
-
/**
|
|
2652
|
-
* Set shared stores after construction
|
|
2653
|
-
*/
|
|
2654
2978
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2655
2979
|
this.sharedStore = sharedStore;
|
|
2656
2980
|
this.sharedVectorStore = sharedVectorStore;
|
|
2657
2981
|
}
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2982
|
+
setQueryRewriter(rewriter) {
|
|
2983
|
+
this.queryRewriter = rewriter;
|
|
2984
|
+
}
|
|
2661
2985
|
async retrieve(query, options = {}) {
|
|
2662
2986
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2663
|
-
const
|
|
2664
|
-
const
|
|
2665
|
-
|
|
2666
|
-
|
|
2987
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
2988
|
+
const fallbackTrace = [];
|
|
2989
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
2990
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
2991
|
+
let current = await this.runStage(query, {
|
|
2992
|
+
strategy: primaryStrategy,
|
|
2993
|
+
topK: opts.topK,
|
|
2667
2994
|
minScore: opts.minScore,
|
|
2668
|
-
sessionId:
|
|
2995
|
+
sessionId: sessionFilter,
|
|
2996
|
+
scope: opts.scope,
|
|
2997
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
2998
|
+
rerankWeights: opts.rerankWeights,
|
|
2999
|
+
decayPolicy: opts.decayPolicy,
|
|
3000
|
+
intentRewrite: opts.intentRewrite === true,
|
|
3001
|
+
graphHop: opts.graphHop,
|
|
3002
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3003
|
+
projectHash: opts.projectHash,
|
|
3004
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2669
3005
|
});
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
3006
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3007
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3008
|
+
current = await this.runStage(query, {
|
|
3009
|
+
strategy: "deep",
|
|
3010
|
+
topK: opts.topK,
|
|
3011
|
+
minScore: opts.minScore,
|
|
3012
|
+
sessionId: sessionFilter,
|
|
3013
|
+
scope: opts.scope,
|
|
3014
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3015
|
+
rerankWeights: opts.rerankWeights,
|
|
3016
|
+
decayPolicy: opts.decayPolicy,
|
|
3017
|
+
graphHop: opts.graphHop,
|
|
3018
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3019
|
+
projectHash: opts.projectHash,
|
|
3020
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3021
|
+
});
|
|
3022
|
+
fallbackTrace.push("fallback:deep");
|
|
3023
|
+
}
|
|
3024
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3025
|
+
current = await this.runStage(query, {
|
|
3026
|
+
strategy: "deep",
|
|
3027
|
+
topK: opts.topK,
|
|
3028
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3029
|
+
sessionId: void 0,
|
|
3030
|
+
scope: void 0,
|
|
3031
|
+
rerankWithKeyword: true,
|
|
3032
|
+
rerankWeights: opts.rerankWeights,
|
|
3033
|
+
decayPolicy: opts.decayPolicy,
|
|
3034
|
+
graphHop: opts.graphHop,
|
|
3035
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3036
|
+
projectHash: opts.projectHash,
|
|
3037
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3038
|
+
});
|
|
3039
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3040
|
+
}
|
|
3041
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3042
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3043
|
+
current = {
|
|
3044
|
+
results: summary,
|
|
3045
|
+
candidateResults: summary,
|
|
3046
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3047
|
+
};
|
|
3048
|
+
fallbackTrace.push("fallback:summary");
|
|
3049
|
+
}
|
|
3050
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2675
3051
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2676
3052
|
return {
|
|
2677
3053
|
memories,
|
|
2678
|
-
matchResult,
|
|
3054
|
+
matchResult: current.matchResult,
|
|
2679
3055
|
totalTokens: this.estimateTokens(context),
|
|
2680
|
-
context
|
|
3056
|
+
context,
|
|
3057
|
+
fallbackTrace,
|
|
3058
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3059
|
+
eventId: r.eventId,
|
|
3060
|
+
score: r.score,
|
|
3061
|
+
semanticScore: r.semanticScore,
|
|
3062
|
+
lexicalScore: r.lexicalScore,
|
|
3063
|
+
recencyScore: r.recencyScore
|
|
3064
|
+
})),
|
|
3065
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3066
|
+
eventId: r.eventId,
|
|
3067
|
+
score: r.score,
|
|
3068
|
+
semanticScore: r.semanticScore,
|
|
3069
|
+
lexicalScore: r.lexicalScore,
|
|
3070
|
+
recencyScore: r.recencyScore
|
|
3071
|
+
}))
|
|
2681
3072
|
};
|
|
2682
3073
|
}
|
|
2683
|
-
/**
|
|
2684
|
-
* Retrieve with unified search (project + shared)
|
|
2685
|
-
*/
|
|
2686
3074
|
async retrieveUnified(query, options = {}) {
|
|
2687
3075
|
const projectResult = await this.retrieve(query, options);
|
|
2688
3076
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2690,22 +3078,19 @@ var Retriever = class {
|
|
|
2690
3078
|
}
|
|
2691
3079
|
try {
|
|
2692
3080
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2693
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
excludeProjectHash: options.projectHash
|
|
2699
|
-
}
|
|
2700
|
-
);
|
|
3081
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3082
|
+
limit: options.topK || 5,
|
|
3083
|
+
minScore: options.minScore || 0.7,
|
|
3084
|
+
excludeProjectHash: options.projectHash
|
|
3085
|
+
});
|
|
2701
3086
|
const sharedMemories = [];
|
|
2702
3087
|
for (const result of sharedVectorResults) {
|
|
2703
3088
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2704
|
-
if (entry)
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
3089
|
+
if (!entry)
|
|
3090
|
+
continue;
|
|
3091
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3092
|
+
sharedMemories.push(entry);
|
|
3093
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2709
3094
|
}
|
|
2710
3095
|
}
|
|
2711
3096
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2720,50 +3105,243 @@ var Retriever = class {
|
|
|
2720
3105
|
return projectResult;
|
|
2721
3106
|
}
|
|
2722
3107
|
}
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
3108
|
+
async runStage(query, input) {
|
|
3109
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3110
|
+
strategy: input.strategy,
|
|
3111
|
+
topK: input.topK,
|
|
3112
|
+
minScore: input.minScore,
|
|
3113
|
+
sessionId: input.sessionId
|
|
3114
|
+
});
|
|
3115
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3116
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3117
|
+
if (rewritten && rewritten !== query) {
|
|
3118
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3119
|
+
strategy: "deep",
|
|
3120
|
+
topK: input.topK,
|
|
3121
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3122
|
+
sessionId: input.sessionId
|
|
3123
|
+
});
|
|
3124
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3128
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3129
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3130
|
+
limit: input.topK * 4
|
|
3131
|
+
});
|
|
3132
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3133
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3134
|
+
scope: input.scope,
|
|
3135
|
+
projectScopeMode: input.projectScopeMode,
|
|
3136
|
+
projectHash: input.projectHash,
|
|
3137
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3138
|
+
});
|
|
3139
|
+
const top = filtered.slice(0, input.topK);
|
|
3140
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3141
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3142
|
+
}
|
|
3143
|
+
mergeResults(primary, secondary, limit) {
|
|
3144
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3145
|
+
for (const row of primary)
|
|
3146
|
+
byId.set(row.eventId, row);
|
|
3147
|
+
for (const row of secondary) {
|
|
3148
|
+
const prev = byId.get(row.eventId);
|
|
3149
|
+
if (!prev || row.score > prev.score) {
|
|
3150
|
+
byId.set(row.eventId, row);
|
|
3151
|
+
}
|
|
3152
|
+
}
|
|
3153
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3154
|
+
}
|
|
3155
|
+
async expandGraphHops(seeds, opts) {
|
|
3156
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3157
|
+
for (const s of seeds)
|
|
3158
|
+
byId.set(s.eventId, s);
|
|
3159
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3160
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3161
|
+
const next = [];
|
|
3162
|
+
for (const f of frontier) {
|
|
3163
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3164
|
+
if (!ev)
|
|
3165
|
+
continue;
|
|
3166
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3167
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3168
|
+
for (const rid of relatedIds) {
|
|
3169
|
+
if (byId.has(rid))
|
|
3170
|
+
continue;
|
|
3171
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3172
|
+
if (!target)
|
|
3173
|
+
continue;
|
|
3174
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3175
|
+
const row = {
|
|
3176
|
+
id: `hop-${hop}-${rid}`,
|
|
3177
|
+
eventId: target.id,
|
|
3178
|
+
content: target.content,
|
|
3179
|
+
score,
|
|
3180
|
+
sessionId: target.sessionId,
|
|
3181
|
+
eventType: target.eventType,
|
|
3182
|
+
timestamp: target.timestamp.toISOString()
|
|
3183
|
+
};
|
|
3184
|
+
byId.set(row.eventId, row);
|
|
3185
|
+
next.push({ row, hop });
|
|
3186
|
+
if (byId.size >= opts.limit)
|
|
3187
|
+
break;
|
|
2744
3188
|
}
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
`;
|
|
3189
|
+
if (byId.size >= opts.limit)
|
|
3190
|
+
break;
|
|
2748
3191
|
}
|
|
3192
|
+
frontier = next;
|
|
3193
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3194
|
+
break;
|
|
2749
3195
|
}
|
|
2750
|
-
return
|
|
3196
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3197
|
+
}
|
|
3198
|
+
shouldFallback(matchResult, results) {
|
|
3199
|
+
if (results.length === 0)
|
|
3200
|
+
return true;
|
|
3201
|
+
if (matchResult.confidence === "none")
|
|
3202
|
+
return true;
|
|
3203
|
+
return false;
|
|
3204
|
+
}
|
|
3205
|
+
async buildSummaryFallback(query, topK) {
|
|
3206
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3207
|
+
const q = this.tokenize(query);
|
|
3208
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3209
|
+
id: `summary-${row.e.id}`,
|
|
3210
|
+
eventId: row.e.id,
|
|
3211
|
+
content: row.e.content,
|
|
3212
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3213
|
+
sessionId: row.e.sessionId,
|
|
3214
|
+
eventType: row.e.eventType,
|
|
3215
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3216
|
+
}));
|
|
3217
|
+
return ranked;
|
|
3218
|
+
}
|
|
3219
|
+
async searchByStrategy(query, input) {
|
|
3220
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3221
|
+
if (strategy === "fast") {
|
|
3222
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3223
|
+
limit: Math.max(5, input.topK * 3),
|
|
3224
|
+
sessionId: input.sessionId
|
|
3225
|
+
});
|
|
3226
|
+
return keyword;
|
|
3227
|
+
}
|
|
3228
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3229
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3230
|
+
limit: Math.max(5, input.topK * 3),
|
|
3231
|
+
minScore: input.minScore,
|
|
3232
|
+
sessionId: input.sessionId
|
|
3233
|
+
});
|
|
3234
|
+
}
|
|
3235
|
+
async searchByKeyword(query, input) {
|
|
3236
|
+
if (this.eventStore.keywordSearch) {
|
|
3237
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3238
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3239
|
+
return filtered2.map((row, idx) => ({
|
|
3240
|
+
id: `kw-${row.event.id}`,
|
|
3241
|
+
eventId: row.event.id,
|
|
3242
|
+
content: row.event.content,
|
|
3243
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3244
|
+
sessionId: row.event.sessionId,
|
|
3245
|
+
eventType: row.event.eventType,
|
|
3246
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3247
|
+
}));
|
|
3248
|
+
}
|
|
3249
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3250
|
+
const tokens = this.tokenize(query);
|
|
3251
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3252
|
+
return filtered.map((row, idx) => ({
|
|
3253
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3254
|
+
eventId: row.e.id,
|
|
3255
|
+
content: row.e.content,
|
|
3256
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3257
|
+
sessionId: row.e.sessionId,
|
|
3258
|
+
eventType: row.e.eventType,
|
|
3259
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3260
|
+
}));
|
|
3261
|
+
}
|
|
3262
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3263
|
+
const q = this.tokenize(query);
|
|
3264
|
+
const now = Date.now();
|
|
3265
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3266
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3267
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3268
|
+
const total = sw + lw + rw || 1;
|
|
3269
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3270
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3271
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3272
|
+
return [...results].map((r) => {
|
|
3273
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3274
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3275
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3276
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3277
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3278
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3279
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3280
|
+
}
|
|
3281
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3282
|
+
}).sort((a, b) => b.score - a.score);
|
|
3283
|
+
}
|
|
3284
|
+
async applyScopeFilters(results, options) {
|
|
3285
|
+
const scope = options?.scope;
|
|
3286
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3287
|
+
const allowedProjectHashes = new Set(
|
|
3288
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3289
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3290
|
+
)
|
|
3291
|
+
);
|
|
3292
|
+
if (!scope && projectScopeMode === "global")
|
|
3293
|
+
return results;
|
|
3294
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3295
|
+
const filtered = [];
|
|
3296
|
+
for (const result of results) {
|
|
3297
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3298
|
+
continue;
|
|
3299
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3300
|
+
continue;
|
|
3301
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3302
|
+
continue;
|
|
3303
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3304
|
+
if (!event)
|
|
3305
|
+
continue;
|
|
3306
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3307
|
+
continue;
|
|
3308
|
+
if (normalizedIncludes.length > 0) {
|
|
3309
|
+
const lc = event.content.toLowerCase();
|
|
3310
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3311
|
+
continue;
|
|
3312
|
+
}
|
|
3313
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3314
|
+
continue;
|
|
3315
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3316
|
+
filtered.push({ result, projectHash });
|
|
3317
|
+
}
|
|
3318
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3319
|
+
return filtered.map((x) => x.result);
|
|
3320
|
+
}
|
|
3321
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3322
|
+
if (projectScopeMode === "strict") {
|
|
3323
|
+
return projectMatched.map((x) => x.result);
|
|
3324
|
+
}
|
|
3325
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3326
|
+
}
|
|
3327
|
+
extractProjectHash(metadata) {
|
|
3328
|
+
if (!metadata || typeof metadata !== "object")
|
|
3329
|
+
return void 0;
|
|
3330
|
+
const scope = metadata.scope;
|
|
3331
|
+
if (!scope || typeof scope !== "object")
|
|
3332
|
+
return void 0;
|
|
3333
|
+
const project = scope.project;
|
|
3334
|
+
if (!project || typeof project !== "object")
|
|
3335
|
+
return void 0;
|
|
3336
|
+
const hash = project.hash;
|
|
3337
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2751
3338
|
}
|
|
2752
|
-
/**
|
|
2753
|
-
* Retrieve memories from a specific session
|
|
2754
|
-
*/
|
|
2755
3339
|
async retrieveFromSession(sessionId) {
|
|
2756
3340
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2757
3341
|
}
|
|
2758
|
-
/**
|
|
2759
|
-
* Get recent memories across all sessions
|
|
2760
|
-
*/
|
|
2761
3342
|
async retrieveRecent(limit = 100) {
|
|
2762
3343
|
return this.eventStore.getRecentEvents(limit);
|
|
2763
3344
|
}
|
|
2764
|
-
/**
|
|
2765
|
-
* Enrich search results with full event data
|
|
2766
|
-
*/
|
|
2767
3345
|
async enrichResults(results, options) {
|
|
2768
3346
|
const memories = [];
|
|
2769
3347
|
for (const result of results) {
|
|
@@ -2771,27 +3349,16 @@ var Retriever = class {
|
|
|
2771
3349
|
if (!event)
|
|
2772
3350
|
continue;
|
|
2773
3351
|
if (this.graduation) {
|
|
2774
|
-
this.graduation.recordAccess(
|
|
2775
|
-
event.id,
|
|
2776
|
-
options.sessionId || "unknown",
|
|
2777
|
-
result.score
|
|
2778
|
-
);
|
|
3352
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2779
3353
|
}
|
|
2780
3354
|
let sessionContext;
|
|
2781
3355
|
if (options.includeSessionContext) {
|
|
2782
3356
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2783
3357
|
}
|
|
2784
|
-
memories.push({
|
|
2785
|
-
event,
|
|
2786
|
-
score: result.score,
|
|
2787
|
-
sessionContext
|
|
2788
|
-
});
|
|
3358
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2789
3359
|
}
|
|
2790
3360
|
return memories;
|
|
2791
3361
|
}
|
|
2792
|
-
/**
|
|
2793
|
-
* Get surrounding context from the same session
|
|
2794
|
-
*/
|
|
2795
3362
|
async getSessionContext(sessionId, eventId) {
|
|
2796
3363
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2797
3364
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2804,55 +3371,86 @@ var Retriever = class {
|
|
|
2804
3371
|
return void 0;
|
|
2805
3372
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2806
3373
|
}
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
3374
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3375
|
+
let context = projectResult.context;
|
|
3376
|
+
if (sharedMemories.length === 0)
|
|
3377
|
+
return context;
|
|
3378
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3379
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3380
|
+
context += `### ${memory.title}
|
|
3381
|
+
`;
|
|
3382
|
+
if (memory.symptoms.length > 0)
|
|
3383
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3384
|
+
`;
|
|
3385
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3386
|
+
`;
|
|
3387
|
+
context += `**Solution:** ${memory.solution}
|
|
3388
|
+
`;
|
|
3389
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3390
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3391
|
+
`;
|
|
3392
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3393
|
+
|
|
3394
|
+
`;
|
|
3395
|
+
}
|
|
3396
|
+
return context;
|
|
3397
|
+
}
|
|
2810
3398
|
buildContext(memories, maxTokens) {
|
|
2811
3399
|
const parts = [];
|
|
2812
3400
|
let currentTokens = 0;
|
|
2813
3401
|
for (const memory of memories) {
|
|
2814
3402
|
const memoryText = this.formatMemory(memory);
|
|
2815
3403
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2816
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3404
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2817
3405
|
break;
|
|
2818
|
-
}
|
|
2819
3406
|
parts.push(memoryText);
|
|
2820
3407
|
currentTokens += memoryTokens;
|
|
2821
3408
|
}
|
|
2822
|
-
if (parts.length === 0)
|
|
3409
|
+
if (parts.length === 0)
|
|
2823
3410
|
return "";
|
|
2824
|
-
}
|
|
2825
3411
|
return `## Relevant Memories
|
|
2826
3412
|
|
|
2827
3413
|
${parts.join("\n\n---\n\n")}`;
|
|
2828
3414
|
}
|
|
2829
|
-
/**
|
|
2830
|
-
* Format a single memory for context
|
|
2831
|
-
*/
|
|
2832
3415
|
formatMemory(memory) {
|
|
2833
3416
|
const { event, score, sessionContext } = memory;
|
|
2834
3417
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2835
3418
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2836
3419
|
${event.content}`;
|
|
2837
|
-
if (sessionContext)
|
|
3420
|
+
if (sessionContext)
|
|
2838
3421
|
text += `
|
|
2839
3422
|
|
|
2840
3423
|
_Context:_ ${sessionContext}`;
|
|
2841
|
-
}
|
|
2842
3424
|
return text;
|
|
2843
3425
|
}
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
3426
|
+
matchesMetadataScope(metadata, expected) {
|
|
3427
|
+
if (!metadata)
|
|
3428
|
+
return false;
|
|
3429
|
+
return Object.entries(expected).every(([path5, value]) => {
|
|
3430
|
+
const actual = path5.split(".").reduce((acc, key) => {
|
|
3431
|
+
if (typeof acc !== "object" || acc === null)
|
|
3432
|
+
return void 0;
|
|
3433
|
+
return acc[key];
|
|
3434
|
+
}, metadata);
|
|
3435
|
+
return actual === value;
|
|
3436
|
+
});
|
|
3437
|
+
}
|
|
3438
|
+
tokenize(text) {
|
|
3439
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3440
|
+
}
|
|
3441
|
+
keywordOverlap(a, b) {
|
|
3442
|
+
if (a.length === 0 || b.length === 0)
|
|
3443
|
+
return 0;
|
|
3444
|
+
const bs = new Set(b);
|
|
3445
|
+
let hit = 0;
|
|
3446
|
+
for (const t of a)
|
|
3447
|
+
if (bs.has(t))
|
|
3448
|
+
hit += 1;
|
|
3449
|
+
return hit / a.length;
|
|
3450
|
+
}
|
|
2847
3451
|
estimateTokens(text) {
|
|
2848
3452
|
return Math.ceil(text.length / 4);
|
|
2849
3453
|
}
|
|
2850
|
-
/**
|
|
2851
|
-
* Get event age in days (for recency scoring)
|
|
2852
|
-
*/
|
|
2853
|
-
getEventAgeDays(eventId) {
|
|
2854
|
-
return 0;
|
|
2855
|
-
}
|
|
2856
3454
|
};
|
|
2857
3455
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2858
3456
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -4142,6 +4740,59 @@ var ConsolidatedStore = class {
|
|
|
4142
4740
|
[memoryId]
|
|
4143
4741
|
);
|
|
4144
4742
|
}
|
|
4743
|
+
/**
|
|
4744
|
+
* Create a long-term rule promoted from stable summaries
|
|
4745
|
+
*/
|
|
4746
|
+
async createRule(input) {
|
|
4747
|
+
const ruleId = randomUUID6();
|
|
4748
|
+
await dbRun(
|
|
4749
|
+
this.db,
|
|
4750
|
+
`INSERT INTO consolidated_rules
|
|
4751
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4752
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4753
|
+
[
|
|
4754
|
+
ruleId,
|
|
4755
|
+
input.rule,
|
|
4756
|
+
JSON.stringify(input.topics),
|
|
4757
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4758
|
+
JSON.stringify(input.sourceEvents),
|
|
4759
|
+
input.confidence
|
|
4760
|
+
]
|
|
4761
|
+
);
|
|
4762
|
+
return ruleId;
|
|
4763
|
+
}
|
|
4764
|
+
async getRules(options) {
|
|
4765
|
+
const limit = options?.limit || 100;
|
|
4766
|
+
const rows = await dbAll(
|
|
4767
|
+
this.db,
|
|
4768
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4769
|
+
[limit]
|
|
4770
|
+
);
|
|
4771
|
+
return rows.map((row) => ({
|
|
4772
|
+
ruleId: row.rule_id,
|
|
4773
|
+
rule: row.rule,
|
|
4774
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4775
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4776
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4777
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4778
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4779
|
+
}));
|
|
4780
|
+
}
|
|
4781
|
+
async countRules() {
|
|
4782
|
+
const result = await dbAll(
|
|
4783
|
+
this.db,
|
|
4784
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4785
|
+
);
|
|
4786
|
+
return result[0]?.count || 0;
|
|
4787
|
+
}
|
|
4788
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4789
|
+
const rows = await dbAll(
|
|
4790
|
+
this.db,
|
|
4791
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4792
|
+
[`%"${memoryId}"%`]
|
|
4793
|
+
);
|
|
4794
|
+
return (rows[0]?.count || 0) > 0;
|
|
4795
|
+
}
|
|
4145
4796
|
/**
|
|
4146
4797
|
* Get count of consolidated memories
|
|
4147
4798
|
*/
|
|
@@ -4291,7 +4942,14 @@ var ConsolidationWorker = class {
|
|
|
4291
4942
|
* Force a consolidation run (manual trigger)
|
|
4292
4943
|
*/
|
|
4293
4944
|
async forceRun() {
|
|
4294
|
-
|
|
4945
|
+
const out = await this.consolidateWithReport();
|
|
4946
|
+
return out.consolidatedCount;
|
|
4947
|
+
}
|
|
4948
|
+
/**
|
|
4949
|
+
* Force a consolidation run and return metrics report
|
|
4950
|
+
*/
|
|
4951
|
+
async forceRunWithReport() {
|
|
4952
|
+
return this.consolidateWithReport();
|
|
4295
4953
|
}
|
|
4296
4954
|
/**
|
|
4297
4955
|
* Schedule the next consolidation check
|
|
@@ -4331,12 +4989,21 @@ var ConsolidationWorker = class {
|
|
|
4331
4989
|
* Perform consolidation
|
|
4332
4990
|
*/
|
|
4333
4991
|
async consolidate() {
|
|
4992
|
+
const out = await this.consolidateWithReport();
|
|
4993
|
+
return out.consolidatedCount;
|
|
4994
|
+
}
|
|
4995
|
+
async consolidateWithReport() {
|
|
4334
4996
|
const workingSet = await this.workingSetStore.get();
|
|
4335
4997
|
if (workingSet.recentEvents.length < 3) {
|
|
4336
|
-
return
|
|
4998
|
+
return {
|
|
4999
|
+
consolidatedCount: 0,
|
|
5000
|
+
promotedRuleCount: 0,
|
|
5001
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
5002
|
+
};
|
|
4337
5003
|
}
|
|
4338
5004
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
4339
5005
|
let consolidatedCount = 0;
|
|
5006
|
+
const createdMemoryIds = [];
|
|
4340
5007
|
for (const group of groups) {
|
|
4341
5008
|
if (group.events.length < 3)
|
|
4342
5009
|
continue;
|
|
@@ -4345,14 +5012,16 @@ var ConsolidationWorker = class {
|
|
|
4345
5012
|
if (alreadyConsolidated)
|
|
4346
5013
|
continue;
|
|
4347
5014
|
const summary = await this.summarize(group);
|
|
4348
|
-
await this.consolidatedStore.create({
|
|
5015
|
+
const memoryId = await this.consolidatedStore.create({
|
|
4349
5016
|
summary,
|
|
4350
5017
|
topics: group.topics,
|
|
4351
5018
|
sourceEvents: eventIds,
|
|
4352
5019
|
confidence: this.calculateConfidence(group)
|
|
4353
5020
|
});
|
|
5021
|
+
createdMemoryIds.push(memoryId);
|
|
4354
5022
|
consolidatedCount++;
|
|
4355
5023
|
}
|
|
5024
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
4356
5025
|
if (consolidatedCount > 0) {
|
|
4357
5026
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
4358
5027
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4366,7 +5035,61 @@ var ConsolidationWorker = class {
|
|
|
4366
5035
|
await this.workingSetStore.prune(oldEventIds);
|
|
4367
5036
|
}
|
|
4368
5037
|
}
|
|
4369
|
-
|
|
5038
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5039
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5040
|
+
}
|
|
5041
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5042
|
+
let promoted = 0;
|
|
5043
|
+
for (const memoryId of memoryIds) {
|
|
5044
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5045
|
+
if (!memory)
|
|
5046
|
+
continue;
|
|
5047
|
+
if (memory.confidence < 0.55)
|
|
5048
|
+
continue;
|
|
5049
|
+
if (memory.sourceEvents.length < 4)
|
|
5050
|
+
continue;
|
|
5051
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5052
|
+
if (exists)
|
|
5053
|
+
continue;
|
|
5054
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5055
|
+
if (!rule)
|
|
5056
|
+
continue;
|
|
5057
|
+
await this.consolidatedStore.createRule({
|
|
5058
|
+
rule,
|
|
5059
|
+
topics: memory.topics,
|
|
5060
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5061
|
+
sourceEvents: memory.sourceEvents,
|
|
5062
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5063
|
+
});
|
|
5064
|
+
promoted++;
|
|
5065
|
+
}
|
|
5066
|
+
return promoted;
|
|
5067
|
+
}
|
|
5068
|
+
buildRuleFromSummary(summary, topics) {
|
|
5069
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5070
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5071
|
+
const seed = bullet || lines[0];
|
|
5072
|
+
if (!seed || seed.length < 8)
|
|
5073
|
+
return null;
|
|
5074
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5075
|
+
return `${topicPrefix}${seed}`;
|
|
5076
|
+
}
|
|
5077
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5078
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5079
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5080
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5081
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5082
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5083
|
+
return {
|
|
5084
|
+
beforeTokenEstimate,
|
|
5085
|
+
afterTokenEstimate,
|
|
5086
|
+
reductionRatio,
|
|
5087
|
+
qualityGuardPassed,
|
|
5088
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5089
|
+
};
|
|
5090
|
+
}
|
|
5091
|
+
estimateTokens(text) {
|
|
5092
|
+
return Math.ceil((text || "").length / 4);
|
|
4370
5093
|
}
|
|
4371
5094
|
/**
|
|
4372
5095
|
* Check if consolidation should run
|
|
@@ -4926,13 +5649,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
4926
5649
|
);
|
|
4927
5650
|
}
|
|
4928
5651
|
|
|
5652
|
+
// src/core/md-mirror.ts
|
|
5653
|
+
import * as fs3 from "node:fs";
|
|
5654
|
+
import * as path2 from "node:path";
|
|
5655
|
+
function sanitizeSegment2(input, fallback) {
|
|
5656
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5657
|
+
return v || fallback;
|
|
5658
|
+
}
|
|
5659
|
+
function getAtPath(obj, dotted) {
|
|
5660
|
+
if (!obj)
|
|
5661
|
+
return void 0;
|
|
5662
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5663
|
+
if (!acc || typeof acc !== "object")
|
|
5664
|
+
return void 0;
|
|
5665
|
+
return acc[key];
|
|
5666
|
+
}, obj);
|
|
5667
|
+
}
|
|
5668
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5669
|
+
const meta = event.metadata;
|
|
5670
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5671
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5672
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5673
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5674
|
+
const d = event.timestamp;
|
|
5675
|
+
const yyyy = d.getFullYear();
|
|
5676
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5677
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5678
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5679
|
+
}
|
|
5680
|
+
var MarkdownMirror2 = class {
|
|
5681
|
+
constructor(rootDir) {
|
|
5682
|
+
this.rootDir = rootDir;
|
|
5683
|
+
}
|
|
5684
|
+
async append(event, eventId) {
|
|
5685
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5686
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5687
|
+
const lines = [
|
|
5688
|
+
"",
|
|
5689
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5690
|
+
`- type: ${event.eventType}`,
|
|
5691
|
+
`- session: ${event.sessionId}`,
|
|
5692
|
+
event.content
|
|
5693
|
+
];
|
|
5694
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5695
|
+
await this.refreshIndex();
|
|
5696
|
+
}
|
|
5697
|
+
async refreshIndex() {
|
|
5698
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5699
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5700
|
+
const files = [];
|
|
5701
|
+
await this.walk(memoryRoot, files);
|
|
5702
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5703
|
+
const index = [
|
|
5704
|
+
"# Memory Index",
|
|
5705
|
+
"",
|
|
5706
|
+
"Generated automatically by MarkdownMirror.",
|
|
5707
|
+
"",
|
|
5708
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5709
|
+
""
|
|
5710
|
+
].join("\n");
|
|
5711
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5712
|
+
}
|
|
5713
|
+
async walk(dir, out) {
|
|
5714
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5715
|
+
for (const e of entries) {
|
|
5716
|
+
const full = path2.join(dir, e.name);
|
|
5717
|
+
if (e.isDirectory()) {
|
|
5718
|
+
await this.walk(full, out);
|
|
5719
|
+
} else {
|
|
5720
|
+
out.push(full);
|
|
5721
|
+
}
|
|
5722
|
+
}
|
|
5723
|
+
}
|
|
5724
|
+
};
|
|
5725
|
+
|
|
5726
|
+
// src/core/ingest-interceptor.ts
|
|
5727
|
+
var IngestInterceptorRegistry = class {
|
|
5728
|
+
before = [];
|
|
5729
|
+
after = [];
|
|
5730
|
+
onError = [];
|
|
5731
|
+
registerBefore(interceptor) {
|
|
5732
|
+
this.before.push(interceptor);
|
|
5733
|
+
return () => {
|
|
5734
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5735
|
+
};
|
|
5736
|
+
}
|
|
5737
|
+
registerAfter(interceptor) {
|
|
5738
|
+
this.after.push(interceptor);
|
|
5739
|
+
return () => {
|
|
5740
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5741
|
+
};
|
|
5742
|
+
}
|
|
5743
|
+
registerOnError(interceptor) {
|
|
5744
|
+
this.onError.push(interceptor);
|
|
5745
|
+
return () => {
|
|
5746
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5747
|
+
};
|
|
5748
|
+
}
|
|
5749
|
+
async run(stage, context) {
|
|
5750
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5751
|
+
for (const interceptor of interceptors) {
|
|
5752
|
+
await interceptor({ ...context, stage });
|
|
5753
|
+
}
|
|
5754
|
+
}
|
|
5755
|
+
};
|
|
5756
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5757
|
+
if (!base && !patch)
|
|
5758
|
+
return void 0;
|
|
5759
|
+
if (!base)
|
|
5760
|
+
return patch;
|
|
5761
|
+
if (!patch)
|
|
5762
|
+
return base;
|
|
5763
|
+
const result = { ...base };
|
|
5764
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5765
|
+
const current = result[key];
|
|
5766
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5767
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5768
|
+
current,
|
|
5769
|
+
value
|
|
5770
|
+
);
|
|
5771
|
+
} else {
|
|
5772
|
+
result[key] = value;
|
|
5773
|
+
}
|
|
5774
|
+
}
|
|
5775
|
+
return result;
|
|
5776
|
+
}
|
|
5777
|
+
|
|
5778
|
+
// src/core/tag-taxonomy.ts
|
|
5779
|
+
var TAG_NAMESPACES = {
|
|
5780
|
+
SYSTEM: "sys:",
|
|
5781
|
+
QUALITY: "q:",
|
|
5782
|
+
PROJECT: "proj:",
|
|
5783
|
+
TOPIC: "topic:",
|
|
5784
|
+
TEMPORAL: "t:",
|
|
5785
|
+
USER: "user:",
|
|
5786
|
+
AGENT: "agent:"
|
|
5787
|
+
};
|
|
5788
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5789
|
+
function parseTag(tag) {
|
|
5790
|
+
const value = (tag || "").trim();
|
|
5791
|
+
const idx = value.indexOf(":");
|
|
5792
|
+
if (idx <= 0)
|
|
5793
|
+
return { value };
|
|
5794
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5795
|
+
const tagValue = value.slice(idx + 1);
|
|
5796
|
+
if (!tagValue)
|
|
5797
|
+
return { value };
|
|
5798
|
+
return { namespace, value: tagValue };
|
|
5799
|
+
}
|
|
5800
|
+
function validateTag(tag) {
|
|
5801
|
+
const normalized = (tag || "").trim();
|
|
5802
|
+
if (!normalized)
|
|
5803
|
+
return false;
|
|
5804
|
+
const { namespace } = parseTag(normalized);
|
|
5805
|
+
if (!namespace)
|
|
5806
|
+
return true;
|
|
5807
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5808
|
+
}
|
|
5809
|
+
function normalizeTags(tags) {
|
|
5810
|
+
if (!Array.isArray(tags))
|
|
5811
|
+
return [];
|
|
5812
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5813
|
+
for (const item of tags) {
|
|
5814
|
+
if (typeof item !== "string")
|
|
5815
|
+
continue;
|
|
5816
|
+
const normalized = item.trim();
|
|
5817
|
+
if (!validateTag(normalized))
|
|
5818
|
+
continue;
|
|
5819
|
+
dedup.add(normalized);
|
|
5820
|
+
}
|
|
5821
|
+
return [...dedup];
|
|
5822
|
+
}
|
|
5823
|
+
|
|
4929
5824
|
// src/services/memory-service.ts
|
|
4930
5825
|
function normalizePath(projectPath) {
|
|
4931
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5826
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
4932
5827
|
try {
|
|
4933
|
-
return
|
|
5828
|
+
return fs4.realpathSync(expanded);
|
|
4934
5829
|
} catch {
|
|
4935
|
-
return
|
|
5830
|
+
return path3.resolve(expanded);
|
|
4936
5831
|
}
|
|
4937
5832
|
}
|
|
4938
5833
|
function hashProjectPath(projectPath) {
|
|
@@ -4941,14 +5836,14 @@ function hashProjectPath(projectPath) {
|
|
|
4941
5836
|
}
|
|
4942
5837
|
function getProjectStoragePath(projectPath) {
|
|
4943
5838
|
const hash = hashProjectPath(projectPath);
|
|
4944
|
-
return
|
|
5839
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
4945
5840
|
}
|
|
4946
|
-
var REGISTRY_PATH =
|
|
4947
|
-
var SHARED_STORAGE_PATH =
|
|
5841
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5842
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
4948
5843
|
function loadSessionRegistry() {
|
|
4949
5844
|
try {
|
|
4950
|
-
if (
|
|
4951
|
-
const data =
|
|
5845
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5846
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
4952
5847
|
return JSON.parse(data);
|
|
4953
5848
|
}
|
|
4954
5849
|
} catch (error) {
|
|
@@ -4974,6 +5869,7 @@ var MemoryService = class {
|
|
|
4974
5869
|
vectorWorker = null;
|
|
4975
5870
|
graduationWorker = null;
|
|
4976
5871
|
initialized = false;
|
|
5872
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
4977
5873
|
// Endless Mode components
|
|
4978
5874
|
workingSetStore = null;
|
|
4979
5875
|
consolidatedStore = null;
|
|
@@ -4987,20 +5883,27 @@ var MemoryService = class {
|
|
|
4987
5883
|
sharedPromoter = null;
|
|
4988
5884
|
sharedStoreConfig = null;
|
|
4989
5885
|
projectHash = null;
|
|
5886
|
+
projectPath = null;
|
|
4990
5887
|
readOnly;
|
|
4991
5888
|
lightweightMode;
|
|
5889
|
+
mdMirror;
|
|
4992
5890
|
constructor(config) {
|
|
4993
5891
|
const storagePath = this.expandPath(config.storagePath);
|
|
4994
5892
|
this.readOnly = config.readOnly ?? false;
|
|
4995
5893
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
4996
|
-
|
|
4997
|
-
|
|
5894
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
5895
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
5896
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
4998
5897
|
}
|
|
4999
5898
|
this.projectHash = config.projectHash || null;
|
|
5899
|
+
this.projectPath = config.projectPath || null;
|
|
5000
5900
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
5001
5901
|
this.sqliteStore = new SQLiteEventStore(
|
|
5002
|
-
|
|
5003
|
-
{
|
|
5902
|
+
path3.join(storagePath, "events.sqlite"),
|
|
5903
|
+
{
|
|
5904
|
+
readonly: this.readOnly,
|
|
5905
|
+
markdownMirrorRoot: storagePath
|
|
5906
|
+
}
|
|
5004
5907
|
);
|
|
5005
5908
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
5006
5909
|
if (!analyticsEnabled) {
|
|
@@ -5008,7 +5911,7 @@ var MemoryService = class {
|
|
|
5008
5911
|
} else if (this.readOnly) {
|
|
5009
5912
|
try {
|
|
5010
5913
|
this.analyticsStore = new EventStore(
|
|
5011
|
-
|
|
5914
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5012
5915
|
{ readOnly: true }
|
|
5013
5916
|
);
|
|
5014
5917
|
} catch {
|
|
@@ -5016,11 +5919,11 @@ var MemoryService = class {
|
|
|
5016
5919
|
}
|
|
5017
5920
|
} else {
|
|
5018
5921
|
this.analyticsStore = new EventStore(
|
|
5019
|
-
|
|
5922
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5020
5923
|
{ readOnly: false }
|
|
5021
5924
|
);
|
|
5022
5925
|
}
|
|
5023
|
-
this.vectorStore = new VectorStore(
|
|
5926
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
5024
5927
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
5025
5928
|
this.matcher = getDefaultMatcher();
|
|
5026
5929
|
this.retriever = createRetriever(
|
|
@@ -5030,6 +5933,7 @@ var MemoryService = class {
|
|
|
5030
5933
|
this.embedder,
|
|
5031
5934
|
this.matcher
|
|
5032
5935
|
);
|
|
5936
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
5033
5937
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
5034
5938
|
}
|
|
5035
5939
|
/**
|
|
@@ -5089,16 +5993,16 @@ var MemoryService = class {
|
|
|
5089
5993
|
*/
|
|
5090
5994
|
async initializeSharedStore() {
|
|
5091
5995
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
5092
|
-
if (!
|
|
5093
|
-
|
|
5996
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
5997
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
5094
5998
|
}
|
|
5095
5999
|
this.sharedEventStore = createSharedEventStore(
|
|
5096
|
-
|
|
6000
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
5097
6001
|
);
|
|
5098
6002
|
await this.sharedEventStore.initialize();
|
|
5099
6003
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
5100
6004
|
this.sharedVectorStore = createSharedVectorStore(
|
|
5101
|
-
|
|
6005
|
+
path3.join(sharedPath, "vectors")
|
|
5102
6006
|
);
|
|
5103
6007
|
await this.sharedVectorStore.initialize();
|
|
5104
6008
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -5109,6 +6013,86 @@ var MemoryService = class {
|
|
|
5109
6013
|
);
|
|
5110
6014
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
5111
6015
|
}
|
|
6016
|
+
registerIngestBefore(interceptor) {
|
|
6017
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6018
|
+
}
|
|
6019
|
+
registerIngestAfter(interceptor) {
|
|
6020
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6021
|
+
}
|
|
6022
|
+
registerIngestOnError(interceptor) {
|
|
6023
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6024
|
+
}
|
|
6025
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6026
|
+
const normalizedInput = {
|
|
6027
|
+
...input,
|
|
6028
|
+
metadata: mergeHierarchicalMetadata(
|
|
6029
|
+
{
|
|
6030
|
+
ingest: {
|
|
6031
|
+
operation,
|
|
6032
|
+
pipeline: "default",
|
|
6033
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6034
|
+
},
|
|
6035
|
+
...this.projectHash ? {
|
|
6036
|
+
scope: {
|
|
6037
|
+
project: {
|
|
6038
|
+
hash: this.projectHash,
|
|
6039
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6040
|
+
}
|
|
6041
|
+
},
|
|
6042
|
+
tags: [`proj:${this.projectHash}`]
|
|
6043
|
+
} : {}
|
|
6044
|
+
},
|
|
6045
|
+
input.metadata
|
|
6046
|
+
)
|
|
6047
|
+
};
|
|
6048
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6049
|
+
const meta = normalizedInput.metadata;
|
|
6050
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6051
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6052
|
+
if (!currentTags.includes(projectTag)) {
|
|
6053
|
+
meta.tags = [...currentTags, projectTag];
|
|
6054
|
+
}
|
|
6055
|
+
}
|
|
6056
|
+
if (normalizedInput.metadata) {
|
|
6057
|
+
const meta = normalizedInput.metadata;
|
|
6058
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6059
|
+
if (normalizedTags.length > 0) {
|
|
6060
|
+
meta.tags = normalizedTags;
|
|
6061
|
+
}
|
|
6062
|
+
}
|
|
6063
|
+
await this.ingestInterceptors.run("before", {
|
|
6064
|
+
operation,
|
|
6065
|
+
sessionId: normalizedInput.sessionId,
|
|
6066
|
+
event: normalizedInput
|
|
6067
|
+
});
|
|
6068
|
+
try {
|
|
6069
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6070
|
+
if (result.success && !result.isDuplicate) {
|
|
6071
|
+
if (onSuccess) {
|
|
6072
|
+
await onSuccess(result.eventId);
|
|
6073
|
+
}
|
|
6074
|
+
try {
|
|
6075
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6076
|
+
} catch {
|
|
6077
|
+
}
|
|
6078
|
+
}
|
|
6079
|
+
await this.ingestInterceptors.run("after", {
|
|
6080
|
+
operation,
|
|
6081
|
+
sessionId: normalizedInput.sessionId,
|
|
6082
|
+
event: normalizedInput
|
|
6083
|
+
});
|
|
6084
|
+
return result;
|
|
6085
|
+
} catch (error) {
|
|
6086
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6087
|
+
await this.ingestInterceptors.run("error", {
|
|
6088
|
+
operation,
|
|
6089
|
+
sessionId: normalizedInput.sessionId,
|
|
6090
|
+
event: normalizedInput,
|
|
6091
|
+
error: normalizedError
|
|
6092
|
+
});
|
|
6093
|
+
throw error;
|
|
6094
|
+
}
|
|
6095
|
+
}
|
|
5112
6096
|
/**
|
|
5113
6097
|
* Start a new session
|
|
5114
6098
|
*/
|
|
@@ -5136,50 +6120,57 @@ var MemoryService = class {
|
|
|
5136
6120
|
*/
|
|
5137
6121
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
5138
6122
|
await this.initialize();
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5149
|
-
|
|
6123
|
+
return this.ingestWithInterceptors(
|
|
6124
|
+
"user_prompt",
|
|
6125
|
+
{
|
|
6126
|
+
eventType: "user_prompt",
|
|
6127
|
+
sessionId,
|
|
6128
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6129
|
+
content,
|
|
6130
|
+
metadata
|
|
6131
|
+
},
|
|
6132
|
+
async (eventId) => {
|
|
6133
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6134
|
+
}
|
|
6135
|
+
);
|
|
5150
6136
|
}
|
|
5151
6137
|
/**
|
|
5152
6138
|
* Store an agent response
|
|
5153
6139
|
*/
|
|
5154
6140
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
5155
6141
|
await this.initialize();
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
6142
|
+
return this.ingestWithInterceptors(
|
|
6143
|
+
"agent_response",
|
|
6144
|
+
{
|
|
6145
|
+
eventType: "agent_response",
|
|
6146
|
+
sessionId,
|
|
6147
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6148
|
+
content,
|
|
6149
|
+
metadata
|
|
6150
|
+
},
|
|
6151
|
+
async (eventId) => {
|
|
6152
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6153
|
+
}
|
|
6154
|
+
);
|
|
5167
6155
|
}
|
|
5168
6156
|
/**
|
|
5169
6157
|
* Store a session summary
|
|
5170
6158
|
*/
|
|
5171
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6159
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
5172
6160
|
await this.initialize();
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
6161
|
+
return this.ingestWithInterceptors(
|
|
6162
|
+
"session_summary",
|
|
6163
|
+
{
|
|
6164
|
+
eventType: "session_summary",
|
|
6165
|
+
sessionId,
|
|
6166
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6167
|
+
content: summary,
|
|
6168
|
+
metadata
|
|
6169
|
+
},
|
|
6170
|
+
async (eventId) => {
|
|
6171
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6172
|
+
}
|
|
6173
|
+
);
|
|
5183
6174
|
}
|
|
5184
6175
|
/**
|
|
5185
6176
|
* Store a tool observation
|
|
@@ -5188,40 +6179,181 @@ var MemoryService = class {
|
|
|
5188
6179
|
await this.initialize();
|
|
5189
6180
|
const content = JSON.stringify(payload);
|
|
5190
6181
|
const turnId = payload.metadata?.turnId;
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
6182
|
+
return this.ingestWithInterceptors(
|
|
6183
|
+
"tool_observation",
|
|
6184
|
+
{
|
|
6185
|
+
eventType: "tool_observation",
|
|
6186
|
+
sessionId,
|
|
6187
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6188
|
+
content,
|
|
6189
|
+
metadata: {
|
|
6190
|
+
toolName: payload.toolName,
|
|
6191
|
+
success: payload.success,
|
|
6192
|
+
...turnId ? { turnId } : {}
|
|
6193
|
+
}
|
|
6194
|
+
},
|
|
6195
|
+
async (eventId) => {
|
|
6196
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6197
|
+
payload.toolName,
|
|
6198
|
+
payload.metadata || {},
|
|
6199
|
+
payload.success
|
|
6200
|
+
);
|
|
6201
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
5200
6202
|
}
|
|
5201
|
-
|
|
5202
|
-
if (result.success && !result.isDuplicate) {
|
|
5203
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
5204
|
-
payload.toolName,
|
|
5205
|
-
payload.metadata || {},
|
|
5206
|
-
payload.success
|
|
5207
|
-
);
|
|
5208
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
5209
|
-
}
|
|
5210
|
-
return result;
|
|
6203
|
+
);
|
|
5211
6204
|
}
|
|
5212
6205
|
/**
|
|
5213
6206
|
* Retrieve relevant memories for a query
|
|
5214
6207
|
*/
|
|
5215
6208
|
async retrieveMemories(query, options) {
|
|
5216
6209
|
await this.initialize();
|
|
6210
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6211
|
+
let result;
|
|
5217
6212
|
if (options?.includeShared && this.sharedStore) {
|
|
5218
|
-
|
|
6213
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
5219
6214
|
...options,
|
|
6215
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6216
|
+
rerankWeights,
|
|
5220
6217
|
includeShared: true,
|
|
5221
|
-
projectHash: this.projectHash || void 0
|
|
6218
|
+
projectHash: this.projectHash || void 0,
|
|
6219
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6220
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6221
|
+
});
|
|
6222
|
+
} else {
|
|
6223
|
+
result = await this.retriever.retrieve(query, {
|
|
6224
|
+
...options,
|
|
6225
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6226
|
+
rerankWeights,
|
|
6227
|
+
projectHash: this.projectHash || void 0,
|
|
6228
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6229
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6230
|
+
});
|
|
6231
|
+
}
|
|
6232
|
+
try {
|
|
6233
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6234
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6235
|
+
eventId: d.eventId,
|
|
6236
|
+
score: d.score,
|
|
6237
|
+
semanticScore: d.semanticScore,
|
|
6238
|
+
lexicalScore: d.lexicalScore,
|
|
6239
|
+
recencyScore: d.recencyScore
|
|
6240
|
+
}));
|
|
6241
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6242
|
+
eventId: d.eventId,
|
|
6243
|
+
score: d.score,
|
|
6244
|
+
semanticScore: d.semanticScore,
|
|
6245
|
+
lexicalScore: d.lexicalScore,
|
|
6246
|
+
recencyScore: d.recencyScore
|
|
6247
|
+
}));
|
|
6248
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6249
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6250
|
+
sessionId: options?.sessionId,
|
|
6251
|
+
projectHash: this.projectHash || void 0,
|
|
6252
|
+
queryText: query,
|
|
6253
|
+
strategy: options?.strategy || "auto",
|
|
6254
|
+
candidateEventIds,
|
|
6255
|
+
selectedEventIds,
|
|
6256
|
+
candidateDetails,
|
|
6257
|
+
selectedDetails,
|
|
6258
|
+
confidence: result.matchResult.confidence,
|
|
6259
|
+
fallbackTrace: result.fallbackTrace || []
|
|
6260
|
+
});
|
|
6261
|
+
} catch {
|
|
6262
|
+
}
|
|
6263
|
+
return result;
|
|
6264
|
+
}
|
|
6265
|
+
getConfiguredRerankWeights() {
|
|
6266
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6267
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6268
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6269
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6270
|
+
if (!allFinite)
|
|
6271
|
+
return void 0;
|
|
6272
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6273
|
+
const total = semantic + lexical + recency;
|
|
6274
|
+
if (!nonNegative || total <= 0)
|
|
6275
|
+
return void 0;
|
|
6276
|
+
return {
|
|
6277
|
+
semantic: semantic / total,
|
|
6278
|
+
lexical: lexical / total,
|
|
6279
|
+
recency: recency / total
|
|
6280
|
+
};
|
|
6281
|
+
}
|
|
6282
|
+
async getRerankWeights(adaptive) {
|
|
6283
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6284
|
+
if (configured)
|
|
6285
|
+
return configured;
|
|
6286
|
+
if (adaptive)
|
|
6287
|
+
return this.getAdaptiveRerankWeights();
|
|
6288
|
+
return void 0;
|
|
6289
|
+
}
|
|
6290
|
+
async rewriteQueryIntent(query) {
|
|
6291
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6292
|
+
return null;
|
|
6293
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6294
|
+
if (!apiUrl)
|
|
6295
|
+
return null;
|
|
6296
|
+
const controller = new AbortController();
|
|
6297
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6298
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6299
|
+
try {
|
|
6300
|
+
const prompt = [
|
|
6301
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6302
|
+
"Return plain text only, one line, no markdown.",
|
|
6303
|
+
`Query: ${query}`
|
|
6304
|
+
].join("\n");
|
|
6305
|
+
const res = await fetch(apiUrl, {
|
|
6306
|
+
method: "POST",
|
|
6307
|
+
headers: {
|
|
6308
|
+
"Content-Type": "application/json",
|
|
6309
|
+
Accept: "*/*",
|
|
6310
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6311
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6312
|
+
},
|
|
6313
|
+
body: JSON.stringify({
|
|
6314
|
+
question: prompt,
|
|
6315
|
+
company_name: null,
|
|
6316
|
+
conversation_id: null
|
|
6317
|
+
}),
|
|
6318
|
+
signal: controller.signal
|
|
5222
6319
|
});
|
|
6320
|
+
const text = (await res.text()).trim();
|
|
6321
|
+
if (!text)
|
|
6322
|
+
return null;
|
|
6323
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6324
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6325
|
+
return null;
|
|
6326
|
+
return oneLine;
|
|
6327
|
+
} catch {
|
|
6328
|
+
return null;
|
|
6329
|
+
} finally {
|
|
6330
|
+
clearTimeout(timeout);
|
|
6331
|
+
}
|
|
6332
|
+
}
|
|
6333
|
+
async getAdaptiveRerankWeights() {
|
|
6334
|
+
try {
|
|
6335
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6336
|
+
if (s.totalEvaluated < 20)
|
|
6337
|
+
return void 0;
|
|
6338
|
+
let semantic = 0.7;
|
|
6339
|
+
let lexical = 0.2;
|
|
6340
|
+
let recency = 0.1;
|
|
6341
|
+
if (s.avgScore < 0.45) {
|
|
6342
|
+
semantic -= 0.1;
|
|
6343
|
+
lexical += 0.1;
|
|
6344
|
+
} else if (s.avgScore > 0.75) {
|
|
6345
|
+
semantic += 0.05;
|
|
6346
|
+
lexical -= 0.05;
|
|
6347
|
+
}
|
|
6348
|
+
if (s.unhelpful > s.helpful) {
|
|
6349
|
+
recency += 0.05;
|
|
6350
|
+
semantic -= 0.03;
|
|
6351
|
+
lexical -= 0.02;
|
|
6352
|
+
}
|
|
6353
|
+
return { semantic, lexical, recency };
|
|
6354
|
+
} catch {
|
|
6355
|
+
return void 0;
|
|
5223
6356
|
}
|
|
5224
|
-
return this.retriever.retrieve(query, options);
|
|
5225
6357
|
}
|
|
5226
6358
|
/**
|
|
5227
6359
|
* Fast keyword search using SQLite FTS5
|
|
@@ -5263,6 +6395,18 @@ var MemoryService = class {
|
|
|
5263
6395
|
/**
|
|
5264
6396
|
* Get memory statistics
|
|
5265
6397
|
*/
|
|
6398
|
+
async getOutboxStats() {
|
|
6399
|
+
await this.initialize();
|
|
6400
|
+
return this.sqliteStore.getOutboxStats();
|
|
6401
|
+
}
|
|
6402
|
+
async getRetrievalTraceStats() {
|
|
6403
|
+
await this.initialize();
|
|
6404
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6405
|
+
}
|
|
6406
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6407
|
+
await this.initialize();
|
|
6408
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6409
|
+
}
|
|
5266
6410
|
async getStats() {
|
|
5267
6411
|
await this.initialize();
|
|
5268
6412
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5753,7 +6897,7 @@ var MemoryService = class {
|
|
|
5753
6897
|
*/
|
|
5754
6898
|
expandPath(p) {
|
|
5755
6899
|
if (p.startsWith("~")) {
|
|
5756
|
-
return
|
|
6900
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5757
6901
|
}
|
|
5758
6902
|
return p;
|
|
5759
6903
|
}
|
|
@@ -5763,10 +6907,11 @@ function getLightweightMemoryService(sessionId) {
|
|
|
5763
6907
|
const projectInfo = getSessionProject(sessionId);
|
|
5764
6908
|
const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : "lightweight_global";
|
|
5765
6909
|
if (!serviceCache.has(key)) {
|
|
5766
|
-
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) :
|
|
6910
|
+
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) : path3.join(os.homedir(), ".claude-code", "memory");
|
|
5767
6911
|
serviceCache.set(key, new MemoryService({
|
|
5768
6912
|
storagePath,
|
|
5769
6913
|
projectHash: projectInfo?.projectHash,
|
|
6914
|
+
projectPath: projectInfo?.projectPath,
|
|
5770
6915
|
lightweightMode: true,
|
|
5771
6916
|
// Skip embedder/vector/workers
|
|
5772
6917
|
analyticsEnabled: false,
|
|
@@ -5777,17 +6922,17 @@ function getLightweightMemoryService(sessionId) {
|
|
|
5777
6922
|
}
|
|
5778
6923
|
|
|
5779
6924
|
// src/core/turn-state.ts
|
|
5780
|
-
import * as
|
|
5781
|
-
import * as
|
|
6925
|
+
import * as fs5 from "fs";
|
|
6926
|
+
import * as path4 from "path";
|
|
5782
6927
|
import * as os2 from "os";
|
|
5783
|
-
var TURN_STATE_DIR =
|
|
6928
|
+
var TURN_STATE_DIR = path4.join(os2.homedir(), ".claude-code", "memory");
|
|
5784
6929
|
function getStatePath(sessionId) {
|
|
5785
|
-
return
|
|
6930
|
+
return path4.join(TURN_STATE_DIR, `.turn-state-${sessionId}.json`);
|
|
5786
6931
|
}
|
|
5787
6932
|
function writeTurnState(sessionId, turnId) {
|
|
5788
6933
|
try {
|
|
5789
|
-
if (!
|
|
5790
|
-
|
|
6934
|
+
if (!fs5.existsSync(TURN_STATE_DIR)) {
|
|
6935
|
+
fs5.mkdirSync(TURN_STATE_DIR, { recursive: true });
|
|
5791
6936
|
}
|
|
5792
6937
|
const state = {
|
|
5793
6938
|
turnId,
|
|
@@ -5796,8 +6941,8 @@ function writeTurnState(sessionId, turnId) {
|
|
|
5796
6941
|
};
|
|
5797
6942
|
const filePath = getStatePath(sessionId);
|
|
5798
6943
|
const tempPath = filePath + ".tmp";
|
|
5799
|
-
|
|
5800
|
-
|
|
6944
|
+
fs5.writeFileSync(tempPath, JSON.stringify(state));
|
|
6945
|
+
fs5.renameSync(tempPath, filePath);
|
|
5801
6946
|
} catch (error) {
|
|
5802
6947
|
if (process.env.CLAUDE_MEMORY_DEBUG) {
|
|
5803
6948
|
console.error("Failed to write turn state:", error);
|