claude-memory-layer 1.0.11 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +2389 -286
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1017 -132
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1347 -202
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1339 -194
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1343 -198
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1351 -206
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1347 -202
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +1436 -211
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +1445 -220
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1345 -199
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +69 -2
- package/dist/ui/index.html +8 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +2 -1
- package/scripts/build.ts +6 -0
- package/scripts/bump-patch-version.sh +18 -0
- package/src/cli/index.ts +281 -2
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +350 -1
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/types.ts +28 -0
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +3 -1
- package/src/server/api/stats.ts +46 -1
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +373 -68
- package/src/services/session-history-importer.ts +53 -25
- package/src/ui/app.js +69 -2
- package/src/ui/index.html +8 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
|
@@ -7,9 +7,9 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
7
7
|
const __dirname = dirname(__filename);
|
|
8
8
|
|
|
9
9
|
// src/services/memory-service.ts
|
|
10
|
-
import * as
|
|
10
|
+
import * as path3 from "path";
|
|
11
11
|
import * as os from "os";
|
|
12
|
-
import * as
|
|
12
|
+
import * as fs4 from "fs";
|
|
13
13
|
import * as crypto2 from "crypto";
|
|
14
14
|
|
|
15
15
|
// src/core/event-store.ts
|
|
@@ -67,11 +67,11 @@ function toDate(value) {
|
|
|
67
67
|
return new Date(value);
|
|
68
68
|
return new Date(String(value));
|
|
69
69
|
}
|
|
70
|
-
function createDatabase(
|
|
70
|
+
function createDatabase(path4, options) {
|
|
71
71
|
if (options?.readOnly) {
|
|
72
|
-
return new duckdb.Database(
|
|
72
|
+
return new duckdb.Database(path4, { access_mode: "READ_ONLY" });
|
|
73
73
|
}
|
|
74
|
-
return new duckdb.Database(
|
|
74
|
+
return new duckdb.Database(path4);
|
|
75
75
|
}
|
|
76
76
|
function dbRun(db, sql, params = []) {
|
|
77
77
|
return new Promise((resolve2, reject) => {
|
|
@@ -335,6 +335,17 @@ var EventStore = class {
|
|
|
335
335
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
336
336
|
)
|
|
337
337
|
`);
|
|
338
|
+
await dbRun(this.db, `
|
|
339
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
340
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
341
|
+
rule TEXT NOT NULL,
|
|
342
|
+
topics JSON,
|
|
343
|
+
source_memory_ids JSON,
|
|
344
|
+
source_events JSON,
|
|
345
|
+
confidence FLOAT DEFAULT 0.5,
|
|
346
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
347
|
+
)
|
|
348
|
+
`);
|
|
338
349
|
await dbRun(this.db, `
|
|
339
350
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
340
351
|
key VARCHAR PRIMARY KEY,
|
|
@@ -354,6 +365,7 @@ var EventStore = class {
|
|
|
354
365
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
355
366
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
356
367
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
368
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
357
369
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
358
370
|
this.initialized = true;
|
|
359
371
|
}
|
|
@@ -743,12 +755,12 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
743
755
|
import Database from "better-sqlite3";
|
|
744
756
|
import * as fs from "fs";
|
|
745
757
|
import * as nodePath from "path";
|
|
746
|
-
function createSQLiteDatabase(
|
|
747
|
-
const dir = nodePath.dirname(
|
|
758
|
+
function createSQLiteDatabase(path4, options) {
|
|
759
|
+
const dir = nodePath.dirname(path4);
|
|
748
760
|
if (!fs.existsSync(dir)) {
|
|
749
761
|
fs.mkdirSync(dir, { recursive: true });
|
|
750
762
|
}
|
|
751
|
-
const db = new Database(
|
|
763
|
+
const db = new Database(path4, {
|
|
752
764
|
readonly: options?.readonly ?? false
|
|
753
765
|
});
|
|
754
766
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -789,6 +801,64 @@ function toSQLiteTimestamp(date) {
|
|
|
789
801
|
return date.toISOString();
|
|
790
802
|
}
|
|
791
803
|
|
|
804
|
+
// src/core/markdown-mirror.ts
|
|
805
|
+
import * as fs2 from "fs/promises";
|
|
806
|
+
import * as path from "path";
|
|
807
|
+
var DEFAULT_NAMESPACE = "default";
|
|
808
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
809
|
+
function sanitizeSegment(input, fallback) {
|
|
810
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
811
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
812
|
+
if (!safe || safe === "." || safe === "..")
|
|
813
|
+
return fallback;
|
|
814
|
+
return safe;
|
|
815
|
+
}
|
|
816
|
+
function getCategorySegments(metadata, eventType) {
|
|
817
|
+
const raw = metadata?.categoryPath;
|
|
818
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
819
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
820
|
+
}
|
|
821
|
+
const single = metadata?.category;
|
|
822
|
+
if (typeof single === "string" && single.trim()) {
|
|
823
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
824
|
+
}
|
|
825
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
826
|
+
}
|
|
827
|
+
function buildMirrorPath(rootDir, event) {
|
|
828
|
+
const metadata = event.metadata;
|
|
829
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
830
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
831
|
+
const d = event.timestamp;
|
|
832
|
+
const yyyy = d.getFullYear();
|
|
833
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
834
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
835
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
836
|
+
}
|
|
837
|
+
function formatMirrorEntry(event) {
|
|
838
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
839
|
+
return [
|
|
840
|
+
"",
|
|
841
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
842
|
+
` id: ${event.id}`,
|
|
843
|
+
` type: ${event.eventType}`,
|
|
844
|
+
` session: ${event.sessionId}`,
|
|
845
|
+
` category: ${category}`,
|
|
846
|
+
" content: |",
|
|
847
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
848
|
+
].join("\n") + "\n";
|
|
849
|
+
}
|
|
850
|
+
var MarkdownMirror = class {
|
|
851
|
+
constructor(rootDir) {
|
|
852
|
+
this.rootDir = rootDir;
|
|
853
|
+
}
|
|
854
|
+
async append(event) {
|
|
855
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
856
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
857
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
858
|
+
return outPath;
|
|
859
|
+
}
|
|
860
|
+
};
|
|
861
|
+
|
|
792
862
|
// src/core/sqlite-event-store.ts
|
|
793
863
|
var SQLiteEventStore = class {
|
|
794
864
|
constructor(dbPath, options) {
|
|
@@ -798,10 +868,12 @@ var SQLiteEventStore = class {
|
|
|
798
868
|
readonly: this.readOnly,
|
|
799
869
|
walMode: !this.readOnly
|
|
800
870
|
});
|
|
871
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
801
872
|
}
|
|
802
873
|
db;
|
|
803
874
|
initialized = false;
|
|
804
875
|
readOnly;
|
|
876
|
+
markdownMirror;
|
|
805
877
|
/**
|
|
806
878
|
* Initialize database schema
|
|
807
879
|
*/
|
|
@@ -1008,6 +1080,17 @@ var SQLiteEventStore = class {
|
|
|
1008
1080
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1009
1081
|
);
|
|
1010
1082
|
|
|
1083
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1084
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1085
|
+
rule_id TEXT PRIMARY KEY,
|
|
1086
|
+
rule TEXT NOT NULL,
|
|
1087
|
+
topics TEXT,
|
|
1088
|
+
source_memory_ids TEXT,
|
|
1089
|
+
source_events TEXT,
|
|
1090
|
+
confidence REAL DEFAULT 0.5,
|
|
1091
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1092
|
+
);
|
|
1093
|
+
|
|
1011
1094
|
-- Endless Mode Config table
|
|
1012
1095
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1013
1096
|
key TEXT PRIMARY KEY,
|
|
@@ -1032,6 +1115,24 @@ var SQLiteEventStore = class {
|
|
|
1032
1115
|
measured_at TEXT
|
|
1033
1116
|
);
|
|
1034
1117
|
|
|
1118
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1119
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1120
|
+
trace_id TEXT PRIMARY KEY,
|
|
1121
|
+
session_id TEXT,
|
|
1122
|
+
project_hash TEXT,
|
|
1123
|
+
query_text TEXT NOT NULL,
|
|
1124
|
+
strategy TEXT,
|
|
1125
|
+
candidate_event_ids TEXT,
|
|
1126
|
+
selected_event_ids TEXT,
|
|
1127
|
+
candidate_details_json TEXT,
|
|
1128
|
+
selected_details_json TEXT,
|
|
1129
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1130
|
+
selected_count INTEGER DEFAULT 0,
|
|
1131
|
+
confidence TEXT,
|
|
1132
|
+
fallback_trace TEXT,
|
|
1133
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1134
|
+
);
|
|
1135
|
+
|
|
1035
1136
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1036
1137
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1037
1138
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1056,10 +1157,14 @@ var SQLiteEventStore = class {
|
|
|
1056
1157
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1057
1158
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1058
1159
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1160
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1059
1161
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1060
1162
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1061
1163
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1062
1164
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1165
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1166
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1167
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1063
1168
|
|
|
1064
1169
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1065
1170
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1083,6 +1188,14 @@ var SQLiteEventStore = class {
|
|
|
1083
1188
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1084
1189
|
END;
|
|
1085
1190
|
`);
|
|
1191
|
+
try {
|
|
1192
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1193
|
+
} catch {
|
|
1194
|
+
}
|
|
1195
|
+
try {
|
|
1196
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1197
|
+
} catch {
|
|
1198
|
+
}
|
|
1086
1199
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1087
1200
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1088
1201
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1182,6 +1295,21 @@ var SQLiteEventStore = class {
|
|
|
1182
1295
|
insertLevel.run(id);
|
|
1183
1296
|
});
|
|
1184
1297
|
transaction();
|
|
1298
|
+
if (this.markdownMirror) {
|
|
1299
|
+
const event = {
|
|
1300
|
+
id,
|
|
1301
|
+
eventType: input.eventType,
|
|
1302
|
+
sessionId: input.sessionId,
|
|
1303
|
+
timestamp: input.timestamp,
|
|
1304
|
+
content: input.content,
|
|
1305
|
+
canonicalKey,
|
|
1306
|
+
dedupeKey,
|
|
1307
|
+
metadata
|
|
1308
|
+
};
|
|
1309
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1310
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1185
1313
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1186
1314
|
} catch (error) {
|
|
1187
1315
|
return {
|
|
@@ -1240,6 +1368,92 @@ var SQLiteEventStore = class {
|
|
|
1240
1368
|
);
|
|
1241
1369
|
return rows.map(this.rowToEvent);
|
|
1242
1370
|
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1373
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1374
|
+
*/
|
|
1375
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1376
|
+
await this.initialize();
|
|
1377
|
+
const rows = sqliteAll(
|
|
1378
|
+
this.db,
|
|
1379
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1380
|
+
[lastRowid, limit]
|
|
1381
|
+
);
|
|
1382
|
+
return rows.map((row) => ({
|
|
1383
|
+
rowid: row._rowid,
|
|
1384
|
+
event: this.rowToEvent(row)
|
|
1385
|
+
}));
|
|
1386
|
+
}
|
|
1387
|
+
/**
|
|
1388
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1389
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1390
|
+
*
|
|
1391
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1392
|
+
*/
|
|
1393
|
+
async importEvents(events) {
|
|
1394
|
+
if (events.length === 0)
|
|
1395
|
+
return { inserted: 0, skipped: 0 };
|
|
1396
|
+
if (this.readOnly)
|
|
1397
|
+
return { inserted: 0, skipped: events.length };
|
|
1398
|
+
await this.initialize();
|
|
1399
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1400
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1401
|
+
const insertEvent = this.db.prepare(`
|
|
1402
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1403
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1404
|
+
`);
|
|
1405
|
+
const insertDedup = this.db.prepare(`
|
|
1406
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1407
|
+
`);
|
|
1408
|
+
const insertLevel = this.db.prepare(`
|
|
1409
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1410
|
+
`);
|
|
1411
|
+
let inserted = 0;
|
|
1412
|
+
let skipped = 0;
|
|
1413
|
+
const insertedEvents = [];
|
|
1414
|
+
const tx = this.db.transaction((batch) => {
|
|
1415
|
+
for (const ev of batch) {
|
|
1416
|
+
const existingById = getById.get(ev.id);
|
|
1417
|
+
if (existingById) {
|
|
1418
|
+
skipped++;
|
|
1419
|
+
continue;
|
|
1420
|
+
}
|
|
1421
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1422
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1423
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1424
|
+
if (existingByDedupe) {
|
|
1425
|
+
skipped++;
|
|
1426
|
+
continue;
|
|
1427
|
+
}
|
|
1428
|
+
const metadata = ev.metadata || {};
|
|
1429
|
+
const turnId = metadata.turnId;
|
|
1430
|
+
insertEvent.run(
|
|
1431
|
+
ev.id,
|
|
1432
|
+
ev.eventType,
|
|
1433
|
+
ev.sessionId,
|
|
1434
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1435
|
+
ev.content,
|
|
1436
|
+
canonicalKey,
|
|
1437
|
+
dedupeKey,
|
|
1438
|
+
JSON.stringify(metadata),
|
|
1439
|
+
turnId ?? null
|
|
1440
|
+
);
|
|
1441
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1442
|
+
insertLevel.run(ev.id);
|
|
1443
|
+
inserted++;
|
|
1444
|
+
insertedEvents.push(ev);
|
|
1445
|
+
}
|
|
1446
|
+
});
|
|
1447
|
+
tx(events);
|
|
1448
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1449
|
+
for (const ev of insertedEvents) {
|
|
1450
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1451
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1452
|
+
});
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
return { inserted, skipped };
|
|
1456
|
+
}
|
|
1243
1457
|
/**
|
|
1244
1458
|
* Create or update session
|
|
1245
1459
|
*/
|
|
@@ -1402,6 +1616,35 @@ var SQLiteEventStore = class {
|
|
|
1402
1616
|
[error, ...ids]
|
|
1403
1617
|
);
|
|
1404
1618
|
}
|
|
1619
|
+
/**
|
|
1620
|
+
* Get embedding/vector outbox health statistics
|
|
1621
|
+
*/
|
|
1622
|
+
async getOutboxStats() {
|
|
1623
|
+
await this.initialize();
|
|
1624
|
+
const embeddingRows = sqliteAll(
|
|
1625
|
+
this.db,
|
|
1626
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1627
|
+
);
|
|
1628
|
+
const vectorRows = sqliteAll(
|
|
1629
|
+
this.db,
|
|
1630
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1631
|
+
);
|
|
1632
|
+
const fromRows = (rows) => {
|
|
1633
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1634
|
+
for (const row of rows) {
|
|
1635
|
+
const key = row.status;
|
|
1636
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1637
|
+
out[key] += row.count;
|
|
1638
|
+
}
|
|
1639
|
+
out.total += row.count;
|
|
1640
|
+
}
|
|
1641
|
+
return out;
|
|
1642
|
+
};
|
|
1643
|
+
return {
|
|
1644
|
+
embedding: fromRows(embeddingRows),
|
|
1645
|
+
vector: fromRows(vectorRows)
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1405
1648
|
/**
|
|
1406
1649
|
* Update memory level
|
|
1407
1650
|
*/
|
|
@@ -1760,6 +2003,79 @@ var SQLiteEventStore = class {
|
|
|
1760
2003
|
getDatabase() {
|
|
1761
2004
|
return this.db;
|
|
1762
2005
|
}
|
|
2006
|
+
async recordRetrievalTrace(input) {
|
|
2007
|
+
await this.initialize();
|
|
2008
|
+
const traceId = randomUUID2();
|
|
2009
|
+
sqliteRun(
|
|
2010
|
+
this.db,
|
|
2011
|
+
`INSERT INTO retrieval_traces (
|
|
2012
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2013
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2014
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2015
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2016
|
+
[
|
|
2017
|
+
traceId,
|
|
2018
|
+
input.sessionId || null,
|
|
2019
|
+
input.projectHash || null,
|
|
2020
|
+
input.queryText,
|
|
2021
|
+
input.strategy || null,
|
|
2022
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2023
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2024
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2025
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2026
|
+
(input.candidateEventIds || []).length,
|
|
2027
|
+
(input.selectedEventIds || []).length,
|
|
2028
|
+
input.confidence || null,
|
|
2029
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2030
|
+
]
|
|
2031
|
+
);
|
|
2032
|
+
}
|
|
2033
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2034
|
+
await this.initialize();
|
|
2035
|
+
const rows = sqliteAll(
|
|
2036
|
+
this.db,
|
|
2037
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2038
|
+
[limit]
|
|
2039
|
+
);
|
|
2040
|
+
return rows.map((row) => ({
|
|
2041
|
+
traceId: row.trace_id,
|
|
2042
|
+
sessionId: row.session_id || void 0,
|
|
2043
|
+
projectHash: row.project_hash || void 0,
|
|
2044
|
+
queryText: row.query_text,
|
|
2045
|
+
strategy: row.strategy || void 0,
|
|
2046
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2047
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2048
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2049
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2050
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2051
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2052
|
+
confidence: row.confidence || void 0,
|
|
2053
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2054
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2055
|
+
}));
|
|
2056
|
+
}
|
|
2057
|
+
async getRetrievalTraceStats() {
|
|
2058
|
+
await this.initialize();
|
|
2059
|
+
const row = sqliteGet(
|
|
2060
|
+
this.db,
|
|
2061
|
+
`SELECT
|
|
2062
|
+
COUNT(*) as total_queries,
|
|
2063
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2064
|
+
AVG(selected_count) as avg_selected_count,
|
|
2065
|
+
CASE
|
|
2066
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2067
|
+
ELSE 0
|
|
2068
|
+
END as selection_rate
|
|
2069
|
+
FROM retrieval_traces`,
|
|
2070
|
+
[]
|
|
2071
|
+
);
|
|
2072
|
+
return {
|
|
2073
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2074
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2075
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2076
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2077
|
+
};
|
|
2078
|
+
}
|
|
1763
2079
|
/**
|
|
1764
2080
|
* Close database connection
|
|
1765
2081
|
*/
|
|
@@ -2621,7 +2937,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2621
2937
|
topK: 5,
|
|
2622
2938
|
minScore: 0.7,
|
|
2623
2939
|
maxTokens: 2e3,
|
|
2624
|
-
includeSessionContext: true
|
|
2940
|
+
includeSessionContext: true,
|
|
2941
|
+
strategy: "auto",
|
|
2942
|
+
rerankWithKeyword: true,
|
|
2943
|
+
decayPolicy: {
|
|
2944
|
+
enabled: true,
|
|
2945
|
+
windowDays: 30,
|
|
2946
|
+
maxPenalty: 0.15
|
|
2947
|
+
},
|
|
2948
|
+
graphHop: {
|
|
2949
|
+
enabled: true,
|
|
2950
|
+
maxHops: 1,
|
|
2951
|
+
hopPenalty: 0.08
|
|
2952
|
+
},
|
|
2953
|
+
projectScopeMode: "global"
|
|
2625
2954
|
};
|
|
2626
2955
|
var Retriever = class {
|
|
2627
2956
|
eventStore;
|
|
@@ -2631,6 +2960,7 @@ var Retriever = class {
|
|
|
2631
2960
|
sharedStore;
|
|
2632
2961
|
sharedVectorStore;
|
|
2633
2962
|
graduation;
|
|
2963
|
+
queryRewriter;
|
|
2634
2964
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2635
2965
|
this.eventStore = eventStore;
|
|
2636
2966
|
this.vectorStore = vectorStore;
|
|
@@ -2639,47 +2969,105 @@ var Retriever = class {
|
|
|
2639
2969
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2640
2970
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2641
2971
|
}
|
|
2642
|
-
/**
|
|
2643
|
-
* Set graduation pipeline for access tracking
|
|
2644
|
-
*/
|
|
2645
2972
|
setGraduationPipeline(graduation) {
|
|
2646
2973
|
this.graduation = graduation;
|
|
2647
2974
|
}
|
|
2648
|
-
/**
|
|
2649
|
-
* Set shared stores after construction
|
|
2650
|
-
*/
|
|
2651
2975
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2652
2976
|
this.sharedStore = sharedStore;
|
|
2653
2977
|
this.sharedVectorStore = sharedVectorStore;
|
|
2654
2978
|
}
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2979
|
+
setQueryRewriter(rewriter) {
|
|
2980
|
+
this.queryRewriter = rewriter;
|
|
2981
|
+
}
|
|
2658
2982
|
async retrieve(query, options = {}) {
|
|
2659
2983
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2660
|
-
const
|
|
2661
|
-
const
|
|
2662
|
-
|
|
2663
|
-
|
|
2984
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
2985
|
+
const fallbackTrace = [];
|
|
2986
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
2987
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
2988
|
+
let current = await this.runStage(query, {
|
|
2989
|
+
strategy: primaryStrategy,
|
|
2990
|
+
topK: opts.topK,
|
|
2664
2991
|
minScore: opts.minScore,
|
|
2665
|
-
sessionId:
|
|
2992
|
+
sessionId: sessionFilter,
|
|
2993
|
+
scope: opts.scope,
|
|
2994
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
2995
|
+
rerankWeights: opts.rerankWeights,
|
|
2996
|
+
decayPolicy: opts.decayPolicy,
|
|
2997
|
+
intentRewrite: opts.intentRewrite === true,
|
|
2998
|
+
graphHop: opts.graphHop,
|
|
2999
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3000
|
+
projectHash: opts.projectHash,
|
|
3001
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2666
3002
|
});
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
3003
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3004
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3005
|
+
current = await this.runStage(query, {
|
|
3006
|
+
strategy: "deep",
|
|
3007
|
+
topK: opts.topK,
|
|
3008
|
+
minScore: opts.minScore,
|
|
3009
|
+
sessionId: sessionFilter,
|
|
3010
|
+
scope: opts.scope,
|
|
3011
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3012
|
+
rerankWeights: opts.rerankWeights,
|
|
3013
|
+
decayPolicy: opts.decayPolicy,
|
|
3014
|
+
graphHop: opts.graphHop,
|
|
3015
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3016
|
+
projectHash: opts.projectHash,
|
|
3017
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3018
|
+
});
|
|
3019
|
+
fallbackTrace.push("fallback:deep");
|
|
3020
|
+
}
|
|
3021
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3022
|
+
current = await this.runStage(query, {
|
|
3023
|
+
strategy: "deep",
|
|
3024
|
+
topK: opts.topK,
|
|
3025
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3026
|
+
sessionId: void 0,
|
|
3027
|
+
scope: void 0,
|
|
3028
|
+
rerankWithKeyword: true,
|
|
3029
|
+
rerankWeights: opts.rerankWeights,
|
|
3030
|
+
decayPolicy: opts.decayPolicy,
|
|
3031
|
+
graphHop: opts.graphHop,
|
|
3032
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3033
|
+
projectHash: opts.projectHash,
|
|
3034
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3035
|
+
});
|
|
3036
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3037
|
+
}
|
|
3038
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3039
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3040
|
+
current = {
|
|
3041
|
+
results: summary,
|
|
3042
|
+
candidateResults: summary,
|
|
3043
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3044
|
+
};
|
|
3045
|
+
fallbackTrace.push("fallback:summary");
|
|
3046
|
+
}
|
|
3047
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2672
3048
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2673
3049
|
return {
|
|
2674
3050
|
memories,
|
|
2675
|
-
matchResult,
|
|
3051
|
+
matchResult: current.matchResult,
|
|
2676
3052
|
totalTokens: this.estimateTokens(context),
|
|
2677
|
-
context
|
|
3053
|
+
context,
|
|
3054
|
+
fallbackTrace,
|
|
3055
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3056
|
+
eventId: r.eventId,
|
|
3057
|
+
score: r.score,
|
|
3058
|
+
semanticScore: r.semanticScore,
|
|
3059
|
+
lexicalScore: r.lexicalScore,
|
|
3060
|
+
recencyScore: r.recencyScore
|
|
3061
|
+
})),
|
|
3062
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3063
|
+
eventId: r.eventId,
|
|
3064
|
+
score: r.score,
|
|
3065
|
+
semanticScore: r.semanticScore,
|
|
3066
|
+
lexicalScore: r.lexicalScore,
|
|
3067
|
+
recencyScore: r.recencyScore
|
|
3068
|
+
}))
|
|
2678
3069
|
};
|
|
2679
3070
|
}
|
|
2680
|
-
/**
|
|
2681
|
-
* Retrieve with unified search (project + shared)
|
|
2682
|
-
*/
|
|
2683
3071
|
async retrieveUnified(query, options = {}) {
|
|
2684
3072
|
const projectResult = await this.retrieve(query, options);
|
|
2685
3073
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2687,22 +3075,19 @@ var Retriever = class {
|
|
|
2687
3075
|
}
|
|
2688
3076
|
try {
|
|
2689
3077
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2690
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
excludeProjectHash: options.projectHash
|
|
2696
|
-
}
|
|
2697
|
-
);
|
|
3078
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3079
|
+
limit: options.topK || 5,
|
|
3080
|
+
minScore: options.minScore || 0.7,
|
|
3081
|
+
excludeProjectHash: options.projectHash
|
|
3082
|
+
});
|
|
2698
3083
|
const sharedMemories = [];
|
|
2699
3084
|
for (const result of sharedVectorResults) {
|
|
2700
3085
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2701
|
-
if (entry)
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
3086
|
+
if (!entry)
|
|
3087
|
+
continue;
|
|
3088
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3089
|
+
sharedMemories.push(entry);
|
|
3090
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2706
3091
|
}
|
|
2707
3092
|
}
|
|
2708
3093
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2717,50 +3102,243 @@ var Retriever = class {
|
|
|
2717
3102
|
return projectResult;
|
|
2718
3103
|
}
|
|
2719
3104
|
}
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
3105
|
+
async runStage(query, input) {
|
|
3106
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3107
|
+
strategy: input.strategy,
|
|
3108
|
+
topK: input.topK,
|
|
3109
|
+
minScore: input.minScore,
|
|
3110
|
+
sessionId: input.sessionId
|
|
3111
|
+
});
|
|
3112
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3113
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3114
|
+
if (rewritten && rewritten !== query) {
|
|
3115
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3116
|
+
strategy: "deep",
|
|
3117
|
+
topK: input.topK,
|
|
3118
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3119
|
+
sessionId: input.sessionId
|
|
3120
|
+
});
|
|
3121
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3122
|
+
}
|
|
3123
|
+
}
|
|
3124
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3125
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3126
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3127
|
+
limit: input.topK * 4
|
|
3128
|
+
});
|
|
3129
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3130
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3131
|
+
scope: input.scope,
|
|
3132
|
+
projectScopeMode: input.projectScopeMode,
|
|
3133
|
+
projectHash: input.projectHash,
|
|
3134
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3135
|
+
});
|
|
3136
|
+
const top = filtered.slice(0, input.topK);
|
|
3137
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3138
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3139
|
+
}
|
|
3140
|
+
mergeResults(primary, secondary, limit) {
|
|
3141
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3142
|
+
for (const row of primary)
|
|
3143
|
+
byId.set(row.eventId, row);
|
|
3144
|
+
for (const row of secondary) {
|
|
3145
|
+
const prev = byId.get(row.eventId);
|
|
3146
|
+
if (!prev || row.score > prev.score) {
|
|
3147
|
+
byId.set(row.eventId, row);
|
|
3148
|
+
}
|
|
3149
|
+
}
|
|
3150
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3151
|
+
}
|
|
3152
|
+
async expandGraphHops(seeds, opts) {
|
|
3153
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3154
|
+
for (const s of seeds)
|
|
3155
|
+
byId.set(s.eventId, s);
|
|
3156
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3157
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3158
|
+
const next = [];
|
|
3159
|
+
for (const f of frontier) {
|
|
3160
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3161
|
+
if (!ev)
|
|
3162
|
+
continue;
|
|
3163
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3164
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3165
|
+
for (const rid of relatedIds) {
|
|
3166
|
+
if (byId.has(rid))
|
|
3167
|
+
continue;
|
|
3168
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3169
|
+
if (!target)
|
|
3170
|
+
continue;
|
|
3171
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3172
|
+
const row = {
|
|
3173
|
+
id: `hop-${hop}-${rid}`,
|
|
3174
|
+
eventId: target.id,
|
|
3175
|
+
content: target.content,
|
|
3176
|
+
score,
|
|
3177
|
+
sessionId: target.sessionId,
|
|
3178
|
+
eventType: target.eventType,
|
|
3179
|
+
timestamp: target.timestamp.toISOString()
|
|
3180
|
+
};
|
|
3181
|
+
byId.set(row.eventId, row);
|
|
3182
|
+
next.push({ row, hop });
|
|
3183
|
+
if (byId.size >= opts.limit)
|
|
3184
|
+
break;
|
|
2741
3185
|
}
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
`;
|
|
3186
|
+
if (byId.size >= opts.limit)
|
|
3187
|
+
break;
|
|
2745
3188
|
}
|
|
3189
|
+
frontier = next;
|
|
3190
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3191
|
+
break;
|
|
2746
3192
|
}
|
|
2747
|
-
return
|
|
3193
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3194
|
+
}
|
|
3195
|
+
shouldFallback(matchResult, results) {
|
|
3196
|
+
if (results.length === 0)
|
|
3197
|
+
return true;
|
|
3198
|
+
if (matchResult.confidence === "none")
|
|
3199
|
+
return true;
|
|
3200
|
+
return false;
|
|
3201
|
+
}
|
|
3202
|
+
async buildSummaryFallback(query, topK) {
|
|
3203
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3204
|
+
const q = this.tokenize(query);
|
|
3205
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3206
|
+
id: `summary-${row.e.id}`,
|
|
3207
|
+
eventId: row.e.id,
|
|
3208
|
+
content: row.e.content,
|
|
3209
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3210
|
+
sessionId: row.e.sessionId,
|
|
3211
|
+
eventType: row.e.eventType,
|
|
3212
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3213
|
+
}));
|
|
3214
|
+
return ranked;
|
|
3215
|
+
}
|
|
3216
|
+
async searchByStrategy(query, input) {
|
|
3217
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3218
|
+
if (strategy === "fast") {
|
|
3219
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3220
|
+
limit: Math.max(5, input.topK * 3),
|
|
3221
|
+
sessionId: input.sessionId
|
|
3222
|
+
});
|
|
3223
|
+
return keyword;
|
|
3224
|
+
}
|
|
3225
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3226
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3227
|
+
limit: Math.max(5, input.topK * 3),
|
|
3228
|
+
minScore: input.minScore,
|
|
3229
|
+
sessionId: input.sessionId
|
|
3230
|
+
});
|
|
3231
|
+
}
|
|
3232
|
+
async searchByKeyword(query, input) {
|
|
3233
|
+
if (this.eventStore.keywordSearch) {
|
|
3234
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3235
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3236
|
+
return filtered2.map((row, idx) => ({
|
|
3237
|
+
id: `kw-${row.event.id}`,
|
|
3238
|
+
eventId: row.event.id,
|
|
3239
|
+
content: row.event.content,
|
|
3240
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3241
|
+
sessionId: row.event.sessionId,
|
|
3242
|
+
eventType: row.event.eventType,
|
|
3243
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3244
|
+
}));
|
|
3245
|
+
}
|
|
3246
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3247
|
+
const tokens = this.tokenize(query);
|
|
3248
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3249
|
+
return filtered.map((row, idx) => ({
|
|
3250
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3251
|
+
eventId: row.e.id,
|
|
3252
|
+
content: row.e.content,
|
|
3253
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3254
|
+
sessionId: row.e.sessionId,
|
|
3255
|
+
eventType: row.e.eventType,
|
|
3256
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3257
|
+
}));
|
|
3258
|
+
}
|
|
3259
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3260
|
+
const q = this.tokenize(query);
|
|
3261
|
+
const now = Date.now();
|
|
3262
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3263
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3264
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3265
|
+
const total = sw + lw + rw || 1;
|
|
3266
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3267
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3268
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3269
|
+
return [...results].map((r) => {
|
|
3270
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3271
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3272
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3273
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3274
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3275
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3276
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3277
|
+
}
|
|
3278
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3279
|
+
}).sort((a, b) => b.score - a.score);
|
|
3280
|
+
}
|
|
3281
|
+
async applyScopeFilters(results, options) {
|
|
3282
|
+
const scope = options?.scope;
|
|
3283
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3284
|
+
const allowedProjectHashes = new Set(
|
|
3285
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3286
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3287
|
+
)
|
|
3288
|
+
);
|
|
3289
|
+
if (!scope && projectScopeMode === "global")
|
|
3290
|
+
return results;
|
|
3291
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3292
|
+
const filtered = [];
|
|
3293
|
+
for (const result of results) {
|
|
3294
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3295
|
+
continue;
|
|
3296
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3297
|
+
continue;
|
|
3298
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3299
|
+
continue;
|
|
3300
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3301
|
+
if (!event)
|
|
3302
|
+
continue;
|
|
3303
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3304
|
+
continue;
|
|
3305
|
+
if (normalizedIncludes.length > 0) {
|
|
3306
|
+
const lc = event.content.toLowerCase();
|
|
3307
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3308
|
+
continue;
|
|
3309
|
+
}
|
|
3310
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3311
|
+
continue;
|
|
3312
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3313
|
+
filtered.push({ result, projectHash });
|
|
3314
|
+
}
|
|
3315
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3316
|
+
return filtered.map((x) => x.result);
|
|
3317
|
+
}
|
|
3318
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3319
|
+
if (projectScopeMode === "strict") {
|
|
3320
|
+
return projectMatched.map((x) => x.result);
|
|
3321
|
+
}
|
|
3322
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3323
|
+
}
|
|
3324
|
+
extractProjectHash(metadata) {
|
|
3325
|
+
if (!metadata || typeof metadata !== "object")
|
|
3326
|
+
return void 0;
|
|
3327
|
+
const scope = metadata.scope;
|
|
3328
|
+
if (!scope || typeof scope !== "object")
|
|
3329
|
+
return void 0;
|
|
3330
|
+
const project = scope.project;
|
|
3331
|
+
if (!project || typeof project !== "object")
|
|
3332
|
+
return void 0;
|
|
3333
|
+
const hash = project.hash;
|
|
3334
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2748
3335
|
}
|
|
2749
|
-
/**
|
|
2750
|
-
* Retrieve memories from a specific session
|
|
2751
|
-
*/
|
|
2752
3336
|
async retrieveFromSession(sessionId) {
|
|
2753
3337
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2754
3338
|
}
|
|
2755
|
-
/**
|
|
2756
|
-
* Get recent memories across all sessions
|
|
2757
|
-
*/
|
|
2758
3339
|
async retrieveRecent(limit = 100) {
|
|
2759
3340
|
return this.eventStore.getRecentEvents(limit);
|
|
2760
3341
|
}
|
|
2761
|
-
/**
|
|
2762
|
-
* Enrich search results with full event data
|
|
2763
|
-
*/
|
|
2764
3342
|
async enrichResults(results, options) {
|
|
2765
3343
|
const memories = [];
|
|
2766
3344
|
for (const result of results) {
|
|
@@ -2768,27 +3346,16 @@ var Retriever = class {
|
|
|
2768
3346
|
if (!event)
|
|
2769
3347
|
continue;
|
|
2770
3348
|
if (this.graduation) {
|
|
2771
|
-
this.graduation.recordAccess(
|
|
2772
|
-
event.id,
|
|
2773
|
-
options.sessionId || "unknown",
|
|
2774
|
-
result.score
|
|
2775
|
-
);
|
|
3349
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2776
3350
|
}
|
|
2777
3351
|
let sessionContext;
|
|
2778
3352
|
if (options.includeSessionContext) {
|
|
2779
3353
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2780
3354
|
}
|
|
2781
|
-
memories.push({
|
|
2782
|
-
event,
|
|
2783
|
-
score: result.score,
|
|
2784
|
-
sessionContext
|
|
2785
|
-
});
|
|
3355
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2786
3356
|
}
|
|
2787
3357
|
return memories;
|
|
2788
3358
|
}
|
|
2789
|
-
/**
|
|
2790
|
-
* Get surrounding context from the same session
|
|
2791
|
-
*/
|
|
2792
3359
|
async getSessionContext(sessionId, eventId) {
|
|
2793
3360
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2794
3361
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2801,55 +3368,86 @@ var Retriever = class {
|
|
|
2801
3368
|
return void 0;
|
|
2802
3369
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2803
3370
|
}
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
3371
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3372
|
+
let context = projectResult.context;
|
|
3373
|
+
if (sharedMemories.length === 0)
|
|
3374
|
+
return context;
|
|
3375
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3376
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3377
|
+
context += `### ${memory.title}
|
|
3378
|
+
`;
|
|
3379
|
+
if (memory.symptoms.length > 0)
|
|
3380
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3381
|
+
`;
|
|
3382
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3383
|
+
`;
|
|
3384
|
+
context += `**Solution:** ${memory.solution}
|
|
3385
|
+
`;
|
|
3386
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3387
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3388
|
+
`;
|
|
3389
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3390
|
+
|
|
3391
|
+
`;
|
|
3392
|
+
}
|
|
3393
|
+
return context;
|
|
3394
|
+
}
|
|
2807
3395
|
buildContext(memories, maxTokens) {
|
|
2808
3396
|
const parts = [];
|
|
2809
3397
|
let currentTokens = 0;
|
|
2810
3398
|
for (const memory of memories) {
|
|
2811
3399
|
const memoryText = this.formatMemory(memory);
|
|
2812
3400
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2813
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3401
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2814
3402
|
break;
|
|
2815
|
-
}
|
|
2816
3403
|
parts.push(memoryText);
|
|
2817
3404
|
currentTokens += memoryTokens;
|
|
2818
3405
|
}
|
|
2819
|
-
if (parts.length === 0)
|
|
3406
|
+
if (parts.length === 0)
|
|
2820
3407
|
return "";
|
|
2821
|
-
}
|
|
2822
3408
|
return `## Relevant Memories
|
|
2823
3409
|
|
|
2824
3410
|
${parts.join("\n\n---\n\n")}`;
|
|
2825
3411
|
}
|
|
2826
|
-
/**
|
|
2827
|
-
* Format a single memory for context
|
|
2828
|
-
*/
|
|
2829
3412
|
formatMemory(memory) {
|
|
2830
3413
|
const { event, score, sessionContext } = memory;
|
|
2831
3414
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2832
3415
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2833
3416
|
${event.content}`;
|
|
2834
|
-
if (sessionContext)
|
|
3417
|
+
if (sessionContext)
|
|
2835
3418
|
text += `
|
|
2836
3419
|
|
|
2837
3420
|
_Context:_ ${sessionContext}`;
|
|
2838
|
-
}
|
|
2839
3421
|
return text;
|
|
2840
3422
|
}
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
3423
|
+
matchesMetadataScope(metadata, expected) {
|
|
3424
|
+
if (!metadata)
|
|
3425
|
+
return false;
|
|
3426
|
+
return Object.entries(expected).every(([path4, value]) => {
|
|
3427
|
+
const actual = path4.split(".").reduce((acc, key) => {
|
|
3428
|
+
if (typeof acc !== "object" || acc === null)
|
|
3429
|
+
return void 0;
|
|
3430
|
+
return acc[key];
|
|
3431
|
+
}, metadata);
|
|
3432
|
+
return actual === value;
|
|
3433
|
+
});
|
|
3434
|
+
}
|
|
3435
|
+
tokenize(text) {
|
|
3436
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3437
|
+
}
|
|
3438
|
+
keywordOverlap(a, b) {
|
|
3439
|
+
if (a.length === 0 || b.length === 0)
|
|
3440
|
+
return 0;
|
|
3441
|
+
const bs = new Set(b);
|
|
3442
|
+
let hit = 0;
|
|
3443
|
+
for (const t of a)
|
|
3444
|
+
if (bs.has(t))
|
|
3445
|
+
hit += 1;
|
|
3446
|
+
return hit / a.length;
|
|
3447
|
+
}
|
|
2844
3448
|
estimateTokens(text) {
|
|
2845
3449
|
return Math.ceil(text.length / 4);
|
|
2846
3450
|
}
|
|
2847
|
-
/**
|
|
2848
|
-
* Get event age in days (for recency scoring)
|
|
2849
|
-
*/
|
|
2850
|
-
getEventAgeDays(eventId) {
|
|
2851
|
-
return 0;
|
|
2852
|
-
}
|
|
2853
3451
|
};
|
|
2854
3452
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2855
3453
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -4139,6 +4737,59 @@ var ConsolidatedStore = class {
|
|
|
4139
4737
|
[memoryId]
|
|
4140
4738
|
);
|
|
4141
4739
|
}
|
|
4740
|
+
/**
|
|
4741
|
+
* Create a long-term rule promoted from stable summaries
|
|
4742
|
+
*/
|
|
4743
|
+
async createRule(input) {
|
|
4744
|
+
const ruleId = randomUUID6();
|
|
4745
|
+
await dbRun(
|
|
4746
|
+
this.db,
|
|
4747
|
+
`INSERT INTO consolidated_rules
|
|
4748
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4749
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4750
|
+
[
|
|
4751
|
+
ruleId,
|
|
4752
|
+
input.rule,
|
|
4753
|
+
JSON.stringify(input.topics),
|
|
4754
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4755
|
+
JSON.stringify(input.sourceEvents),
|
|
4756
|
+
input.confidence
|
|
4757
|
+
]
|
|
4758
|
+
);
|
|
4759
|
+
return ruleId;
|
|
4760
|
+
}
|
|
4761
|
+
async getRules(options) {
|
|
4762
|
+
const limit = options?.limit || 100;
|
|
4763
|
+
const rows = await dbAll(
|
|
4764
|
+
this.db,
|
|
4765
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4766
|
+
[limit]
|
|
4767
|
+
);
|
|
4768
|
+
return rows.map((row) => ({
|
|
4769
|
+
ruleId: row.rule_id,
|
|
4770
|
+
rule: row.rule,
|
|
4771
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4772
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4773
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4774
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4775
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4776
|
+
}));
|
|
4777
|
+
}
|
|
4778
|
+
async countRules() {
|
|
4779
|
+
const result = await dbAll(
|
|
4780
|
+
this.db,
|
|
4781
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4782
|
+
);
|
|
4783
|
+
return result[0]?.count || 0;
|
|
4784
|
+
}
|
|
4785
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4786
|
+
const rows = await dbAll(
|
|
4787
|
+
this.db,
|
|
4788
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4789
|
+
[`%"${memoryId}"%`]
|
|
4790
|
+
);
|
|
4791
|
+
return (rows[0]?.count || 0) > 0;
|
|
4792
|
+
}
|
|
4142
4793
|
/**
|
|
4143
4794
|
* Get count of consolidated memories
|
|
4144
4795
|
*/
|
|
@@ -4288,7 +4939,14 @@ var ConsolidationWorker = class {
|
|
|
4288
4939
|
* Force a consolidation run (manual trigger)
|
|
4289
4940
|
*/
|
|
4290
4941
|
async forceRun() {
|
|
4291
|
-
|
|
4942
|
+
const out = await this.consolidateWithReport();
|
|
4943
|
+
return out.consolidatedCount;
|
|
4944
|
+
}
|
|
4945
|
+
/**
|
|
4946
|
+
* Force a consolidation run and return metrics report
|
|
4947
|
+
*/
|
|
4948
|
+
async forceRunWithReport() {
|
|
4949
|
+
return this.consolidateWithReport();
|
|
4292
4950
|
}
|
|
4293
4951
|
/**
|
|
4294
4952
|
* Schedule the next consolidation check
|
|
@@ -4328,12 +4986,21 @@ var ConsolidationWorker = class {
|
|
|
4328
4986
|
* Perform consolidation
|
|
4329
4987
|
*/
|
|
4330
4988
|
async consolidate() {
|
|
4989
|
+
const out = await this.consolidateWithReport();
|
|
4990
|
+
return out.consolidatedCount;
|
|
4991
|
+
}
|
|
4992
|
+
async consolidateWithReport() {
|
|
4331
4993
|
const workingSet = await this.workingSetStore.get();
|
|
4332
4994
|
if (workingSet.recentEvents.length < 3) {
|
|
4333
|
-
return
|
|
4995
|
+
return {
|
|
4996
|
+
consolidatedCount: 0,
|
|
4997
|
+
promotedRuleCount: 0,
|
|
4998
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
4999
|
+
};
|
|
4334
5000
|
}
|
|
4335
5001
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
4336
5002
|
let consolidatedCount = 0;
|
|
5003
|
+
const createdMemoryIds = [];
|
|
4337
5004
|
for (const group of groups) {
|
|
4338
5005
|
if (group.events.length < 3)
|
|
4339
5006
|
continue;
|
|
@@ -4342,14 +5009,16 @@ var ConsolidationWorker = class {
|
|
|
4342
5009
|
if (alreadyConsolidated)
|
|
4343
5010
|
continue;
|
|
4344
5011
|
const summary = await this.summarize(group);
|
|
4345
|
-
await this.consolidatedStore.create({
|
|
5012
|
+
const memoryId = await this.consolidatedStore.create({
|
|
4346
5013
|
summary,
|
|
4347
5014
|
topics: group.topics,
|
|
4348
5015
|
sourceEvents: eventIds,
|
|
4349
5016
|
confidence: this.calculateConfidence(group)
|
|
4350
5017
|
});
|
|
5018
|
+
createdMemoryIds.push(memoryId);
|
|
4351
5019
|
consolidatedCount++;
|
|
4352
5020
|
}
|
|
5021
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
4353
5022
|
if (consolidatedCount > 0) {
|
|
4354
5023
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
4355
5024
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4363,7 +5032,61 @@ var ConsolidationWorker = class {
|
|
|
4363
5032
|
await this.workingSetStore.prune(oldEventIds);
|
|
4364
5033
|
}
|
|
4365
5034
|
}
|
|
4366
|
-
|
|
5035
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5036
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5037
|
+
}
|
|
5038
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5039
|
+
let promoted = 0;
|
|
5040
|
+
for (const memoryId of memoryIds) {
|
|
5041
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5042
|
+
if (!memory)
|
|
5043
|
+
continue;
|
|
5044
|
+
if (memory.confidence < 0.55)
|
|
5045
|
+
continue;
|
|
5046
|
+
if (memory.sourceEvents.length < 4)
|
|
5047
|
+
continue;
|
|
5048
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5049
|
+
if (exists)
|
|
5050
|
+
continue;
|
|
5051
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5052
|
+
if (!rule)
|
|
5053
|
+
continue;
|
|
5054
|
+
await this.consolidatedStore.createRule({
|
|
5055
|
+
rule,
|
|
5056
|
+
topics: memory.topics,
|
|
5057
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5058
|
+
sourceEvents: memory.sourceEvents,
|
|
5059
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5060
|
+
});
|
|
5061
|
+
promoted++;
|
|
5062
|
+
}
|
|
5063
|
+
return promoted;
|
|
5064
|
+
}
|
|
5065
|
+
buildRuleFromSummary(summary, topics) {
|
|
5066
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5067
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5068
|
+
const seed = bullet || lines[0];
|
|
5069
|
+
if (!seed || seed.length < 8)
|
|
5070
|
+
return null;
|
|
5071
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5072
|
+
return `${topicPrefix}${seed}`;
|
|
5073
|
+
}
|
|
5074
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5075
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5076
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5077
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5078
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5079
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5080
|
+
return {
|
|
5081
|
+
beforeTokenEstimate,
|
|
5082
|
+
afterTokenEstimate,
|
|
5083
|
+
reductionRatio,
|
|
5084
|
+
qualityGuardPassed,
|
|
5085
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5086
|
+
};
|
|
5087
|
+
}
|
|
5088
|
+
estimateTokens(text) {
|
|
5089
|
+
return Math.ceil((text || "").length / 4);
|
|
4367
5090
|
}
|
|
4368
5091
|
/**
|
|
4369
5092
|
* Check if consolidation should run
|
|
@@ -4923,13 +5646,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
4923
5646
|
);
|
|
4924
5647
|
}
|
|
4925
5648
|
|
|
5649
|
+
// src/core/md-mirror.ts
|
|
5650
|
+
import * as fs3 from "node:fs";
|
|
5651
|
+
import * as path2 from "node:path";
|
|
5652
|
+
function sanitizeSegment2(input, fallback) {
|
|
5653
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5654
|
+
return v || fallback;
|
|
5655
|
+
}
|
|
5656
|
+
function getAtPath(obj, dotted) {
|
|
5657
|
+
if (!obj)
|
|
5658
|
+
return void 0;
|
|
5659
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5660
|
+
if (!acc || typeof acc !== "object")
|
|
5661
|
+
return void 0;
|
|
5662
|
+
return acc[key];
|
|
5663
|
+
}, obj);
|
|
5664
|
+
}
|
|
5665
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5666
|
+
const meta = event.metadata;
|
|
5667
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5668
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5669
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5670
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5671
|
+
const d = event.timestamp;
|
|
5672
|
+
const yyyy = d.getFullYear();
|
|
5673
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5674
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5675
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5676
|
+
}
|
|
5677
|
+
var MarkdownMirror2 = class {
|
|
5678
|
+
constructor(rootDir) {
|
|
5679
|
+
this.rootDir = rootDir;
|
|
5680
|
+
}
|
|
5681
|
+
async append(event, eventId) {
|
|
5682
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5683
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5684
|
+
const lines = [
|
|
5685
|
+
"",
|
|
5686
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5687
|
+
`- type: ${event.eventType}`,
|
|
5688
|
+
`- session: ${event.sessionId}`,
|
|
5689
|
+
event.content
|
|
5690
|
+
];
|
|
5691
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5692
|
+
await this.refreshIndex();
|
|
5693
|
+
}
|
|
5694
|
+
async refreshIndex() {
|
|
5695
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5696
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5697
|
+
const files = [];
|
|
5698
|
+
await this.walk(memoryRoot, files);
|
|
5699
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5700
|
+
const index = [
|
|
5701
|
+
"# Memory Index",
|
|
5702
|
+
"",
|
|
5703
|
+
"Generated automatically by MarkdownMirror.",
|
|
5704
|
+
"",
|
|
5705
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5706
|
+
""
|
|
5707
|
+
].join("\n");
|
|
5708
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5709
|
+
}
|
|
5710
|
+
async walk(dir, out) {
|
|
5711
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5712
|
+
for (const e of entries) {
|
|
5713
|
+
const full = path2.join(dir, e.name);
|
|
5714
|
+
if (e.isDirectory()) {
|
|
5715
|
+
await this.walk(full, out);
|
|
5716
|
+
} else {
|
|
5717
|
+
out.push(full);
|
|
5718
|
+
}
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
};
|
|
5722
|
+
|
|
5723
|
+
// src/core/ingest-interceptor.ts
|
|
5724
|
+
var IngestInterceptorRegistry = class {
|
|
5725
|
+
before = [];
|
|
5726
|
+
after = [];
|
|
5727
|
+
onError = [];
|
|
5728
|
+
registerBefore(interceptor) {
|
|
5729
|
+
this.before.push(interceptor);
|
|
5730
|
+
return () => {
|
|
5731
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5732
|
+
};
|
|
5733
|
+
}
|
|
5734
|
+
registerAfter(interceptor) {
|
|
5735
|
+
this.after.push(interceptor);
|
|
5736
|
+
return () => {
|
|
5737
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5738
|
+
};
|
|
5739
|
+
}
|
|
5740
|
+
registerOnError(interceptor) {
|
|
5741
|
+
this.onError.push(interceptor);
|
|
5742
|
+
return () => {
|
|
5743
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5744
|
+
};
|
|
5745
|
+
}
|
|
5746
|
+
async run(stage, context) {
|
|
5747
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5748
|
+
for (const interceptor of interceptors) {
|
|
5749
|
+
await interceptor({ ...context, stage });
|
|
5750
|
+
}
|
|
5751
|
+
}
|
|
5752
|
+
};
|
|
5753
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5754
|
+
if (!base && !patch)
|
|
5755
|
+
return void 0;
|
|
5756
|
+
if (!base)
|
|
5757
|
+
return patch;
|
|
5758
|
+
if (!patch)
|
|
5759
|
+
return base;
|
|
5760
|
+
const result = { ...base };
|
|
5761
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5762
|
+
const current = result[key];
|
|
5763
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5764
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5765
|
+
current,
|
|
5766
|
+
value
|
|
5767
|
+
);
|
|
5768
|
+
} else {
|
|
5769
|
+
result[key] = value;
|
|
5770
|
+
}
|
|
5771
|
+
}
|
|
5772
|
+
return result;
|
|
5773
|
+
}
|
|
5774
|
+
|
|
5775
|
+
// src/core/tag-taxonomy.ts
|
|
5776
|
+
var TAG_NAMESPACES = {
|
|
5777
|
+
SYSTEM: "sys:",
|
|
5778
|
+
QUALITY: "q:",
|
|
5779
|
+
PROJECT: "proj:",
|
|
5780
|
+
TOPIC: "topic:",
|
|
5781
|
+
TEMPORAL: "t:",
|
|
5782
|
+
USER: "user:",
|
|
5783
|
+
AGENT: "agent:"
|
|
5784
|
+
};
|
|
5785
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5786
|
+
function parseTag(tag) {
|
|
5787
|
+
const value = (tag || "").trim();
|
|
5788
|
+
const idx = value.indexOf(":");
|
|
5789
|
+
if (idx <= 0)
|
|
5790
|
+
return { value };
|
|
5791
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5792
|
+
const tagValue = value.slice(idx + 1);
|
|
5793
|
+
if (!tagValue)
|
|
5794
|
+
return { value };
|
|
5795
|
+
return { namespace, value: tagValue };
|
|
5796
|
+
}
|
|
5797
|
+
function validateTag(tag) {
|
|
5798
|
+
const normalized = (tag || "").trim();
|
|
5799
|
+
if (!normalized)
|
|
5800
|
+
return false;
|
|
5801
|
+
const { namespace } = parseTag(normalized);
|
|
5802
|
+
if (!namespace)
|
|
5803
|
+
return true;
|
|
5804
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5805
|
+
}
|
|
5806
|
+
function normalizeTags(tags) {
|
|
5807
|
+
if (!Array.isArray(tags))
|
|
5808
|
+
return [];
|
|
5809
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5810
|
+
for (const item of tags) {
|
|
5811
|
+
if (typeof item !== "string")
|
|
5812
|
+
continue;
|
|
5813
|
+
const normalized = item.trim();
|
|
5814
|
+
if (!validateTag(normalized))
|
|
5815
|
+
continue;
|
|
5816
|
+
dedup.add(normalized);
|
|
5817
|
+
}
|
|
5818
|
+
return [...dedup];
|
|
5819
|
+
}
|
|
5820
|
+
|
|
4926
5821
|
// src/services/memory-service.ts
|
|
4927
5822
|
function normalizePath(projectPath) {
|
|
4928
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5823
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
4929
5824
|
try {
|
|
4930
|
-
return
|
|
5825
|
+
return fs4.realpathSync(expanded);
|
|
4931
5826
|
} catch {
|
|
4932
|
-
return
|
|
5827
|
+
return path3.resolve(expanded);
|
|
4933
5828
|
}
|
|
4934
5829
|
}
|
|
4935
5830
|
function hashProjectPath(projectPath) {
|
|
@@ -4938,14 +5833,14 @@ function hashProjectPath(projectPath) {
|
|
|
4938
5833
|
}
|
|
4939
5834
|
function getProjectStoragePath(projectPath) {
|
|
4940
5835
|
const hash = hashProjectPath(projectPath);
|
|
4941
|
-
return
|
|
5836
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
4942
5837
|
}
|
|
4943
|
-
var REGISTRY_PATH =
|
|
4944
|
-
var SHARED_STORAGE_PATH =
|
|
5838
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5839
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
4945
5840
|
function loadSessionRegistry() {
|
|
4946
5841
|
try {
|
|
4947
|
-
if (
|
|
4948
|
-
const data =
|
|
5842
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5843
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
4949
5844
|
return JSON.parse(data);
|
|
4950
5845
|
}
|
|
4951
5846
|
} catch (error) {
|
|
@@ -4971,6 +5866,7 @@ var MemoryService = class {
|
|
|
4971
5866
|
vectorWorker = null;
|
|
4972
5867
|
graduationWorker = null;
|
|
4973
5868
|
initialized = false;
|
|
5869
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
4974
5870
|
// Endless Mode components
|
|
4975
5871
|
workingSetStore = null;
|
|
4976
5872
|
consolidatedStore = null;
|
|
@@ -4984,20 +5880,27 @@ var MemoryService = class {
|
|
|
4984
5880
|
sharedPromoter = null;
|
|
4985
5881
|
sharedStoreConfig = null;
|
|
4986
5882
|
projectHash = null;
|
|
5883
|
+
projectPath = null;
|
|
4987
5884
|
readOnly;
|
|
4988
5885
|
lightweightMode;
|
|
5886
|
+
mdMirror;
|
|
4989
5887
|
constructor(config) {
|
|
4990
5888
|
const storagePath = this.expandPath(config.storagePath);
|
|
4991
5889
|
this.readOnly = config.readOnly ?? false;
|
|
4992
5890
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
4993
|
-
|
|
4994
|
-
|
|
5891
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
5892
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
5893
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
4995
5894
|
}
|
|
4996
5895
|
this.projectHash = config.projectHash || null;
|
|
5896
|
+
this.projectPath = config.projectPath || null;
|
|
4997
5897
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
4998
5898
|
this.sqliteStore = new SQLiteEventStore(
|
|
4999
|
-
|
|
5000
|
-
{
|
|
5899
|
+
path3.join(storagePath, "events.sqlite"),
|
|
5900
|
+
{
|
|
5901
|
+
readonly: this.readOnly,
|
|
5902
|
+
markdownMirrorRoot: storagePath
|
|
5903
|
+
}
|
|
5001
5904
|
);
|
|
5002
5905
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
5003
5906
|
if (!analyticsEnabled) {
|
|
@@ -5005,7 +5908,7 @@ var MemoryService = class {
|
|
|
5005
5908
|
} else if (this.readOnly) {
|
|
5006
5909
|
try {
|
|
5007
5910
|
this.analyticsStore = new EventStore(
|
|
5008
|
-
|
|
5911
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5009
5912
|
{ readOnly: true }
|
|
5010
5913
|
);
|
|
5011
5914
|
} catch {
|
|
@@ -5013,11 +5916,11 @@ var MemoryService = class {
|
|
|
5013
5916
|
}
|
|
5014
5917
|
} else {
|
|
5015
5918
|
this.analyticsStore = new EventStore(
|
|
5016
|
-
|
|
5919
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5017
5920
|
{ readOnly: false }
|
|
5018
5921
|
);
|
|
5019
5922
|
}
|
|
5020
|
-
this.vectorStore = new VectorStore(
|
|
5923
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
5021
5924
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
5022
5925
|
this.matcher = getDefaultMatcher();
|
|
5023
5926
|
this.retriever = createRetriever(
|
|
@@ -5027,6 +5930,7 @@ var MemoryService = class {
|
|
|
5027
5930
|
this.embedder,
|
|
5028
5931
|
this.matcher
|
|
5029
5932
|
);
|
|
5933
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
5030
5934
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
5031
5935
|
}
|
|
5032
5936
|
/**
|
|
@@ -5086,16 +5990,16 @@ var MemoryService = class {
|
|
|
5086
5990
|
*/
|
|
5087
5991
|
async initializeSharedStore() {
|
|
5088
5992
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
5089
|
-
if (!
|
|
5090
|
-
|
|
5993
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
5994
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
5091
5995
|
}
|
|
5092
5996
|
this.sharedEventStore = createSharedEventStore(
|
|
5093
|
-
|
|
5997
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
5094
5998
|
);
|
|
5095
5999
|
await this.sharedEventStore.initialize();
|
|
5096
6000
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
5097
6001
|
this.sharedVectorStore = createSharedVectorStore(
|
|
5098
|
-
|
|
6002
|
+
path3.join(sharedPath, "vectors")
|
|
5099
6003
|
);
|
|
5100
6004
|
await this.sharedVectorStore.initialize();
|
|
5101
6005
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -5106,6 +6010,86 @@ var MemoryService = class {
|
|
|
5106
6010
|
);
|
|
5107
6011
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
5108
6012
|
}
|
|
6013
|
+
registerIngestBefore(interceptor) {
|
|
6014
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6015
|
+
}
|
|
6016
|
+
registerIngestAfter(interceptor) {
|
|
6017
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6018
|
+
}
|
|
6019
|
+
registerIngestOnError(interceptor) {
|
|
6020
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6021
|
+
}
|
|
6022
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6023
|
+
const normalizedInput = {
|
|
6024
|
+
...input,
|
|
6025
|
+
metadata: mergeHierarchicalMetadata(
|
|
6026
|
+
{
|
|
6027
|
+
ingest: {
|
|
6028
|
+
operation,
|
|
6029
|
+
pipeline: "default",
|
|
6030
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6031
|
+
},
|
|
6032
|
+
...this.projectHash ? {
|
|
6033
|
+
scope: {
|
|
6034
|
+
project: {
|
|
6035
|
+
hash: this.projectHash,
|
|
6036
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6037
|
+
}
|
|
6038
|
+
},
|
|
6039
|
+
tags: [`proj:${this.projectHash}`]
|
|
6040
|
+
} : {}
|
|
6041
|
+
},
|
|
6042
|
+
input.metadata
|
|
6043
|
+
)
|
|
6044
|
+
};
|
|
6045
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6046
|
+
const meta = normalizedInput.metadata;
|
|
6047
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6048
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6049
|
+
if (!currentTags.includes(projectTag)) {
|
|
6050
|
+
meta.tags = [...currentTags, projectTag];
|
|
6051
|
+
}
|
|
6052
|
+
}
|
|
6053
|
+
if (normalizedInput.metadata) {
|
|
6054
|
+
const meta = normalizedInput.metadata;
|
|
6055
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6056
|
+
if (normalizedTags.length > 0) {
|
|
6057
|
+
meta.tags = normalizedTags;
|
|
6058
|
+
}
|
|
6059
|
+
}
|
|
6060
|
+
await this.ingestInterceptors.run("before", {
|
|
6061
|
+
operation,
|
|
6062
|
+
sessionId: normalizedInput.sessionId,
|
|
6063
|
+
event: normalizedInput
|
|
6064
|
+
});
|
|
6065
|
+
try {
|
|
6066
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6067
|
+
if (result.success && !result.isDuplicate) {
|
|
6068
|
+
if (onSuccess) {
|
|
6069
|
+
await onSuccess(result.eventId);
|
|
6070
|
+
}
|
|
6071
|
+
try {
|
|
6072
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6073
|
+
} catch {
|
|
6074
|
+
}
|
|
6075
|
+
}
|
|
6076
|
+
await this.ingestInterceptors.run("after", {
|
|
6077
|
+
operation,
|
|
6078
|
+
sessionId: normalizedInput.sessionId,
|
|
6079
|
+
event: normalizedInput
|
|
6080
|
+
});
|
|
6081
|
+
return result;
|
|
6082
|
+
} catch (error) {
|
|
6083
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6084
|
+
await this.ingestInterceptors.run("error", {
|
|
6085
|
+
operation,
|
|
6086
|
+
sessionId: normalizedInput.sessionId,
|
|
6087
|
+
event: normalizedInput,
|
|
6088
|
+
error: normalizedError
|
|
6089
|
+
});
|
|
6090
|
+
throw error;
|
|
6091
|
+
}
|
|
6092
|
+
}
|
|
5109
6093
|
/**
|
|
5110
6094
|
* Start a new session
|
|
5111
6095
|
*/
|
|
@@ -5133,50 +6117,57 @@ var MemoryService = class {
|
|
|
5133
6117
|
*/
|
|
5134
6118
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
5135
6119
|
await this.initialize();
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
6120
|
+
return this.ingestWithInterceptors(
|
|
6121
|
+
"user_prompt",
|
|
6122
|
+
{
|
|
6123
|
+
eventType: "user_prompt",
|
|
6124
|
+
sessionId,
|
|
6125
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6126
|
+
content,
|
|
6127
|
+
metadata
|
|
6128
|
+
},
|
|
6129
|
+
async (eventId) => {
|
|
6130
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6131
|
+
}
|
|
6132
|
+
);
|
|
5147
6133
|
}
|
|
5148
6134
|
/**
|
|
5149
6135
|
* Store an agent response
|
|
5150
6136
|
*/
|
|
5151
6137
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
5152
6138
|
await this.initialize();
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
6139
|
+
return this.ingestWithInterceptors(
|
|
6140
|
+
"agent_response",
|
|
6141
|
+
{
|
|
6142
|
+
eventType: "agent_response",
|
|
6143
|
+
sessionId,
|
|
6144
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6145
|
+
content,
|
|
6146
|
+
metadata
|
|
6147
|
+
},
|
|
6148
|
+
async (eventId) => {
|
|
6149
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6150
|
+
}
|
|
6151
|
+
);
|
|
5164
6152
|
}
|
|
5165
6153
|
/**
|
|
5166
6154
|
* Store a session summary
|
|
5167
6155
|
*/
|
|
5168
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6156
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
5169
6157
|
await this.initialize();
|
|
5170
|
-
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
6158
|
+
return this.ingestWithInterceptors(
|
|
6159
|
+
"session_summary",
|
|
6160
|
+
{
|
|
6161
|
+
eventType: "session_summary",
|
|
6162
|
+
sessionId,
|
|
6163
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6164
|
+
content: summary,
|
|
6165
|
+
metadata
|
|
6166
|
+
},
|
|
6167
|
+
async (eventId) => {
|
|
6168
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6169
|
+
}
|
|
6170
|
+
);
|
|
5180
6171
|
}
|
|
5181
6172
|
/**
|
|
5182
6173
|
* Store a tool observation
|
|
@@ -5185,40 +6176,181 @@ var MemoryService = class {
|
|
|
5185
6176
|
await this.initialize();
|
|
5186
6177
|
const content = JSON.stringify(payload);
|
|
5187
6178
|
const turnId = payload.metadata?.turnId;
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
6179
|
+
return this.ingestWithInterceptors(
|
|
6180
|
+
"tool_observation",
|
|
6181
|
+
{
|
|
6182
|
+
eventType: "tool_observation",
|
|
6183
|
+
sessionId,
|
|
6184
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6185
|
+
content,
|
|
6186
|
+
metadata: {
|
|
6187
|
+
toolName: payload.toolName,
|
|
6188
|
+
success: payload.success,
|
|
6189
|
+
...turnId ? { turnId } : {}
|
|
6190
|
+
}
|
|
6191
|
+
},
|
|
6192
|
+
async (eventId) => {
|
|
6193
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6194
|
+
payload.toolName,
|
|
6195
|
+
payload.metadata || {},
|
|
6196
|
+
payload.success
|
|
6197
|
+
);
|
|
6198
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
5197
6199
|
}
|
|
5198
|
-
|
|
5199
|
-
if (result.success && !result.isDuplicate) {
|
|
5200
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
5201
|
-
payload.toolName,
|
|
5202
|
-
payload.metadata || {},
|
|
5203
|
-
payload.success
|
|
5204
|
-
);
|
|
5205
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
5206
|
-
}
|
|
5207
|
-
return result;
|
|
6200
|
+
);
|
|
5208
6201
|
}
|
|
5209
6202
|
/**
|
|
5210
6203
|
* Retrieve relevant memories for a query
|
|
5211
6204
|
*/
|
|
5212
6205
|
async retrieveMemories(query, options) {
|
|
5213
6206
|
await this.initialize();
|
|
6207
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6208
|
+
let result;
|
|
5214
6209
|
if (options?.includeShared && this.sharedStore) {
|
|
5215
|
-
|
|
6210
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
5216
6211
|
...options,
|
|
6212
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6213
|
+
rerankWeights,
|
|
5217
6214
|
includeShared: true,
|
|
5218
|
-
projectHash: this.projectHash || void 0
|
|
6215
|
+
projectHash: this.projectHash || void 0,
|
|
6216
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6217
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6218
|
+
});
|
|
6219
|
+
} else {
|
|
6220
|
+
result = await this.retriever.retrieve(query, {
|
|
6221
|
+
...options,
|
|
6222
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6223
|
+
rerankWeights,
|
|
6224
|
+
projectHash: this.projectHash || void 0,
|
|
6225
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6226
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6227
|
+
});
|
|
6228
|
+
}
|
|
6229
|
+
try {
|
|
6230
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6231
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6232
|
+
eventId: d.eventId,
|
|
6233
|
+
score: d.score,
|
|
6234
|
+
semanticScore: d.semanticScore,
|
|
6235
|
+
lexicalScore: d.lexicalScore,
|
|
6236
|
+
recencyScore: d.recencyScore
|
|
6237
|
+
}));
|
|
6238
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6239
|
+
eventId: d.eventId,
|
|
6240
|
+
score: d.score,
|
|
6241
|
+
semanticScore: d.semanticScore,
|
|
6242
|
+
lexicalScore: d.lexicalScore,
|
|
6243
|
+
recencyScore: d.recencyScore
|
|
6244
|
+
}));
|
|
6245
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6246
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6247
|
+
sessionId: options?.sessionId,
|
|
6248
|
+
projectHash: this.projectHash || void 0,
|
|
6249
|
+
queryText: query,
|
|
6250
|
+
strategy: options?.strategy || "auto",
|
|
6251
|
+
candidateEventIds,
|
|
6252
|
+
selectedEventIds,
|
|
6253
|
+
candidateDetails,
|
|
6254
|
+
selectedDetails,
|
|
6255
|
+
confidence: result.matchResult.confidence,
|
|
6256
|
+
fallbackTrace: result.fallbackTrace || []
|
|
6257
|
+
});
|
|
6258
|
+
} catch {
|
|
6259
|
+
}
|
|
6260
|
+
return result;
|
|
6261
|
+
}
|
|
6262
|
+
getConfiguredRerankWeights() {
|
|
6263
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6264
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6265
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6266
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6267
|
+
if (!allFinite)
|
|
6268
|
+
return void 0;
|
|
6269
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6270
|
+
const total = semantic + lexical + recency;
|
|
6271
|
+
if (!nonNegative || total <= 0)
|
|
6272
|
+
return void 0;
|
|
6273
|
+
return {
|
|
6274
|
+
semantic: semantic / total,
|
|
6275
|
+
lexical: lexical / total,
|
|
6276
|
+
recency: recency / total
|
|
6277
|
+
};
|
|
6278
|
+
}
|
|
6279
|
+
async getRerankWeights(adaptive) {
|
|
6280
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6281
|
+
if (configured)
|
|
6282
|
+
return configured;
|
|
6283
|
+
if (adaptive)
|
|
6284
|
+
return this.getAdaptiveRerankWeights();
|
|
6285
|
+
return void 0;
|
|
6286
|
+
}
|
|
6287
|
+
async rewriteQueryIntent(query) {
|
|
6288
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6289
|
+
return null;
|
|
6290
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6291
|
+
if (!apiUrl)
|
|
6292
|
+
return null;
|
|
6293
|
+
const controller = new AbortController();
|
|
6294
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6295
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6296
|
+
try {
|
|
6297
|
+
const prompt = [
|
|
6298
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6299
|
+
"Return plain text only, one line, no markdown.",
|
|
6300
|
+
`Query: ${query}`
|
|
6301
|
+
].join("\n");
|
|
6302
|
+
const res = await fetch(apiUrl, {
|
|
6303
|
+
method: "POST",
|
|
6304
|
+
headers: {
|
|
6305
|
+
"Content-Type": "application/json",
|
|
6306
|
+
Accept: "*/*",
|
|
6307
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6308
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6309
|
+
},
|
|
6310
|
+
body: JSON.stringify({
|
|
6311
|
+
question: prompt,
|
|
6312
|
+
company_name: null,
|
|
6313
|
+
conversation_id: null
|
|
6314
|
+
}),
|
|
6315
|
+
signal: controller.signal
|
|
5219
6316
|
});
|
|
6317
|
+
const text = (await res.text()).trim();
|
|
6318
|
+
if (!text)
|
|
6319
|
+
return null;
|
|
6320
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6321
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6322
|
+
return null;
|
|
6323
|
+
return oneLine;
|
|
6324
|
+
} catch {
|
|
6325
|
+
return null;
|
|
6326
|
+
} finally {
|
|
6327
|
+
clearTimeout(timeout);
|
|
6328
|
+
}
|
|
6329
|
+
}
|
|
6330
|
+
async getAdaptiveRerankWeights() {
|
|
6331
|
+
try {
|
|
6332
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6333
|
+
if (s.totalEvaluated < 20)
|
|
6334
|
+
return void 0;
|
|
6335
|
+
let semantic = 0.7;
|
|
6336
|
+
let lexical = 0.2;
|
|
6337
|
+
let recency = 0.1;
|
|
6338
|
+
if (s.avgScore < 0.45) {
|
|
6339
|
+
semantic -= 0.1;
|
|
6340
|
+
lexical += 0.1;
|
|
6341
|
+
} else if (s.avgScore > 0.75) {
|
|
6342
|
+
semantic += 0.05;
|
|
6343
|
+
lexical -= 0.05;
|
|
6344
|
+
}
|
|
6345
|
+
if (s.unhelpful > s.helpful) {
|
|
6346
|
+
recency += 0.05;
|
|
6347
|
+
semantic -= 0.03;
|
|
6348
|
+
lexical -= 0.02;
|
|
6349
|
+
}
|
|
6350
|
+
return { semantic, lexical, recency };
|
|
6351
|
+
} catch {
|
|
6352
|
+
return void 0;
|
|
5220
6353
|
}
|
|
5221
|
-
return this.retriever.retrieve(query, options);
|
|
5222
6354
|
}
|
|
5223
6355
|
/**
|
|
5224
6356
|
* Fast keyword search using SQLite FTS5
|
|
@@ -5260,6 +6392,18 @@ var MemoryService = class {
|
|
|
5260
6392
|
/**
|
|
5261
6393
|
* Get memory statistics
|
|
5262
6394
|
*/
|
|
6395
|
+
async getOutboxStats() {
|
|
6396
|
+
await this.initialize();
|
|
6397
|
+
return this.sqliteStore.getOutboxStats();
|
|
6398
|
+
}
|
|
6399
|
+
async getRetrievalTraceStats() {
|
|
6400
|
+
await this.initialize();
|
|
6401
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6402
|
+
}
|
|
6403
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6404
|
+
await this.initialize();
|
|
6405
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6406
|
+
}
|
|
5263
6407
|
async getStats() {
|
|
5264
6408
|
await this.initialize();
|
|
5265
6409
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5750,7 +6894,7 @@ var MemoryService = class {
|
|
|
5750
6894
|
*/
|
|
5751
6895
|
expandPath(p) {
|
|
5752
6896
|
if (p.startsWith("~")) {
|
|
5753
|
-
return
|
|
6897
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5754
6898
|
}
|
|
5755
6899
|
return p;
|
|
5756
6900
|
}
|
|
@@ -5760,10 +6904,11 @@ function getLightweightMemoryService(sessionId) {
|
|
|
5760
6904
|
const projectInfo = getSessionProject(sessionId);
|
|
5761
6905
|
const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : "lightweight_global";
|
|
5762
6906
|
if (!serviceCache.has(key)) {
|
|
5763
|
-
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) :
|
|
6907
|
+
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) : path3.join(os.homedir(), ".claude-code", "memory");
|
|
5764
6908
|
serviceCache.set(key, new MemoryService({
|
|
5765
6909
|
storagePath,
|
|
5766
6910
|
projectHash: projectInfo?.projectHash,
|
|
6911
|
+
projectPath: projectInfo?.projectPath,
|
|
5767
6912
|
lightweightMode: true,
|
|
5768
6913
|
// Skip embedder/vector/workers
|
|
5769
6914
|
analyticsEnabled: false,
|