claude-memory-layer 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +3577 -389
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1383 -138
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1917 -214
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1813 -231
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1802 -205
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1909 -248
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1861 -206
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +2341 -217
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +2350 -226
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1805 -206
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +1447 -55
- package/dist/ui/index.html +318 -147
- package/dist/ui/style.css +892 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/docs/OPERATIONS.md +18 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +9 -2
- package/scripts/build.ts +6 -0
- package/scripts/fix-sync-gap.js +32 -0
- package/scripts/heartbeat-memory-orchestrator.sh +28 -0
- package/scripts/report-sync-gap.js +26 -0
- package/scripts/review-queue-auto-resolve.js +21 -0
- package/scripts/sync-gap-auto-heal.sh +17 -0
- package/specs/20260207-dashboard-upgrade/context.md +38 -0
- package/specs/20260207-dashboard-upgrade/spec.md +96 -0
- package/src/cli/index.ts +391 -60
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +794 -7
- package/src/core/sqlite-wrapper.ts +8 -0
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/turn-state.ts +159 -0
- package/src/core/types.ts +51 -8
- package/src/core/vector-store.ts +21 -3
- package/src/hooks/post-tool-use.ts +68 -23
- package/src/hooks/session-end.ts +8 -3
- package/src/hooks/stop.ts +96 -25
- package/src/hooks/user-prompt-submit.ts +44 -5
- package/src/server/api/chat.ts +244 -0
- package/src/server/api/citations.ts +3 -3
- package/src/server/api/events.ts +30 -5
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +9 -1
- package/src/server/api/projects.ts +74 -0
- package/src/server/api/search.ts +3 -3
- package/src/server/api/sessions.ts +3 -3
- package/src/server/api/stats.ts +89 -8
- package/src/server/api/turns.ts +143 -0
- package/src/server/api/utils.ts +46 -0
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +508 -71
- package/src/services/session-history-importer.ts +215 -51
- package/src/ui/app.js +1447 -55
- package/src/ui/index.html +318 -147
- package/src/ui/style.css +892 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
- package/.claude/settings.local.json +0 -27
- package/.claude-memory/test.sqlite +0 -0
- package/.history/package_20260201112328.json +0 -45
- package/.history/package_20260201113602.json +0 -45
- package/.history/package_20260201113713.json +0 -45
- package/.history/package_20260201114110.json +0 -45
- package/.history/package_20260201114632.json +0 -46
- package/.history/package_20260201133143.json +0 -45
- package/.history/package_20260201134319.json +0 -45
- package/.history/package_20260201134326.json +0 -45
- package/.history/package_20260201134334.json +0 -45
- package/.history/package_20260201134912.json +0 -45
- package/.history/package_20260201142928.json +0 -46
- package/.history/package_20260201192048.json +0 -47
- package/.history/package_20260202114053.json +0 -49
- package/.history/package_20260202121115.json +0 -49
- package/test_access.js +0 -49
package/dist/hooks/stop.js
CHANGED
|
@@ -6,10 +6,14 @@ const require = createRequire(import.meta.url);
|
|
|
6
6
|
const __filename = fileURLToPath(import.meta.url);
|
|
7
7
|
const __dirname = dirname(__filename);
|
|
8
8
|
|
|
9
|
+
// src/hooks/stop.ts
|
|
10
|
+
import * as fs6 from "fs";
|
|
11
|
+
import * as readline from "readline";
|
|
12
|
+
|
|
9
13
|
// src/services/memory-service.ts
|
|
10
|
-
import * as
|
|
14
|
+
import * as path3 from "path";
|
|
11
15
|
import * as os from "os";
|
|
12
|
-
import * as
|
|
16
|
+
import * as fs4 from "fs";
|
|
13
17
|
import * as crypto2 from "crypto";
|
|
14
18
|
|
|
15
19
|
// src/core/event-store.ts
|
|
@@ -67,11 +71,11 @@ function toDate(value) {
|
|
|
67
71
|
return new Date(value);
|
|
68
72
|
return new Date(String(value));
|
|
69
73
|
}
|
|
70
|
-
function createDatabase(
|
|
74
|
+
function createDatabase(path5, options) {
|
|
71
75
|
if (options?.readOnly) {
|
|
72
|
-
return new duckdb.Database(
|
|
76
|
+
return new duckdb.Database(path5, { access_mode: "READ_ONLY" });
|
|
73
77
|
}
|
|
74
|
-
return new duckdb.Database(
|
|
78
|
+
return new duckdb.Database(path5);
|
|
75
79
|
}
|
|
76
80
|
function dbRun(db, sql, params = []) {
|
|
77
81
|
return new Promise((resolve2, reject) => {
|
|
@@ -335,6 +339,17 @@ var EventStore = class {
|
|
|
335
339
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
336
340
|
)
|
|
337
341
|
`);
|
|
342
|
+
await dbRun(this.db, `
|
|
343
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
344
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
345
|
+
rule TEXT NOT NULL,
|
|
346
|
+
topics JSON,
|
|
347
|
+
source_memory_ids JSON,
|
|
348
|
+
source_events JSON,
|
|
349
|
+
confidence FLOAT DEFAULT 0.5,
|
|
350
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
351
|
+
)
|
|
352
|
+
`);
|
|
338
353
|
await dbRun(this.db, `
|
|
339
354
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
340
355
|
key VARCHAR PRIMARY KEY,
|
|
@@ -354,6 +369,7 @@ var EventStore = class {
|
|
|
354
369
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
355
370
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
356
371
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
372
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
357
373
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
358
374
|
this.initialized = true;
|
|
359
375
|
}
|
|
@@ -741,8 +757,14 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
741
757
|
|
|
742
758
|
// src/core/sqlite-wrapper.ts
|
|
743
759
|
import Database from "better-sqlite3";
|
|
744
|
-
|
|
745
|
-
|
|
760
|
+
import * as fs from "fs";
|
|
761
|
+
import * as nodePath from "path";
|
|
762
|
+
function createSQLiteDatabase(path5, options) {
|
|
763
|
+
const dir = nodePath.dirname(path5);
|
|
764
|
+
if (!fs.existsSync(dir)) {
|
|
765
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
766
|
+
}
|
|
767
|
+
const db = new Database(path5, {
|
|
746
768
|
readonly: options?.readonly ?? false
|
|
747
769
|
});
|
|
748
770
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -783,6 +805,64 @@ function toSQLiteTimestamp(date) {
|
|
|
783
805
|
return date.toISOString();
|
|
784
806
|
}
|
|
785
807
|
|
|
808
|
+
// src/core/markdown-mirror.ts
|
|
809
|
+
import * as fs2 from "fs/promises";
|
|
810
|
+
import * as path from "path";
|
|
811
|
+
var DEFAULT_NAMESPACE = "default";
|
|
812
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
813
|
+
function sanitizeSegment(input, fallback) {
|
|
814
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
815
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
816
|
+
if (!safe || safe === "." || safe === "..")
|
|
817
|
+
return fallback;
|
|
818
|
+
return safe;
|
|
819
|
+
}
|
|
820
|
+
function getCategorySegments(metadata, eventType) {
|
|
821
|
+
const raw = metadata?.categoryPath;
|
|
822
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
823
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
824
|
+
}
|
|
825
|
+
const single = metadata?.category;
|
|
826
|
+
if (typeof single === "string" && single.trim()) {
|
|
827
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
828
|
+
}
|
|
829
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
830
|
+
}
|
|
831
|
+
function buildMirrorPath(rootDir, event) {
|
|
832
|
+
const metadata = event.metadata;
|
|
833
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
834
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
835
|
+
const d = event.timestamp;
|
|
836
|
+
const yyyy = d.getFullYear();
|
|
837
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
838
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
839
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
840
|
+
}
|
|
841
|
+
function formatMirrorEntry(event) {
|
|
842
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
843
|
+
return [
|
|
844
|
+
"",
|
|
845
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
846
|
+
` id: ${event.id}`,
|
|
847
|
+
` type: ${event.eventType}`,
|
|
848
|
+
` session: ${event.sessionId}`,
|
|
849
|
+
` category: ${category}`,
|
|
850
|
+
" content: |",
|
|
851
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
852
|
+
].join("\n") + "\n";
|
|
853
|
+
}
|
|
854
|
+
var MarkdownMirror = class {
|
|
855
|
+
constructor(rootDir) {
|
|
856
|
+
this.rootDir = rootDir;
|
|
857
|
+
}
|
|
858
|
+
async append(event) {
|
|
859
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
860
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
861
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
862
|
+
return outPath;
|
|
863
|
+
}
|
|
864
|
+
};
|
|
865
|
+
|
|
786
866
|
// src/core/sqlite-event-store.ts
|
|
787
867
|
var SQLiteEventStore = class {
|
|
788
868
|
constructor(dbPath, options) {
|
|
@@ -792,10 +872,12 @@ var SQLiteEventStore = class {
|
|
|
792
872
|
readonly: this.readOnly,
|
|
793
873
|
walMode: !this.readOnly
|
|
794
874
|
});
|
|
875
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
795
876
|
}
|
|
796
877
|
db;
|
|
797
878
|
initialized = false;
|
|
798
879
|
readOnly;
|
|
880
|
+
markdownMirror;
|
|
799
881
|
/**
|
|
800
882
|
* Initialize database schema
|
|
801
883
|
*/
|
|
@@ -1002,6 +1084,17 @@ var SQLiteEventStore = class {
|
|
|
1002
1084
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1003
1085
|
);
|
|
1004
1086
|
|
|
1087
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1088
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1089
|
+
rule_id TEXT PRIMARY KEY,
|
|
1090
|
+
rule TEXT NOT NULL,
|
|
1091
|
+
topics TEXT,
|
|
1092
|
+
source_memory_ids TEXT,
|
|
1093
|
+
source_events TEXT,
|
|
1094
|
+
confidence REAL DEFAULT 0.5,
|
|
1095
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1096
|
+
);
|
|
1097
|
+
|
|
1005
1098
|
-- Endless Mode Config table
|
|
1006
1099
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1007
1100
|
key TEXT PRIMARY KEY,
|
|
@@ -1009,6 +1102,41 @@ var SQLiteEventStore = class {
|
|
|
1009
1102
|
updated_at TEXT DEFAULT (datetime('now'))
|
|
1010
1103
|
);
|
|
1011
1104
|
|
|
1105
|
+
-- Memory Helpfulness tracking
|
|
1106
|
+
CREATE TABLE IF NOT EXISTS memory_helpfulness (
|
|
1107
|
+
id TEXT PRIMARY KEY,
|
|
1108
|
+
event_id TEXT NOT NULL,
|
|
1109
|
+
session_id TEXT NOT NULL,
|
|
1110
|
+
retrieval_score REAL DEFAULT 0,
|
|
1111
|
+
query_preview TEXT,
|
|
1112
|
+
session_continued INTEGER DEFAULT 0,
|
|
1113
|
+
prompt_count_after INTEGER DEFAULT 0,
|
|
1114
|
+
tool_success_count INTEGER DEFAULT 0,
|
|
1115
|
+
tool_total_count INTEGER DEFAULT 0,
|
|
1116
|
+
was_reasked INTEGER DEFAULT 0,
|
|
1117
|
+
helpfulness_score REAL DEFAULT 0.5,
|
|
1118
|
+
created_at TEXT DEFAULT (datetime('now')),
|
|
1119
|
+
measured_at TEXT
|
|
1120
|
+
);
|
|
1121
|
+
|
|
1122
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1123
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1124
|
+
trace_id TEXT PRIMARY KEY,
|
|
1125
|
+
session_id TEXT,
|
|
1126
|
+
project_hash TEXT,
|
|
1127
|
+
query_text TEXT NOT NULL,
|
|
1128
|
+
strategy TEXT,
|
|
1129
|
+
candidate_event_ids TEXT,
|
|
1130
|
+
selected_event_ids TEXT,
|
|
1131
|
+
candidate_details_json TEXT,
|
|
1132
|
+
selected_details_json TEXT,
|
|
1133
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1134
|
+
selected_count INTEGER DEFAULT 0,
|
|
1135
|
+
confidence TEXT,
|
|
1136
|
+
fallback_trace TEXT,
|
|
1137
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1138
|
+
);
|
|
1139
|
+
|
|
1012
1140
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1013
1141
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1014
1142
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1033,7 +1161,14 @@ var SQLiteEventStore = class {
|
|
|
1033
1161
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1034
1162
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1035
1163
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1164
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1036
1165
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1166
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1167
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1168
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1169
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1170
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1171
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1037
1172
|
|
|
1038
1173
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1039
1174
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1057,6 +1192,14 @@ var SQLiteEventStore = class {
|
|
|
1057
1192
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1058
1193
|
END;
|
|
1059
1194
|
`);
|
|
1195
|
+
try {
|
|
1196
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1197
|
+
} catch {
|
|
1198
|
+
}
|
|
1199
|
+
try {
|
|
1200
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1201
|
+
} catch {
|
|
1202
|
+
}
|
|
1060
1203
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1061
1204
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1062
1205
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1077,6 +1220,15 @@ var SQLiteEventStore = class {
|
|
|
1077
1220
|
console.error("Error adding last_accessed_at column:", err);
|
|
1078
1221
|
}
|
|
1079
1222
|
}
|
|
1223
|
+
if (!columnNames.includes("turn_id")) {
|
|
1224
|
+
try {
|
|
1225
|
+
sqliteExec(this.db, `
|
|
1226
|
+
ALTER TABLE events ADD COLUMN turn_id TEXT;
|
|
1227
|
+
`);
|
|
1228
|
+
} catch (err) {
|
|
1229
|
+
console.error("Error adding turn_id column:", err);
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1080
1232
|
try {
|
|
1081
1233
|
sqliteExec(this.db, `
|
|
1082
1234
|
CREATE INDEX IF NOT EXISTS idx_events_access_count ON events(access_count DESC);
|
|
@@ -1089,6 +1241,12 @@ var SQLiteEventStore = class {
|
|
|
1089
1241
|
`);
|
|
1090
1242
|
} catch (err) {
|
|
1091
1243
|
}
|
|
1244
|
+
try {
|
|
1245
|
+
sqliteExec(this.db, `
|
|
1246
|
+
CREATE INDEX IF NOT EXISTS idx_events_turn_id ON events(turn_id);
|
|
1247
|
+
`);
|
|
1248
|
+
} catch (err) {
|
|
1249
|
+
}
|
|
1092
1250
|
this.initialized = true;
|
|
1093
1251
|
}
|
|
1094
1252
|
/**
|
|
@@ -1113,9 +1271,11 @@ var SQLiteEventStore = class {
|
|
|
1113
1271
|
const id = randomUUID2();
|
|
1114
1272
|
const timestamp = toSQLiteTimestamp(input.timestamp);
|
|
1115
1273
|
try {
|
|
1274
|
+
const metadata = input.metadata || {};
|
|
1275
|
+
const turnId = metadata.turnId || null;
|
|
1116
1276
|
const insertEvent = this.db.prepare(`
|
|
1117
|
-
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata)
|
|
1118
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1277
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1278
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1119
1279
|
`);
|
|
1120
1280
|
const insertDedup = this.db.prepare(`
|
|
1121
1281
|
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
@@ -1132,12 +1292,28 @@ var SQLiteEventStore = class {
|
|
|
1132
1292
|
input.content,
|
|
1133
1293
|
canonicalKey,
|
|
1134
1294
|
dedupeKey,
|
|
1135
|
-
JSON.stringify(
|
|
1295
|
+
JSON.stringify(metadata),
|
|
1296
|
+
turnId
|
|
1136
1297
|
);
|
|
1137
1298
|
insertDedup.run(dedupeKey, id);
|
|
1138
1299
|
insertLevel.run(id);
|
|
1139
1300
|
});
|
|
1140
1301
|
transaction();
|
|
1302
|
+
if (this.markdownMirror) {
|
|
1303
|
+
const event = {
|
|
1304
|
+
id,
|
|
1305
|
+
eventType: input.eventType,
|
|
1306
|
+
sessionId: input.sessionId,
|
|
1307
|
+
timestamp: input.timestamp,
|
|
1308
|
+
content: input.content,
|
|
1309
|
+
canonicalKey,
|
|
1310
|
+
dedupeKey,
|
|
1311
|
+
metadata
|
|
1312
|
+
};
|
|
1313
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1314
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1315
|
+
});
|
|
1316
|
+
}
|
|
1141
1317
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1142
1318
|
} catch (error) {
|
|
1143
1319
|
return {
|
|
@@ -1196,6 +1372,92 @@ var SQLiteEventStore = class {
|
|
|
1196
1372
|
);
|
|
1197
1373
|
return rows.map(this.rowToEvent);
|
|
1198
1374
|
}
|
|
1375
|
+
/**
|
|
1376
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1377
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1378
|
+
*/
|
|
1379
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1380
|
+
await this.initialize();
|
|
1381
|
+
const rows = sqliteAll(
|
|
1382
|
+
this.db,
|
|
1383
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1384
|
+
[lastRowid, limit]
|
|
1385
|
+
);
|
|
1386
|
+
return rows.map((row) => ({
|
|
1387
|
+
rowid: row._rowid,
|
|
1388
|
+
event: this.rowToEvent(row)
|
|
1389
|
+
}));
|
|
1390
|
+
}
|
|
1391
|
+
/**
|
|
1392
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1393
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1394
|
+
*
|
|
1395
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1396
|
+
*/
|
|
1397
|
+
async importEvents(events) {
|
|
1398
|
+
if (events.length === 0)
|
|
1399
|
+
return { inserted: 0, skipped: 0 };
|
|
1400
|
+
if (this.readOnly)
|
|
1401
|
+
return { inserted: 0, skipped: events.length };
|
|
1402
|
+
await this.initialize();
|
|
1403
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1404
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1405
|
+
const insertEvent = this.db.prepare(`
|
|
1406
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1407
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1408
|
+
`);
|
|
1409
|
+
const insertDedup = this.db.prepare(`
|
|
1410
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1411
|
+
`);
|
|
1412
|
+
const insertLevel = this.db.prepare(`
|
|
1413
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1414
|
+
`);
|
|
1415
|
+
let inserted = 0;
|
|
1416
|
+
let skipped = 0;
|
|
1417
|
+
const insertedEvents = [];
|
|
1418
|
+
const tx = this.db.transaction((batch) => {
|
|
1419
|
+
for (const ev of batch) {
|
|
1420
|
+
const existingById = getById.get(ev.id);
|
|
1421
|
+
if (existingById) {
|
|
1422
|
+
skipped++;
|
|
1423
|
+
continue;
|
|
1424
|
+
}
|
|
1425
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1426
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1427
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1428
|
+
if (existingByDedupe) {
|
|
1429
|
+
skipped++;
|
|
1430
|
+
continue;
|
|
1431
|
+
}
|
|
1432
|
+
const metadata = ev.metadata || {};
|
|
1433
|
+
const turnId = metadata.turnId;
|
|
1434
|
+
insertEvent.run(
|
|
1435
|
+
ev.id,
|
|
1436
|
+
ev.eventType,
|
|
1437
|
+
ev.sessionId,
|
|
1438
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1439
|
+
ev.content,
|
|
1440
|
+
canonicalKey,
|
|
1441
|
+
dedupeKey,
|
|
1442
|
+
JSON.stringify(metadata),
|
|
1443
|
+
turnId ?? null
|
|
1444
|
+
);
|
|
1445
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1446
|
+
insertLevel.run(ev.id);
|
|
1447
|
+
inserted++;
|
|
1448
|
+
insertedEvents.push(ev);
|
|
1449
|
+
}
|
|
1450
|
+
});
|
|
1451
|
+
tx(events);
|
|
1452
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1453
|
+
for (const ev of insertedEvents) {
|
|
1454
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1455
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1456
|
+
});
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1459
|
+
return { inserted, skipped };
|
|
1460
|
+
}
|
|
1199
1461
|
/**
|
|
1200
1462
|
* Create or update session
|
|
1201
1463
|
*/
|
|
@@ -1358,6 +1620,35 @@ var SQLiteEventStore = class {
|
|
|
1358
1620
|
[error, ...ids]
|
|
1359
1621
|
);
|
|
1360
1622
|
}
|
|
1623
|
+
/**
|
|
1624
|
+
* Get embedding/vector outbox health statistics
|
|
1625
|
+
*/
|
|
1626
|
+
async getOutboxStats() {
|
|
1627
|
+
await this.initialize();
|
|
1628
|
+
const embeddingRows = sqliteAll(
|
|
1629
|
+
this.db,
|
|
1630
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1631
|
+
);
|
|
1632
|
+
const vectorRows = sqliteAll(
|
|
1633
|
+
this.db,
|
|
1634
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1635
|
+
);
|
|
1636
|
+
const fromRows = (rows) => {
|
|
1637
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1638
|
+
for (const row of rows) {
|
|
1639
|
+
const key = row.status;
|
|
1640
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1641
|
+
out[key] += row.count;
|
|
1642
|
+
}
|
|
1643
|
+
out.total += row.count;
|
|
1644
|
+
}
|
|
1645
|
+
return out;
|
|
1646
|
+
};
|
|
1647
|
+
return {
|
|
1648
|
+
embedding: fromRows(embeddingRows),
|
|
1649
|
+
vector: fromRows(vectorRows)
|
|
1650
|
+
};
|
|
1651
|
+
}
|
|
1361
1652
|
/**
|
|
1362
1653
|
* Update memory level
|
|
1363
1654
|
*/
|
|
@@ -1482,11 +1773,11 @@ var SQLiteEventStore = class {
|
|
|
1482
1773
|
);
|
|
1483
1774
|
}
|
|
1484
1775
|
/**
|
|
1485
|
-
* Get most accessed memories
|
|
1776
|
+
* Get most accessed memories (falls back to recent events if none accessed)
|
|
1486
1777
|
*/
|
|
1487
1778
|
async getMostAccessed(limit = 10) {
|
|
1488
1779
|
await this.initialize();
|
|
1489
|
-
|
|
1780
|
+
let rows = sqliteAll(
|
|
1490
1781
|
this.db,
|
|
1491
1782
|
`SELECT * FROM events
|
|
1492
1783
|
WHERE access_count > 0
|
|
@@ -1494,8 +1785,166 @@ var SQLiteEventStore = class {
|
|
|
1494
1785
|
LIMIT ?`,
|
|
1495
1786
|
[limit]
|
|
1496
1787
|
);
|
|
1788
|
+
if (rows.length === 0) {
|
|
1789
|
+
rows = sqliteAll(
|
|
1790
|
+
this.db,
|
|
1791
|
+
`SELECT * FROM events
|
|
1792
|
+
ORDER BY timestamp DESC
|
|
1793
|
+
LIMIT ?`,
|
|
1794
|
+
[limit]
|
|
1795
|
+
);
|
|
1796
|
+
}
|
|
1497
1797
|
return rows.map((row) => this.rowToEvent(row));
|
|
1498
1798
|
}
|
|
1799
|
+
/**
|
|
1800
|
+
* Record a memory retrieval for helpfulness tracking
|
|
1801
|
+
*/
|
|
1802
|
+
async recordRetrieval(eventId, sessionId, score, query) {
|
|
1803
|
+
if (this.readOnly)
|
|
1804
|
+
return;
|
|
1805
|
+
await this.initialize();
|
|
1806
|
+
const id = randomUUID2();
|
|
1807
|
+
sqliteRun(
|
|
1808
|
+
this.db,
|
|
1809
|
+
`INSERT INTO memory_helpfulness (id, event_id, session_id, retrieval_score, query_preview, created_at)
|
|
1810
|
+
VALUES (?, ?, ?, ?, ?, datetime('now'))`,
|
|
1811
|
+
[id, eventId, sessionId, score, query.slice(0, 100)]
|
|
1812
|
+
);
|
|
1813
|
+
}
|
|
1814
|
+
/**
|
|
1815
|
+
* Evaluate helpfulness for all retrievals in a session
|
|
1816
|
+
* Called at session end - uses behavioral signals to compute score
|
|
1817
|
+
*/
|
|
1818
|
+
async evaluateSessionHelpfulness(sessionId) {
|
|
1819
|
+
if (this.readOnly)
|
|
1820
|
+
return;
|
|
1821
|
+
await this.initialize();
|
|
1822
|
+
const retrievals = sqliteAll(
|
|
1823
|
+
this.db,
|
|
1824
|
+
`SELECT * FROM memory_helpfulness WHERE session_id = ? AND measured_at IS NULL`,
|
|
1825
|
+
[sessionId]
|
|
1826
|
+
);
|
|
1827
|
+
if (retrievals.length === 0)
|
|
1828
|
+
return;
|
|
1829
|
+
const sessionEvents = sqliteAll(
|
|
1830
|
+
this.db,
|
|
1831
|
+
`SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
|
|
1832
|
+
[sessionId]
|
|
1833
|
+
);
|
|
1834
|
+
const promptEvents = sessionEvents.filter((e) => e.event_type === "user_prompt");
|
|
1835
|
+
const toolEvents = sessionEvents.filter((e) => e.event_type === "tool_observation");
|
|
1836
|
+
let toolSuccessCount = 0;
|
|
1837
|
+
let toolTotalCount = toolEvents.length;
|
|
1838
|
+
for (const t of toolEvents) {
|
|
1839
|
+
try {
|
|
1840
|
+
const content = JSON.parse(t.content);
|
|
1841
|
+
if (content.success !== false)
|
|
1842
|
+
toolSuccessCount++;
|
|
1843
|
+
} catch {
|
|
1844
|
+
toolSuccessCount++;
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
const toolSuccessRatio = toolTotalCount > 0 ? toolSuccessCount / toolTotalCount : 0.5;
|
|
1848
|
+
for (const retrieval of retrievals) {
|
|
1849
|
+
const retrievalTime = retrieval.created_at;
|
|
1850
|
+
const eventsAfter = sessionEvents.filter((e) => e.timestamp > retrievalTime);
|
|
1851
|
+
const sessionContinued = eventsAfter.length > 0 ? 1 : 0;
|
|
1852
|
+
const promptsAfter = promptEvents.filter((e) => e.timestamp > retrievalTime);
|
|
1853
|
+
const promptCountAfter = promptsAfter.length;
|
|
1854
|
+
const queryWords = new Set((retrieval.query_preview || "").toLowerCase().split(/\s+/).filter((w) => w.length > 2));
|
|
1855
|
+
let wasReasked = 0;
|
|
1856
|
+
for (const p of promptsAfter) {
|
|
1857
|
+
const pWords = new Set(p.content.toLowerCase().split(/\s+/).filter((w) => w.length > 2));
|
|
1858
|
+
let overlap = 0;
|
|
1859
|
+
for (const w of queryWords) {
|
|
1860
|
+
if (pWords.has(w))
|
|
1861
|
+
overlap++;
|
|
1862
|
+
}
|
|
1863
|
+
if (queryWords.size > 0 && overlap / queryWords.size > 0.5) {
|
|
1864
|
+
wasReasked = 1;
|
|
1865
|
+
break;
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
const retrievalScore = retrieval.retrieval_score || 0;
|
|
1869
|
+
const helpfulnessScore = 0.3 * Math.min(retrievalScore, 1) + 0.25 * (sessionContinued ? 1 : 0) + 0.25 * toolSuccessRatio + 0.2 * (wasReasked ? 0 : 1);
|
|
1870
|
+
sqliteRun(
|
|
1871
|
+
this.db,
|
|
1872
|
+
`UPDATE memory_helpfulness
|
|
1873
|
+
SET session_continued = ?, prompt_count_after = ?,
|
|
1874
|
+
tool_success_count = ?, tool_total_count = ?,
|
|
1875
|
+
was_reasked = ?, helpfulness_score = ?,
|
|
1876
|
+
measured_at = datetime('now')
|
|
1877
|
+
WHERE id = ?`,
|
|
1878
|
+
[
|
|
1879
|
+
sessionContinued,
|
|
1880
|
+
promptCountAfter,
|
|
1881
|
+
toolSuccessCount,
|
|
1882
|
+
toolTotalCount,
|
|
1883
|
+
wasReasked,
|
|
1884
|
+
helpfulnessScore,
|
|
1885
|
+
retrieval.id
|
|
1886
|
+
]
|
|
1887
|
+
);
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
/**
|
|
1891
|
+
* Get most helpful memories ranked by helpfulness score
|
|
1892
|
+
*/
|
|
1893
|
+
async getHelpfulMemories(limit = 10) {
|
|
1894
|
+
await this.initialize();
|
|
1895
|
+
const rows = sqliteAll(
|
|
1896
|
+
this.db,
|
|
1897
|
+
`SELECT
|
|
1898
|
+
mh.event_id,
|
|
1899
|
+
AVG(mh.helpfulness_score) as avg_score,
|
|
1900
|
+
COUNT(*) as eval_count,
|
|
1901
|
+
e.content,
|
|
1902
|
+
e.access_count
|
|
1903
|
+
FROM memory_helpfulness mh
|
|
1904
|
+
JOIN events e ON e.id = mh.event_id
|
|
1905
|
+
WHERE mh.measured_at IS NOT NULL
|
|
1906
|
+
GROUP BY mh.event_id
|
|
1907
|
+
ORDER BY avg_score DESC
|
|
1908
|
+
LIMIT ?`,
|
|
1909
|
+
[limit]
|
|
1910
|
+
);
|
|
1911
|
+
return rows.map((r) => ({
|
|
1912
|
+
eventId: r.event_id,
|
|
1913
|
+
summary: r.content.substring(0, 200) + (r.content.length > 200 ? "..." : ""),
|
|
1914
|
+
helpfulnessScore: Math.round(r.avg_score * 100) / 100,
|
|
1915
|
+
accessCount: r.access_count || 0,
|
|
1916
|
+
evaluationCount: r.eval_count
|
|
1917
|
+
}));
|
|
1918
|
+
}
|
|
1919
|
+
/**
|
|
1920
|
+
* Get helpfulness statistics for dashboard
|
|
1921
|
+
*/
|
|
1922
|
+
async getHelpfulnessStats() {
|
|
1923
|
+
await this.initialize();
|
|
1924
|
+
const stats = sqliteGet(
|
|
1925
|
+
this.db,
|
|
1926
|
+
`SELECT
|
|
1927
|
+
AVG(helpfulness_score) as avg_score,
|
|
1928
|
+
COUNT(*) as total_evaluated,
|
|
1929
|
+
SUM(CASE WHEN helpfulness_score >= 0.7 THEN 1 ELSE 0 END) as helpful,
|
|
1930
|
+
SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
|
|
1931
|
+
SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
|
|
1932
|
+
FROM memory_helpfulness
|
|
1933
|
+
WHERE measured_at IS NOT NULL`
|
|
1934
|
+
);
|
|
1935
|
+
const totalRow = sqliteGet(
|
|
1936
|
+
this.db,
|
|
1937
|
+
`SELECT COUNT(*) as total FROM memory_helpfulness`
|
|
1938
|
+
);
|
|
1939
|
+
return {
|
|
1940
|
+
avgScore: Math.round((stats?.avg_score || 0) * 100) / 100,
|
|
1941
|
+
totalEvaluated: stats?.total_evaluated || 0,
|
|
1942
|
+
totalRetrievals: totalRow?.total || 0,
|
|
1943
|
+
helpful: stats?.helpful || 0,
|
|
1944
|
+
neutral: stats?.neutral || 0,
|
|
1945
|
+
unhelpful: stats?.unhelpful || 0
|
|
1946
|
+
};
|
|
1947
|
+
}
|
|
1499
1948
|
/**
|
|
1500
1949
|
* Fast keyword search using FTS5
|
|
1501
1950
|
* Returns events matching the search query, ranked by relevance
|
|
@@ -1558,12 +2007,222 @@ var SQLiteEventStore = class {
|
|
|
1558
2007
|
getDatabase() {
|
|
1559
2008
|
return this.db;
|
|
1560
2009
|
}
|
|
2010
|
+
async recordRetrievalTrace(input) {
|
|
2011
|
+
await this.initialize();
|
|
2012
|
+
const traceId = randomUUID2();
|
|
2013
|
+
sqliteRun(
|
|
2014
|
+
this.db,
|
|
2015
|
+
`INSERT INTO retrieval_traces (
|
|
2016
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2017
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2018
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2019
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2020
|
+
[
|
|
2021
|
+
traceId,
|
|
2022
|
+
input.sessionId || null,
|
|
2023
|
+
input.projectHash || null,
|
|
2024
|
+
input.queryText,
|
|
2025
|
+
input.strategy || null,
|
|
2026
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2027
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2028
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2029
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2030
|
+
(input.candidateEventIds || []).length,
|
|
2031
|
+
(input.selectedEventIds || []).length,
|
|
2032
|
+
input.confidence || null,
|
|
2033
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2034
|
+
]
|
|
2035
|
+
);
|
|
2036
|
+
}
|
|
2037
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2038
|
+
await this.initialize();
|
|
2039
|
+
const rows = sqliteAll(
|
|
2040
|
+
this.db,
|
|
2041
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2042
|
+
[limit]
|
|
2043
|
+
);
|
|
2044
|
+
return rows.map((row) => ({
|
|
2045
|
+
traceId: row.trace_id,
|
|
2046
|
+
sessionId: row.session_id || void 0,
|
|
2047
|
+
projectHash: row.project_hash || void 0,
|
|
2048
|
+
queryText: row.query_text,
|
|
2049
|
+
strategy: row.strategy || void 0,
|
|
2050
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2051
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2052
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2053
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2054
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2055
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2056
|
+
confidence: row.confidence || void 0,
|
|
2057
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2058
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2059
|
+
}));
|
|
2060
|
+
}
|
|
2061
|
+
async getRetrievalTraceStats() {
|
|
2062
|
+
await this.initialize();
|
|
2063
|
+
const row = sqliteGet(
|
|
2064
|
+
this.db,
|
|
2065
|
+
`SELECT
|
|
2066
|
+
COUNT(*) as total_queries,
|
|
2067
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2068
|
+
AVG(selected_count) as avg_selected_count,
|
|
2069
|
+
CASE
|
|
2070
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2071
|
+
ELSE 0
|
|
2072
|
+
END as selection_rate
|
|
2073
|
+
FROM retrieval_traces`,
|
|
2074
|
+
[]
|
|
2075
|
+
);
|
|
2076
|
+
return {
|
|
2077
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2078
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2079
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2080
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2081
|
+
};
|
|
2082
|
+
}
|
|
1561
2083
|
/**
|
|
1562
2084
|
* Close database connection
|
|
1563
2085
|
*/
|
|
1564
2086
|
async close() {
|
|
1565
2087
|
sqliteClose(this.db);
|
|
1566
2088
|
}
|
|
2089
|
+
/**
|
|
2090
|
+
* Get events grouped by turn_id for a session
|
|
2091
|
+
* Returns turns ordered by first event timestamp (newest first)
|
|
2092
|
+
*/
|
|
2093
|
+
async getSessionTurns(sessionId, options) {
|
|
2094
|
+
await this.initialize();
|
|
2095
|
+
const limit = options?.limit || 20;
|
|
2096
|
+
const offset = options?.offset || 0;
|
|
2097
|
+
const turnRows = sqliteAll(
|
|
2098
|
+
this.db,
|
|
2099
|
+
`SELECT turn_id, MIN(timestamp) as min_ts
|
|
2100
|
+
FROM events
|
|
2101
|
+
WHERE session_id = ? AND turn_id IS NOT NULL
|
|
2102
|
+
GROUP BY turn_id
|
|
2103
|
+
ORDER BY min_ts DESC
|
|
2104
|
+
LIMIT ? OFFSET ?`,
|
|
2105
|
+
[sessionId, limit, offset]
|
|
2106
|
+
);
|
|
2107
|
+
const turns = [];
|
|
2108
|
+
for (const turnRow of turnRows) {
|
|
2109
|
+
const events = await this.getEventsByTurn(turnRow.turn_id);
|
|
2110
|
+
const promptEvent = events.find((e) => e.eventType === "user_prompt");
|
|
2111
|
+
const toolEvents = events.filter((e) => e.eventType === "tool_observation");
|
|
2112
|
+
const hasResponse = events.some((e) => e.eventType === "agent_response");
|
|
2113
|
+
turns.push({
|
|
2114
|
+
turnId: turnRow.turn_id,
|
|
2115
|
+
events,
|
|
2116
|
+
startedAt: toDateFromSQLite(turnRow.min_ts),
|
|
2117
|
+
promptPreview: promptEvent ? promptEvent.content.slice(0, 200) + (promptEvent.content.length > 200 ? "..." : "") : "(no prompt)",
|
|
2118
|
+
eventCount: events.length,
|
|
2119
|
+
toolCount: toolEvents.length,
|
|
2120
|
+
hasResponse
|
|
2121
|
+
});
|
|
2122
|
+
}
|
|
2123
|
+
return turns;
|
|
2124
|
+
}
|
|
2125
|
+
/**
|
|
2126
|
+
* Get all events for a specific turn_id
|
|
2127
|
+
*/
|
|
2128
|
+
async getEventsByTurn(turnId) {
|
|
2129
|
+
await this.initialize();
|
|
2130
|
+
const rows = sqliteAll(
|
|
2131
|
+
this.db,
|
|
2132
|
+
`SELECT * FROM events WHERE turn_id = ? ORDER BY timestamp ASC`,
|
|
2133
|
+
[turnId]
|
|
2134
|
+
);
|
|
2135
|
+
return rows.map(this.rowToEvent);
|
|
2136
|
+
}
|
|
2137
|
+
/**
|
|
2138
|
+
* Count total turns for a session
|
|
2139
|
+
*/
|
|
2140
|
+
async countSessionTurns(sessionId) {
|
|
2141
|
+
await this.initialize();
|
|
2142
|
+
const row = sqliteGet(
|
|
2143
|
+
this.db,
|
|
2144
|
+
`SELECT COUNT(DISTINCT turn_id) as count
|
|
2145
|
+
FROM events
|
|
2146
|
+
WHERE session_id = ? AND turn_id IS NOT NULL`,
|
|
2147
|
+
[sessionId]
|
|
2148
|
+
);
|
|
2149
|
+
return row?.count || 0;
|
|
2150
|
+
}
|
|
2151
|
+
/**
|
|
2152
|
+
* Migrate existing events: backfill turn_id for events that have turnId in metadata
|
|
2153
|
+
* but no turn_id column value (for events stored before this migration)
|
|
2154
|
+
*/
|
|
2155
|
+
async backfillTurnIds() {
|
|
2156
|
+
await this.initialize();
|
|
2157
|
+
const rows = sqliteAll(
|
|
2158
|
+
this.db,
|
|
2159
|
+
`SELECT id, metadata FROM events
|
|
2160
|
+
WHERE turn_id IS NULL AND metadata IS NOT NULL AND metadata LIKE '%turnId%'`
|
|
2161
|
+
);
|
|
2162
|
+
let updated = 0;
|
|
2163
|
+
for (const row of rows) {
|
|
2164
|
+
try {
|
|
2165
|
+
const metadata = JSON.parse(row.metadata);
|
|
2166
|
+
if (metadata.turnId) {
|
|
2167
|
+
sqliteRun(
|
|
2168
|
+
this.db,
|
|
2169
|
+
`UPDATE events SET turn_id = ? WHERE id = ?`,
|
|
2170
|
+
[metadata.turnId, row.id]
|
|
2171
|
+
);
|
|
2172
|
+
updated++;
|
|
2173
|
+
}
|
|
2174
|
+
} catch {
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
return updated;
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* Delete all events for a session (for force reimport)
|
|
2181
|
+
*/
|
|
2182
|
+
async deleteSessionEvents(sessionId) {
|
|
2183
|
+
await this.initialize();
|
|
2184
|
+
const events = sqliteAll(
|
|
2185
|
+
this.db,
|
|
2186
|
+
`SELECT id FROM events WHERE session_id = ?`,
|
|
2187
|
+
[sessionId]
|
|
2188
|
+
);
|
|
2189
|
+
if (events.length === 0)
|
|
2190
|
+
return 0;
|
|
2191
|
+
const eventIds = events.map((e) => e.id);
|
|
2192
|
+
const placeholders = eventIds.map(() => "?").join(",");
|
|
2193
|
+
const ftsTriggersDropped = [];
|
|
2194
|
+
for (const triggerName of ["events_fts_delete", "events_fts_update", "events_fts_insert"]) {
|
|
2195
|
+
try {
|
|
2196
|
+
sqliteRun(this.db, `DROP TRIGGER IF EXISTS ${triggerName}`);
|
|
2197
|
+
ftsTriggersDropped.push(triggerName);
|
|
2198
|
+
} catch {
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
for (const table of ["event_dedup", "memory_levels", "embedding_queue", "embedding_outbox", "vector_outbox"]) {
|
|
2202
|
+
try {
|
|
2203
|
+
sqliteRun(this.db, `DELETE FROM ${table} WHERE event_id IN (${placeholders})`, eventIds);
|
|
2204
|
+
} catch {
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
const result = sqliteRun(this.db, `DELETE FROM events WHERE session_id = ?`, [sessionId]);
|
|
2208
|
+
if (ftsTriggersDropped.length > 0) {
|
|
2209
|
+
try {
|
|
2210
|
+
sqliteRun(this.db, `INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
|
|
2211
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
|
|
2212
|
+
INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
|
|
2213
|
+
END`);
|
|
2214
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
|
|
2215
|
+
INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
|
|
2216
|
+
END`);
|
|
2217
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
|
|
2218
|
+
INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
|
|
2219
|
+
INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
|
|
2220
|
+
END`);
|
|
2221
|
+
} catch {
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
return result.changes || 0;
|
|
2225
|
+
}
|
|
1567
2226
|
/**
|
|
1568
2227
|
* Convert database row to MemoryEvent
|
|
1569
2228
|
*/
|
|
@@ -1584,6 +2243,9 @@ var SQLiteEventStore = class {
|
|
|
1584
2243
|
if (row.last_accessed_at !== void 0) {
|
|
1585
2244
|
event.last_accessed_at = row.last_accessed_at;
|
|
1586
2245
|
}
|
|
2246
|
+
if (row.turn_id !== void 0 && row.turn_id !== null) {
|
|
2247
|
+
event.turn_id = row.turn_id;
|
|
2248
|
+
}
|
|
1587
2249
|
return event;
|
|
1588
2250
|
}
|
|
1589
2251
|
};
|
|
@@ -1795,7 +2457,16 @@ var VectorStore = class {
|
|
|
1795
2457
|
metadata: JSON.stringify(record.metadata || {})
|
|
1796
2458
|
};
|
|
1797
2459
|
if (!this.table) {
|
|
1798
|
-
|
|
2460
|
+
try {
|
|
2461
|
+
this.table = await this.db.createTable(this.tableName, [data]);
|
|
2462
|
+
} catch (e) {
|
|
2463
|
+
if (e?.message?.includes("already exists")) {
|
|
2464
|
+
this.table = await this.db.openTable(this.tableName);
|
|
2465
|
+
await this.table.add([data]);
|
|
2466
|
+
} else {
|
|
2467
|
+
throw e;
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
1799
2470
|
} else {
|
|
1800
2471
|
await this.table.add([data]);
|
|
1801
2472
|
}
|
|
@@ -1821,8 +2492,17 @@ var VectorStore = class {
|
|
|
1821
2492
|
metadata: JSON.stringify(record.metadata || {})
|
|
1822
2493
|
}));
|
|
1823
2494
|
if (!this.table) {
|
|
1824
|
-
|
|
1825
|
-
|
|
2495
|
+
try {
|
|
2496
|
+
this.table = await this.db.createTable(this.tableName, data);
|
|
2497
|
+
} catch (e) {
|
|
2498
|
+
if (e?.message?.includes("already exists")) {
|
|
2499
|
+
this.table = await this.db.openTable(this.tableName);
|
|
2500
|
+
await this.table.add(data);
|
|
2501
|
+
} else {
|
|
2502
|
+
throw e;
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
} else {
|
|
1826
2506
|
await this.table.add(data);
|
|
1827
2507
|
}
|
|
1828
2508
|
}
|
|
@@ -2261,7 +2941,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2261
2941
|
topK: 5,
|
|
2262
2942
|
minScore: 0.7,
|
|
2263
2943
|
maxTokens: 2e3,
|
|
2264
|
-
includeSessionContext: true
|
|
2944
|
+
includeSessionContext: true,
|
|
2945
|
+
strategy: "auto",
|
|
2946
|
+
rerankWithKeyword: true,
|
|
2947
|
+
decayPolicy: {
|
|
2948
|
+
enabled: true,
|
|
2949
|
+
windowDays: 30,
|
|
2950
|
+
maxPenalty: 0.15
|
|
2951
|
+
},
|
|
2952
|
+
graphHop: {
|
|
2953
|
+
enabled: true,
|
|
2954
|
+
maxHops: 1,
|
|
2955
|
+
hopPenalty: 0.08
|
|
2956
|
+
},
|
|
2957
|
+
projectScopeMode: "global"
|
|
2265
2958
|
};
|
|
2266
2959
|
var Retriever = class {
|
|
2267
2960
|
eventStore;
|
|
@@ -2271,6 +2964,7 @@ var Retriever = class {
|
|
|
2271
2964
|
sharedStore;
|
|
2272
2965
|
sharedVectorStore;
|
|
2273
2966
|
graduation;
|
|
2967
|
+
queryRewriter;
|
|
2274
2968
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2275
2969
|
this.eventStore = eventStore;
|
|
2276
2970
|
this.vectorStore = vectorStore;
|
|
@@ -2279,47 +2973,105 @@ var Retriever = class {
|
|
|
2279
2973
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2280
2974
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2281
2975
|
}
|
|
2282
|
-
/**
|
|
2283
|
-
* Set graduation pipeline for access tracking
|
|
2284
|
-
*/
|
|
2285
2976
|
setGraduationPipeline(graduation) {
|
|
2286
2977
|
this.graduation = graduation;
|
|
2287
2978
|
}
|
|
2288
|
-
/**
|
|
2289
|
-
* Set shared stores after construction
|
|
2290
|
-
*/
|
|
2291
2979
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2292
2980
|
this.sharedStore = sharedStore;
|
|
2293
2981
|
this.sharedVectorStore = sharedVectorStore;
|
|
2294
2982
|
}
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2983
|
+
setQueryRewriter(rewriter) {
|
|
2984
|
+
this.queryRewriter = rewriter;
|
|
2985
|
+
}
|
|
2298
2986
|
async retrieve(query, options = {}) {
|
|
2299
2987
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2300
|
-
const
|
|
2301
|
-
const
|
|
2302
|
-
|
|
2303
|
-
|
|
2988
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
2989
|
+
const fallbackTrace = [];
|
|
2990
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
2991
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
2992
|
+
let current = await this.runStage(query, {
|
|
2993
|
+
strategy: primaryStrategy,
|
|
2994
|
+
topK: opts.topK,
|
|
2304
2995
|
minScore: opts.minScore,
|
|
2305
|
-
sessionId:
|
|
2996
|
+
sessionId: sessionFilter,
|
|
2997
|
+
scope: opts.scope,
|
|
2998
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
2999
|
+
rerankWeights: opts.rerankWeights,
|
|
3000
|
+
decayPolicy: opts.decayPolicy,
|
|
3001
|
+
intentRewrite: opts.intentRewrite === true,
|
|
3002
|
+
graphHop: opts.graphHop,
|
|
3003
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3004
|
+
projectHash: opts.projectHash,
|
|
3005
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2306
3006
|
});
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
3007
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3008
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3009
|
+
current = await this.runStage(query, {
|
|
3010
|
+
strategy: "deep",
|
|
3011
|
+
topK: opts.topK,
|
|
3012
|
+
minScore: opts.minScore,
|
|
3013
|
+
sessionId: sessionFilter,
|
|
3014
|
+
scope: opts.scope,
|
|
3015
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3016
|
+
rerankWeights: opts.rerankWeights,
|
|
3017
|
+
decayPolicy: opts.decayPolicy,
|
|
3018
|
+
graphHop: opts.graphHop,
|
|
3019
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3020
|
+
projectHash: opts.projectHash,
|
|
3021
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3022
|
+
});
|
|
3023
|
+
fallbackTrace.push("fallback:deep");
|
|
3024
|
+
}
|
|
3025
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3026
|
+
current = await this.runStage(query, {
|
|
3027
|
+
strategy: "deep",
|
|
3028
|
+
topK: opts.topK,
|
|
3029
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3030
|
+
sessionId: void 0,
|
|
3031
|
+
scope: void 0,
|
|
3032
|
+
rerankWithKeyword: true,
|
|
3033
|
+
rerankWeights: opts.rerankWeights,
|
|
3034
|
+
decayPolicy: opts.decayPolicy,
|
|
3035
|
+
graphHop: opts.graphHop,
|
|
3036
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3037
|
+
projectHash: opts.projectHash,
|
|
3038
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3039
|
+
});
|
|
3040
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3041
|
+
}
|
|
3042
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3043
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3044
|
+
current = {
|
|
3045
|
+
results: summary,
|
|
3046
|
+
candidateResults: summary,
|
|
3047
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3048
|
+
};
|
|
3049
|
+
fallbackTrace.push("fallback:summary");
|
|
3050
|
+
}
|
|
3051
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2312
3052
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2313
3053
|
return {
|
|
2314
3054
|
memories,
|
|
2315
|
-
matchResult,
|
|
3055
|
+
matchResult: current.matchResult,
|
|
2316
3056
|
totalTokens: this.estimateTokens(context),
|
|
2317
|
-
context
|
|
3057
|
+
context,
|
|
3058
|
+
fallbackTrace,
|
|
3059
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3060
|
+
eventId: r.eventId,
|
|
3061
|
+
score: r.score,
|
|
3062
|
+
semanticScore: r.semanticScore,
|
|
3063
|
+
lexicalScore: r.lexicalScore,
|
|
3064
|
+
recencyScore: r.recencyScore
|
|
3065
|
+
})),
|
|
3066
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3067
|
+
eventId: r.eventId,
|
|
3068
|
+
score: r.score,
|
|
3069
|
+
semanticScore: r.semanticScore,
|
|
3070
|
+
lexicalScore: r.lexicalScore,
|
|
3071
|
+
recencyScore: r.recencyScore
|
|
3072
|
+
}))
|
|
2318
3073
|
};
|
|
2319
3074
|
}
|
|
2320
|
-
/**
|
|
2321
|
-
* Retrieve with unified search (project + shared)
|
|
2322
|
-
*/
|
|
2323
3075
|
async retrieveUnified(query, options = {}) {
|
|
2324
3076
|
const projectResult = await this.retrieve(query, options);
|
|
2325
3077
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2327,22 +3079,19 @@ var Retriever = class {
|
|
|
2327
3079
|
}
|
|
2328
3080
|
try {
|
|
2329
3081
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2330
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
excludeProjectHash: options.projectHash
|
|
2336
|
-
}
|
|
2337
|
-
);
|
|
3082
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3083
|
+
limit: options.topK || 5,
|
|
3084
|
+
minScore: options.minScore || 0.7,
|
|
3085
|
+
excludeProjectHash: options.projectHash
|
|
3086
|
+
});
|
|
2338
3087
|
const sharedMemories = [];
|
|
2339
3088
|
for (const result of sharedVectorResults) {
|
|
2340
3089
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2341
|
-
if (entry)
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
3090
|
+
if (!entry)
|
|
3091
|
+
continue;
|
|
3092
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3093
|
+
sharedMemories.push(entry);
|
|
3094
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2346
3095
|
}
|
|
2347
3096
|
}
|
|
2348
3097
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2357,50 +3106,243 @@ var Retriever = class {
|
|
|
2357
3106
|
return projectResult;
|
|
2358
3107
|
}
|
|
2359
3108
|
}
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
3109
|
+
async runStage(query, input) {
|
|
3110
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3111
|
+
strategy: input.strategy,
|
|
3112
|
+
topK: input.topK,
|
|
3113
|
+
minScore: input.minScore,
|
|
3114
|
+
sessionId: input.sessionId
|
|
3115
|
+
});
|
|
3116
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3117
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3118
|
+
if (rewritten && rewritten !== query) {
|
|
3119
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3120
|
+
strategy: "deep",
|
|
3121
|
+
topK: input.topK,
|
|
3122
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3123
|
+
sessionId: input.sessionId
|
|
3124
|
+
});
|
|
3125
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3126
|
+
}
|
|
3127
|
+
}
|
|
3128
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3129
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3130
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3131
|
+
limit: input.topK * 4
|
|
3132
|
+
});
|
|
3133
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3134
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3135
|
+
scope: input.scope,
|
|
3136
|
+
projectScopeMode: input.projectScopeMode,
|
|
3137
|
+
projectHash: input.projectHash,
|
|
3138
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3139
|
+
});
|
|
3140
|
+
const top = filtered.slice(0, input.topK);
|
|
3141
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3142
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3143
|
+
}
|
|
3144
|
+
mergeResults(primary, secondary, limit) {
|
|
3145
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3146
|
+
for (const row of primary)
|
|
3147
|
+
byId.set(row.eventId, row);
|
|
3148
|
+
for (const row of secondary) {
|
|
3149
|
+
const prev = byId.get(row.eventId);
|
|
3150
|
+
if (!prev || row.score > prev.score) {
|
|
3151
|
+
byId.set(row.eventId, row);
|
|
3152
|
+
}
|
|
3153
|
+
}
|
|
3154
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3155
|
+
}
|
|
3156
|
+
async expandGraphHops(seeds, opts) {
|
|
3157
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3158
|
+
for (const s of seeds)
|
|
3159
|
+
byId.set(s.eventId, s);
|
|
3160
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3161
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3162
|
+
const next = [];
|
|
3163
|
+
for (const f of frontier) {
|
|
3164
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3165
|
+
if (!ev)
|
|
3166
|
+
continue;
|
|
3167
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3168
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3169
|
+
for (const rid of relatedIds) {
|
|
3170
|
+
if (byId.has(rid))
|
|
3171
|
+
continue;
|
|
3172
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3173
|
+
if (!target)
|
|
3174
|
+
continue;
|
|
3175
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3176
|
+
const row = {
|
|
3177
|
+
id: `hop-${hop}-${rid}`,
|
|
3178
|
+
eventId: target.id,
|
|
3179
|
+
content: target.content,
|
|
3180
|
+
score,
|
|
3181
|
+
sessionId: target.sessionId,
|
|
3182
|
+
eventType: target.eventType,
|
|
3183
|
+
timestamp: target.timestamp.toISOString()
|
|
3184
|
+
};
|
|
3185
|
+
byId.set(row.eventId, row);
|
|
3186
|
+
next.push({ row, hop });
|
|
3187
|
+
if (byId.size >= opts.limit)
|
|
3188
|
+
break;
|
|
2381
3189
|
}
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
`;
|
|
3190
|
+
if (byId.size >= opts.limit)
|
|
3191
|
+
break;
|
|
2385
3192
|
}
|
|
3193
|
+
frontier = next;
|
|
3194
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3195
|
+
break;
|
|
2386
3196
|
}
|
|
2387
|
-
return
|
|
3197
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3198
|
+
}
|
|
3199
|
+
shouldFallback(matchResult, results) {
|
|
3200
|
+
if (results.length === 0)
|
|
3201
|
+
return true;
|
|
3202
|
+
if (matchResult.confidence === "none")
|
|
3203
|
+
return true;
|
|
3204
|
+
return false;
|
|
3205
|
+
}
|
|
3206
|
+
async buildSummaryFallback(query, topK) {
|
|
3207
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3208
|
+
const q = this.tokenize(query);
|
|
3209
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3210
|
+
id: `summary-${row.e.id}`,
|
|
3211
|
+
eventId: row.e.id,
|
|
3212
|
+
content: row.e.content,
|
|
3213
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3214
|
+
sessionId: row.e.sessionId,
|
|
3215
|
+
eventType: row.e.eventType,
|
|
3216
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3217
|
+
}));
|
|
3218
|
+
return ranked;
|
|
3219
|
+
}
|
|
3220
|
+
async searchByStrategy(query, input) {
|
|
3221
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3222
|
+
if (strategy === "fast") {
|
|
3223
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3224
|
+
limit: Math.max(5, input.topK * 3),
|
|
3225
|
+
sessionId: input.sessionId
|
|
3226
|
+
});
|
|
3227
|
+
return keyword;
|
|
3228
|
+
}
|
|
3229
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3230
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3231
|
+
limit: Math.max(5, input.topK * 3),
|
|
3232
|
+
minScore: input.minScore,
|
|
3233
|
+
sessionId: input.sessionId
|
|
3234
|
+
});
|
|
3235
|
+
}
|
|
3236
|
+
async searchByKeyword(query, input) {
|
|
3237
|
+
if (this.eventStore.keywordSearch) {
|
|
3238
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3239
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3240
|
+
return filtered2.map((row, idx) => ({
|
|
3241
|
+
id: `kw-${row.event.id}`,
|
|
3242
|
+
eventId: row.event.id,
|
|
3243
|
+
content: row.event.content,
|
|
3244
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3245
|
+
sessionId: row.event.sessionId,
|
|
3246
|
+
eventType: row.event.eventType,
|
|
3247
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3248
|
+
}));
|
|
3249
|
+
}
|
|
3250
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3251
|
+
const tokens = this.tokenize(query);
|
|
3252
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3253
|
+
return filtered.map((row, idx) => ({
|
|
3254
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3255
|
+
eventId: row.e.id,
|
|
3256
|
+
content: row.e.content,
|
|
3257
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3258
|
+
sessionId: row.e.sessionId,
|
|
3259
|
+
eventType: row.e.eventType,
|
|
3260
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3261
|
+
}));
|
|
3262
|
+
}
|
|
3263
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3264
|
+
const q = this.tokenize(query);
|
|
3265
|
+
const now = Date.now();
|
|
3266
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3267
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3268
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3269
|
+
const total = sw + lw + rw || 1;
|
|
3270
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3271
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3272
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3273
|
+
return [...results].map((r) => {
|
|
3274
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3275
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3276
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3277
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3278
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3279
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3280
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3281
|
+
}
|
|
3282
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3283
|
+
}).sort((a, b) => b.score - a.score);
|
|
3284
|
+
}
|
|
3285
|
+
async applyScopeFilters(results, options) {
|
|
3286
|
+
const scope = options?.scope;
|
|
3287
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3288
|
+
const allowedProjectHashes = new Set(
|
|
3289
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3290
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3291
|
+
)
|
|
3292
|
+
);
|
|
3293
|
+
if (!scope && projectScopeMode === "global")
|
|
3294
|
+
return results;
|
|
3295
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3296
|
+
const filtered = [];
|
|
3297
|
+
for (const result of results) {
|
|
3298
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3299
|
+
continue;
|
|
3300
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3301
|
+
continue;
|
|
3302
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3303
|
+
continue;
|
|
3304
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3305
|
+
if (!event)
|
|
3306
|
+
continue;
|
|
3307
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3308
|
+
continue;
|
|
3309
|
+
if (normalizedIncludes.length > 0) {
|
|
3310
|
+
const lc = event.content.toLowerCase();
|
|
3311
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3312
|
+
continue;
|
|
3313
|
+
}
|
|
3314
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3315
|
+
continue;
|
|
3316
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3317
|
+
filtered.push({ result, projectHash });
|
|
3318
|
+
}
|
|
3319
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3320
|
+
return filtered.map((x) => x.result);
|
|
3321
|
+
}
|
|
3322
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3323
|
+
if (projectScopeMode === "strict") {
|
|
3324
|
+
return projectMatched.map((x) => x.result);
|
|
3325
|
+
}
|
|
3326
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3327
|
+
}
|
|
3328
|
+
extractProjectHash(metadata) {
|
|
3329
|
+
if (!metadata || typeof metadata !== "object")
|
|
3330
|
+
return void 0;
|
|
3331
|
+
const scope = metadata.scope;
|
|
3332
|
+
if (!scope || typeof scope !== "object")
|
|
3333
|
+
return void 0;
|
|
3334
|
+
const project = scope.project;
|
|
3335
|
+
if (!project || typeof project !== "object")
|
|
3336
|
+
return void 0;
|
|
3337
|
+
const hash = project.hash;
|
|
3338
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2388
3339
|
}
|
|
2389
|
-
/**
|
|
2390
|
-
* Retrieve memories from a specific session
|
|
2391
|
-
*/
|
|
2392
3340
|
async retrieveFromSession(sessionId) {
|
|
2393
3341
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2394
3342
|
}
|
|
2395
|
-
/**
|
|
2396
|
-
* Get recent memories across all sessions
|
|
2397
|
-
*/
|
|
2398
3343
|
async retrieveRecent(limit = 100) {
|
|
2399
3344
|
return this.eventStore.getRecentEvents(limit);
|
|
2400
3345
|
}
|
|
2401
|
-
/**
|
|
2402
|
-
* Enrich search results with full event data
|
|
2403
|
-
*/
|
|
2404
3346
|
async enrichResults(results, options) {
|
|
2405
3347
|
const memories = [];
|
|
2406
3348
|
for (const result of results) {
|
|
@@ -2408,27 +3350,16 @@ var Retriever = class {
|
|
|
2408
3350
|
if (!event)
|
|
2409
3351
|
continue;
|
|
2410
3352
|
if (this.graduation) {
|
|
2411
|
-
this.graduation.recordAccess(
|
|
2412
|
-
event.id,
|
|
2413
|
-
options.sessionId || "unknown",
|
|
2414
|
-
result.score
|
|
2415
|
-
);
|
|
3353
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2416
3354
|
}
|
|
2417
3355
|
let sessionContext;
|
|
2418
3356
|
if (options.includeSessionContext) {
|
|
2419
3357
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2420
3358
|
}
|
|
2421
|
-
memories.push({
|
|
2422
|
-
event,
|
|
2423
|
-
score: result.score,
|
|
2424
|
-
sessionContext
|
|
2425
|
-
});
|
|
3359
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2426
3360
|
}
|
|
2427
3361
|
return memories;
|
|
2428
3362
|
}
|
|
2429
|
-
/**
|
|
2430
|
-
* Get surrounding context from the same session
|
|
2431
|
-
*/
|
|
2432
3363
|
async getSessionContext(sessionId, eventId) {
|
|
2433
3364
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2434
3365
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2441,55 +3372,86 @@ var Retriever = class {
|
|
|
2441
3372
|
return void 0;
|
|
2442
3373
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2443
3374
|
}
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
3375
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3376
|
+
let context = projectResult.context;
|
|
3377
|
+
if (sharedMemories.length === 0)
|
|
3378
|
+
return context;
|
|
3379
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3380
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3381
|
+
context += `### ${memory.title}
|
|
3382
|
+
`;
|
|
3383
|
+
if (memory.symptoms.length > 0)
|
|
3384
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3385
|
+
`;
|
|
3386
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3387
|
+
`;
|
|
3388
|
+
context += `**Solution:** ${memory.solution}
|
|
3389
|
+
`;
|
|
3390
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3391
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3392
|
+
`;
|
|
3393
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3394
|
+
|
|
3395
|
+
`;
|
|
3396
|
+
}
|
|
3397
|
+
return context;
|
|
3398
|
+
}
|
|
2447
3399
|
buildContext(memories, maxTokens) {
|
|
2448
3400
|
const parts = [];
|
|
2449
3401
|
let currentTokens = 0;
|
|
2450
3402
|
for (const memory of memories) {
|
|
2451
3403
|
const memoryText = this.formatMemory(memory);
|
|
2452
3404
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2453
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3405
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2454
3406
|
break;
|
|
2455
|
-
}
|
|
2456
3407
|
parts.push(memoryText);
|
|
2457
3408
|
currentTokens += memoryTokens;
|
|
2458
3409
|
}
|
|
2459
|
-
if (parts.length === 0)
|
|
3410
|
+
if (parts.length === 0)
|
|
2460
3411
|
return "";
|
|
2461
|
-
}
|
|
2462
3412
|
return `## Relevant Memories
|
|
2463
3413
|
|
|
2464
3414
|
${parts.join("\n\n---\n\n")}`;
|
|
2465
3415
|
}
|
|
2466
|
-
/**
|
|
2467
|
-
* Format a single memory for context
|
|
2468
|
-
*/
|
|
2469
3416
|
formatMemory(memory) {
|
|
2470
3417
|
const { event, score, sessionContext } = memory;
|
|
2471
3418
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2472
3419
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2473
3420
|
${event.content}`;
|
|
2474
|
-
if (sessionContext)
|
|
3421
|
+
if (sessionContext)
|
|
2475
3422
|
text += `
|
|
2476
3423
|
|
|
2477
3424
|
_Context:_ ${sessionContext}`;
|
|
2478
|
-
}
|
|
2479
3425
|
return text;
|
|
2480
3426
|
}
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
3427
|
+
matchesMetadataScope(metadata, expected) {
|
|
3428
|
+
if (!metadata)
|
|
3429
|
+
return false;
|
|
3430
|
+
return Object.entries(expected).every(([path5, value]) => {
|
|
3431
|
+
const actual = path5.split(".").reduce((acc, key) => {
|
|
3432
|
+
if (typeof acc !== "object" || acc === null)
|
|
3433
|
+
return void 0;
|
|
3434
|
+
return acc[key];
|
|
3435
|
+
}, metadata);
|
|
3436
|
+
return actual === value;
|
|
3437
|
+
});
|
|
3438
|
+
}
|
|
3439
|
+
tokenize(text) {
|
|
3440
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3441
|
+
}
|
|
3442
|
+
keywordOverlap(a, b) {
|
|
3443
|
+
if (a.length === 0 || b.length === 0)
|
|
3444
|
+
return 0;
|
|
3445
|
+
const bs = new Set(b);
|
|
3446
|
+
let hit = 0;
|
|
3447
|
+
for (const t of a)
|
|
3448
|
+
if (bs.has(t))
|
|
3449
|
+
hit += 1;
|
|
3450
|
+
return hit / a.length;
|
|
3451
|
+
}
|
|
2484
3452
|
estimateTokens(text) {
|
|
2485
3453
|
return Math.ceil(text.length / 4);
|
|
2486
3454
|
}
|
|
2487
|
-
/**
|
|
2488
|
-
* Get event age in days (for recency scoring)
|
|
2489
|
-
*/
|
|
2490
|
-
getEventAgeDays(eventId) {
|
|
2491
|
-
return 0;
|
|
2492
|
-
}
|
|
2493
3455
|
};
|
|
2494
3456
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2495
3457
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -3779,6 +4741,59 @@ var ConsolidatedStore = class {
|
|
|
3779
4741
|
[memoryId]
|
|
3780
4742
|
);
|
|
3781
4743
|
}
|
|
4744
|
+
/**
|
|
4745
|
+
* Create a long-term rule promoted from stable summaries
|
|
4746
|
+
*/
|
|
4747
|
+
async createRule(input) {
|
|
4748
|
+
const ruleId = randomUUID6();
|
|
4749
|
+
await dbRun(
|
|
4750
|
+
this.db,
|
|
4751
|
+
`INSERT INTO consolidated_rules
|
|
4752
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4753
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4754
|
+
[
|
|
4755
|
+
ruleId,
|
|
4756
|
+
input.rule,
|
|
4757
|
+
JSON.stringify(input.topics),
|
|
4758
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4759
|
+
JSON.stringify(input.sourceEvents),
|
|
4760
|
+
input.confidence
|
|
4761
|
+
]
|
|
4762
|
+
);
|
|
4763
|
+
return ruleId;
|
|
4764
|
+
}
|
|
4765
|
+
async getRules(options) {
|
|
4766
|
+
const limit = options?.limit || 100;
|
|
4767
|
+
const rows = await dbAll(
|
|
4768
|
+
this.db,
|
|
4769
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4770
|
+
[limit]
|
|
4771
|
+
);
|
|
4772
|
+
return rows.map((row) => ({
|
|
4773
|
+
ruleId: row.rule_id,
|
|
4774
|
+
rule: row.rule,
|
|
4775
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4776
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4777
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4778
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4779
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4780
|
+
}));
|
|
4781
|
+
}
|
|
4782
|
+
async countRules() {
|
|
4783
|
+
const result = await dbAll(
|
|
4784
|
+
this.db,
|
|
4785
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4786
|
+
);
|
|
4787
|
+
return result[0]?.count || 0;
|
|
4788
|
+
}
|
|
4789
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4790
|
+
const rows = await dbAll(
|
|
4791
|
+
this.db,
|
|
4792
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4793
|
+
[`%"${memoryId}"%`]
|
|
4794
|
+
);
|
|
4795
|
+
return (rows[0]?.count || 0) > 0;
|
|
4796
|
+
}
|
|
3782
4797
|
/**
|
|
3783
4798
|
* Get count of consolidated memories
|
|
3784
4799
|
*/
|
|
@@ -3928,7 +4943,14 @@ var ConsolidationWorker = class {
|
|
|
3928
4943
|
* Force a consolidation run (manual trigger)
|
|
3929
4944
|
*/
|
|
3930
4945
|
async forceRun() {
|
|
3931
|
-
|
|
4946
|
+
const out = await this.consolidateWithReport();
|
|
4947
|
+
return out.consolidatedCount;
|
|
4948
|
+
}
|
|
4949
|
+
/**
|
|
4950
|
+
* Force a consolidation run and return metrics report
|
|
4951
|
+
*/
|
|
4952
|
+
async forceRunWithReport() {
|
|
4953
|
+
return this.consolidateWithReport();
|
|
3932
4954
|
}
|
|
3933
4955
|
/**
|
|
3934
4956
|
* Schedule the next consolidation check
|
|
@@ -3968,12 +4990,21 @@ var ConsolidationWorker = class {
|
|
|
3968
4990
|
* Perform consolidation
|
|
3969
4991
|
*/
|
|
3970
4992
|
async consolidate() {
|
|
4993
|
+
const out = await this.consolidateWithReport();
|
|
4994
|
+
return out.consolidatedCount;
|
|
4995
|
+
}
|
|
4996
|
+
async consolidateWithReport() {
|
|
3971
4997
|
const workingSet = await this.workingSetStore.get();
|
|
3972
4998
|
if (workingSet.recentEvents.length < 3) {
|
|
3973
|
-
return
|
|
4999
|
+
return {
|
|
5000
|
+
consolidatedCount: 0,
|
|
5001
|
+
promotedRuleCount: 0,
|
|
5002
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
5003
|
+
};
|
|
3974
5004
|
}
|
|
3975
5005
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
3976
5006
|
let consolidatedCount = 0;
|
|
5007
|
+
const createdMemoryIds = [];
|
|
3977
5008
|
for (const group of groups) {
|
|
3978
5009
|
if (group.events.length < 3)
|
|
3979
5010
|
continue;
|
|
@@ -3982,14 +5013,16 @@ var ConsolidationWorker = class {
|
|
|
3982
5013
|
if (alreadyConsolidated)
|
|
3983
5014
|
continue;
|
|
3984
5015
|
const summary = await this.summarize(group);
|
|
3985
|
-
await this.consolidatedStore.create({
|
|
5016
|
+
const memoryId = await this.consolidatedStore.create({
|
|
3986
5017
|
summary,
|
|
3987
5018
|
topics: group.topics,
|
|
3988
5019
|
sourceEvents: eventIds,
|
|
3989
5020
|
confidence: this.calculateConfidence(group)
|
|
3990
5021
|
});
|
|
5022
|
+
createdMemoryIds.push(memoryId);
|
|
3991
5023
|
consolidatedCount++;
|
|
3992
5024
|
}
|
|
5025
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
3993
5026
|
if (consolidatedCount > 0) {
|
|
3994
5027
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
3995
5028
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4003,7 +5036,61 @@ var ConsolidationWorker = class {
|
|
|
4003
5036
|
await this.workingSetStore.prune(oldEventIds);
|
|
4004
5037
|
}
|
|
4005
5038
|
}
|
|
4006
|
-
|
|
5039
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5040
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5041
|
+
}
|
|
5042
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5043
|
+
let promoted = 0;
|
|
5044
|
+
for (const memoryId of memoryIds) {
|
|
5045
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5046
|
+
if (!memory)
|
|
5047
|
+
continue;
|
|
5048
|
+
if (memory.confidence < 0.55)
|
|
5049
|
+
continue;
|
|
5050
|
+
if (memory.sourceEvents.length < 4)
|
|
5051
|
+
continue;
|
|
5052
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5053
|
+
if (exists)
|
|
5054
|
+
continue;
|
|
5055
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5056
|
+
if (!rule)
|
|
5057
|
+
continue;
|
|
5058
|
+
await this.consolidatedStore.createRule({
|
|
5059
|
+
rule,
|
|
5060
|
+
topics: memory.topics,
|
|
5061
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5062
|
+
sourceEvents: memory.sourceEvents,
|
|
5063
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5064
|
+
});
|
|
5065
|
+
promoted++;
|
|
5066
|
+
}
|
|
5067
|
+
return promoted;
|
|
5068
|
+
}
|
|
5069
|
+
buildRuleFromSummary(summary, topics) {
|
|
5070
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5071
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5072
|
+
const seed = bullet || lines[0];
|
|
5073
|
+
if (!seed || seed.length < 8)
|
|
5074
|
+
return null;
|
|
5075
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5076
|
+
return `${topicPrefix}${seed}`;
|
|
5077
|
+
}
|
|
5078
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5079
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5080
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5081
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5082
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5083
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5084
|
+
return {
|
|
5085
|
+
beforeTokenEstimate,
|
|
5086
|
+
afterTokenEstimate,
|
|
5087
|
+
reductionRatio,
|
|
5088
|
+
qualityGuardPassed,
|
|
5089
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5090
|
+
};
|
|
5091
|
+
}
|
|
5092
|
+
estimateTokens(text) {
|
|
5093
|
+
return Math.ceil((text || "").length / 4);
|
|
4007
5094
|
}
|
|
4008
5095
|
/**
|
|
4009
5096
|
* Check if consolidation should run
|
|
@@ -4563,13 +5650,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
4563
5650
|
);
|
|
4564
5651
|
}
|
|
4565
5652
|
|
|
5653
|
+
// src/core/md-mirror.ts
|
|
5654
|
+
import * as fs3 from "node:fs";
|
|
5655
|
+
import * as path2 from "node:path";
|
|
5656
|
+
function sanitizeSegment2(input, fallback) {
|
|
5657
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5658
|
+
return v || fallback;
|
|
5659
|
+
}
|
|
5660
|
+
function getAtPath(obj, dotted) {
|
|
5661
|
+
if (!obj)
|
|
5662
|
+
return void 0;
|
|
5663
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5664
|
+
if (!acc || typeof acc !== "object")
|
|
5665
|
+
return void 0;
|
|
5666
|
+
return acc[key];
|
|
5667
|
+
}, obj);
|
|
5668
|
+
}
|
|
5669
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5670
|
+
const meta = event.metadata;
|
|
5671
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5672
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5673
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5674
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5675
|
+
const d = event.timestamp;
|
|
5676
|
+
const yyyy = d.getFullYear();
|
|
5677
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5678
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5679
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5680
|
+
}
|
|
5681
|
+
var MarkdownMirror2 = class {
|
|
5682
|
+
constructor(rootDir) {
|
|
5683
|
+
this.rootDir = rootDir;
|
|
5684
|
+
}
|
|
5685
|
+
async append(event, eventId) {
|
|
5686
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5687
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5688
|
+
const lines = [
|
|
5689
|
+
"",
|
|
5690
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5691
|
+
`- type: ${event.eventType}`,
|
|
5692
|
+
`- session: ${event.sessionId}`,
|
|
5693
|
+
event.content
|
|
5694
|
+
];
|
|
5695
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5696
|
+
await this.refreshIndex();
|
|
5697
|
+
}
|
|
5698
|
+
async refreshIndex() {
|
|
5699
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5700
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5701
|
+
const files = [];
|
|
5702
|
+
await this.walk(memoryRoot, files);
|
|
5703
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5704
|
+
const index = [
|
|
5705
|
+
"# Memory Index",
|
|
5706
|
+
"",
|
|
5707
|
+
"Generated automatically by MarkdownMirror.",
|
|
5708
|
+
"",
|
|
5709
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5710
|
+
""
|
|
5711
|
+
].join("\n");
|
|
5712
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5713
|
+
}
|
|
5714
|
+
async walk(dir, out) {
|
|
5715
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5716
|
+
for (const e of entries) {
|
|
5717
|
+
const full = path2.join(dir, e.name);
|
|
5718
|
+
if (e.isDirectory()) {
|
|
5719
|
+
await this.walk(full, out);
|
|
5720
|
+
} else {
|
|
5721
|
+
out.push(full);
|
|
5722
|
+
}
|
|
5723
|
+
}
|
|
5724
|
+
}
|
|
5725
|
+
};
|
|
5726
|
+
|
|
5727
|
+
// src/core/ingest-interceptor.ts
|
|
5728
|
+
var IngestInterceptorRegistry = class {
|
|
5729
|
+
before = [];
|
|
5730
|
+
after = [];
|
|
5731
|
+
onError = [];
|
|
5732
|
+
registerBefore(interceptor) {
|
|
5733
|
+
this.before.push(interceptor);
|
|
5734
|
+
return () => {
|
|
5735
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5736
|
+
};
|
|
5737
|
+
}
|
|
5738
|
+
registerAfter(interceptor) {
|
|
5739
|
+
this.after.push(interceptor);
|
|
5740
|
+
return () => {
|
|
5741
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5742
|
+
};
|
|
5743
|
+
}
|
|
5744
|
+
registerOnError(interceptor) {
|
|
5745
|
+
this.onError.push(interceptor);
|
|
5746
|
+
return () => {
|
|
5747
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5748
|
+
};
|
|
5749
|
+
}
|
|
5750
|
+
async run(stage, context) {
|
|
5751
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5752
|
+
for (const interceptor of interceptors) {
|
|
5753
|
+
await interceptor({ ...context, stage });
|
|
5754
|
+
}
|
|
5755
|
+
}
|
|
5756
|
+
};
|
|
5757
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5758
|
+
if (!base && !patch)
|
|
5759
|
+
return void 0;
|
|
5760
|
+
if (!base)
|
|
5761
|
+
return patch;
|
|
5762
|
+
if (!patch)
|
|
5763
|
+
return base;
|
|
5764
|
+
const result = { ...base };
|
|
5765
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5766
|
+
const current = result[key];
|
|
5767
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5768
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5769
|
+
current,
|
|
5770
|
+
value
|
|
5771
|
+
);
|
|
5772
|
+
} else {
|
|
5773
|
+
result[key] = value;
|
|
5774
|
+
}
|
|
5775
|
+
}
|
|
5776
|
+
return result;
|
|
5777
|
+
}
|
|
5778
|
+
|
|
5779
|
+
// src/core/tag-taxonomy.ts
|
|
5780
|
+
var TAG_NAMESPACES = {
|
|
5781
|
+
SYSTEM: "sys:",
|
|
5782
|
+
QUALITY: "q:",
|
|
5783
|
+
PROJECT: "proj:",
|
|
5784
|
+
TOPIC: "topic:",
|
|
5785
|
+
TEMPORAL: "t:",
|
|
5786
|
+
USER: "user:",
|
|
5787
|
+
AGENT: "agent:"
|
|
5788
|
+
};
|
|
5789
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5790
|
+
function parseTag(tag) {
|
|
5791
|
+
const value = (tag || "").trim();
|
|
5792
|
+
const idx = value.indexOf(":");
|
|
5793
|
+
if (idx <= 0)
|
|
5794
|
+
return { value };
|
|
5795
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5796
|
+
const tagValue = value.slice(idx + 1);
|
|
5797
|
+
if (!tagValue)
|
|
5798
|
+
return { value };
|
|
5799
|
+
return { namespace, value: tagValue };
|
|
5800
|
+
}
|
|
5801
|
+
function validateTag(tag) {
|
|
5802
|
+
const normalized = (tag || "").trim();
|
|
5803
|
+
if (!normalized)
|
|
5804
|
+
return false;
|
|
5805
|
+
const { namespace } = parseTag(normalized);
|
|
5806
|
+
if (!namespace)
|
|
5807
|
+
return true;
|
|
5808
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5809
|
+
}
|
|
5810
|
+
function normalizeTags(tags) {
|
|
5811
|
+
if (!Array.isArray(tags))
|
|
5812
|
+
return [];
|
|
5813
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5814
|
+
for (const item of tags) {
|
|
5815
|
+
if (typeof item !== "string")
|
|
5816
|
+
continue;
|
|
5817
|
+
const normalized = item.trim();
|
|
5818
|
+
if (!validateTag(normalized))
|
|
5819
|
+
continue;
|
|
5820
|
+
dedup.add(normalized);
|
|
5821
|
+
}
|
|
5822
|
+
return [...dedup];
|
|
5823
|
+
}
|
|
5824
|
+
|
|
4566
5825
|
// src/services/memory-service.ts
|
|
4567
5826
|
function normalizePath(projectPath) {
|
|
4568
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5827
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
4569
5828
|
try {
|
|
4570
|
-
return
|
|
5829
|
+
return fs4.realpathSync(expanded);
|
|
4571
5830
|
} catch {
|
|
4572
|
-
return
|
|
5831
|
+
return path3.resolve(expanded);
|
|
4573
5832
|
}
|
|
4574
5833
|
}
|
|
4575
5834
|
function hashProjectPath(projectPath) {
|
|
@@ -4578,14 +5837,14 @@ function hashProjectPath(projectPath) {
|
|
|
4578
5837
|
}
|
|
4579
5838
|
function getProjectStoragePath(projectPath) {
|
|
4580
5839
|
const hash = hashProjectPath(projectPath);
|
|
4581
|
-
return
|
|
5840
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
4582
5841
|
}
|
|
4583
|
-
var REGISTRY_PATH =
|
|
4584
|
-
var SHARED_STORAGE_PATH =
|
|
5842
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5843
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
4585
5844
|
function loadSessionRegistry() {
|
|
4586
5845
|
try {
|
|
4587
|
-
if (
|
|
4588
|
-
const data =
|
|
5846
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5847
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
4589
5848
|
return JSON.parse(data);
|
|
4590
5849
|
}
|
|
4591
5850
|
} catch (error) {
|
|
@@ -4611,6 +5870,7 @@ var MemoryService = class {
|
|
|
4611
5870
|
vectorWorker = null;
|
|
4612
5871
|
graduationWorker = null;
|
|
4613
5872
|
initialized = false;
|
|
5873
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
4614
5874
|
// Endless Mode components
|
|
4615
5875
|
workingSetStore = null;
|
|
4616
5876
|
consolidatedStore = null;
|
|
@@ -4624,20 +5884,27 @@ var MemoryService = class {
|
|
|
4624
5884
|
sharedPromoter = null;
|
|
4625
5885
|
sharedStoreConfig = null;
|
|
4626
5886
|
projectHash = null;
|
|
5887
|
+
projectPath = null;
|
|
4627
5888
|
readOnly;
|
|
4628
5889
|
lightweightMode;
|
|
5890
|
+
mdMirror;
|
|
4629
5891
|
constructor(config) {
|
|
4630
5892
|
const storagePath = this.expandPath(config.storagePath);
|
|
4631
5893
|
this.readOnly = config.readOnly ?? false;
|
|
4632
5894
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
4633
|
-
|
|
4634
|
-
|
|
5895
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
5896
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
5897
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
4635
5898
|
}
|
|
4636
5899
|
this.projectHash = config.projectHash || null;
|
|
5900
|
+
this.projectPath = config.projectPath || null;
|
|
4637
5901
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
4638
5902
|
this.sqliteStore = new SQLiteEventStore(
|
|
4639
|
-
|
|
4640
|
-
{
|
|
5903
|
+
path3.join(storagePath, "events.sqlite"),
|
|
5904
|
+
{
|
|
5905
|
+
readonly: this.readOnly,
|
|
5906
|
+
markdownMirrorRoot: storagePath
|
|
5907
|
+
}
|
|
4641
5908
|
);
|
|
4642
5909
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
4643
5910
|
if (!analyticsEnabled) {
|
|
@@ -4645,7 +5912,7 @@ var MemoryService = class {
|
|
|
4645
5912
|
} else if (this.readOnly) {
|
|
4646
5913
|
try {
|
|
4647
5914
|
this.analyticsStore = new EventStore(
|
|
4648
|
-
|
|
5915
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
4649
5916
|
{ readOnly: true }
|
|
4650
5917
|
);
|
|
4651
5918
|
} catch {
|
|
@@ -4653,11 +5920,11 @@ var MemoryService = class {
|
|
|
4653
5920
|
}
|
|
4654
5921
|
} else {
|
|
4655
5922
|
this.analyticsStore = new EventStore(
|
|
4656
|
-
|
|
5923
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
4657
5924
|
{ readOnly: false }
|
|
4658
5925
|
);
|
|
4659
5926
|
}
|
|
4660
|
-
this.vectorStore = new VectorStore(
|
|
5927
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
4661
5928
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
4662
5929
|
this.matcher = getDefaultMatcher();
|
|
4663
5930
|
this.retriever = createRetriever(
|
|
@@ -4667,6 +5934,7 @@ var MemoryService = class {
|
|
|
4667
5934
|
this.embedder,
|
|
4668
5935
|
this.matcher
|
|
4669
5936
|
);
|
|
5937
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
4670
5938
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
4671
5939
|
}
|
|
4672
5940
|
/**
|
|
@@ -4726,16 +5994,16 @@ var MemoryService = class {
|
|
|
4726
5994
|
*/
|
|
4727
5995
|
async initializeSharedStore() {
|
|
4728
5996
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
4729
|
-
if (!
|
|
4730
|
-
|
|
5997
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
5998
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
4731
5999
|
}
|
|
4732
6000
|
this.sharedEventStore = createSharedEventStore(
|
|
4733
|
-
|
|
6001
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
4734
6002
|
);
|
|
4735
6003
|
await this.sharedEventStore.initialize();
|
|
4736
6004
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
4737
6005
|
this.sharedVectorStore = createSharedVectorStore(
|
|
4738
|
-
|
|
6006
|
+
path3.join(sharedPath, "vectors")
|
|
4739
6007
|
);
|
|
4740
6008
|
await this.sharedVectorStore.initialize();
|
|
4741
6009
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -4746,6 +6014,86 @@ var MemoryService = class {
|
|
|
4746
6014
|
);
|
|
4747
6015
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
4748
6016
|
}
|
|
6017
|
+
registerIngestBefore(interceptor) {
|
|
6018
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6019
|
+
}
|
|
6020
|
+
registerIngestAfter(interceptor) {
|
|
6021
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6022
|
+
}
|
|
6023
|
+
registerIngestOnError(interceptor) {
|
|
6024
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6025
|
+
}
|
|
6026
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6027
|
+
const normalizedInput = {
|
|
6028
|
+
...input,
|
|
6029
|
+
metadata: mergeHierarchicalMetadata(
|
|
6030
|
+
{
|
|
6031
|
+
ingest: {
|
|
6032
|
+
operation,
|
|
6033
|
+
pipeline: "default",
|
|
6034
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6035
|
+
},
|
|
6036
|
+
...this.projectHash ? {
|
|
6037
|
+
scope: {
|
|
6038
|
+
project: {
|
|
6039
|
+
hash: this.projectHash,
|
|
6040
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6041
|
+
}
|
|
6042
|
+
},
|
|
6043
|
+
tags: [`proj:${this.projectHash}`]
|
|
6044
|
+
} : {}
|
|
6045
|
+
},
|
|
6046
|
+
input.metadata
|
|
6047
|
+
)
|
|
6048
|
+
};
|
|
6049
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6050
|
+
const meta = normalizedInput.metadata;
|
|
6051
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6052
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6053
|
+
if (!currentTags.includes(projectTag)) {
|
|
6054
|
+
meta.tags = [...currentTags, projectTag];
|
|
6055
|
+
}
|
|
6056
|
+
}
|
|
6057
|
+
if (normalizedInput.metadata) {
|
|
6058
|
+
const meta = normalizedInput.metadata;
|
|
6059
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6060
|
+
if (normalizedTags.length > 0) {
|
|
6061
|
+
meta.tags = normalizedTags;
|
|
6062
|
+
}
|
|
6063
|
+
}
|
|
6064
|
+
await this.ingestInterceptors.run("before", {
|
|
6065
|
+
operation,
|
|
6066
|
+
sessionId: normalizedInput.sessionId,
|
|
6067
|
+
event: normalizedInput
|
|
6068
|
+
});
|
|
6069
|
+
try {
|
|
6070
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6071
|
+
if (result.success && !result.isDuplicate) {
|
|
6072
|
+
if (onSuccess) {
|
|
6073
|
+
await onSuccess(result.eventId);
|
|
6074
|
+
}
|
|
6075
|
+
try {
|
|
6076
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6077
|
+
} catch {
|
|
6078
|
+
}
|
|
6079
|
+
}
|
|
6080
|
+
await this.ingestInterceptors.run("after", {
|
|
6081
|
+
operation,
|
|
6082
|
+
sessionId: normalizedInput.sessionId,
|
|
6083
|
+
event: normalizedInput
|
|
6084
|
+
});
|
|
6085
|
+
return result;
|
|
6086
|
+
} catch (error) {
|
|
6087
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6088
|
+
await this.ingestInterceptors.run("error", {
|
|
6089
|
+
operation,
|
|
6090
|
+
sessionId: normalizedInput.sessionId,
|
|
6091
|
+
event: normalizedInput,
|
|
6092
|
+
error: normalizedError
|
|
6093
|
+
});
|
|
6094
|
+
throw error;
|
|
6095
|
+
}
|
|
6096
|
+
}
|
|
4749
6097
|
/**
|
|
4750
6098
|
* Start a new session
|
|
4751
6099
|
*/
|
|
@@ -4773,50 +6121,57 @@ var MemoryService = class {
|
|
|
4773
6121
|
*/
|
|
4774
6122
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
4775
6123
|
await this.initialize();
|
|
4776
|
-
|
|
4777
|
-
|
|
4778
|
-
|
|
4779
|
-
|
|
4780
|
-
|
|
4781
|
-
|
|
4782
|
-
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
|
|
6124
|
+
return this.ingestWithInterceptors(
|
|
6125
|
+
"user_prompt",
|
|
6126
|
+
{
|
|
6127
|
+
eventType: "user_prompt",
|
|
6128
|
+
sessionId,
|
|
6129
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6130
|
+
content,
|
|
6131
|
+
metadata
|
|
6132
|
+
},
|
|
6133
|
+
async (eventId) => {
|
|
6134
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6135
|
+
}
|
|
6136
|
+
);
|
|
4787
6137
|
}
|
|
4788
6138
|
/**
|
|
4789
6139
|
* Store an agent response
|
|
4790
6140
|
*/
|
|
4791
6141
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
4792
6142
|
await this.initialize();
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
4799
|
-
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
6143
|
+
return this.ingestWithInterceptors(
|
|
6144
|
+
"agent_response",
|
|
6145
|
+
{
|
|
6146
|
+
eventType: "agent_response",
|
|
6147
|
+
sessionId,
|
|
6148
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6149
|
+
content,
|
|
6150
|
+
metadata
|
|
6151
|
+
},
|
|
6152
|
+
async (eventId) => {
|
|
6153
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6154
|
+
}
|
|
6155
|
+
);
|
|
4804
6156
|
}
|
|
4805
6157
|
/**
|
|
4806
6158
|
* Store a session summary
|
|
4807
6159
|
*/
|
|
4808
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6160
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
4809
6161
|
await this.initialize();
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
|
|
4813
|
-
|
|
4814
|
-
|
|
4815
|
-
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
6162
|
+
return this.ingestWithInterceptors(
|
|
6163
|
+
"session_summary",
|
|
6164
|
+
{
|
|
6165
|
+
eventType: "session_summary",
|
|
6166
|
+
sessionId,
|
|
6167
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6168
|
+
content: summary,
|
|
6169
|
+
metadata
|
|
6170
|
+
},
|
|
6171
|
+
async (eventId) => {
|
|
6172
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6173
|
+
}
|
|
6174
|
+
);
|
|
4820
6175
|
}
|
|
4821
6176
|
/**
|
|
4822
6177
|
* Store a tool observation
|
|
@@ -4824,39 +6179,182 @@ var MemoryService = class {
|
|
|
4824
6179
|
async storeToolObservation(sessionId, payload) {
|
|
4825
6180
|
await this.initialize();
|
|
4826
6181
|
const content = JSON.stringify(payload);
|
|
4827
|
-
const
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
4834
|
-
|
|
6182
|
+
const turnId = payload.metadata?.turnId;
|
|
6183
|
+
return this.ingestWithInterceptors(
|
|
6184
|
+
"tool_observation",
|
|
6185
|
+
{
|
|
6186
|
+
eventType: "tool_observation",
|
|
6187
|
+
sessionId,
|
|
6188
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6189
|
+
content,
|
|
6190
|
+
metadata: {
|
|
6191
|
+
toolName: payload.toolName,
|
|
6192
|
+
success: payload.success,
|
|
6193
|
+
...turnId ? { turnId } : {}
|
|
6194
|
+
}
|
|
6195
|
+
},
|
|
6196
|
+
async (eventId) => {
|
|
6197
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6198
|
+
payload.toolName,
|
|
6199
|
+
payload.metadata || {},
|
|
6200
|
+
payload.success
|
|
6201
|
+
);
|
|
6202
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
4835
6203
|
}
|
|
4836
|
-
|
|
4837
|
-
if (result.success && !result.isDuplicate) {
|
|
4838
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
4839
|
-
payload.toolName,
|
|
4840
|
-
payload.metadata || {},
|
|
4841
|
-
payload.success
|
|
4842
|
-
);
|
|
4843
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
4844
|
-
}
|
|
4845
|
-
return result;
|
|
6204
|
+
);
|
|
4846
6205
|
}
|
|
4847
6206
|
/**
|
|
4848
6207
|
* Retrieve relevant memories for a query
|
|
4849
6208
|
*/
|
|
4850
6209
|
async retrieveMemories(query, options) {
|
|
4851
6210
|
await this.initialize();
|
|
6211
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6212
|
+
let result;
|
|
4852
6213
|
if (options?.includeShared && this.sharedStore) {
|
|
4853
|
-
|
|
6214
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
4854
6215
|
...options,
|
|
6216
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6217
|
+
rerankWeights,
|
|
4855
6218
|
includeShared: true,
|
|
4856
|
-
projectHash: this.projectHash || void 0
|
|
6219
|
+
projectHash: this.projectHash || void 0,
|
|
6220
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6221
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6222
|
+
});
|
|
6223
|
+
} else {
|
|
6224
|
+
result = await this.retriever.retrieve(query, {
|
|
6225
|
+
...options,
|
|
6226
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6227
|
+
rerankWeights,
|
|
6228
|
+
projectHash: this.projectHash || void 0,
|
|
6229
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6230
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
4857
6231
|
});
|
|
4858
6232
|
}
|
|
4859
|
-
|
|
6233
|
+
try {
|
|
6234
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6235
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6236
|
+
eventId: d.eventId,
|
|
6237
|
+
score: d.score,
|
|
6238
|
+
semanticScore: d.semanticScore,
|
|
6239
|
+
lexicalScore: d.lexicalScore,
|
|
6240
|
+
recencyScore: d.recencyScore
|
|
6241
|
+
}));
|
|
6242
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6243
|
+
eventId: d.eventId,
|
|
6244
|
+
score: d.score,
|
|
6245
|
+
semanticScore: d.semanticScore,
|
|
6246
|
+
lexicalScore: d.lexicalScore,
|
|
6247
|
+
recencyScore: d.recencyScore
|
|
6248
|
+
}));
|
|
6249
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6250
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6251
|
+
sessionId: options?.sessionId,
|
|
6252
|
+
projectHash: this.projectHash || void 0,
|
|
6253
|
+
queryText: query,
|
|
6254
|
+
strategy: options?.strategy || "auto",
|
|
6255
|
+
candidateEventIds,
|
|
6256
|
+
selectedEventIds,
|
|
6257
|
+
candidateDetails,
|
|
6258
|
+
selectedDetails,
|
|
6259
|
+
confidence: result.matchResult.confidence,
|
|
6260
|
+
fallbackTrace: result.fallbackTrace || []
|
|
6261
|
+
});
|
|
6262
|
+
} catch {
|
|
6263
|
+
}
|
|
6264
|
+
return result;
|
|
6265
|
+
}
|
|
6266
|
+
getConfiguredRerankWeights() {
|
|
6267
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6268
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6269
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6270
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6271
|
+
if (!allFinite)
|
|
6272
|
+
return void 0;
|
|
6273
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6274
|
+
const total = semantic + lexical + recency;
|
|
6275
|
+
if (!nonNegative || total <= 0)
|
|
6276
|
+
return void 0;
|
|
6277
|
+
return {
|
|
6278
|
+
semantic: semantic / total,
|
|
6279
|
+
lexical: lexical / total,
|
|
6280
|
+
recency: recency / total
|
|
6281
|
+
};
|
|
6282
|
+
}
|
|
6283
|
+
async getRerankWeights(adaptive) {
|
|
6284
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6285
|
+
if (configured)
|
|
6286
|
+
return configured;
|
|
6287
|
+
if (adaptive)
|
|
6288
|
+
return this.getAdaptiveRerankWeights();
|
|
6289
|
+
return void 0;
|
|
6290
|
+
}
|
|
6291
|
+
async rewriteQueryIntent(query) {
|
|
6292
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6293
|
+
return null;
|
|
6294
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6295
|
+
if (!apiUrl)
|
|
6296
|
+
return null;
|
|
6297
|
+
const controller = new AbortController();
|
|
6298
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6299
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6300
|
+
try {
|
|
6301
|
+
const prompt = [
|
|
6302
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6303
|
+
"Return plain text only, one line, no markdown.",
|
|
6304
|
+
`Query: ${query}`
|
|
6305
|
+
].join("\n");
|
|
6306
|
+
const res = await fetch(apiUrl, {
|
|
6307
|
+
method: "POST",
|
|
6308
|
+
headers: {
|
|
6309
|
+
"Content-Type": "application/json",
|
|
6310
|
+
Accept: "*/*",
|
|
6311
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6312
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6313
|
+
},
|
|
6314
|
+
body: JSON.stringify({
|
|
6315
|
+
question: prompt,
|
|
6316
|
+
company_name: null,
|
|
6317
|
+
conversation_id: null
|
|
6318
|
+
}),
|
|
6319
|
+
signal: controller.signal
|
|
6320
|
+
});
|
|
6321
|
+
const text = (await res.text()).trim();
|
|
6322
|
+
if (!text)
|
|
6323
|
+
return null;
|
|
6324
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6325
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6326
|
+
return null;
|
|
6327
|
+
return oneLine;
|
|
6328
|
+
} catch {
|
|
6329
|
+
return null;
|
|
6330
|
+
} finally {
|
|
6331
|
+
clearTimeout(timeout);
|
|
6332
|
+
}
|
|
6333
|
+
}
|
|
6334
|
+
async getAdaptiveRerankWeights() {
|
|
6335
|
+
try {
|
|
6336
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6337
|
+
if (s.totalEvaluated < 20)
|
|
6338
|
+
return void 0;
|
|
6339
|
+
let semantic = 0.7;
|
|
6340
|
+
let lexical = 0.2;
|
|
6341
|
+
let recency = 0.1;
|
|
6342
|
+
if (s.avgScore < 0.45) {
|
|
6343
|
+
semantic -= 0.1;
|
|
6344
|
+
lexical += 0.1;
|
|
6345
|
+
} else if (s.avgScore > 0.75) {
|
|
6346
|
+
semantic += 0.05;
|
|
6347
|
+
lexical -= 0.05;
|
|
6348
|
+
}
|
|
6349
|
+
if (s.unhelpful > s.helpful) {
|
|
6350
|
+
recency += 0.05;
|
|
6351
|
+
semantic -= 0.03;
|
|
6352
|
+
lexical -= 0.02;
|
|
6353
|
+
}
|
|
6354
|
+
return { semantic, lexical, recency };
|
|
6355
|
+
} catch {
|
|
6356
|
+
return void 0;
|
|
6357
|
+
}
|
|
4860
6358
|
}
|
|
4861
6359
|
/**
|
|
4862
6360
|
* Fast keyword search using SQLite FTS5
|
|
@@ -4898,6 +6396,18 @@ var MemoryService = class {
|
|
|
4898
6396
|
/**
|
|
4899
6397
|
* Get memory statistics
|
|
4900
6398
|
*/
|
|
6399
|
+
async getOutboxStats() {
|
|
6400
|
+
await this.initialize();
|
|
6401
|
+
return this.sqliteStore.getOutboxStats();
|
|
6402
|
+
}
|
|
6403
|
+
async getRetrievalTraceStats() {
|
|
6404
|
+
await this.initialize();
|
|
6405
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6406
|
+
}
|
|
6407
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6408
|
+
await this.initialize();
|
|
6409
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6410
|
+
}
|
|
4901
6411
|
async getStats() {
|
|
4902
6412
|
await this.initialize();
|
|
4903
6413
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5114,6 +6624,31 @@ var MemoryService = class {
|
|
|
5114
6624
|
return [];
|
|
5115
6625
|
return this.consolidatedStore.getAll({ limit });
|
|
5116
6626
|
}
|
|
6627
|
+
/**
|
|
6628
|
+
* Extract topic keywords from event content (markdown headings and key terms)
|
|
6629
|
+
*/
|
|
6630
|
+
extractTopicsFromContent(content) {
|
|
6631
|
+
const topics = /* @__PURE__ */ new Set();
|
|
6632
|
+
const headings = content.match(/^#{1,3}\s+(.+)$/gm);
|
|
6633
|
+
if (headings) {
|
|
6634
|
+
for (const h of headings.slice(0, 5)) {
|
|
6635
|
+
const text = h.replace(/^#+\s+/, "").replace(/[*_`#]/g, "").trim();
|
|
6636
|
+
if (text.length > 2 && text.length < 50) {
|
|
6637
|
+
topics.add(text);
|
|
6638
|
+
}
|
|
6639
|
+
}
|
|
6640
|
+
}
|
|
6641
|
+
const boldTerms = content.match(/\*\*([^*]+)\*\*/g);
|
|
6642
|
+
if (boldTerms) {
|
|
6643
|
+
for (const b of boldTerms.slice(0, 5)) {
|
|
6644
|
+
const text = b.replace(/\*\*/g, "").trim();
|
|
6645
|
+
if (text.length > 2 && text.length < 30) {
|
|
6646
|
+
topics.add(text);
|
|
6647
|
+
}
|
|
6648
|
+
}
|
|
6649
|
+
}
|
|
6650
|
+
return Array.from(topics).slice(0, 5);
|
|
6651
|
+
}
|
|
5117
6652
|
/**
|
|
5118
6653
|
* Increment access count for memories that were used in prompts
|
|
5119
6654
|
*/
|
|
@@ -5137,8 +6672,7 @@ var MemoryService = class {
|
|
|
5137
6672
|
return events.map((event) => ({
|
|
5138
6673
|
memoryId: event.id,
|
|
5139
6674
|
summary: event.content.substring(0, 200) + (event.content.length > 200 ? "..." : ""),
|
|
5140
|
-
topics:
|
|
5141
|
-
// Could extract topics from content if needed
|
|
6675
|
+
topics: this.extractTopicsFromContent(event.content),
|
|
5142
6676
|
accessCount: event.access_count || 0,
|
|
5143
6677
|
lastAccessed: event.last_accessed_at || null,
|
|
5144
6678
|
confidence: 1,
|
|
@@ -5159,6 +6693,34 @@ var MemoryService = class {
|
|
|
5159
6693
|
}
|
|
5160
6694
|
return [];
|
|
5161
6695
|
}
|
|
6696
|
+
/**
|
|
6697
|
+
* Record a memory retrieval for helpfulness tracking
|
|
6698
|
+
*/
|
|
6699
|
+
async recordRetrieval(eventId, sessionId, score, query) {
|
|
6700
|
+
await this.initialize();
|
|
6701
|
+
await this.sqliteStore.recordRetrieval(eventId, sessionId, score, query);
|
|
6702
|
+
}
|
|
6703
|
+
/**
|
|
6704
|
+
* Evaluate helpfulness of retrievals in a session (called at session end)
|
|
6705
|
+
*/
|
|
6706
|
+
async evaluateSessionHelpfulness(sessionId) {
|
|
6707
|
+
await this.initialize();
|
|
6708
|
+
await this.sqliteStore.evaluateSessionHelpfulness(sessionId);
|
|
6709
|
+
}
|
|
6710
|
+
/**
|
|
6711
|
+
* Get most helpful memories ranked by helpfulness score
|
|
6712
|
+
*/
|
|
6713
|
+
async getHelpfulMemories(limit = 10) {
|
|
6714
|
+
await this.initialize();
|
|
6715
|
+
return this.sqliteStore.getHelpfulMemories(limit);
|
|
6716
|
+
}
|
|
6717
|
+
/**
|
|
6718
|
+
* Get helpfulness statistics for dashboard
|
|
6719
|
+
*/
|
|
6720
|
+
async getHelpfulnessStats() {
|
|
6721
|
+
await this.initialize();
|
|
6722
|
+
return this.sqliteStore.getHelpfulnessStats();
|
|
6723
|
+
}
|
|
5162
6724
|
/**
|
|
5163
6725
|
* Mark a consolidated memory as accessed
|
|
5164
6726
|
*/
|
|
@@ -5222,6 +6784,44 @@ var MemoryService = class {
|
|
|
5222
6784
|
lastConsolidation
|
|
5223
6785
|
};
|
|
5224
6786
|
}
|
|
6787
|
+
// ============================================================
|
|
6788
|
+
// Turn Grouping Methods
|
|
6789
|
+
// ============================================================
|
|
6790
|
+
/**
|
|
6791
|
+
* Get events grouped by turn for a session
|
|
6792
|
+
*/
|
|
6793
|
+
async getSessionTurns(sessionId, options) {
|
|
6794
|
+
await this.initialize();
|
|
6795
|
+
return this.sqliteStore.getSessionTurns(sessionId, options);
|
|
6796
|
+
}
|
|
6797
|
+
/**
|
|
6798
|
+
* Get all events for a specific turn
|
|
6799
|
+
*/
|
|
6800
|
+
async getEventsByTurn(turnId) {
|
|
6801
|
+
await this.initialize();
|
|
6802
|
+
return this.sqliteStore.getEventsByTurn(turnId);
|
|
6803
|
+
}
|
|
6804
|
+
/**
|
|
6805
|
+
* Count total turns for a session
|
|
6806
|
+
*/
|
|
6807
|
+
async countSessionTurns(sessionId) {
|
|
6808
|
+
await this.initialize();
|
|
6809
|
+
return this.sqliteStore.countSessionTurns(sessionId);
|
|
6810
|
+
}
|
|
6811
|
+
/**
|
|
6812
|
+
* Backfill turn_ids from metadata for events stored before the migration
|
|
6813
|
+
*/
|
|
6814
|
+
async backfillTurnIds() {
|
|
6815
|
+
await this.initialize();
|
|
6816
|
+
return this.sqliteStore.backfillTurnIds();
|
|
6817
|
+
}
|
|
6818
|
+
/**
|
|
6819
|
+
* Delete all events for a session (for force reimport)
|
|
6820
|
+
*/
|
|
6821
|
+
async deleteSessionEvents(sessionId) {
|
|
6822
|
+
await this.initialize();
|
|
6823
|
+
return this.sqliteStore.deleteSessionEvents(sessionId);
|
|
6824
|
+
}
|
|
5225
6825
|
/**
|
|
5226
6826
|
* Format Endless Mode context for Claude
|
|
5227
6827
|
*/
|
|
@@ -5298,46 +6898,28 @@ var MemoryService = class {
|
|
|
5298
6898
|
*/
|
|
5299
6899
|
expandPath(p) {
|
|
5300
6900
|
if (p.startsWith("~")) {
|
|
5301
|
-
return
|
|
6901
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5302
6902
|
}
|
|
5303
6903
|
return p;
|
|
5304
6904
|
}
|
|
5305
6905
|
};
|
|
5306
6906
|
var serviceCache = /* @__PURE__ */ new Map();
|
|
5307
|
-
|
|
5308
|
-
|
|
5309
|
-
|
|
5310
|
-
|
|
5311
|
-
|
|
6907
|
+
function getLightweightMemoryService(sessionId) {
|
|
6908
|
+
const projectInfo = getSessionProject(sessionId);
|
|
6909
|
+
const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : "lightweight_global";
|
|
6910
|
+
if (!serviceCache.has(key)) {
|
|
6911
|
+
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) : path3.join(os.homedir(), ".claude-code", "memory");
|
|
6912
|
+
serviceCache.set(key, new MemoryService({
|
|
6913
|
+
storagePath,
|
|
6914
|
+
projectHash: projectInfo?.projectHash,
|
|
6915
|
+
projectPath: projectInfo?.projectPath,
|
|
6916
|
+
lightweightMode: true,
|
|
6917
|
+
// Skip embedder/vector/workers
|
|
5312
6918
|
analyticsEnabled: false,
|
|
5313
|
-
// Hooks don't need DuckDB
|
|
5314
6919
|
sharedStoreConfig: { enabled: false }
|
|
5315
|
-
// Shared store uses DuckDB too
|
|
5316
6920
|
}));
|
|
5317
6921
|
}
|
|
5318
|
-
return serviceCache.get(
|
|
5319
|
-
}
|
|
5320
|
-
function getMemoryServiceForProject(projectPath, sharedStoreConfig) {
|
|
5321
|
-
const hash = hashProjectPath(projectPath);
|
|
5322
|
-
if (!serviceCache.has(hash)) {
|
|
5323
|
-
const storagePath = getProjectStoragePath(projectPath);
|
|
5324
|
-
serviceCache.set(hash, new MemoryService({
|
|
5325
|
-
storagePath,
|
|
5326
|
-
projectHash: hash,
|
|
5327
|
-
// Override shared store config - hooks don't need DuckDB
|
|
5328
|
-
sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
|
|
5329
|
-
analyticsEnabled: false
|
|
5330
|
-
// Hooks don't need DuckDB
|
|
5331
|
-
}));
|
|
5332
|
-
}
|
|
5333
|
-
return serviceCache.get(hash);
|
|
5334
|
-
}
|
|
5335
|
-
function getMemoryServiceForSession(sessionId) {
|
|
5336
|
-
const projectInfo = getSessionProject(sessionId);
|
|
5337
|
-
if (projectInfo) {
|
|
5338
|
-
return getMemoryServiceForProject(projectInfo.projectPath);
|
|
5339
|
-
}
|
|
5340
|
-
return getDefaultMemoryService();
|
|
6922
|
+
return serviceCache.get(key);
|
|
5341
6923
|
}
|
|
5342
6924
|
|
|
5343
6925
|
// src/core/privacy/tag-parser.ts
|
|
@@ -5485,6 +7067,52 @@ function applyPrivacyFilter(content, config) {
|
|
|
5485
7067
|
};
|
|
5486
7068
|
}
|
|
5487
7069
|
|
|
7070
|
+
// src/core/turn-state.ts
|
|
7071
|
+
import * as fs5 from "fs";
|
|
7072
|
+
import * as path4 from "path";
|
|
7073
|
+
import * as os2 from "os";
|
|
7074
|
+
var TURN_STATE_DIR = path4.join(os2.homedir(), ".claude-code", "memory");
|
|
7075
|
+
function getStatePath(sessionId) {
|
|
7076
|
+
return path4.join(TURN_STATE_DIR, `.turn-state-${sessionId}.json`);
|
|
7077
|
+
}
|
|
7078
|
+
function readTurnState(sessionId) {
|
|
7079
|
+
try {
|
|
7080
|
+
const filePath = getStatePath(sessionId);
|
|
7081
|
+
if (!fs5.existsSync(filePath)) {
|
|
7082
|
+
return null;
|
|
7083
|
+
}
|
|
7084
|
+
const data = fs5.readFileSync(filePath, "utf-8");
|
|
7085
|
+
const state = JSON.parse(data);
|
|
7086
|
+
if (state.sessionId !== sessionId) {
|
|
7087
|
+
return null;
|
|
7088
|
+
}
|
|
7089
|
+
const createdAt = new Date(state.createdAt).getTime();
|
|
7090
|
+
const now = Date.now();
|
|
7091
|
+
if (now - createdAt > 30 * 60 * 1e3) {
|
|
7092
|
+
clearTurnState(sessionId);
|
|
7093
|
+
return null;
|
|
7094
|
+
}
|
|
7095
|
+
return state.turnId;
|
|
7096
|
+
} catch (error) {
|
|
7097
|
+
if (process.env.CLAUDE_MEMORY_DEBUG) {
|
|
7098
|
+
console.error("Failed to read turn state:", error);
|
|
7099
|
+
}
|
|
7100
|
+
return null;
|
|
7101
|
+
}
|
|
7102
|
+
}
|
|
7103
|
+
function clearTurnState(sessionId) {
|
|
7104
|
+
try {
|
|
7105
|
+
const filePath = getStatePath(sessionId);
|
|
7106
|
+
if (fs5.existsSync(filePath)) {
|
|
7107
|
+
fs5.unlinkSync(filePath);
|
|
7108
|
+
}
|
|
7109
|
+
} catch (error) {
|
|
7110
|
+
if (process.env.CLAUDE_MEMORY_DEBUG) {
|
|
7111
|
+
console.error("Failed to clear turn state:", error);
|
|
7112
|
+
}
|
|
7113
|
+
}
|
|
7114
|
+
}
|
|
7115
|
+
|
|
5488
7116
|
// src/hooks/stop.ts
|
|
5489
7117
|
var DEFAULT_PRIVACY_CONFIG = {
|
|
5490
7118
|
excludePatterns: ["password", "secret", "api_key", "token", "bearer"],
|
|
@@ -5496,32 +7124,65 @@ var DEFAULT_PRIVACY_CONFIG = {
|
|
|
5496
7124
|
supportedFormats: ["xml"]
|
|
5497
7125
|
}
|
|
5498
7126
|
};
|
|
7127
|
+
async function extractAssistantMessages(transcriptPath) {
|
|
7128
|
+
if (!fs6.existsSync(transcriptPath))
|
|
7129
|
+
return [];
|
|
7130
|
+
const messages = [];
|
|
7131
|
+
const stats = fs6.statSync(transcriptPath);
|
|
7132
|
+
const readStart = Math.max(0, stats.size - 200 * 1024);
|
|
7133
|
+
const stream = fs6.createReadStream(transcriptPath, {
|
|
7134
|
+
start: readStart,
|
|
7135
|
+
encoding: "utf8"
|
|
7136
|
+
});
|
|
7137
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
7138
|
+
for await (const line of rl) {
|
|
7139
|
+
try {
|
|
7140
|
+
const entry = JSON.parse(line);
|
|
7141
|
+
if (entry.type !== "assistant")
|
|
7142
|
+
continue;
|
|
7143
|
+
const content = entry.message?.content;
|
|
7144
|
+
if (!Array.isArray(content))
|
|
7145
|
+
continue;
|
|
7146
|
+
const textParts = content.filter((c) => c.type === "text").map((c) => c.text).filter(Boolean);
|
|
7147
|
+
if (textParts.length > 0) {
|
|
7148
|
+
messages.push(textParts.join("\n"));
|
|
7149
|
+
}
|
|
7150
|
+
} catch {
|
|
7151
|
+
}
|
|
7152
|
+
}
|
|
7153
|
+
return messages;
|
|
7154
|
+
}
|
|
5499
7155
|
async function main() {
|
|
5500
7156
|
const inputData = await readStdin();
|
|
5501
7157
|
const input = JSON.parse(inputData);
|
|
5502
|
-
const memoryService =
|
|
7158
|
+
const memoryService = getLightweightMemoryService(input.session_id);
|
|
5503
7159
|
try {
|
|
5504
|
-
|
|
5505
|
-
|
|
5506
|
-
|
|
5507
|
-
|
|
5508
|
-
|
|
5509
|
-
|
|
5510
|
-
|
|
5511
|
-
await memoryService.storeAgentResponse(
|
|
5512
|
-
input.session_id,
|
|
5513
|
-
content,
|
|
5514
|
-
{
|
|
5515
|
-
stopReason: input.stop_reason,
|
|
5516
|
-
privacy: filterResult.metadata
|
|
5517
|
-
}
|
|
5518
|
-
);
|
|
7160
|
+
const turnId = readTurnState(input.session_id);
|
|
7161
|
+
const assistantMessages = await extractAssistantMessages(input.transcript_path);
|
|
7162
|
+
for (const text of assistantMessages) {
|
|
7163
|
+
const filterResult = applyPrivacyFilter(text, DEFAULT_PRIVACY_CONFIG);
|
|
7164
|
+
let content = filterResult.content;
|
|
7165
|
+
if (content.length > 5e3) {
|
|
7166
|
+
content = content.slice(0, 5e3) + "...[truncated]";
|
|
5519
7167
|
}
|
|
7168
|
+
if (content.trim().length < 10)
|
|
7169
|
+
continue;
|
|
7170
|
+
await memoryService.storeAgentResponse(
|
|
7171
|
+
input.session_id,
|
|
7172
|
+
content,
|
|
7173
|
+
{
|
|
7174
|
+
privacy: filterResult.metadata,
|
|
7175
|
+
...turnId ? { turnId } : {}
|
|
7176
|
+
}
|
|
7177
|
+
);
|
|
5520
7178
|
}
|
|
7179
|
+
clearTurnState(input.session_id);
|
|
5521
7180
|
await memoryService.processPendingEmbeddings();
|
|
5522
7181
|
console.log(JSON.stringify({}));
|
|
5523
7182
|
} catch (error) {
|
|
5524
|
-
|
|
7183
|
+
if (process.env.CLAUDE_MEMORY_DEBUG) {
|
|
7184
|
+
console.error("Stop hook error:", error);
|
|
7185
|
+
}
|
|
5525
7186
|
console.log(JSON.stringify({}));
|
|
5526
7187
|
}
|
|
5527
7188
|
}
|