claude-memory-layer 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +3577 -389
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1383 -138
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1917 -214
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1813 -231
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1802 -205
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1909 -248
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1861 -206
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +2341 -217
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +2350 -226
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1805 -206
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +1447 -55
- package/dist/ui/index.html +318 -147
- package/dist/ui/style.css +892 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/docs/OPERATIONS.md +18 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +9 -2
- package/scripts/build.ts +6 -0
- package/scripts/fix-sync-gap.js +32 -0
- package/scripts/heartbeat-memory-orchestrator.sh +28 -0
- package/scripts/report-sync-gap.js +26 -0
- package/scripts/review-queue-auto-resolve.js +21 -0
- package/scripts/sync-gap-auto-heal.sh +17 -0
- package/specs/20260207-dashboard-upgrade/context.md +38 -0
- package/specs/20260207-dashboard-upgrade/spec.md +96 -0
- package/src/cli/index.ts +391 -60
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +794 -7
- package/src/core/sqlite-wrapper.ts +8 -0
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/turn-state.ts +159 -0
- package/src/core/types.ts +51 -8
- package/src/core/vector-store.ts +21 -3
- package/src/hooks/post-tool-use.ts +68 -23
- package/src/hooks/session-end.ts +8 -3
- package/src/hooks/stop.ts +96 -25
- package/src/hooks/user-prompt-submit.ts +44 -5
- package/src/server/api/chat.ts +244 -0
- package/src/server/api/citations.ts +3 -3
- package/src/server/api/events.ts +30 -5
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +9 -1
- package/src/server/api/projects.ts +74 -0
- package/src/server/api/search.ts +3 -3
- package/src/server/api/sessions.ts +3 -3
- package/src/server/api/stats.ts +89 -8
- package/src/server/api/turns.ts +143 -0
- package/src/server/api/utils.ts +46 -0
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +508 -71
- package/src/services/session-history-importer.ts +215 -51
- package/src/ui/app.js +1447 -55
- package/src/ui/index.html +318 -147
- package/src/ui/style.css +892 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
- package/.claude/settings.local.json +0 -27
- package/.claude-memory/test.sqlite +0 -0
- package/.history/package_20260201112328.json +0 -45
- package/.history/package_20260201113602.json +0 -45
- package/.history/package_20260201113713.json +0 -45
- package/.history/package_20260201114110.json +0 -45
- package/.history/package_20260201114632.json +0 -46
- package/.history/package_20260201133143.json +0 -45
- package/.history/package_20260201134319.json +0 -45
- package/.history/package_20260201134326.json +0 -45
- package/.history/package_20260201134334.json +0 -45
- package/.history/package_20260201134912.json +0 -45
- package/.history/package_20260201142928.json +0 -46
- package/.history/package_20260201192048.json +0 -47
- package/.history/package_20260202114053.json +0 -49
- package/.history/package_20260202121115.json +0 -49
- package/test_access.js +0 -49
|
@@ -6,9 +6,9 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
6
6
|
const __dirname = dirname(__filename);
|
|
7
7
|
|
|
8
8
|
// src/services/memory-service.ts
|
|
9
|
-
import * as
|
|
9
|
+
import * as path3 from "path";
|
|
10
10
|
import * as os from "os";
|
|
11
|
-
import * as
|
|
11
|
+
import * as fs4 from "fs";
|
|
12
12
|
import * as crypto2 from "crypto";
|
|
13
13
|
|
|
14
14
|
// src/core/event-store.ts
|
|
@@ -66,11 +66,11 @@ function toDate(value) {
|
|
|
66
66
|
return new Date(value);
|
|
67
67
|
return new Date(String(value));
|
|
68
68
|
}
|
|
69
|
-
function createDatabase(
|
|
69
|
+
function createDatabase(path4, options) {
|
|
70
70
|
if (options?.readOnly) {
|
|
71
|
-
return new duckdb.Database(
|
|
71
|
+
return new duckdb.Database(path4, { access_mode: "READ_ONLY" });
|
|
72
72
|
}
|
|
73
|
-
return new duckdb.Database(
|
|
73
|
+
return new duckdb.Database(path4);
|
|
74
74
|
}
|
|
75
75
|
function dbRun(db, sql, params = []) {
|
|
76
76
|
return new Promise((resolve2, reject) => {
|
|
@@ -334,6 +334,17 @@ var EventStore = class {
|
|
|
334
334
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
335
335
|
)
|
|
336
336
|
`);
|
|
337
|
+
await dbRun(this.db, `
|
|
338
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
339
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
340
|
+
rule TEXT NOT NULL,
|
|
341
|
+
topics JSON,
|
|
342
|
+
source_memory_ids JSON,
|
|
343
|
+
source_events JSON,
|
|
344
|
+
confidence FLOAT DEFAULT 0.5,
|
|
345
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
346
|
+
)
|
|
347
|
+
`);
|
|
337
348
|
await dbRun(this.db, `
|
|
338
349
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
339
350
|
key VARCHAR PRIMARY KEY,
|
|
@@ -353,6 +364,7 @@ var EventStore = class {
|
|
|
353
364
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
354
365
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
355
366
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
367
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
356
368
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
357
369
|
this.initialized = true;
|
|
358
370
|
}
|
|
@@ -740,8 +752,14 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
740
752
|
|
|
741
753
|
// src/core/sqlite-wrapper.ts
|
|
742
754
|
import Database from "better-sqlite3";
|
|
743
|
-
|
|
744
|
-
|
|
755
|
+
import * as fs from "fs";
|
|
756
|
+
import * as nodePath from "path";
|
|
757
|
+
function createSQLiteDatabase(path4, options) {
|
|
758
|
+
const dir = nodePath.dirname(path4);
|
|
759
|
+
if (!fs.existsSync(dir)) {
|
|
760
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
761
|
+
}
|
|
762
|
+
const db = new Database(path4, {
|
|
745
763
|
readonly: options?.readonly ?? false
|
|
746
764
|
});
|
|
747
765
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -782,6 +800,64 @@ function toSQLiteTimestamp(date) {
|
|
|
782
800
|
return date.toISOString();
|
|
783
801
|
}
|
|
784
802
|
|
|
803
|
+
// src/core/markdown-mirror.ts
|
|
804
|
+
import * as fs2 from "fs/promises";
|
|
805
|
+
import * as path from "path";
|
|
806
|
+
var DEFAULT_NAMESPACE = "default";
|
|
807
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
808
|
+
function sanitizeSegment(input, fallback) {
|
|
809
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
810
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
811
|
+
if (!safe || safe === "." || safe === "..")
|
|
812
|
+
return fallback;
|
|
813
|
+
return safe;
|
|
814
|
+
}
|
|
815
|
+
function getCategorySegments(metadata, eventType) {
|
|
816
|
+
const raw = metadata?.categoryPath;
|
|
817
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
818
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
819
|
+
}
|
|
820
|
+
const single = metadata?.category;
|
|
821
|
+
if (typeof single === "string" && single.trim()) {
|
|
822
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
823
|
+
}
|
|
824
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
825
|
+
}
|
|
826
|
+
function buildMirrorPath(rootDir, event) {
|
|
827
|
+
const metadata = event.metadata;
|
|
828
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
829
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
830
|
+
const d = event.timestamp;
|
|
831
|
+
const yyyy = d.getFullYear();
|
|
832
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
833
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
834
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
835
|
+
}
|
|
836
|
+
function formatMirrorEntry(event) {
|
|
837
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
838
|
+
return [
|
|
839
|
+
"",
|
|
840
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
841
|
+
` id: ${event.id}`,
|
|
842
|
+
` type: ${event.eventType}`,
|
|
843
|
+
` session: ${event.sessionId}`,
|
|
844
|
+
` category: ${category}`,
|
|
845
|
+
" content: |",
|
|
846
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
847
|
+
].join("\n") + "\n";
|
|
848
|
+
}
|
|
849
|
+
var MarkdownMirror = class {
|
|
850
|
+
constructor(rootDir) {
|
|
851
|
+
this.rootDir = rootDir;
|
|
852
|
+
}
|
|
853
|
+
async append(event) {
|
|
854
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
855
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
856
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
857
|
+
return outPath;
|
|
858
|
+
}
|
|
859
|
+
};
|
|
860
|
+
|
|
785
861
|
// src/core/sqlite-event-store.ts
|
|
786
862
|
var SQLiteEventStore = class {
|
|
787
863
|
constructor(dbPath, options) {
|
|
@@ -791,10 +867,12 @@ var SQLiteEventStore = class {
|
|
|
791
867
|
readonly: this.readOnly,
|
|
792
868
|
walMode: !this.readOnly
|
|
793
869
|
});
|
|
870
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
794
871
|
}
|
|
795
872
|
db;
|
|
796
873
|
initialized = false;
|
|
797
874
|
readOnly;
|
|
875
|
+
markdownMirror;
|
|
798
876
|
/**
|
|
799
877
|
* Initialize database schema
|
|
800
878
|
*/
|
|
@@ -1001,6 +1079,17 @@ var SQLiteEventStore = class {
|
|
|
1001
1079
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1002
1080
|
);
|
|
1003
1081
|
|
|
1082
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1083
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1084
|
+
rule_id TEXT PRIMARY KEY,
|
|
1085
|
+
rule TEXT NOT NULL,
|
|
1086
|
+
topics TEXT,
|
|
1087
|
+
source_memory_ids TEXT,
|
|
1088
|
+
source_events TEXT,
|
|
1089
|
+
confidence REAL DEFAULT 0.5,
|
|
1090
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1091
|
+
);
|
|
1092
|
+
|
|
1004
1093
|
-- Endless Mode Config table
|
|
1005
1094
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1006
1095
|
key TEXT PRIMARY KEY,
|
|
@@ -1008,6 +1097,41 @@ var SQLiteEventStore = class {
|
|
|
1008
1097
|
updated_at TEXT DEFAULT (datetime('now'))
|
|
1009
1098
|
);
|
|
1010
1099
|
|
|
1100
|
+
-- Memory Helpfulness tracking
|
|
1101
|
+
CREATE TABLE IF NOT EXISTS memory_helpfulness (
|
|
1102
|
+
id TEXT PRIMARY KEY,
|
|
1103
|
+
event_id TEXT NOT NULL,
|
|
1104
|
+
session_id TEXT NOT NULL,
|
|
1105
|
+
retrieval_score REAL DEFAULT 0,
|
|
1106
|
+
query_preview TEXT,
|
|
1107
|
+
session_continued INTEGER DEFAULT 0,
|
|
1108
|
+
prompt_count_after INTEGER DEFAULT 0,
|
|
1109
|
+
tool_success_count INTEGER DEFAULT 0,
|
|
1110
|
+
tool_total_count INTEGER DEFAULT 0,
|
|
1111
|
+
was_reasked INTEGER DEFAULT 0,
|
|
1112
|
+
helpfulness_score REAL DEFAULT 0.5,
|
|
1113
|
+
created_at TEXT DEFAULT (datetime('now')),
|
|
1114
|
+
measured_at TEXT
|
|
1115
|
+
);
|
|
1116
|
+
|
|
1117
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1118
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1119
|
+
trace_id TEXT PRIMARY KEY,
|
|
1120
|
+
session_id TEXT,
|
|
1121
|
+
project_hash TEXT,
|
|
1122
|
+
query_text TEXT NOT NULL,
|
|
1123
|
+
strategy TEXT,
|
|
1124
|
+
candidate_event_ids TEXT,
|
|
1125
|
+
selected_event_ids TEXT,
|
|
1126
|
+
candidate_details_json TEXT,
|
|
1127
|
+
selected_details_json TEXT,
|
|
1128
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1129
|
+
selected_count INTEGER DEFAULT 0,
|
|
1130
|
+
confidence TEXT,
|
|
1131
|
+
fallback_trace TEXT,
|
|
1132
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1133
|
+
);
|
|
1134
|
+
|
|
1011
1135
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1012
1136
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1013
1137
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1032,7 +1156,14 @@ var SQLiteEventStore = class {
|
|
|
1032
1156
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1033
1157
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1034
1158
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1159
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1035
1160
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1161
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1162
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1163
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1164
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1165
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1166
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1036
1167
|
|
|
1037
1168
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1038
1169
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1056,6 +1187,14 @@ var SQLiteEventStore = class {
|
|
|
1056
1187
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1057
1188
|
END;
|
|
1058
1189
|
`);
|
|
1190
|
+
try {
|
|
1191
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1192
|
+
} catch {
|
|
1193
|
+
}
|
|
1194
|
+
try {
|
|
1195
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1196
|
+
} catch {
|
|
1197
|
+
}
|
|
1059
1198
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1060
1199
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1061
1200
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1076,6 +1215,15 @@ var SQLiteEventStore = class {
|
|
|
1076
1215
|
console.error("Error adding last_accessed_at column:", err);
|
|
1077
1216
|
}
|
|
1078
1217
|
}
|
|
1218
|
+
if (!columnNames.includes("turn_id")) {
|
|
1219
|
+
try {
|
|
1220
|
+
sqliteExec(this.db, `
|
|
1221
|
+
ALTER TABLE events ADD COLUMN turn_id TEXT;
|
|
1222
|
+
`);
|
|
1223
|
+
} catch (err) {
|
|
1224
|
+
console.error("Error adding turn_id column:", err);
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1079
1227
|
try {
|
|
1080
1228
|
sqliteExec(this.db, `
|
|
1081
1229
|
CREATE INDEX IF NOT EXISTS idx_events_access_count ON events(access_count DESC);
|
|
@@ -1088,6 +1236,12 @@ var SQLiteEventStore = class {
|
|
|
1088
1236
|
`);
|
|
1089
1237
|
} catch (err) {
|
|
1090
1238
|
}
|
|
1239
|
+
try {
|
|
1240
|
+
sqliteExec(this.db, `
|
|
1241
|
+
CREATE INDEX IF NOT EXISTS idx_events_turn_id ON events(turn_id);
|
|
1242
|
+
`);
|
|
1243
|
+
} catch (err) {
|
|
1244
|
+
}
|
|
1091
1245
|
this.initialized = true;
|
|
1092
1246
|
}
|
|
1093
1247
|
/**
|
|
@@ -1112,9 +1266,11 @@ var SQLiteEventStore = class {
|
|
|
1112
1266
|
const id = randomUUID2();
|
|
1113
1267
|
const timestamp = toSQLiteTimestamp(input.timestamp);
|
|
1114
1268
|
try {
|
|
1269
|
+
const metadata = input.metadata || {};
|
|
1270
|
+
const turnId = metadata.turnId || null;
|
|
1115
1271
|
const insertEvent = this.db.prepare(`
|
|
1116
|
-
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata)
|
|
1117
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1272
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1273
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1118
1274
|
`);
|
|
1119
1275
|
const insertDedup = this.db.prepare(`
|
|
1120
1276
|
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
@@ -1131,12 +1287,28 @@ var SQLiteEventStore = class {
|
|
|
1131
1287
|
input.content,
|
|
1132
1288
|
canonicalKey,
|
|
1133
1289
|
dedupeKey,
|
|
1134
|
-
JSON.stringify(
|
|
1290
|
+
JSON.stringify(metadata),
|
|
1291
|
+
turnId
|
|
1135
1292
|
);
|
|
1136
1293
|
insertDedup.run(dedupeKey, id);
|
|
1137
1294
|
insertLevel.run(id);
|
|
1138
1295
|
});
|
|
1139
1296
|
transaction();
|
|
1297
|
+
if (this.markdownMirror) {
|
|
1298
|
+
const event = {
|
|
1299
|
+
id,
|
|
1300
|
+
eventType: input.eventType,
|
|
1301
|
+
sessionId: input.sessionId,
|
|
1302
|
+
timestamp: input.timestamp,
|
|
1303
|
+
content: input.content,
|
|
1304
|
+
canonicalKey,
|
|
1305
|
+
dedupeKey,
|
|
1306
|
+
metadata
|
|
1307
|
+
};
|
|
1308
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1309
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1310
|
+
});
|
|
1311
|
+
}
|
|
1140
1312
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1141
1313
|
} catch (error) {
|
|
1142
1314
|
return {
|
|
@@ -1195,6 +1367,92 @@ var SQLiteEventStore = class {
|
|
|
1195
1367
|
);
|
|
1196
1368
|
return rows.map(this.rowToEvent);
|
|
1197
1369
|
}
|
|
1370
|
+
/**
|
|
1371
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1372
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1373
|
+
*/
|
|
1374
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1375
|
+
await this.initialize();
|
|
1376
|
+
const rows = sqliteAll(
|
|
1377
|
+
this.db,
|
|
1378
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1379
|
+
[lastRowid, limit]
|
|
1380
|
+
);
|
|
1381
|
+
return rows.map((row) => ({
|
|
1382
|
+
rowid: row._rowid,
|
|
1383
|
+
event: this.rowToEvent(row)
|
|
1384
|
+
}));
|
|
1385
|
+
}
|
|
1386
|
+
/**
|
|
1387
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1388
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1389
|
+
*
|
|
1390
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1391
|
+
*/
|
|
1392
|
+
async importEvents(events) {
|
|
1393
|
+
if (events.length === 0)
|
|
1394
|
+
return { inserted: 0, skipped: 0 };
|
|
1395
|
+
if (this.readOnly)
|
|
1396
|
+
return { inserted: 0, skipped: events.length };
|
|
1397
|
+
await this.initialize();
|
|
1398
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1399
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1400
|
+
const insertEvent = this.db.prepare(`
|
|
1401
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1402
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1403
|
+
`);
|
|
1404
|
+
const insertDedup = this.db.prepare(`
|
|
1405
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1406
|
+
`);
|
|
1407
|
+
const insertLevel = this.db.prepare(`
|
|
1408
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1409
|
+
`);
|
|
1410
|
+
let inserted = 0;
|
|
1411
|
+
let skipped = 0;
|
|
1412
|
+
const insertedEvents = [];
|
|
1413
|
+
const tx = this.db.transaction((batch) => {
|
|
1414
|
+
for (const ev of batch) {
|
|
1415
|
+
const existingById = getById.get(ev.id);
|
|
1416
|
+
if (existingById) {
|
|
1417
|
+
skipped++;
|
|
1418
|
+
continue;
|
|
1419
|
+
}
|
|
1420
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1421
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1422
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1423
|
+
if (existingByDedupe) {
|
|
1424
|
+
skipped++;
|
|
1425
|
+
continue;
|
|
1426
|
+
}
|
|
1427
|
+
const metadata = ev.metadata || {};
|
|
1428
|
+
const turnId = metadata.turnId;
|
|
1429
|
+
insertEvent.run(
|
|
1430
|
+
ev.id,
|
|
1431
|
+
ev.eventType,
|
|
1432
|
+
ev.sessionId,
|
|
1433
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1434
|
+
ev.content,
|
|
1435
|
+
canonicalKey,
|
|
1436
|
+
dedupeKey,
|
|
1437
|
+
JSON.stringify(metadata),
|
|
1438
|
+
turnId ?? null
|
|
1439
|
+
);
|
|
1440
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1441
|
+
insertLevel.run(ev.id);
|
|
1442
|
+
inserted++;
|
|
1443
|
+
insertedEvents.push(ev);
|
|
1444
|
+
}
|
|
1445
|
+
});
|
|
1446
|
+
tx(events);
|
|
1447
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1448
|
+
for (const ev of insertedEvents) {
|
|
1449
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1450
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
return { inserted, skipped };
|
|
1455
|
+
}
|
|
1198
1456
|
/**
|
|
1199
1457
|
* Create or update session
|
|
1200
1458
|
*/
|
|
@@ -1357,6 +1615,35 @@ var SQLiteEventStore = class {
|
|
|
1357
1615
|
[error, ...ids]
|
|
1358
1616
|
);
|
|
1359
1617
|
}
|
|
1618
|
+
/**
|
|
1619
|
+
* Get embedding/vector outbox health statistics
|
|
1620
|
+
*/
|
|
1621
|
+
async getOutboxStats() {
|
|
1622
|
+
await this.initialize();
|
|
1623
|
+
const embeddingRows = sqliteAll(
|
|
1624
|
+
this.db,
|
|
1625
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1626
|
+
);
|
|
1627
|
+
const vectorRows = sqliteAll(
|
|
1628
|
+
this.db,
|
|
1629
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1630
|
+
);
|
|
1631
|
+
const fromRows = (rows) => {
|
|
1632
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1633
|
+
for (const row of rows) {
|
|
1634
|
+
const key = row.status;
|
|
1635
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1636
|
+
out[key] += row.count;
|
|
1637
|
+
}
|
|
1638
|
+
out.total += row.count;
|
|
1639
|
+
}
|
|
1640
|
+
return out;
|
|
1641
|
+
};
|
|
1642
|
+
return {
|
|
1643
|
+
embedding: fromRows(embeddingRows),
|
|
1644
|
+
vector: fromRows(vectorRows)
|
|
1645
|
+
};
|
|
1646
|
+
}
|
|
1360
1647
|
/**
|
|
1361
1648
|
* Update memory level
|
|
1362
1649
|
*/
|
|
@@ -1481,11 +1768,11 @@ var SQLiteEventStore = class {
|
|
|
1481
1768
|
);
|
|
1482
1769
|
}
|
|
1483
1770
|
/**
|
|
1484
|
-
* Get most accessed memories
|
|
1771
|
+
* Get most accessed memories (falls back to recent events if none accessed)
|
|
1485
1772
|
*/
|
|
1486
1773
|
async getMostAccessed(limit = 10) {
|
|
1487
1774
|
await this.initialize();
|
|
1488
|
-
|
|
1775
|
+
let rows = sqliteAll(
|
|
1489
1776
|
this.db,
|
|
1490
1777
|
`SELECT * FROM events
|
|
1491
1778
|
WHERE access_count > 0
|
|
@@ -1493,8 +1780,166 @@ var SQLiteEventStore = class {
|
|
|
1493
1780
|
LIMIT ?`,
|
|
1494
1781
|
[limit]
|
|
1495
1782
|
);
|
|
1783
|
+
if (rows.length === 0) {
|
|
1784
|
+
rows = sqliteAll(
|
|
1785
|
+
this.db,
|
|
1786
|
+
`SELECT * FROM events
|
|
1787
|
+
ORDER BY timestamp DESC
|
|
1788
|
+
LIMIT ?`,
|
|
1789
|
+
[limit]
|
|
1790
|
+
);
|
|
1791
|
+
}
|
|
1496
1792
|
return rows.map((row) => this.rowToEvent(row));
|
|
1497
1793
|
}
|
|
1794
|
+
/**
|
|
1795
|
+
* Record a memory retrieval for helpfulness tracking
|
|
1796
|
+
*/
|
|
1797
|
+
async recordRetrieval(eventId, sessionId, score, query) {
|
|
1798
|
+
if (this.readOnly)
|
|
1799
|
+
return;
|
|
1800
|
+
await this.initialize();
|
|
1801
|
+
const id = randomUUID2();
|
|
1802
|
+
sqliteRun(
|
|
1803
|
+
this.db,
|
|
1804
|
+
`INSERT INTO memory_helpfulness (id, event_id, session_id, retrieval_score, query_preview, created_at)
|
|
1805
|
+
VALUES (?, ?, ?, ?, ?, datetime('now'))`,
|
|
1806
|
+
[id, eventId, sessionId, score, query.slice(0, 100)]
|
|
1807
|
+
);
|
|
1808
|
+
}
|
|
1809
|
+
/**
|
|
1810
|
+
* Evaluate helpfulness for all retrievals in a session
|
|
1811
|
+
* Called at session end - uses behavioral signals to compute score
|
|
1812
|
+
*/
|
|
1813
|
+
async evaluateSessionHelpfulness(sessionId) {
|
|
1814
|
+
if (this.readOnly)
|
|
1815
|
+
return;
|
|
1816
|
+
await this.initialize();
|
|
1817
|
+
const retrievals = sqliteAll(
|
|
1818
|
+
this.db,
|
|
1819
|
+
`SELECT * FROM memory_helpfulness WHERE session_id = ? AND measured_at IS NULL`,
|
|
1820
|
+
[sessionId]
|
|
1821
|
+
);
|
|
1822
|
+
if (retrievals.length === 0)
|
|
1823
|
+
return;
|
|
1824
|
+
const sessionEvents = sqliteAll(
|
|
1825
|
+
this.db,
|
|
1826
|
+
`SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
|
|
1827
|
+
[sessionId]
|
|
1828
|
+
);
|
|
1829
|
+
const promptEvents = sessionEvents.filter((e) => e.event_type === "user_prompt");
|
|
1830
|
+
const toolEvents = sessionEvents.filter((e) => e.event_type === "tool_observation");
|
|
1831
|
+
let toolSuccessCount = 0;
|
|
1832
|
+
let toolTotalCount = toolEvents.length;
|
|
1833
|
+
for (const t of toolEvents) {
|
|
1834
|
+
try {
|
|
1835
|
+
const content = JSON.parse(t.content);
|
|
1836
|
+
if (content.success !== false)
|
|
1837
|
+
toolSuccessCount++;
|
|
1838
|
+
} catch {
|
|
1839
|
+
toolSuccessCount++;
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
const toolSuccessRatio = toolTotalCount > 0 ? toolSuccessCount / toolTotalCount : 0.5;
|
|
1843
|
+
for (const retrieval of retrievals) {
|
|
1844
|
+
const retrievalTime = retrieval.created_at;
|
|
1845
|
+
const eventsAfter = sessionEvents.filter((e) => e.timestamp > retrievalTime);
|
|
1846
|
+
const sessionContinued = eventsAfter.length > 0 ? 1 : 0;
|
|
1847
|
+
const promptsAfter = promptEvents.filter((e) => e.timestamp > retrievalTime);
|
|
1848
|
+
const promptCountAfter = promptsAfter.length;
|
|
1849
|
+
const queryWords = new Set((retrieval.query_preview || "").toLowerCase().split(/\s+/).filter((w) => w.length > 2));
|
|
1850
|
+
let wasReasked = 0;
|
|
1851
|
+
for (const p of promptsAfter) {
|
|
1852
|
+
const pWords = new Set(p.content.toLowerCase().split(/\s+/).filter((w) => w.length > 2));
|
|
1853
|
+
let overlap = 0;
|
|
1854
|
+
for (const w of queryWords) {
|
|
1855
|
+
if (pWords.has(w))
|
|
1856
|
+
overlap++;
|
|
1857
|
+
}
|
|
1858
|
+
if (queryWords.size > 0 && overlap / queryWords.size > 0.5) {
|
|
1859
|
+
wasReasked = 1;
|
|
1860
|
+
break;
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
const retrievalScore = retrieval.retrieval_score || 0;
|
|
1864
|
+
const helpfulnessScore = 0.3 * Math.min(retrievalScore, 1) + 0.25 * (sessionContinued ? 1 : 0) + 0.25 * toolSuccessRatio + 0.2 * (wasReasked ? 0 : 1);
|
|
1865
|
+
sqliteRun(
|
|
1866
|
+
this.db,
|
|
1867
|
+
`UPDATE memory_helpfulness
|
|
1868
|
+
SET session_continued = ?, prompt_count_after = ?,
|
|
1869
|
+
tool_success_count = ?, tool_total_count = ?,
|
|
1870
|
+
was_reasked = ?, helpfulness_score = ?,
|
|
1871
|
+
measured_at = datetime('now')
|
|
1872
|
+
WHERE id = ?`,
|
|
1873
|
+
[
|
|
1874
|
+
sessionContinued,
|
|
1875
|
+
promptCountAfter,
|
|
1876
|
+
toolSuccessCount,
|
|
1877
|
+
toolTotalCount,
|
|
1878
|
+
wasReasked,
|
|
1879
|
+
helpfulnessScore,
|
|
1880
|
+
retrieval.id
|
|
1881
|
+
]
|
|
1882
|
+
);
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
/**
|
|
1886
|
+
* Get most helpful memories ranked by helpfulness score
|
|
1887
|
+
*/
|
|
1888
|
+
async getHelpfulMemories(limit = 10) {
|
|
1889
|
+
await this.initialize();
|
|
1890
|
+
const rows = sqliteAll(
|
|
1891
|
+
this.db,
|
|
1892
|
+
`SELECT
|
|
1893
|
+
mh.event_id,
|
|
1894
|
+
AVG(mh.helpfulness_score) as avg_score,
|
|
1895
|
+
COUNT(*) as eval_count,
|
|
1896
|
+
e.content,
|
|
1897
|
+
e.access_count
|
|
1898
|
+
FROM memory_helpfulness mh
|
|
1899
|
+
JOIN events e ON e.id = mh.event_id
|
|
1900
|
+
WHERE mh.measured_at IS NOT NULL
|
|
1901
|
+
GROUP BY mh.event_id
|
|
1902
|
+
ORDER BY avg_score DESC
|
|
1903
|
+
LIMIT ?`,
|
|
1904
|
+
[limit]
|
|
1905
|
+
);
|
|
1906
|
+
return rows.map((r) => ({
|
|
1907
|
+
eventId: r.event_id,
|
|
1908
|
+
summary: r.content.substring(0, 200) + (r.content.length > 200 ? "..." : ""),
|
|
1909
|
+
helpfulnessScore: Math.round(r.avg_score * 100) / 100,
|
|
1910
|
+
accessCount: r.access_count || 0,
|
|
1911
|
+
evaluationCount: r.eval_count
|
|
1912
|
+
}));
|
|
1913
|
+
}
|
|
1914
|
+
/**
|
|
1915
|
+
* Get helpfulness statistics for dashboard
|
|
1916
|
+
*/
|
|
1917
|
+
async getHelpfulnessStats() {
|
|
1918
|
+
await this.initialize();
|
|
1919
|
+
const stats = sqliteGet(
|
|
1920
|
+
this.db,
|
|
1921
|
+
`SELECT
|
|
1922
|
+
AVG(helpfulness_score) as avg_score,
|
|
1923
|
+
COUNT(*) as total_evaluated,
|
|
1924
|
+
SUM(CASE WHEN helpfulness_score >= 0.7 THEN 1 ELSE 0 END) as helpful,
|
|
1925
|
+
SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
|
|
1926
|
+
SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
|
|
1927
|
+
FROM memory_helpfulness
|
|
1928
|
+
WHERE measured_at IS NOT NULL`
|
|
1929
|
+
);
|
|
1930
|
+
const totalRow = sqliteGet(
|
|
1931
|
+
this.db,
|
|
1932
|
+
`SELECT COUNT(*) as total FROM memory_helpfulness`
|
|
1933
|
+
);
|
|
1934
|
+
return {
|
|
1935
|
+
avgScore: Math.round((stats?.avg_score || 0) * 100) / 100,
|
|
1936
|
+
totalEvaluated: stats?.total_evaluated || 0,
|
|
1937
|
+
totalRetrievals: totalRow?.total || 0,
|
|
1938
|
+
helpful: stats?.helpful || 0,
|
|
1939
|
+
neutral: stats?.neutral || 0,
|
|
1940
|
+
unhelpful: stats?.unhelpful || 0
|
|
1941
|
+
};
|
|
1942
|
+
}
|
|
1498
1943
|
/**
|
|
1499
1944
|
* Fast keyword search using FTS5
|
|
1500
1945
|
* Returns events matching the search query, ranked by relevance
|
|
@@ -1557,12 +2002,222 @@ var SQLiteEventStore = class {
|
|
|
1557
2002
|
getDatabase() {
|
|
1558
2003
|
return this.db;
|
|
1559
2004
|
}
|
|
2005
|
+
async recordRetrievalTrace(input) {
|
|
2006
|
+
await this.initialize();
|
|
2007
|
+
const traceId = randomUUID2();
|
|
2008
|
+
sqliteRun(
|
|
2009
|
+
this.db,
|
|
2010
|
+
`INSERT INTO retrieval_traces (
|
|
2011
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2012
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2013
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2014
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2015
|
+
[
|
|
2016
|
+
traceId,
|
|
2017
|
+
input.sessionId || null,
|
|
2018
|
+
input.projectHash || null,
|
|
2019
|
+
input.queryText,
|
|
2020
|
+
input.strategy || null,
|
|
2021
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2022
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2023
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2024
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2025
|
+
(input.candidateEventIds || []).length,
|
|
2026
|
+
(input.selectedEventIds || []).length,
|
|
2027
|
+
input.confidence || null,
|
|
2028
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2029
|
+
]
|
|
2030
|
+
);
|
|
2031
|
+
}
|
|
2032
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2033
|
+
await this.initialize();
|
|
2034
|
+
const rows = sqliteAll(
|
|
2035
|
+
this.db,
|
|
2036
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2037
|
+
[limit]
|
|
2038
|
+
);
|
|
2039
|
+
return rows.map((row) => ({
|
|
2040
|
+
traceId: row.trace_id,
|
|
2041
|
+
sessionId: row.session_id || void 0,
|
|
2042
|
+
projectHash: row.project_hash || void 0,
|
|
2043
|
+
queryText: row.query_text,
|
|
2044
|
+
strategy: row.strategy || void 0,
|
|
2045
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2046
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2047
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2048
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2049
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2050
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2051
|
+
confidence: row.confidence || void 0,
|
|
2052
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2053
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2054
|
+
}));
|
|
2055
|
+
}
|
|
2056
|
+
async getRetrievalTraceStats() {
|
|
2057
|
+
await this.initialize();
|
|
2058
|
+
const row = sqliteGet(
|
|
2059
|
+
this.db,
|
|
2060
|
+
`SELECT
|
|
2061
|
+
COUNT(*) as total_queries,
|
|
2062
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2063
|
+
AVG(selected_count) as avg_selected_count,
|
|
2064
|
+
CASE
|
|
2065
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2066
|
+
ELSE 0
|
|
2067
|
+
END as selection_rate
|
|
2068
|
+
FROM retrieval_traces`,
|
|
2069
|
+
[]
|
|
2070
|
+
);
|
|
2071
|
+
return {
|
|
2072
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2073
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2074
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2075
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2076
|
+
};
|
|
2077
|
+
}
|
|
1560
2078
|
/**
|
|
1561
2079
|
* Close database connection
|
|
1562
2080
|
*/
|
|
1563
2081
|
async close() {
|
|
1564
2082
|
sqliteClose(this.db);
|
|
1565
2083
|
}
|
|
2084
|
+
/**
|
|
2085
|
+
* Get events grouped by turn_id for a session
|
|
2086
|
+
* Returns turns ordered by first event timestamp (newest first)
|
|
2087
|
+
*/
|
|
2088
|
+
async getSessionTurns(sessionId, options) {
|
|
2089
|
+
await this.initialize();
|
|
2090
|
+
const limit = options?.limit || 20;
|
|
2091
|
+
const offset = options?.offset || 0;
|
|
2092
|
+
const turnRows = sqliteAll(
|
|
2093
|
+
this.db,
|
|
2094
|
+
`SELECT turn_id, MIN(timestamp) as min_ts
|
|
2095
|
+
FROM events
|
|
2096
|
+
WHERE session_id = ? AND turn_id IS NOT NULL
|
|
2097
|
+
GROUP BY turn_id
|
|
2098
|
+
ORDER BY min_ts DESC
|
|
2099
|
+
LIMIT ? OFFSET ?`,
|
|
2100
|
+
[sessionId, limit, offset]
|
|
2101
|
+
);
|
|
2102
|
+
const turns = [];
|
|
2103
|
+
for (const turnRow of turnRows) {
|
|
2104
|
+
const events = await this.getEventsByTurn(turnRow.turn_id);
|
|
2105
|
+
const promptEvent = events.find((e) => e.eventType === "user_prompt");
|
|
2106
|
+
const toolEvents = events.filter((e) => e.eventType === "tool_observation");
|
|
2107
|
+
const hasResponse = events.some((e) => e.eventType === "agent_response");
|
|
2108
|
+
turns.push({
|
|
2109
|
+
turnId: turnRow.turn_id,
|
|
2110
|
+
events,
|
|
2111
|
+
startedAt: toDateFromSQLite(turnRow.min_ts),
|
|
2112
|
+
promptPreview: promptEvent ? promptEvent.content.slice(0, 200) + (promptEvent.content.length > 200 ? "..." : "") : "(no prompt)",
|
|
2113
|
+
eventCount: events.length,
|
|
2114
|
+
toolCount: toolEvents.length,
|
|
2115
|
+
hasResponse
|
|
2116
|
+
});
|
|
2117
|
+
}
|
|
2118
|
+
return turns;
|
|
2119
|
+
}
|
|
2120
|
+
/**
|
|
2121
|
+
* Get all events for a specific turn_id
|
|
2122
|
+
*/
|
|
2123
|
+
async getEventsByTurn(turnId) {
|
|
2124
|
+
await this.initialize();
|
|
2125
|
+
const rows = sqliteAll(
|
|
2126
|
+
this.db,
|
|
2127
|
+
`SELECT * FROM events WHERE turn_id = ? ORDER BY timestamp ASC`,
|
|
2128
|
+
[turnId]
|
|
2129
|
+
);
|
|
2130
|
+
return rows.map(this.rowToEvent);
|
|
2131
|
+
}
|
|
2132
|
+
/**
|
|
2133
|
+
* Count total turns for a session
|
|
2134
|
+
*/
|
|
2135
|
+
async countSessionTurns(sessionId) {
|
|
2136
|
+
await this.initialize();
|
|
2137
|
+
const row = sqliteGet(
|
|
2138
|
+
this.db,
|
|
2139
|
+
`SELECT COUNT(DISTINCT turn_id) as count
|
|
2140
|
+
FROM events
|
|
2141
|
+
WHERE session_id = ? AND turn_id IS NOT NULL`,
|
|
2142
|
+
[sessionId]
|
|
2143
|
+
);
|
|
2144
|
+
return row?.count || 0;
|
|
2145
|
+
}
|
|
2146
|
+
/**
|
|
2147
|
+
* Migrate existing events: backfill turn_id for events that have turnId in metadata
|
|
2148
|
+
* but no turn_id column value (for events stored before this migration)
|
|
2149
|
+
*/
|
|
2150
|
+
async backfillTurnIds() {
|
|
2151
|
+
await this.initialize();
|
|
2152
|
+
const rows = sqliteAll(
|
|
2153
|
+
this.db,
|
|
2154
|
+
`SELECT id, metadata FROM events
|
|
2155
|
+
WHERE turn_id IS NULL AND metadata IS NOT NULL AND metadata LIKE '%turnId%'`
|
|
2156
|
+
);
|
|
2157
|
+
let updated = 0;
|
|
2158
|
+
for (const row of rows) {
|
|
2159
|
+
try {
|
|
2160
|
+
const metadata = JSON.parse(row.metadata);
|
|
2161
|
+
if (metadata.turnId) {
|
|
2162
|
+
sqliteRun(
|
|
2163
|
+
this.db,
|
|
2164
|
+
`UPDATE events SET turn_id = ? WHERE id = ?`,
|
|
2165
|
+
[metadata.turnId, row.id]
|
|
2166
|
+
);
|
|
2167
|
+
updated++;
|
|
2168
|
+
}
|
|
2169
|
+
} catch {
|
|
2170
|
+
}
|
|
2171
|
+
}
|
|
2172
|
+
return updated;
|
|
2173
|
+
}
|
|
2174
|
+
/**
|
|
2175
|
+
* Delete all events for a session (for force reimport)
|
|
2176
|
+
*/
|
|
2177
|
+
async deleteSessionEvents(sessionId) {
|
|
2178
|
+
await this.initialize();
|
|
2179
|
+
const events = sqliteAll(
|
|
2180
|
+
this.db,
|
|
2181
|
+
`SELECT id FROM events WHERE session_id = ?`,
|
|
2182
|
+
[sessionId]
|
|
2183
|
+
);
|
|
2184
|
+
if (events.length === 0)
|
|
2185
|
+
return 0;
|
|
2186
|
+
const eventIds = events.map((e) => e.id);
|
|
2187
|
+
const placeholders = eventIds.map(() => "?").join(",");
|
|
2188
|
+
const ftsTriggersDropped = [];
|
|
2189
|
+
for (const triggerName of ["events_fts_delete", "events_fts_update", "events_fts_insert"]) {
|
|
2190
|
+
try {
|
|
2191
|
+
sqliteRun(this.db, `DROP TRIGGER IF EXISTS ${triggerName}`);
|
|
2192
|
+
ftsTriggersDropped.push(triggerName);
|
|
2193
|
+
} catch {
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
for (const table of ["event_dedup", "memory_levels", "embedding_queue", "embedding_outbox", "vector_outbox"]) {
|
|
2197
|
+
try {
|
|
2198
|
+
sqliteRun(this.db, `DELETE FROM ${table} WHERE event_id IN (${placeholders})`, eventIds);
|
|
2199
|
+
} catch {
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
const result = sqliteRun(this.db, `DELETE FROM events WHERE session_id = ?`, [sessionId]);
|
|
2203
|
+
if (ftsTriggersDropped.length > 0) {
|
|
2204
|
+
try {
|
|
2205
|
+
sqliteRun(this.db, `INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
|
|
2206
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
|
|
2207
|
+
INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
|
|
2208
|
+
END`);
|
|
2209
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
|
|
2210
|
+
INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
|
|
2211
|
+
END`);
|
|
2212
|
+
sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
|
|
2213
|
+
INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
|
|
2214
|
+
INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
|
|
2215
|
+
END`);
|
|
2216
|
+
} catch {
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
return result.changes || 0;
|
|
2220
|
+
}
|
|
1566
2221
|
/**
|
|
1567
2222
|
* Convert database row to MemoryEvent
|
|
1568
2223
|
*/
|
|
@@ -1583,6 +2238,9 @@ var SQLiteEventStore = class {
|
|
|
1583
2238
|
if (row.last_accessed_at !== void 0) {
|
|
1584
2239
|
event.last_accessed_at = row.last_accessed_at;
|
|
1585
2240
|
}
|
|
2241
|
+
if (row.turn_id !== void 0 && row.turn_id !== null) {
|
|
2242
|
+
event.turn_id = row.turn_id;
|
|
2243
|
+
}
|
|
1586
2244
|
return event;
|
|
1587
2245
|
}
|
|
1588
2246
|
};
|
|
@@ -1794,7 +2452,16 @@ var VectorStore = class {
|
|
|
1794
2452
|
metadata: JSON.stringify(record.metadata || {})
|
|
1795
2453
|
};
|
|
1796
2454
|
if (!this.table) {
|
|
1797
|
-
|
|
2455
|
+
try {
|
|
2456
|
+
this.table = await this.db.createTable(this.tableName, [data]);
|
|
2457
|
+
} catch (e) {
|
|
2458
|
+
if (e?.message?.includes("already exists")) {
|
|
2459
|
+
this.table = await this.db.openTable(this.tableName);
|
|
2460
|
+
await this.table.add([data]);
|
|
2461
|
+
} else {
|
|
2462
|
+
throw e;
|
|
2463
|
+
}
|
|
2464
|
+
}
|
|
1798
2465
|
} else {
|
|
1799
2466
|
await this.table.add([data]);
|
|
1800
2467
|
}
|
|
@@ -1820,7 +2487,16 @@ var VectorStore = class {
|
|
|
1820
2487
|
metadata: JSON.stringify(record.metadata || {})
|
|
1821
2488
|
}));
|
|
1822
2489
|
if (!this.table) {
|
|
1823
|
-
|
|
2490
|
+
try {
|
|
2491
|
+
this.table = await this.db.createTable(this.tableName, data);
|
|
2492
|
+
} catch (e) {
|
|
2493
|
+
if (e?.message?.includes("already exists")) {
|
|
2494
|
+
this.table = await this.db.openTable(this.tableName);
|
|
2495
|
+
await this.table.add(data);
|
|
2496
|
+
} else {
|
|
2497
|
+
throw e;
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
1824
2500
|
} else {
|
|
1825
2501
|
await this.table.add(data);
|
|
1826
2502
|
}
|
|
@@ -2260,7 +2936,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2260
2936
|
topK: 5,
|
|
2261
2937
|
minScore: 0.7,
|
|
2262
2938
|
maxTokens: 2e3,
|
|
2263
|
-
includeSessionContext: true
|
|
2939
|
+
includeSessionContext: true,
|
|
2940
|
+
strategy: "auto",
|
|
2941
|
+
rerankWithKeyword: true,
|
|
2942
|
+
decayPolicy: {
|
|
2943
|
+
enabled: true,
|
|
2944
|
+
windowDays: 30,
|
|
2945
|
+
maxPenalty: 0.15
|
|
2946
|
+
},
|
|
2947
|
+
graphHop: {
|
|
2948
|
+
enabled: true,
|
|
2949
|
+
maxHops: 1,
|
|
2950
|
+
hopPenalty: 0.08
|
|
2951
|
+
},
|
|
2952
|
+
projectScopeMode: "global"
|
|
2264
2953
|
};
|
|
2265
2954
|
var Retriever = class {
|
|
2266
2955
|
eventStore;
|
|
@@ -2270,6 +2959,7 @@ var Retriever = class {
|
|
|
2270
2959
|
sharedStore;
|
|
2271
2960
|
sharedVectorStore;
|
|
2272
2961
|
graduation;
|
|
2962
|
+
queryRewriter;
|
|
2273
2963
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2274
2964
|
this.eventStore = eventStore;
|
|
2275
2965
|
this.vectorStore = vectorStore;
|
|
@@ -2278,47 +2968,105 @@ var Retriever = class {
|
|
|
2278
2968
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2279
2969
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2280
2970
|
}
|
|
2281
|
-
/**
|
|
2282
|
-
* Set graduation pipeline for access tracking
|
|
2283
|
-
*/
|
|
2284
2971
|
setGraduationPipeline(graduation) {
|
|
2285
2972
|
this.graduation = graduation;
|
|
2286
2973
|
}
|
|
2287
|
-
/**
|
|
2288
|
-
* Set shared stores after construction
|
|
2289
|
-
*/
|
|
2290
2974
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2291
2975
|
this.sharedStore = sharedStore;
|
|
2292
2976
|
this.sharedVectorStore = sharedVectorStore;
|
|
2293
2977
|
}
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2978
|
+
setQueryRewriter(rewriter) {
|
|
2979
|
+
this.queryRewriter = rewriter;
|
|
2980
|
+
}
|
|
2297
2981
|
async retrieve(query, options = {}) {
|
|
2298
2982
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2299
|
-
const
|
|
2300
|
-
const
|
|
2301
|
-
|
|
2302
|
-
|
|
2983
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
2984
|
+
const fallbackTrace = [];
|
|
2985
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
2986
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
2987
|
+
let current = await this.runStage(query, {
|
|
2988
|
+
strategy: primaryStrategy,
|
|
2989
|
+
topK: opts.topK,
|
|
2303
2990
|
minScore: opts.minScore,
|
|
2304
|
-
sessionId:
|
|
2991
|
+
sessionId: sessionFilter,
|
|
2992
|
+
scope: opts.scope,
|
|
2993
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
2994
|
+
rerankWeights: opts.rerankWeights,
|
|
2995
|
+
decayPolicy: opts.decayPolicy,
|
|
2996
|
+
intentRewrite: opts.intentRewrite === true,
|
|
2997
|
+
graphHop: opts.graphHop,
|
|
2998
|
+
projectScopeMode: opts.projectScopeMode,
|
|
2999
|
+
projectHash: opts.projectHash,
|
|
3000
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2305
3001
|
});
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
3002
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3003
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3004
|
+
current = await this.runStage(query, {
|
|
3005
|
+
strategy: "deep",
|
|
3006
|
+
topK: opts.topK,
|
|
3007
|
+
minScore: opts.minScore,
|
|
3008
|
+
sessionId: sessionFilter,
|
|
3009
|
+
scope: opts.scope,
|
|
3010
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3011
|
+
rerankWeights: opts.rerankWeights,
|
|
3012
|
+
decayPolicy: opts.decayPolicy,
|
|
3013
|
+
graphHop: opts.graphHop,
|
|
3014
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3015
|
+
projectHash: opts.projectHash,
|
|
3016
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3017
|
+
});
|
|
3018
|
+
fallbackTrace.push("fallback:deep");
|
|
3019
|
+
}
|
|
3020
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3021
|
+
current = await this.runStage(query, {
|
|
3022
|
+
strategy: "deep",
|
|
3023
|
+
topK: opts.topK,
|
|
3024
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3025
|
+
sessionId: void 0,
|
|
3026
|
+
scope: void 0,
|
|
3027
|
+
rerankWithKeyword: true,
|
|
3028
|
+
rerankWeights: opts.rerankWeights,
|
|
3029
|
+
decayPolicy: opts.decayPolicy,
|
|
3030
|
+
graphHop: opts.graphHop,
|
|
3031
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3032
|
+
projectHash: opts.projectHash,
|
|
3033
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3034
|
+
});
|
|
3035
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3036
|
+
}
|
|
3037
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3038
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3039
|
+
current = {
|
|
3040
|
+
results: summary,
|
|
3041
|
+
candidateResults: summary,
|
|
3042
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3043
|
+
};
|
|
3044
|
+
fallbackTrace.push("fallback:summary");
|
|
3045
|
+
}
|
|
3046
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2311
3047
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2312
3048
|
return {
|
|
2313
3049
|
memories,
|
|
2314
|
-
matchResult,
|
|
3050
|
+
matchResult: current.matchResult,
|
|
2315
3051
|
totalTokens: this.estimateTokens(context),
|
|
2316
|
-
context
|
|
3052
|
+
context,
|
|
3053
|
+
fallbackTrace,
|
|
3054
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3055
|
+
eventId: r.eventId,
|
|
3056
|
+
score: r.score,
|
|
3057
|
+
semanticScore: r.semanticScore,
|
|
3058
|
+
lexicalScore: r.lexicalScore,
|
|
3059
|
+
recencyScore: r.recencyScore
|
|
3060
|
+
})),
|
|
3061
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3062
|
+
eventId: r.eventId,
|
|
3063
|
+
score: r.score,
|
|
3064
|
+
semanticScore: r.semanticScore,
|
|
3065
|
+
lexicalScore: r.lexicalScore,
|
|
3066
|
+
recencyScore: r.recencyScore
|
|
3067
|
+
}))
|
|
2317
3068
|
};
|
|
2318
3069
|
}
|
|
2319
|
-
/**
|
|
2320
|
-
* Retrieve with unified search (project + shared)
|
|
2321
|
-
*/
|
|
2322
3070
|
async retrieveUnified(query, options = {}) {
|
|
2323
3071
|
const projectResult = await this.retrieve(query, options);
|
|
2324
3072
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2326,22 +3074,19 @@ var Retriever = class {
|
|
|
2326
3074
|
}
|
|
2327
3075
|
try {
|
|
2328
3076
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2329
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
excludeProjectHash: options.projectHash
|
|
2335
|
-
}
|
|
2336
|
-
);
|
|
3077
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3078
|
+
limit: options.topK || 5,
|
|
3079
|
+
minScore: options.minScore || 0.7,
|
|
3080
|
+
excludeProjectHash: options.projectHash
|
|
3081
|
+
});
|
|
2337
3082
|
const sharedMemories = [];
|
|
2338
3083
|
for (const result of sharedVectorResults) {
|
|
2339
3084
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2340
|
-
if (entry)
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
3085
|
+
if (!entry)
|
|
3086
|
+
continue;
|
|
3087
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3088
|
+
sharedMemories.push(entry);
|
|
3089
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2345
3090
|
}
|
|
2346
3091
|
}
|
|
2347
3092
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2356,50 +3101,243 @@ var Retriever = class {
|
|
|
2356
3101
|
return projectResult;
|
|
2357
3102
|
}
|
|
2358
3103
|
}
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
3104
|
+
async runStage(query, input) {
|
|
3105
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3106
|
+
strategy: input.strategy,
|
|
3107
|
+
topK: input.topK,
|
|
3108
|
+
minScore: input.minScore,
|
|
3109
|
+
sessionId: input.sessionId
|
|
3110
|
+
});
|
|
3111
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3112
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3113
|
+
if (rewritten && rewritten !== query) {
|
|
3114
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3115
|
+
strategy: "deep",
|
|
3116
|
+
topK: input.topK,
|
|
3117
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3118
|
+
sessionId: input.sessionId
|
|
3119
|
+
});
|
|
3120
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3121
|
+
}
|
|
3122
|
+
}
|
|
3123
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3124
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3125
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3126
|
+
limit: input.topK * 4
|
|
3127
|
+
});
|
|
3128
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3129
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3130
|
+
scope: input.scope,
|
|
3131
|
+
projectScopeMode: input.projectScopeMode,
|
|
3132
|
+
projectHash: input.projectHash,
|
|
3133
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3134
|
+
});
|
|
3135
|
+
const top = filtered.slice(0, input.topK);
|
|
3136
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3137
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3138
|
+
}
|
|
3139
|
+
mergeResults(primary, secondary, limit) {
|
|
3140
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3141
|
+
for (const row of primary)
|
|
3142
|
+
byId.set(row.eventId, row);
|
|
3143
|
+
for (const row of secondary) {
|
|
3144
|
+
const prev = byId.get(row.eventId);
|
|
3145
|
+
if (!prev || row.score > prev.score) {
|
|
3146
|
+
byId.set(row.eventId, row);
|
|
3147
|
+
}
|
|
3148
|
+
}
|
|
3149
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3150
|
+
}
|
|
3151
|
+
async expandGraphHops(seeds, opts) {
|
|
3152
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3153
|
+
for (const s of seeds)
|
|
3154
|
+
byId.set(s.eventId, s);
|
|
3155
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3156
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3157
|
+
const next = [];
|
|
3158
|
+
for (const f of frontier) {
|
|
3159
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3160
|
+
if (!ev)
|
|
3161
|
+
continue;
|
|
3162
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3163
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3164
|
+
for (const rid of relatedIds) {
|
|
3165
|
+
if (byId.has(rid))
|
|
3166
|
+
continue;
|
|
3167
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3168
|
+
if (!target)
|
|
3169
|
+
continue;
|
|
3170
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3171
|
+
const row = {
|
|
3172
|
+
id: `hop-${hop}-${rid}`,
|
|
3173
|
+
eventId: target.id,
|
|
3174
|
+
content: target.content,
|
|
3175
|
+
score,
|
|
3176
|
+
sessionId: target.sessionId,
|
|
3177
|
+
eventType: target.eventType,
|
|
3178
|
+
timestamp: target.timestamp.toISOString()
|
|
3179
|
+
};
|
|
3180
|
+
byId.set(row.eventId, row);
|
|
3181
|
+
next.push({ row, hop });
|
|
3182
|
+
if (byId.size >= opts.limit)
|
|
3183
|
+
break;
|
|
2380
3184
|
}
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
`;
|
|
3185
|
+
if (byId.size >= opts.limit)
|
|
3186
|
+
break;
|
|
2384
3187
|
}
|
|
3188
|
+
frontier = next;
|
|
3189
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3190
|
+
break;
|
|
2385
3191
|
}
|
|
2386
|
-
return
|
|
3192
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3193
|
+
}
|
|
3194
|
+
shouldFallback(matchResult, results) {
|
|
3195
|
+
if (results.length === 0)
|
|
3196
|
+
return true;
|
|
3197
|
+
if (matchResult.confidence === "none")
|
|
3198
|
+
return true;
|
|
3199
|
+
return false;
|
|
3200
|
+
}
|
|
3201
|
+
async buildSummaryFallback(query, topK) {
|
|
3202
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3203
|
+
const q = this.tokenize(query);
|
|
3204
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3205
|
+
id: `summary-${row.e.id}`,
|
|
3206
|
+
eventId: row.e.id,
|
|
3207
|
+
content: row.e.content,
|
|
3208
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3209
|
+
sessionId: row.e.sessionId,
|
|
3210
|
+
eventType: row.e.eventType,
|
|
3211
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3212
|
+
}));
|
|
3213
|
+
return ranked;
|
|
3214
|
+
}
|
|
3215
|
+
async searchByStrategy(query, input) {
|
|
3216
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3217
|
+
if (strategy === "fast") {
|
|
3218
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3219
|
+
limit: Math.max(5, input.topK * 3),
|
|
3220
|
+
sessionId: input.sessionId
|
|
3221
|
+
});
|
|
3222
|
+
return keyword;
|
|
3223
|
+
}
|
|
3224
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3225
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3226
|
+
limit: Math.max(5, input.topK * 3),
|
|
3227
|
+
minScore: input.minScore,
|
|
3228
|
+
sessionId: input.sessionId
|
|
3229
|
+
});
|
|
3230
|
+
}
|
|
3231
|
+
async searchByKeyword(query, input) {
|
|
3232
|
+
if (this.eventStore.keywordSearch) {
|
|
3233
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3234
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3235
|
+
return filtered2.map((row, idx) => ({
|
|
3236
|
+
id: `kw-${row.event.id}`,
|
|
3237
|
+
eventId: row.event.id,
|
|
3238
|
+
content: row.event.content,
|
|
3239
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3240
|
+
sessionId: row.event.sessionId,
|
|
3241
|
+
eventType: row.event.eventType,
|
|
3242
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3243
|
+
}));
|
|
3244
|
+
}
|
|
3245
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3246
|
+
const tokens = this.tokenize(query);
|
|
3247
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3248
|
+
return filtered.map((row, idx) => ({
|
|
3249
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3250
|
+
eventId: row.e.id,
|
|
3251
|
+
content: row.e.content,
|
|
3252
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3253
|
+
sessionId: row.e.sessionId,
|
|
3254
|
+
eventType: row.e.eventType,
|
|
3255
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3256
|
+
}));
|
|
3257
|
+
}
|
|
3258
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3259
|
+
const q = this.tokenize(query);
|
|
3260
|
+
const now = Date.now();
|
|
3261
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3262
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3263
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3264
|
+
const total = sw + lw + rw || 1;
|
|
3265
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3266
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3267
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3268
|
+
return [...results].map((r) => {
|
|
3269
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3270
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3271
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3272
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3273
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3274
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3275
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3276
|
+
}
|
|
3277
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3278
|
+
}).sort((a, b) => b.score - a.score);
|
|
3279
|
+
}
|
|
3280
|
+
async applyScopeFilters(results, options) {
|
|
3281
|
+
const scope = options?.scope;
|
|
3282
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3283
|
+
const allowedProjectHashes = new Set(
|
|
3284
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3285
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3286
|
+
)
|
|
3287
|
+
);
|
|
3288
|
+
if (!scope && projectScopeMode === "global")
|
|
3289
|
+
return results;
|
|
3290
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3291
|
+
const filtered = [];
|
|
3292
|
+
for (const result of results) {
|
|
3293
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3294
|
+
continue;
|
|
3295
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3296
|
+
continue;
|
|
3297
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3298
|
+
continue;
|
|
3299
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3300
|
+
if (!event)
|
|
3301
|
+
continue;
|
|
3302
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3303
|
+
continue;
|
|
3304
|
+
if (normalizedIncludes.length > 0) {
|
|
3305
|
+
const lc = event.content.toLowerCase();
|
|
3306
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3307
|
+
continue;
|
|
3308
|
+
}
|
|
3309
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3310
|
+
continue;
|
|
3311
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3312
|
+
filtered.push({ result, projectHash });
|
|
3313
|
+
}
|
|
3314
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3315
|
+
return filtered.map((x) => x.result);
|
|
3316
|
+
}
|
|
3317
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3318
|
+
if (projectScopeMode === "strict") {
|
|
3319
|
+
return projectMatched.map((x) => x.result);
|
|
3320
|
+
}
|
|
3321
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3322
|
+
}
|
|
3323
|
+
extractProjectHash(metadata) {
|
|
3324
|
+
if (!metadata || typeof metadata !== "object")
|
|
3325
|
+
return void 0;
|
|
3326
|
+
const scope = metadata.scope;
|
|
3327
|
+
if (!scope || typeof scope !== "object")
|
|
3328
|
+
return void 0;
|
|
3329
|
+
const project = scope.project;
|
|
3330
|
+
if (!project || typeof project !== "object")
|
|
3331
|
+
return void 0;
|
|
3332
|
+
const hash = project.hash;
|
|
3333
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2387
3334
|
}
|
|
2388
|
-
/**
|
|
2389
|
-
* Retrieve memories from a specific session
|
|
2390
|
-
*/
|
|
2391
3335
|
async retrieveFromSession(sessionId) {
|
|
2392
3336
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2393
3337
|
}
|
|
2394
|
-
/**
|
|
2395
|
-
* Get recent memories across all sessions
|
|
2396
|
-
*/
|
|
2397
3338
|
async retrieveRecent(limit = 100) {
|
|
2398
3339
|
return this.eventStore.getRecentEvents(limit);
|
|
2399
3340
|
}
|
|
2400
|
-
/**
|
|
2401
|
-
* Enrich search results with full event data
|
|
2402
|
-
*/
|
|
2403
3341
|
async enrichResults(results, options) {
|
|
2404
3342
|
const memories = [];
|
|
2405
3343
|
for (const result of results) {
|
|
@@ -2407,27 +3345,16 @@ var Retriever = class {
|
|
|
2407
3345
|
if (!event)
|
|
2408
3346
|
continue;
|
|
2409
3347
|
if (this.graduation) {
|
|
2410
|
-
this.graduation.recordAccess(
|
|
2411
|
-
event.id,
|
|
2412
|
-
options.sessionId || "unknown",
|
|
2413
|
-
result.score
|
|
2414
|
-
);
|
|
3348
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2415
3349
|
}
|
|
2416
3350
|
let sessionContext;
|
|
2417
3351
|
if (options.includeSessionContext) {
|
|
2418
3352
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2419
3353
|
}
|
|
2420
|
-
memories.push({
|
|
2421
|
-
event,
|
|
2422
|
-
score: result.score,
|
|
2423
|
-
sessionContext
|
|
2424
|
-
});
|
|
3354
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2425
3355
|
}
|
|
2426
3356
|
return memories;
|
|
2427
3357
|
}
|
|
2428
|
-
/**
|
|
2429
|
-
* Get surrounding context from the same session
|
|
2430
|
-
*/
|
|
2431
3358
|
async getSessionContext(sessionId, eventId) {
|
|
2432
3359
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2433
3360
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2440,55 +3367,86 @@ var Retriever = class {
|
|
|
2440
3367
|
return void 0;
|
|
2441
3368
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2442
3369
|
}
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
3370
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3371
|
+
let context = projectResult.context;
|
|
3372
|
+
if (sharedMemories.length === 0)
|
|
3373
|
+
return context;
|
|
3374
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3375
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3376
|
+
context += `### ${memory.title}
|
|
3377
|
+
`;
|
|
3378
|
+
if (memory.symptoms.length > 0)
|
|
3379
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3380
|
+
`;
|
|
3381
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3382
|
+
`;
|
|
3383
|
+
context += `**Solution:** ${memory.solution}
|
|
3384
|
+
`;
|
|
3385
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3386
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3387
|
+
`;
|
|
3388
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3389
|
+
|
|
3390
|
+
`;
|
|
3391
|
+
}
|
|
3392
|
+
return context;
|
|
3393
|
+
}
|
|
2446
3394
|
buildContext(memories, maxTokens) {
|
|
2447
3395
|
const parts = [];
|
|
2448
3396
|
let currentTokens = 0;
|
|
2449
3397
|
for (const memory of memories) {
|
|
2450
3398
|
const memoryText = this.formatMemory(memory);
|
|
2451
3399
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2452
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3400
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2453
3401
|
break;
|
|
2454
|
-
}
|
|
2455
3402
|
parts.push(memoryText);
|
|
2456
3403
|
currentTokens += memoryTokens;
|
|
2457
3404
|
}
|
|
2458
|
-
if (parts.length === 0)
|
|
3405
|
+
if (parts.length === 0)
|
|
2459
3406
|
return "";
|
|
2460
|
-
}
|
|
2461
3407
|
return `## Relevant Memories
|
|
2462
3408
|
|
|
2463
3409
|
${parts.join("\n\n---\n\n")}`;
|
|
2464
3410
|
}
|
|
2465
|
-
/**
|
|
2466
|
-
* Format a single memory for context
|
|
2467
|
-
*/
|
|
2468
3411
|
formatMemory(memory) {
|
|
2469
3412
|
const { event, score, sessionContext } = memory;
|
|
2470
3413
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2471
3414
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2472
3415
|
${event.content}`;
|
|
2473
|
-
if (sessionContext)
|
|
3416
|
+
if (sessionContext)
|
|
2474
3417
|
text += `
|
|
2475
3418
|
|
|
2476
3419
|
_Context:_ ${sessionContext}`;
|
|
2477
|
-
}
|
|
2478
3420
|
return text;
|
|
2479
3421
|
}
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
3422
|
+
matchesMetadataScope(metadata, expected) {
|
|
3423
|
+
if (!metadata)
|
|
3424
|
+
return false;
|
|
3425
|
+
return Object.entries(expected).every(([path4, value]) => {
|
|
3426
|
+
const actual = path4.split(".").reduce((acc, key) => {
|
|
3427
|
+
if (typeof acc !== "object" || acc === null)
|
|
3428
|
+
return void 0;
|
|
3429
|
+
return acc[key];
|
|
3430
|
+
}, metadata);
|
|
3431
|
+
return actual === value;
|
|
3432
|
+
});
|
|
3433
|
+
}
|
|
3434
|
+
tokenize(text) {
|
|
3435
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3436
|
+
}
|
|
3437
|
+
keywordOverlap(a, b) {
|
|
3438
|
+
if (a.length === 0 || b.length === 0)
|
|
3439
|
+
return 0;
|
|
3440
|
+
const bs = new Set(b);
|
|
3441
|
+
let hit = 0;
|
|
3442
|
+
for (const t of a)
|
|
3443
|
+
if (bs.has(t))
|
|
3444
|
+
hit += 1;
|
|
3445
|
+
return hit / a.length;
|
|
3446
|
+
}
|
|
2483
3447
|
estimateTokens(text) {
|
|
2484
3448
|
return Math.ceil(text.length / 4);
|
|
2485
3449
|
}
|
|
2486
|
-
/**
|
|
2487
|
-
* Get event age in days (for recency scoring)
|
|
2488
|
-
*/
|
|
2489
|
-
getEventAgeDays(eventId) {
|
|
2490
|
-
return 0;
|
|
2491
|
-
}
|
|
2492
3450
|
};
|
|
2493
3451
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2494
3452
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -3778,6 +4736,59 @@ var ConsolidatedStore = class {
|
|
|
3778
4736
|
[memoryId]
|
|
3779
4737
|
);
|
|
3780
4738
|
}
|
|
4739
|
+
/**
|
|
4740
|
+
* Create a long-term rule promoted from stable summaries
|
|
4741
|
+
*/
|
|
4742
|
+
async createRule(input) {
|
|
4743
|
+
const ruleId = randomUUID6();
|
|
4744
|
+
await dbRun(
|
|
4745
|
+
this.db,
|
|
4746
|
+
`INSERT INTO consolidated_rules
|
|
4747
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4748
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4749
|
+
[
|
|
4750
|
+
ruleId,
|
|
4751
|
+
input.rule,
|
|
4752
|
+
JSON.stringify(input.topics),
|
|
4753
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4754
|
+
JSON.stringify(input.sourceEvents),
|
|
4755
|
+
input.confidence
|
|
4756
|
+
]
|
|
4757
|
+
);
|
|
4758
|
+
return ruleId;
|
|
4759
|
+
}
|
|
4760
|
+
async getRules(options) {
|
|
4761
|
+
const limit = options?.limit || 100;
|
|
4762
|
+
const rows = await dbAll(
|
|
4763
|
+
this.db,
|
|
4764
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4765
|
+
[limit]
|
|
4766
|
+
);
|
|
4767
|
+
return rows.map((row) => ({
|
|
4768
|
+
ruleId: row.rule_id,
|
|
4769
|
+
rule: row.rule,
|
|
4770
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4771
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4772
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4773
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4774
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4775
|
+
}));
|
|
4776
|
+
}
|
|
4777
|
+
async countRules() {
|
|
4778
|
+
const result = await dbAll(
|
|
4779
|
+
this.db,
|
|
4780
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4781
|
+
);
|
|
4782
|
+
return result[0]?.count || 0;
|
|
4783
|
+
}
|
|
4784
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4785
|
+
const rows = await dbAll(
|
|
4786
|
+
this.db,
|
|
4787
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4788
|
+
[`%"${memoryId}"%`]
|
|
4789
|
+
);
|
|
4790
|
+
return (rows[0]?.count || 0) > 0;
|
|
4791
|
+
}
|
|
3781
4792
|
/**
|
|
3782
4793
|
* Get count of consolidated memories
|
|
3783
4794
|
*/
|
|
@@ -3927,7 +4938,14 @@ var ConsolidationWorker = class {
|
|
|
3927
4938
|
* Force a consolidation run (manual trigger)
|
|
3928
4939
|
*/
|
|
3929
4940
|
async forceRun() {
|
|
3930
|
-
|
|
4941
|
+
const out = await this.consolidateWithReport();
|
|
4942
|
+
return out.consolidatedCount;
|
|
4943
|
+
}
|
|
4944
|
+
/**
|
|
4945
|
+
* Force a consolidation run and return metrics report
|
|
4946
|
+
*/
|
|
4947
|
+
async forceRunWithReport() {
|
|
4948
|
+
return this.consolidateWithReport();
|
|
3931
4949
|
}
|
|
3932
4950
|
/**
|
|
3933
4951
|
* Schedule the next consolidation check
|
|
@@ -3967,12 +4985,21 @@ var ConsolidationWorker = class {
|
|
|
3967
4985
|
* Perform consolidation
|
|
3968
4986
|
*/
|
|
3969
4987
|
async consolidate() {
|
|
4988
|
+
const out = await this.consolidateWithReport();
|
|
4989
|
+
return out.consolidatedCount;
|
|
4990
|
+
}
|
|
4991
|
+
async consolidateWithReport() {
|
|
3970
4992
|
const workingSet = await this.workingSetStore.get();
|
|
3971
4993
|
if (workingSet.recentEvents.length < 3) {
|
|
3972
|
-
return
|
|
4994
|
+
return {
|
|
4995
|
+
consolidatedCount: 0,
|
|
4996
|
+
promotedRuleCount: 0,
|
|
4997
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
4998
|
+
};
|
|
3973
4999
|
}
|
|
3974
5000
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
3975
5001
|
let consolidatedCount = 0;
|
|
5002
|
+
const createdMemoryIds = [];
|
|
3976
5003
|
for (const group of groups) {
|
|
3977
5004
|
if (group.events.length < 3)
|
|
3978
5005
|
continue;
|
|
@@ -3981,14 +5008,16 @@ var ConsolidationWorker = class {
|
|
|
3981
5008
|
if (alreadyConsolidated)
|
|
3982
5009
|
continue;
|
|
3983
5010
|
const summary = await this.summarize(group);
|
|
3984
|
-
await this.consolidatedStore.create({
|
|
5011
|
+
const memoryId = await this.consolidatedStore.create({
|
|
3985
5012
|
summary,
|
|
3986
5013
|
topics: group.topics,
|
|
3987
5014
|
sourceEvents: eventIds,
|
|
3988
5015
|
confidence: this.calculateConfidence(group)
|
|
3989
5016
|
});
|
|
5017
|
+
createdMemoryIds.push(memoryId);
|
|
3990
5018
|
consolidatedCount++;
|
|
3991
5019
|
}
|
|
5020
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
3992
5021
|
if (consolidatedCount > 0) {
|
|
3993
5022
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
3994
5023
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4002,7 +5031,61 @@ var ConsolidationWorker = class {
|
|
|
4002
5031
|
await this.workingSetStore.prune(oldEventIds);
|
|
4003
5032
|
}
|
|
4004
5033
|
}
|
|
4005
|
-
|
|
5034
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5035
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5036
|
+
}
|
|
5037
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5038
|
+
let promoted = 0;
|
|
5039
|
+
for (const memoryId of memoryIds) {
|
|
5040
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5041
|
+
if (!memory)
|
|
5042
|
+
continue;
|
|
5043
|
+
if (memory.confidence < 0.55)
|
|
5044
|
+
continue;
|
|
5045
|
+
if (memory.sourceEvents.length < 4)
|
|
5046
|
+
continue;
|
|
5047
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5048
|
+
if (exists)
|
|
5049
|
+
continue;
|
|
5050
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5051
|
+
if (!rule)
|
|
5052
|
+
continue;
|
|
5053
|
+
await this.consolidatedStore.createRule({
|
|
5054
|
+
rule,
|
|
5055
|
+
topics: memory.topics,
|
|
5056
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5057
|
+
sourceEvents: memory.sourceEvents,
|
|
5058
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5059
|
+
});
|
|
5060
|
+
promoted++;
|
|
5061
|
+
}
|
|
5062
|
+
return promoted;
|
|
5063
|
+
}
|
|
5064
|
+
buildRuleFromSummary(summary, topics) {
|
|
5065
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5066
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5067
|
+
const seed = bullet || lines[0];
|
|
5068
|
+
if (!seed || seed.length < 8)
|
|
5069
|
+
return null;
|
|
5070
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5071
|
+
return `${topicPrefix}${seed}`;
|
|
5072
|
+
}
|
|
5073
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5074
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5075
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5076
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5077
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5078
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5079
|
+
return {
|
|
5080
|
+
beforeTokenEstimate,
|
|
5081
|
+
afterTokenEstimate,
|
|
5082
|
+
reductionRatio,
|
|
5083
|
+
qualityGuardPassed,
|
|
5084
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5085
|
+
};
|
|
5086
|
+
}
|
|
5087
|
+
estimateTokens(text) {
|
|
5088
|
+
return Math.ceil((text || "").length / 4);
|
|
4006
5089
|
}
|
|
4007
5090
|
/**
|
|
4008
5091
|
* Check if consolidation should run
|
|
@@ -4562,13 +5645,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
4562
5645
|
);
|
|
4563
5646
|
}
|
|
4564
5647
|
|
|
5648
|
+
// src/core/md-mirror.ts
|
|
5649
|
+
import * as fs3 from "node:fs";
|
|
5650
|
+
import * as path2 from "node:path";
|
|
5651
|
+
function sanitizeSegment2(input, fallback) {
|
|
5652
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5653
|
+
return v || fallback;
|
|
5654
|
+
}
|
|
5655
|
+
function getAtPath(obj, dotted) {
|
|
5656
|
+
if (!obj)
|
|
5657
|
+
return void 0;
|
|
5658
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5659
|
+
if (!acc || typeof acc !== "object")
|
|
5660
|
+
return void 0;
|
|
5661
|
+
return acc[key];
|
|
5662
|
+
}, obj);
|
|
5663
|
+
}
|
|
5664
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5665
|
+
const meta = event.metadata;
|
|
5666
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5667
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5668
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5669
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5670
|
+
const d = event.timestamp;
|
|
5671
|
+
const yyyy = d.getFullYear();
|
|
5672
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5673
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5674
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5675
|
+
}
|
|
5676
|
+
var MarkdownMirror2 = class {
|
|
5677
|
+
constructor(rootDir) {
|
|
5678
|
+
this.rootDir = rootDir;
|
|
5679
|
+
}
|
|
5680
|
+
async append(event, eventId) {
|
|
5681
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5682
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5683
|
+
const lines = [
|
|
5684
|
+
"",
|
|
5685
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5686
|
+
`- type: ${event.eventType}`,
|
|
5687
|
+
`- session: ${event.sessionId}`,
|
|
5688
|
+
event.content
|
|
5689
|
+
];
|
|
5690
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5691
|
+
await this.refreshIndex();
|
|
5692
|
+
}
|
|
5693
|
+
async refreshIndex() {
|
|
5694
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5695
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5696
|
+
const files = [];
|
|
5697
|
+
await this.walk(memoryRoot, files);
|
|
5698
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5699
|
+
const index = [
|
|
5700
|
+
"# Memory Index",
|
|
5701
|
+
"",
|
|
5702
|
+
"Generated automatically by MarkdownMirror.",
|
|
5703
|
+
"",
|
|
5704
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5705
|
+
""
|
|
5706
|
+
].join("\n");
|
|
5707
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5708
|
+
}
|
|
5709
|
+
async walk(dir, out) {
|
|
5710
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5711
|
+
for (const e of entries) {
|
|
5712
|
+
const full = path2.join(dir, e.name);
|
|
5713
|
+
if (e.isDirectory()) {
|
|
5714
|
+
await this.walk(full, out);
|
|
5715
|
+
} else {
|
|
5716
|
+
out.push(full);
|
|
5717
|
+
}
|
|
5718
|
+
}
|
|
5719
|
+
}
|
|
5720
|
+
};
|
|
5721
|
+
|
|
5722
|
+
// src/core/ingest-interceptor.ts
|
|
5723
|
+
var IngestInterceptorRegistry = class {
|
|
5724
|
+
before = [];
|
|
5725
|
+
after = [];
|
|
5726
|
+
onError = [];
|
|
5727
|
+
registerBefore(interceptor) {
|
|
5728
|
+
this.before.push(interceptor);
|
|
5729
|
+
return () => {
|
|
5730
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5731
|
+
};
|
|
5732
|
+
}
|
|
5733
|
+
registerAfter(interceptor) {
|
|
5734
|
+
this.after.push(interceptor);
|
|
5735
|
+
return () => {
|
|
5736
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5737
|
+
};
|
|
5738
|
+
}
|
|
5739
|
+
registerOnError(interceptor) {
|
|
5740
|
+
this.onError.push(interceptor);
|
|
5741
|
+
return () => {
|
|
5742
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5743
|
+
};
|
|
5744
|
+
}
|
|
5745
|
+
async run(stage, context) {
|
|
5746
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5747
|
+
for (const interceptor of interceptors) {
|
|
5748
|
+
await interceptor({ ...context, stage });
|
|
5749
|
+
}
|
|
5750
|
+
}
|
|
5751
|
+
};
|
|
5752
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5753
|
+
if (!base && !patch)
|
|
5754
|
+
return void 0;
|
|
5755
|
+
if (!base)
|
|
5756
|
+
return patch;
|
|
5757
|
+
if (!patch)
|
|
5758
|
+
return base;
|
|
5759
|
+
const result = { ...base };
|
|
5760
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5761
|
+
const current = result[key];
|
|
5762
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5763
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5764
|
+
current,
|
|
5765
|
+
value
|
|
5766
|
+
);
|
|
5767
|
+
} else {
|
|
5768
|
+
result[key] = value;
|
|
5769
|
+
}
|
|
5770
|
+
}
|
|
5771
|
+
return result;
|
|
5772
|
+
}
|
|
5773
|
+
|
|
5774
|
+
// src/core/tag-taxonomy.ts
|
|
5775
|
+
var TAG_NAMESPACES = {
|
|
5776
|
+
SYSTEM: "sys:",
|
|
5777
|
+
QUALITY: "q:",
|
|
5778
|
+
PROJECT: "proj:",
|
|
5779
|
+
TOPIC: "topic:",
|
|
5780
|
+
TEMPORAL: "t:",
|
|
5781
|
+
USER: "user:",
|
|
5782
|
+
AGENT: "agent:"
|
|
5783
|
+
};
|
|
5784
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5785
|
+
function parseTag(tag) {
|
|
5786
|
+
const value = (tag || "").trim();
|
|
5787
|
+
const idx = value.indexOf(":");
|
|
5788
|
+
if (idx <= 0)
|
|
5789
|
+
return { value };
|
|
5790
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5791
|
+
const tagValue = value.slice(idx + 1);
|
|
5792
|
+
if (!tagValue)
|
|
5793
|
+
return { value };
|
|
5794
|
+
return { namespace, value: tagValue };
|
|
5795
|
+
}
|
|
5796
|
+
function validateTag(tag) {
|
|
5797
|
+
const normalized = (tag || "").trim();
|
|
5798
|
+
if (!normalized)
|
|
5799
|
+
return false;
|
|
5800
|
+
const { namespace } = parseTag(normalized);
|
|
5801
|
+
if (!namespace)
|
|
5802
|
+
return true;
|
|
5803
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5804
|
+
}
|
|
5805
|
+
function normalizeTags(tags) {
|
|
5806
|
+
if (!Array.isArray(tags))
|
|
5807
|
+
return [];
|
|
5808
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5809
|
+
for (const item of tags) {
|
|
5810
|
+
if (typeof item !== "string")
|
|
5811
|
+
continue;
|
|
5812
|
+
const normalized = item.trim();
|
|
5813
|
+
if (!validateTag(normalized))
|
|
5814
|
+
continue;
|
|
5815
|
+
dedup.add(normalized);
|
|
5816
|
+
}
|
|
5817
|
+
return [...dedup];
|
|
5818
|
+
}
|
|
5819
|
+
|
|
4565
5820
|
// src/services/memory-service.ts
|
|
4566
5821
|
function normalizePath(projectPath) {
|
|
4567
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5822
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
4568
5823
|
try {
|
|
4569
|
-
return
|
|
5824
|
+
return fs4.realpathSync(expanded);
|
|
4570
5825
|
} catch {
|
|
4571
|
-
return
|
|
5826
|
+
return path3.resolve(expanded);
|
|
4572
5827
|
}
|
|
4573
5828
|
}
|
|
4574
5829
|
function hashProjectPath(projectPath) {
|
|
@@ -4577,14 +5832,14 @@ function hashProjectPath(projectPath) {
|
|
|
4577
5832
|
}
|
|
4578
5833
|
function getProjectStoragePath(projectPath) {
|
|
4579
5834
|
const hash = hashProjectPath(projectPath);
|
|
4580
|
-
return
|
|
5835
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
4581
5836
|
}
|
|
4582
|
-
var REGISTRY_PATH =
|
|
4583
|
-
var SHARED_STORAGE_PATH =
|
|
5837
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5838
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
4584
5839
|
function loadSessionRegistry() {
|
|
4585
5840
|
try {
|
|
4586
|
-
if (
|
|
4587
|
-
const data =
|
|
5841
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5842
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
4588
5843
|
return JSON.parse(data);
|
|
4589
5844
|
}
|
|
4590
5845
|
} catch (error) {
|
|
@@ -4593,13 +5848,13 @@ function loadSessionRegistry() {
|
|
|
4593
5848
|
return { version: 1, sessions: {} };
|
|
4594
5849
|
}
|
|
4595
5850
|
function saveSessionRegistry(registry) {
|
|
4596
|
-
const dir =
|
|
4597
|
-
if (!
|
|
4598
|
-
|
|
5851
|
+
const dir = path3.dirname(REGISTRY_PATH);
|
|
5852
|
+
if (!fs4.existsSync(dir)) {
|
|
5853
|
+
fs4.mkdirSync(dir, { recursive: true });
|
|
4599
5854
|
}
|
|
4600
5855
|
const tempPath = REGISTRY_PATH + ".tmp";
|
|
4601
|
-
|
|
4602
|
-
|
|
5856
|
+
fs4.writeFileSync(tempPath, JSON.stringify(registry, null, 2));
|
|
5857
|
+
fs4.renameSync(tempPath, REGISTRY_PATH);
|
|
4603
5858
|
}
|
|
4604
5859
|
function registerSession(sessionId, projectPath) {
|
|
4605
5860
|
const registry = loadSessionRegistry();
|
|
@@ -4635,6 +5890,7 @@ var MemoryService = class {
|
|
|
4635
5890
|
vectorWorker = null;
|
|
4636
5891
|
graduationWorker = null;
|
|
4637
5892
|
initialized = false;
|
|
5893
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
4638
5894
|
// Endless Mode components
|
|
4639
5895
|
workingSetStore = null;
|
|
4640
5896
|
consolidatedStore = null;
|
|
@@ -4648,20 +5904,27 @@ var MemoryService = class {
|
|
|
4648
5904
|
sharedPromoter = null;
|
|
4649
5905
|
sharedStoreConfig = null;
|
|
4650
5906
|
projectHash = null;
|
|
5907
|
+
projectPath = null;
|
|
4651
5908
|
readOnly;
|
|
4652
5909
|
lightweightMode;
|
|
5910
|
+
mdMirror;
|
|
4653
5911
|
constructor(config) {
|
|
4654
5912
|
const storagePath = this.expandPath(config.storagePath);
|
|
4655
5913
|
this.readOnly = config.readOnly ?? false;
|
|
4656
5914
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
4657
|
-
|
|
4658
|
-
|
|
5915
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
5916
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
5917
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
4659
5918
|
}
|
|
4660
5919
|
this.projectHash = config.projectHash || null;
|
|
5920
|
+
this.projectPath = config.projectPath || null;
|
|
4661
5921
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
4662
5922
|
this.sqliteStore = new SQLiteEventStore(
|
|
4663
|
-
|
|
4664
|
-
{
|
|
5923
|
+
path3.join(storagePath, "events.sqlite"),
|
|
5924
|
+
{
|
|
5925
|
+
readonly: this.readOnly,
|
|
5926
|
+
markdownMirrorRoot: storagePath
|
|
5927
|
+
}
|
|
4665
5928
|
);
|
|
4666
5929
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
4667
5930
|
if (!analyticsEnabled) {
|
|
@@ -4669,7 +5932,7 @@ var MemoryService = class {
|
|
|
4669
5932
|
} else if (this.readOnly) {
|
|
4670
5933
|
try {
|
|
4671
5934
|
this.analyticsStore = new EventStore(
|
|
4672
|
-
|
|
5935
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
4673
5936
|
{ readOnly: true }
|
|
4674
5937
|
);
|
|
4675
5938
|
} catch {
|
|
@@ -4677,11 +5940,11 @@ var MemoryService = class {
|
|
|
4677
5940
|
}
|
|
4678
5941
|
} else {
|
|
4679
5942
|
this.analyticsStore = new EventStore(
|
|
4680
|
-
|
|
5943
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
4681
5944
|
{ readOnly: false }
|
|
4682
5945
|
);
|
|
4683
5946
|
}
|
|
4684
|
-
this.vectorStore = new VectorStore(
|
|
5947
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
4685
5948
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
4686
5949
|
this.matcher = getDefaultMatcher();
|
|
4687
5950
|
this.retriever = createRetriever(
|
|
@@ -4691,6 +5954,7 @@ var MemoryService = class {
|
|
|
4691
5954
|
this.embedder,
|
|
4692
5955
|
this.matcher
|
|
4693
5956
|
);
|
|
5957
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
4694
5958
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
4695
5959
|
}
|
|
4696
5960
|
/**
|
|
@@ -4750,16 +6014,16 @@ var MemoryService = class {
|
|
|
4750
6014
|
*/
|
|
4751
6015
|
async initializeSharedStore() {
|
|
4752
6016
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
4753
|
-
if (!
|
|
4754
|
-
|
|
6017
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
6018
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
4755
6019
|
}
|
|
4756
6020
|
this.sharedEventStore = createSharedEventStore(
|
|
4757
|
-
|
|
6021
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
4758
6022
|
);
|
|
4759
6023
|
await this.sharedEventStore.initialize();
|
|
4760
6024
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
4761
6025
|
this.sharedVectorStore = createSharedVectorStore(
|
|
4762
|
-
|
|
6026
|
+
path3.join(sharedPath, "vectors")
|
|
4763
6027
|
);
|
|
4764
6028
|
await this.sharedVectorStore.initialize();
|
|
4765
6029
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -4770,6 +6034,86 @@ var MemoryService = class {
|
|
|
4770
6034
|
);
|
|
4771
6035
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
4772
6036
|
}
|
|
6037
|
+
registerIngestBefore(interceptor) {
|
|
6038
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6039
|
+
}
|
|
6040
|
+
registerIngestAfter(interceptor) {
|
|
6041
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6042
|
+
}
|
|
6043
|
+
registerIngestOnError(interceptor) {
|
|
6044
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6045
|
+
}
|
|
6046
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6047
|
+
const normalizedInput = {
|
|
6048
|
+
...input,
|
|
6049
|
+
metadata: mergeHierarchicalMetadata(
|
|
6050
|
+
{
|
|
6051
|
+
ingest: {
|
|
6052
|
+
operation,
|
|
6053
|
+
pipeline: "default",
|
|
6054
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6055
|
+
},
|
|
6056
|
+
...this.projectHash ? {
|
|
6057
|
+
scope: {
|
|
6058
|
+
project: {
|
|
6059
|
+
hash: this.projectHash,
|
|
6060
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6061
|
+
}
|
|
6062
|
+
},
|
|
6063
|
+
tags: [`proj:${this.projectHash}`]
|
|
6064
|
+
} : {}
|
|
6065
|
+
},
|
|
6066
|
+
input.metadata
|
|
6067
|
+
)
|
|
6068
|
+
};
|
|
6069
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6070
|
+
const meta = normalizedInput.metadata;
|
|
6071
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6072
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6073
|
+
if (!currentTags.includes(projectTag)) {
|
|
6074
|
+
meta.tags = [...currentTags, projectTag];
|
|
6075
|
+
}
|
|
6076
|
+
}
|
|
6077
|
+
if (normalizedInput.metadata) {
|
|
6078
|
+
const meta = normalizedInput.metadata;
|
|
6079
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6080
|
+
if (normalizedTags.length > 0) {
|
|
6081
|
+
meta.tags = normalizedTags;
|
|
6082
|
+
}
|
|
6083
|
+
}
|
|
6084
|
+
await this.ingestInterceptors.run("before", {
|
|
6085
|
+
operation,
|
|
6086
|
+
sessionId: normalizedInput.sessionId,
|
|
6087
|
+
event: normalizedInput
|
|
6088
|
+
});
|
|
6089
|
+
try {
|
|
6090
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6091
|
+
if (result.success && !result.isDuplicate) {
|
|
6092
|
+
if (onSuccess) {
|
|
6093
|
+
await onSuccess(result.eventId);
|
|
6094
|
+
}
|
|
6095
|
+
try {
|
|
6096
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6097
|
+
} catch {
|
|
6098
|
+
}
|
|
6099
|
+
}
|
|
6100
|
+
await this.ingestInterceptors.run("after", {
|
|
6101
|
+
operation,
|
|
6102
|
+
sessionId: normalizedInput.sessionId,
|
|
6103
|
+
event: normalizedInput
|
|
6104
|
+
});
|
|
6105
|
+
return result;
|
|
6106
|
+
} catch (error) {
|
|
6107
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6108
|
+
await this.ingestInterceptors.run("error", {
|
|
6109
|
+
operation,
|
|
6110
|
+
sessionId: normalizedInput.sessionId,
|
|
6111
|
+
event: normalizedInput,
|
|
6112
|
+
error: normalizedError
|
|
6113
|
+
});
|
|
6114
|
+
throw error;
|
|
6115
|
+
}
|
|
6116
|
+
}
|
|
4773
6117
|
/**
|
|
4774
6118
|
* Start a new session
|
|
4775
6119
|
*/
|
|
@@ -4797,50 +6141,57 @@ var MemoryService = class {
|
|
|
4797
6141
|
*/
|
|
4798
6142
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
4799
6143
|
await this.initialize();
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4810
|
-
|
|
6144
|
+
return this.ingestWithInterceptors(
|
|
6145
|
+
"user_prompt",
|
|
6146
|
+
{
|
|
6147
|
+
eventType: "user_prompt",
|
|
6148
|
+
sessionId,
|
|
6149
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6150
|
+
content,
|
|
6151
|
+
metadata
|
|
6152
|
+
},
|
|
6153
|
+
async (eventId) => {
|
|
6154
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6155
|
+
}
|
|
6156
|
+
);
|
|
4811
6157
|
}
|
|
4812
6158
|
/**
|
|
4813
6159
|
* Store an agent response
|
|
4814
6160
|
*/
|
|
4815
6161
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
4816
6162
|
await this.initialize();
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
|
|
6163
|
+
return this.ingestWithInterceptors(
|
|
6164
|
+
"agent_response",
|
|
6165
|
+
{
|
|
6166
|
+
eventType: "agent_response",
|
|
6167
|
+
sessionId,
|
|
6168
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6169
|
+
content,
|
|
6170
|
+
metadata
|
|
6171
|
+
},
|
|
6172
|
+
async (eventId) => {
|
|
6173
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6174
|
+
}
|
|
6175
|
+
);
|
|
4828
6176
|
}
|
|
4829
6177
|
/**
|
|
4830
6178
|
* Store a session summary
|
|
4831
6179
|
*/
|
|
4832
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6180
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
4833
6181
|
await this.initialize();
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
|
|
4838
|
-
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
|
|
4842
|
-
|
|
4843
|
-
|
|
6182
|
+
return this.ingestWithInterceptors(
|
|
6183
|
+
"session_summary",
|
|
6184
|
+
{
|
|
6185
|
+
eventType: "session_summary",
|
|
6186
|
+
sessionId,
|
|
6187
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6188
|
+
content: summary,
|
|
6189
|
+
metadata
|
|
6190
|
+
},
|
|
6191
|
+
async (eventId) => {
|
|
6192
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6193
|
+
}
|
|
6194
|
+
);
|
|
4844
6195
|
}
|
|
4845
6196
|
/**
|
|
4846
6197
|
* Store a tool observation
|
|
@@ -4848,39 +6199,182 @@ var MemoryService = class {
|
|
|
4848
6199
|
async storeToolObservation(sessionId, payload) {
|
|
4849
6200
|
await this.initialize();
|
|
4850
6201
|
const content = JSON.stringify(payload);
|
|
4851
|
-
const
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
|
|
4857
|
-
|
|
4858
|
-
|
|
6202
|
+
const turnId = payload.metadata?.turnId;
|
|
6203
|
+
return this.ingestWithInterceptors(
|
|
6204
|
+
"tool_observation",
|
|
6205
|
+
{
|
|
6206
|
+
eventType: "tool_observation",
|
|
6207
|
+
sessionId,
|
|
6208
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6209
|
+
content,
|
|
6210
|
+
metadata: {
|
|
6211
|
+
toolName: payload.toolName,
|
|
6212
|
+
success: payload.success,
|
|
6213
|
+
...turnId ? { turnId } : {}
|
|
6214
|
+
}
|
|
6215
|
+
},
|
|
6216
|
+
async (eventId) => {
|
|
6217
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6218
|
+
payload.toolName,
|
|
6219
|
+
payload.metadata || {},
|
|
6220
|
+
payload.success
|
|
6221
|
+
);
|
|
6222
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
4859
6223
|
}
|
|
4860
|
-
|
|
4861
|
-
if (result.success && !result.isDuplicate) {
|
|
4862
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
4863
|
-
payload.toolName,
|
|
4864
|
-
payload.metadata || {},
|
|
4865
|
-
payload.success
|
|
4866
|
-
);
|
|
4867
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
4868
|
-
}
|
|
4869
|
-
return result;
|
|
6224
|
+
);
|
|
4870
6225
|
}
|
|
4871
6226
|
/**
|
|
4872
6227
|
* Retrieve relevant memories for a query
|
|
4873
6228
|
*/
|
|
4874
6229
|
async retrieveMemories(query, options) {
|
|
4875
6230
|
await this.initialize();
|
|
6231
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6232
|
+
let result;
|
|
4876
6233
|
if (options?.includeShared && this.sharedStore) {
|
|
4877
|
-
|
|
6234
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
4878
6235
|
...options,
|
|
6236
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6237
|
+
rerankWeights,
|
|
4879
6238
|
includeShared: true,
|
|
4880
|
-
projectHash: this.projectHash || void 0
|
|
6239
|
+
projectHash: this.projectHash || void 0,
|
|
6240
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6241
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6242
|
+
});
|
|
6243
|
+
} else {
|
|
6244
|
+
result = await this.retriever.retrieve(query, {
|
|
6245
|
+
...options,
|
|
6246
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6247
|
+
rerankWeights,
|
|
6248
|
+
projectHash: this.projectHash || void 0,
|
|
6249
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6250
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6251
|
+
});
|
|
6252
|
+
}
|
|
6253
|
+
try {
|
|
6254
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6255
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6256
|
+
eventId: d.eventId,
|
|
6257
|
+
score: d.score,
|
|
6258
|
+
semanticScore: d.semanticScore,
|
|
6259
|
+
lexicalScore: d.lexicalScore,
|
|
6260
|
+
recencyScore: d.recencyScore
|
|
6261
|
+
}));
|
|
6262
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6263
|
+
eventId: d.eventId,
|
|
6264
|
+
score: d.score,
|
|
6265
|
+
semanticScore: d.semanticScore,
|
|
6266
|
+
lexicalScore: d.lexicalScore,
|
|
6267
|
+
recencyScore: d.recencyScore
|
|
6268
|
+
}));
|
|
6269
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6270
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6271
|
+
sessionId: options?.sessionId,
|
|
6272
|
+
projectHash: this.projectHash || void 0,
|
|
6273
|
+
queryText: query,
|
|
6274
|
+
strategy: options?.strategy || "auto",
|
|
6275
|
+
candidateEventIds,
|
|
6276
|
+
selectedEventIds,
|
|
6277
|
+
candidateDetails,
|
|
6278
|
+
selectedDetails,
|
|
6279
|
+
confidence: result.matchResult.confidence,
|
|
6280
|
+
fallbackTrace: result.fallbackTrace || []
|
|
6281
|
+
});
|
|
6282
|
+
} catch {
|
|
6283
|
+
}
|
|
6284
|
+
return result;
|
|
6285
|
+
}
|
|
6286
|
+
getConfiguredRerankWeights() {
|
|
6287
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6288
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6289
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6290
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6291
|
+
if (!allFinite)
|
|
6292
|
+
return void 0;
|
|
6293
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6294
|
+
const total = semantic + lexical + recency;
|
|
6295
|
+
if (!nonNegative || total <= 0)
|
|
6296
|
+
return void 0;
|
|
6297
|
+
return {
|
|
6298
|
+
semantic: semantic / total,
|
|
6299
|
+
lexical: lexical / total,
|
|
6300
|
+
recency: recency / total
|
|
6301
|
+
};
|
|
6302
|
+
}
|
|
6303
|
+
async getRerankWeights(adaptive) {
|
|
6304
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6305
|
+
if (configured)
|
|
6306
|
+
return configured;
|
|
6307
|
+
if (adaptive)
|
|
6308
|
+
return this.getAdaptiveRerankWeights();
|
|
6309
|
+
return void 0;
|
|
6310
|
+
}
|
|
6311
|
+
async rewriteQueryIntent(query) {
|
|
6312
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6313
|
+
return null;
|
|
6314
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6315
|
+
if (!apiUrl)
|
|
6316
|
+
return null;
|
|
6317
|
+
const controller = new AbortController();
|
|
6318
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6319
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6320
|
+
try {
|
|
6321
|
+
const prompt = [
|
|
6322
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6323
|
+
"Return plain text only, one line, no markdown.",
|
|
6324
|
+
`Query: ${query}`
|
|
6325
|
+
].join("\n");
|
|
6326
|
+
const res = await fetch(apiUrl, {
|
|
6327
|
+
method: "POST",
|
|
6328
|
+
headers: {
|
|
6329
|
+
"Content-Type": "application/json",
|
|
6330
|
+
Accept: "*/*",
|
|
6331
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6332
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6333
|
+
},
|
|
6334
|
+
body: JSON.stringify({
|
|
6335
|
+
question: prompt,
|
|
6336
|
+
company_name: null,
|
|
6337
|
+
conversation_id: null
|
|
6338
|
+
}),
|
|
6339
|
+
signal: controller.signal
|
|
4881
6340
|
});
|
|
6341
|
+
const text = (await res.text()).trim();
|
|
6342
|
+
if (!text)
|
|
6343
|
+
return null;
|
|
6344
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6345
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6346
|
+
return null;
|
|
6347
|
+
return oneLine;
|
|
6348
|
+
} catch {
|
|
6349
|
+
return null;
|
|
6350
|
+
} finally {
|
|
6351
|
+
clearTimeout(timeout);
|
|
6352
|
+
}
|
|
6353
|
+
}
|
|
6354
|
+
async getAdaptiveRerankWeights() {
|
|
6355
|
+
try {
|
|
6356
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6357
|
+
if (s.totalEvaluated < 20)
|
|
6358
|
+
return void 0;
|
|
6359
|
+
let semantic = 0.7;
|
|
6360
|
+
let lexical = 0.2;
|
|
6361
|
+
let recency = 0.1;
|
|
6362
|
+
if (s.avgScore < 0.45) {
|
|
6363
|
+
semantic -= 0.1;
|
|
6364
|
+
lexical += 0.1;
|
|
6365
|
+
} else if (s.avgScore > 0.75) {
|
|
6366
|
+
semantic += 0.05;
|
|
6367
|
+
lexical -= 0.05;
|
|
6368
|
+
}
|
|
6369
|
+
if (s.unhelpful > s.helpful) {
|
|
6370
|
+
recency += 0.05;
|
|
6371
|
+
semantic -= 0.03;
|
|
6372
|
+
lexical -= 0.02;
|
|
6373
|
+
}
|
|
6374
|
+
return { semantic, lexical, recency };
|
|
6375
|
+
} catch {
|
|
6376
|
+
return void 0;
|
|
4882
6377
|
}
|
|
4883
|
-
return this.retriever.retrieve(query, options);
|
|
4884
6378
|
}
|
|
4885
6379
|
/**
|
|
4886
6380
|
* Fast keyword search using SQLite FTS5
|
|
@@ -4922,6 +6416,18 @@ var MemoryService = class {
|
|
|
4922
6416
|
/**
|
|
4923
6417
|
* Get memory statistics
|
|
4924
6418
|
*/
|
|
6419
|
+
async getOutboxStats() {
|
|
6420
|
+
await this.initialize();
|
|
6421
|
+
return this.sqliteStore.getOutboxStats();
|
|
6422
|
+
}
|
|
6423
|
+
async getRetrievalTraceStats() {
|
|
6424
|
+
await this.initialize();
|
|
6425
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6426
|
+
}
|
|
6427
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6428
|
+
await this.initialize();
|
|
6429
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6430
|
+
}
|
|
4925
6431
|
async getStats() {
|
|
4926
6432
|
await this.initialize();
|
|
4927
6433
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5138,6 +6644,31 @@ var MemoryService = class {
|
|
|
5138
6644
|
return [];
|
|
5139
6645
|
return this.consolidatedStore.getAll({ limit });
|
|
5140
6646
|
}
|
|
6647
|
+
/**
|
|
6648
|
+
* Extract topic keywords from event content (markdown headings and key terms)
|
|
6649
|
+
*/
|
|
6650
|
+
extractTopicsFromContent(content) {
|
|
6651
|
+
const topics = /* @__PURE__ */ new Set();
|
|
6652
|
+
const headings = content.match(/^#{1,3}\s+(.+)$/gm);
|
|
6653
|
+
if (headings) {
|
|
6654
|
+
for (const h of headings.slice(0, 5)) {
|
|
6655
|
+
const text = h.replace(/^#+\s+/, "").replace(/[*_`#]/g, "").trim();
|
|
6656
|
+
if (text.length > 2 && text.length < 50) {
|
|
6657
|
+
topics.add(text);
|
|
6658
|
+
}
|
|
6659
|
+
}
|
|
6660
|
+
}
|
|
6661
|
+
const boldTerms = content.match(/\*\*([^*]+)\*\*/g);
|
|
6662
|
+
if (boldTerms) {
|
|
6663
|
+
for (const b of boldTerms.slice(0, 5)) {
|
|
6664
|
+
const text = b.replace(/\*\*/g, "").trim();
|
|
6665
|
+
if (text.length > 2 && text.length < 30) {
|
|
6666
|
+
topics.add(text);
|
|
6667
|
+
}
|
|
6668
|
+
}
|
|
6669
|
+
}
|
|
6670
|
+
return Array.from(topics).slice(0, 5);
|
|
6671
|
+
}
|
|
5141
6672
|
/**
|
|
5142
6673
|
* Increment access count for memories that were used in prompts
|
|
5143
6674
|
*/
|
|
@@ -5161,8 +6692,7 @@ var MemoryService = class {
|
|
|
5161
6692
|
return events.map((event) => ({
|
|
5162
6693
|
memoryId: event.id,
|
|
5163
6694
|
summary: event.content.substring(0, 200) + (event.content.length > 200 ? "..." : ""),
|
|
5164
|
-
topics:
|
|
5165
|
-
// Could extract topics from content if needed
|
|
6695
|
+
topics: this.extractTopicsFromContent(event.content),
|
|
5166
6696
|
accessCount: event.access_count || 0,
|
|
5167
6697
|
lastAccessed: event.last_accessed_at || null,
|
|
5168
6698
|
confidence: 1,
|
|
@@ -5183,6 +6713,34 @@ var MemoryService = class {
|
|
|
5183
6713
|
}
|
|
5184
6714
|
return [];
|
|
5185
6715
|
}
|
|
6716
|
+
/**
|
|
6717
|
+
* Record a memory retrieval for helpfulness tracking
|
|
6718
|
+
*/
|
|
6719
|
+
async recordRetrieval(eventId, sessionId, score, query) {
|
|
6720
|
+
await this.initialize();
|
|
6721
|
+
await this.sqliteStore.recordRetrieval(eventId, sessionId, score, query);
|
|
6722
|
+
}
|
|
6723
|
+
/**
|
|
6724
|
+
* Evaluate helpfulness of retrievals in a session (called at session end)
|
|
6725
|
+
*/
|
|
6726
|
+
async evaluateSessionHelpfulness(sessionId) {
|
|
6727
|
+
await this.initialize();
|
|
6728
|
+
await this.sqliteStore.evaluateSessionHelpfulness(sessionId);
|
|
6729
|
+
}
|
|
6730
|
+
/**
|
|
6731
|
+
* Get most helpful memories ranked by helpfulness score
|
|
6732
|
+
*/
|
|
6733
|
+
async getHelpfulMemories(limit = 10) {
|
|
6734
|
+
await this.initialize();
|
|
6735
|
+
return this.sqliteStore.getHelpfulMemories(limit);
|
|
6736
|
+
}
|
|
6737
|
+
/**
|
|
6738
|
+
* Get helpfulness statistics for dashboard
|
|
6739
|
+
*/
|
|
6740
|
+
async getHelpfulnessStats() {
|
|
6741
|
+
await this.initialize();
|
|
6742
|
+
return this.sqliteStore.getHelpfulnessStats();
|
|
6743
|
+
}
|
|
5186
6744
|
/**
|
|
5187
6745
|
* Mark a consolidated memory as accessed
|
|
5188
6746
|
*/
|
|
@@ -5246,6 +6804,44 @@ var MemoryService = class {
|
|
|
5246
6804
|
lastConsolidation
|
|
5247
6805
|
};
|
|
5248
6806
|
}
|
|
6807
|
+
// ============================================================
|
|
6808
|
+
// Turn Grouping Methods
|
|
6809
|
+
// ============================================================
|
|
6810
|
+
/**
|
|
6811
|
+
* Get events grouped by turn for a session
|
|
6812
|
+
*/
|
|
6813
|
+
async getSessionTurns(sessionId, options) {
|
|
6814
|
+
await this.initialize();
|
|
6815
|
+
return this.sqliteStore.getSessionTurns(sessionId, options);
|
|
6816
|
+
}
|
|
6817
|
+
/**
|
|
6818
|
+
* Get all events for a specific turn
|
|
6819
|
+
*/
|
|
6820
|
+
async getEventsByTurn(turnId) {
|
|
6821
|
+
await this.initialize();
|
|
6822
|
+
return this.sqliteStore.getEventsByTurn(turnId);
|
|
6823
|
+
}
|
|
6824
|
+
/**
|
|
6825
|
+
* Count total turns for a session
|
|
6826
|
+
*/
|
|
6827
|
+
async countSessionTurns(sessionId) {
|
|
6828
|
+
await this.initialize();
|
|
6829
|
+
return this.sqliteStore.countSessionTurns(sessionId);
|
|
6830
|
+
}
|
|
6831
|
+
/**
|
|
6832
|
+
* Backfill turn_ids from metadata for events stored before the migration
|
|
6833
|
+
*/
|
|
6834
|
+
async backfillTurnIds() {
|
|
6835
|
+
await this.initialize();
|
|
6836
|
+
return this.sqliteStore.backfillTurnIds();
|
|
6837
|
+
}
|
|
6838
|
+
/**
|
|
6839
|
+
* Delete all events for a session (for force reimport)
|
|
6840
|
+
*/
|
|
6841
|
+
async deleteSessionEvents(sessionId) {
|
|
6842
|
+
await this.initialize();
|
|
6843
|
+
return this.sqliteStore.deleteSessionEvents(sessionId);
|
|
6844
|
+
}
|
|
5249
6845
|
/**
|
|
5250
6846
|
* Format Endless Mode context for Claude
|
|
5251
6847
|
*/
|
|
@@ -5322,7 +6918,7 @@ var MemoryService = class {
|
|
|
5322
6918
|
*/
|
|
5323
6919
|
expandPath(p) {
|
|
5324
6920
|
if (p.startsWith("~")) {
|
|
5325
|
-
return
|
|
6921
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5326
6922
|
}
|
|
5327
6923
|
return p;
|
|
5328
6924
|
}
|
|
@@ -5358,6 +6954,7 @@ function getMemoryServiceForProject(projectPath, sharedStoreConfig) {
|
|
|
5358
6954
|
serviceCache.set(hash, new MemoryService({
|
|
5359
6955
|
storagePath,
|
|
5360
6956
|
projectHash: hash,
|
|
6957
|
+
projectPath,
|
|
5361
6958
|
// Override shared store config - hooks don't need DuckDB
|
|
5362
6959
|
sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
|
|
5363
6960
|
analyticsEnabled: false
|
|
@@ -5377,10 +6974,11 @@ function getLightweightMemoryService(sessionId) {
|
|
|
5377
6974
|
const projectInfo = getSessionProject(sessionId);
|
|
5378
6975
|
const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : "lightweight_global";
|
|
5379
6976
|
if (!serviceCache.has(key)) {
|
|
5380
|
-
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) :
|
|
6977
|
+
const storagePath = projectInfo ? getProjectStoragePath(projectInfo.projectPath) : path3.join(os.homedir(), ".claude-code", "memory");
|
|
5381
6978
|
serviceCache.set(key, new MemoryService({
|
|
5382
6979
|
storagePath,
|
|
5383
6980
|
projectHash: projectInfo?.projectHash,
|
|
6981
|
+
projectPath: projectInfo?.projectPath,
|
|
5384
6982
|
lightweightMode: true,
|
|
5385
6983
|
// Skip embedder/vector/workers
|
|
5386
6984
|
analyticsEnabled: false,
|
|
@@ -5403,6 +7001,7 @@ export {
|
|
|
5403
7001
|
getReadOnlyMemoryService,
|
|
5404
7002
|
getSessionProject,
|
|
5405
7003
|
hashProjectPath,
|
|
7004
|
+
loadSessionRegistry,
|
|
5406
7005
|
registerSession
|
|
5407
7006
|
};
|
|
5408
7007
|
//# sourceMappingURL=memory-service.js.map
|