claude-memory-layer 1.0.11 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +60 -0
- package/README.md +166 -2
- package/bootstrap-kb/decisions/decisions.md +244 -0
- package/bootstrap-kb/glossary/glossary.md +46 -0
- package/bootstrap-kb/modules/.claude-plugin.md +22 -0
- package/bootstrap-kb/modules/agents.md.md +15 -0
- package/bootstrap-kb/modules/claude.md.md +15 -0
- package/bootstrap-kb/modules/context.md.md +15 -0
- package/bootstrap-kb/modules/docs.md +18 -0
- package/bootstrap-kb/modules/handoff.md.md +15 -0
- package/bootstrap-kb/modules/package-lock.json.md +15 -0
- package/bootstrap-kb/modules/package.json.md +15 -0
- package/bootstrap-kb/modules/plan.md.md +15 -0
- package/bootstrap-kb/modules/readme.md.md +15 -0
- package/bootstrap-kb/modules/scripts.md +26 -0
- package/bootstrap-kb/modules/spec.md.md +15 -0
- package/bootstrap-kb/modules/specs.md +20 -0
- package/bootstrap-kb/modules/src.md +51 -0
- package/bootstrap-kb/modules/tests.md +42 -0
- package/bootstrap-kb/modules/tsconfig.json.md +15 -0
- package/bootstrap-kb/modules/vitest.config.ts.md +15 -0
- package/bootstrap-kb/overview/overview.md +40 -0
- package/bootstrap-kb/sources/manifest.json +950 -0
- package/bootstrap-kb/sources/manifest.md +227 -0
- package/bootstrap-kb/timeline/timeline.md +57 -0
- package/d.sh +3 -0
- package/deploy.sh +3 -0
- package/dist/cli/index.js +2389 -286
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +1017 -132
- package/dist/core/index.js.map +4 -4
- package/dist/hooks/post-tool-use.js +1347 -202
- package/dist/hooks/post-tool-use.js.map +4 -4
- package/dist/hooks/session-end.js +1339 -194
- package/dist/hooks/session-end.js.map +4 -4
- package/dist/hooks/session-start.js +1343 -198
- package/dist/hooks/session-start.js.map +4 -4
- package/dist/hooks/stop.js +1351 -206
- package/dist/hooks/stop.js.map +4 -4
- package/dist/hooks/user-prompt-submit.js +1347 -202
- package/dist/hooks/user-prompt-submit.js.map +4 -4
- package/dist/server/api/index.js +1436 -211
- package/dist/server/api/index.js.map +4 -4
- package/dist/server/index.js +1445 -220
- package/dist/server/index.js.map +4 -4
- package/dist/services/memory-service.js +1345 -199
- package/dist/services/memory-service.js.map +4 -4
- package/dist/ui/app.js +69 -2
- package/dist/ui/index.html +8 -0
- package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +271 -0
- package/docs/MEMU_ADOPTION.md +40 -0
- package/memory/.claude-plugin/commands/2026-02-25.md +263 -0
- package/memory/_index.md +405 -0
- package/memory/default/uncategorized/2026-02-25.md +4839 -0
- package/memory/specs/20260207-dashboard-upgrade/2026-02-25.md +142 -0
- package/memory/specs/citations-system/2026-02-25.md +1121 -0
- package/memory/specs/endless-mode/2026-02-25.md +1392 -0
- package/memory/specs/entity-edge-model/2026-02-25.md +1263 -0
- package/memory/specs/evidence-aligner-v2/2026-02-25.md +1028 -0
- package/memory/specs/mcp-desktop-integration/2026-02-25.md +1334 -0
- package/memory/specs/post-tool-use-hook/2026-02-25.md +1164 -0
- package/memory/specs/private-tags/2026-02-25.md +1057 -0
- package/memory/specs/progressive-disclosure/2026-02-25.md +1436 -0
- package/memory/specs/task-entity-system/2026-02-25.md +924 -0
- package/memory/specs/vector-outbox-v2/2026-02-25.md +1510 -0
- package/memory/specs/web-viewer-ui/2026-02-25.md +1709 -0
- package/package.json +2 -1
- package/scripts/build.ts +6 -0
- package/src/cli/index.ts +281 -2
- package/src/core/consolidated-store.ts +63 -1
- package/src/core/consolidation-worker.ts +115 -6
- package/src/core/event-store.ts +14 -0
- package/src/core/index.ts +1 -0
- package/src/core/ingest-interceptor.ts +80 -0
- package/src/core/markdown-mirror.ts +70 -0
- package/src/core/md-mirror.ts +92 -0
- package/src/core/mongo-sync-config.ts +165 -0
- package/src/core/mongo-sync-worker.ts +381 -0
- package/src/core/retriever.ts +540 -150
- package/src/core/sqlite-event-store.ts +350 -1
- package/src/core/tag-taxonomy.ts +51 -0
- package/src/core/types.ts +28 -0
- package/src/server/api/health.ts +53 -0
- package/src/server/api/index.ts +3 -1
- package/src/server/api/stats.ts +46 -1
- package/src/services/bootstrap-organizer.ts +443 -0
- package/src/services/codex-session-history-importer.ts +474 -0
- package/src/services/memory-service.ts +373 -68
- package/src/ui/app.js +69 -2
- package/src/ui/index.html +8 -0
- package/tests/bootstrap-organizer.test.ts +111 -0
- package/tests/consolidation-worker.test.ts +75 -0
- package/tests/ingest-interceptor.test.ts +38 -0
- package/tests/markdown-mirror.test.ts +85 -0
- package/tests/md-mirror.test.ts +50 -0
- package/tests/retriever-fallback-chain.test.ts +223 -0
- package/tests/retriever-strategy-scope.test.ts +97 -0
- package/tests/retriever.memu-adoption.test.ts +122 -0
- package/tests/sqlite-event-store-replication.test.ts +92 -0
package/dist/cli/index.js
CHANGED
|
@@ -16,14 +16,14 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
|
|
|
16
16
|
// src/cli/index.ts
|
|
17
17
|
import { Command } from "commander";
|
|
18
18
|
import { exec } from "child_process";
|
|
19
|
-
import * as
|
|
20
|
-
import * as
|
|
21
|
-
import * as
|
|
19
|
+
import * as fs9 from "fs";
|
|
20
|
+
import * as path9 from "path";
|
|
21
|
+
import * as os6 from "os";
|
|
22
22
|
|
|
23
23
|
// src/services/memory-service.ts
|
|
24
|
-
import * as
|
|
24
|
+
import * as path3 from "path";
|
|
25
25
|
import * as os from "os";
|
|
26
|
-
import * as
|
|
26
|
+
import * as fs4 from "fs";
|
|
27
27
|
import * as crypto2 from "crypto";
|
|
28
28
|
|
|
29
29
|
// src/core/event-store.ts
|
|
@@ -81,57 +81,57 @@ function toDate(value) {
|
|
|
81
81
|
return new Date(value);
|
|
82
82
|
return new Date(String(value));
|
|
83
83
|
}
|
|
84
|
-
function createDatabase(
|
|
84
|
+
function createDatabase(path10, options) {
|
|
85
85
|
if (options?.readOnly) {
|
|
86
|
-
return new duckdb.Database(
|
|
86
|
+
return new duckdb.Database(path10, { access_mode: "READ_ONLY" });
|
|
87
87
|
}
|
|
88
|
-
return new duckdb.Database(
|
|
88
|
+
return new duckdb.Database(path10);
|
|
89
89
|
}
|
|
90
90
|
function dbRun(db, sql, params = []) {
|
|
91
|
-
return new Promise((
|
|
91
|
+
return new Promise((resolve4, reject) => {
|
|
92
92
|
if (params.length === 0) {
|
|
93
93
|
db.run(sql, (err) => {
|
|
94
94
|
if (err)
|
|
95
95
|
reject(err);
|
|
96
96
|
else
|
|
97
|
-
|
|
97
|
+
resolve4();
|
|
98
98
|
});
|
|
99
99
|
} else {
|
|
100
100
|
db.run(sql, ...params, (err) => {
|
|
101
101
|
if (err)
|
|
102
102
|
reject(err);
|
|
103
103
|
else
|
|
104
|
-
|
|
104
|
+
resolve4();
|
|
105
105
|
});
|
|
106
106
|
}
|
|
107
107
|
});
|
|
108
108
|
}
|
|
109
109
|
function dbAll(db, sql, params = []) {
|
|
110
|
-
return new Promise((
|
|
110
|
+
return new Promise((resolve4, reject) => {
|
|
111
111
|
if (params.length === 0) {
|
|
112
112
|
db.all(sql, (err, rows) => {
|
|
113
113
|
if (err)
|
|
114
114
|
reject(err);
|
|
115
115
|
else
|
|
116
|
-
|
|
116
|
+
resolve4(convertBigInts(rows || []));
|
|
117
117
|
});
|
|
118
118
|
} else {
|
|
119
119
|
db.all(sql, ...params, (err, rows) => {
|
|
120
120
|
if (err)
|
|
121
121
|
reject(err);
|
|
122
122
|
else
|
|
123
|
-
|
|
123
|
+
resolve4(convertBigInts(rows || []));
|
|
124
124
|
});
|
|
125
125
|
}
|
|
126
126
|
});
|
|
127
127
|
}
|
|
128
128
|
function dbClose(db) {
|
|
129
|
-
return new Promise((
|
|
129
|
+
return new Promise((resolve4, reject) => {
|
|
130
130
|
db.close((err) => {
|
|
131
131
|
if (err)
|
|
132
132
|
reject(err);
|
|
133
133
|
else
|
|
134
|
-
|
|
134
|
+
resolve4();
|
|
135
135
|
});
|
|
136
136
|
});
|
|
137
137
|
}
|
|
@@ -349,6 +349,17 @@ var EventStore = class {
|
|
|
349
349
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
350
350
|
)
|
|
351
351
|
`);
|
|
352
|
+
await dbRun(this.db, `
|
|
353
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
354
|
+
rule_id VARCHAR PRIMARY KEY,
|
|
355
|
+
rule TEXT NOT NULL,
|
|
356
|
+
topics JSON,
|
|
357
|
+
source_memory_ids JSON,
|
|
358
|
+
source_events JSON,
|
|
359
|
+
confidence FLOAT DEFAULT 0.5,
|
|
360
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
361
|
+
)
|
|
362
|
+
`);
|
|
352
363
|
await dbRun(this.db, `
|
|
353
364
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
354
365
|
key VARCHAR PRIMARY KEY,
|
|
@@ -368,6 +379,7 @@ var EventStore = class {
|
|
|
368
379
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
|
|
369
380
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
|
|
370
381
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
|
|
382
|
+
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
|
|
371
383
|
await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
|
|
372
384
|
this.initialized = true;
|
|
373
385
|
}
|
|
@@ -757,12 +769,12 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
757
769
|
import Database from "better-sqlite3";
|
|
758
770
|
import * as fs from "fs";
|
|
759
771
|
import * as nodePath from "path";
|
|
760
|
-
function createSQLiteDatabase(
|
|
761
|
-
const dir = nodePath.dirname(
|
|
772
|
+
function createSQLiteDatabase(path10, options) {
|
|
773
|
+
const dir = nodePath.dirname(path10);
|
|
762
774
|
if (!fs.existsSync(dir)) {
|
|
763
775
|
fs.mkdirSync(dir, { recursive: true });
|
|
764
776
|
}
|
|
765
|
-
const db = new Database(
|
|
777
|
+
const db = new Database(path10, {
|
|
766
778
|
readonly: options?.readonly ?? false
|
|
767
779
|
});
|
|
768
780
|
if (!options?.readonly && (options?.walMode ?? true)) {
|
|
@@ -803,6 +815,64 @@ function toSQLiteTimestamp(date) {
|
|
|
803
815
|
return date.toISOString();
|
|
804
816
|
}
|
|
805
817
|
|
|
818
|
+
// src/core/markdown-mirror.ts
|
|
819
|
+
import * as fs2 from "fs/promises";
|
|
820
|
+
import * as path from "path";
|
|
821
|
+
var DEFAULT_NAMESPACE = "default";
|
|
822
|
+
var DEFAULT_CATEGORY = "uncategorized";
|
|
823
|
+
function sanitizeSegment(input, fallback) {
|
|
824
|
+
const raw = String(input ?? "").trim().toLowerCase();
|
|
825
|
+
const safe = raw.normalize("NFKD").replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
826
|
+
if (!safe || safe === "." || safe === "..")
|
|
827
|
+
return fallback;
|
|
828
|
+
return safe;
|
|
829
|
+
}
|
|
830
|
+
function getCategorySegments(metadata, eventType) {
|
|
831
|
+
const raw = metadata?.categoryPath;
|
|
832
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
833
|
+
return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
|
|
834
|
+
}
|
|
835
|
+
const single = metadata?.category;
|
|
836
|
+
if (typeof single === "string" && single.trim()) {
|
|
837
|
+
return [sanitizeSegment(single, DEFAULT_CATEGORY)];
|
|
838
|
+
}
|
|
839
|
+
return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
|
|
840
|
+
}
|
|
841
|
+
function buildMirrorPath(rootDir, event) {
|
|
842
|
+
const metadata = event.metadata;
|
|
843
|
+
const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
|
|
844
|
+
const categories = getCategorySegments(metadata, event.eventType);
|
|
845
|
+
const d = event.timestamp;
|
|
846
|
+
const yyyy = d.getFullYear();
|
|
847
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
848
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
849
|
+
return path.join(rootDir, "memory", namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
|
|
850
|
+
}
|
|
851
|
+
function formatMirrorEntry(event) {
|
|
852
|
+
const category = Array.isArray(event.metadata?.categoryPath) ? event.metadata.categoryPath.join("/") : String(event.metadata?.category ?? event.eventType);
|
|
853
|
+
return [
|
|
854
|
+
"",
|
|
855
|
+
`- ts: ${event.timestamp.toISOString()}`,
|
|
856
|
+
` id: ${event.id}`,
|
|
857
|
+
` type: ${event.eventType}`,
|
|
858
|
+
` session: ${event.sessionId}`,
|
|
859
|
+
` category: ${category}`,
|
|
860
|
+
" content: |",
|
|
861
|
+
...event.content.split("\n").map((line) => ` ${line}`)
|
|
862
|
+
].join("\n") + "\n";
|
|
863
|
+
}
|
|
864
|
+
var MarkdownMirror = class {
|
|
865
|
+
constructor(rootDir) {
|
|
866
|
+
this.rootDir = rootDir;
|
|
867
|
+
}
|
|
868
|
+
async append(event) {
|
|
869
|
+
const outPath = buildMirrorPath(this.rootDir, event);
|
|
870
|
+
await fs2.mkdir(path.dirname(outPath), { recursive: true });
|
|
871
|
+
await fs2.appendFile(outPath, formatMirrorEntry(event), "utf8");
|
|
872
|
+
return outPath;
|
|
873
|
+
}
|
|
874
|
+
};
|
|
875
|
+
|
|
806
876
|
// src/core/sqlite-event-store.ts
|
|
807
877
|
var SQLiteEventStore = class {
|
|
808
878
|
constructor(dbPath, options) {
|
|
@@ -812,10 +882,12 @@ var SQLiteEventStore = class {
|
|
|
812
882
|
readonly: this.readOnly,
|
|
813
883
|
walMode: !this.readOnly
|
|
814
884
|
});
|
|
885
|
+
this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot ? null : new MarkdownMirror(options.markdownMirrorRoot);
|
|
815
886
|
}
|
|
816
887
|
db;
|
|
817
888
|
initialized = false;
|
|
818
889
|
readOnly;
|
|
890
|
+
markdownMirror;
|
|
819
891
|
/**
|
|
820
892
|
* Initialize database schema
|
|
821
893
|
*/
|
|
@@ -1022,6 +1094,17 @@ var SQLiteEventStore = class {
|
|
|
1022
1094
|
created_at TEXT DEFAULT (datetime('now'))
|
|
1023
1095
|
);
|
|
1024
1096
|
|
|
1097
|
+
-- Consolidated Rules table (long-term stable memory)
|
|
1098
|
+
CREATE TABLE IF NOT EXISTS consolidated_rules (
|
|
1099
|
+
rule_id TEXT PRIMARY KEY,
|
|
1100
|
+
rule TEXT NOT NULL,
|
|
1101
|
+
topics TEXT,
|
|
1102
|
+
source_memory_ids TEXT,
|
|
1103
|
+
source_events TEXT,
|
|
1104
|
+
confidence REAL DEFAULT 0.5,
|
|
1105
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1106
|
+
);
|
|
1107
|
+
|
|
1025
1108
|
-- Endless Mode Config table
|
|
1026
1109
|
CREATE TABLE IF NOT EXISTS endless_config (
|
|
1027
1110
|
key TEXT PRIMARY KEY,
|
|
@@ -1046,6 +1129,24 @@ var SQLiteEventStore = class {
|
|
|
1046
1129
|
measured_at TEXT
|
|
1047
1130
|
);
|
|
1048
1131
|
|
|
1132
|
+
-- Retrieval trace log (query -> candidates -> selected for context)
|
|
1133
|
+
CREATE TABLE IF NOT EXISTS retrieval_traces (
|
|
1134
|
+
trace_id TEXT PRIMARY KEY,
|
|
1135
|
+
session_id TEXT,
|
|
1136
|
+
project_hash TEXT,
|
|
1137
|
+
query_text TEXT NOT NULL,
|
|
1138
|
+
strategy TEXT,
|
|
1139
|
+
candidate_event_ids TEXT,
|
|
1140
|
+
selected_event_ids TEXT,
|
|
1141
|
+
candidate_details_json TEXT,
|
|
1142
|
+
selected_details_json TEXT,
|
|
1143
|
+
candidate_count INTEGER DEFAULT 0,
|
|
1144
|
+
selected_count INTEGER DEFAULT 0,
|
|
1145
|
+
confidence TEXT,
|
|
1146
|
+
fallback_trace TEXT,
|
|
1147
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
1148
|
+
);
|
|
1149
|
+
|
|
1049
1150
|
-- Sync position tracking (for SQLite -> DuckDB sync)
|
|
1050
1151
|
CREATE TABLE IF NOT EXISTS sync_positions (
|
|
1051
1152
|
target_name TEXT PRIMARY KEY,
|
|
@@ -1070,10 +1171,14 @@ var SQLiteEventStore = class {
|
|
|
1070
1171
|
CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
|
|
1071
1172
|
CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
|
|
1072
1173
|
CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
|
|
1174
|
+
CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
|
|
1073
1175
|
CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
|
|
1074
1176
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
1075
1177
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
1076
1178
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
1179
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
1180
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
1181
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
1077
1182
|
|
|
1078
1183
|
-- FTS5 Full-Text Search for fast keyword search
|
|
1079
1184
|
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
|
|
@@ -1097,6 +1202,14 @@ var SQLiteEventStore = class {
|
|
|
1097
1202
|
INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
|
|
1098
1203
|
END;
|
|
1099
1204
|
`);
|
|
1205
|
+
try {
|
|
1206
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
|
|
1207
|
+
} catch {
|
|
1208
|
+
}
|
|
1209
|
+
try {
|
|
1210
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
1211
|
+
} catch {
|
|
1212
|
+
}
|
|
1100
1213
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
1101
1214
|
const columnNames = tableInfo.map((col) => col.name);
|
|
1102
1215
|
if (!columnNames.includes("access_count")) {
|
|
@@ -1196,6 +1309,21 @@ var SQLiteEventStore = class {
|
|
|
1196
1309
|
insertLevel.run(id);
|
|
1197
1310
|
});
|
|
1198
1311
|
transaction();
|
|
1312
|
+
if (this.markdownMirror) {
|
|
1313
|
+
const event = {
|
|
1314
|
+
id,
|
|
1315
|
+
eventType: input.eventType,
|
|
1316
|
+
sessionId: input.sessionId,
|
|
1317
|
+
timestamp: input.timestamp,
|
|
1318
|
+
content: input.content,
|
|
1319
|
+
canonicalKey,
|
|
1320
|
+
dedupeKey,
|
|
1321
|
+
metadata
|
|
1322
|
+
};
|
|
1323
|
+
this.markdownMirror.append(event).catch((err) => {
|
|
1324
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1325
|
+
});
|
|
1326
|
+
}
|
|
1199
1327
|
return { success: true, eventId: id, isDuplicate: false };
|
|
1200
1328
|
} catch (error) {
|
|
1201
1329
|
return {
|
|
@@ -1254,6 +1382,92 @@ var SQLiteEventStore = class {
|
|
|
1254
1382
|
);
|
|
1255
1383
|
return rows.map(this.rowToEvent);
|
|
1256
1384
|
}
|
|
1385
|
+
/**
|
|
1386
|
+
* Get events since a SQLite rowid (for robust incremental replication).
|
|
1387
|
+
* Rowid is monotonic for append-only tables, independent of client timestamps.
|
|
1388
|
+
*/
|
|
1389
|
+
async getEventsSinceRowid(lastRowid, limit = 1e3) {
|
|
1390
|
+
await this.initialize();
|
|
1391
|
+
const rows = sqliteAll(
|
|
1392
|
+
this.db,
|
|
1393
|
+
`SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
|
|
1394
|
+
[lastRowid, limit]
|
|
1395
|
+
);
|
|
1396
|
+
return rows.map((row) => ({
|
|
1397
|
+
rowid: row._rowid,
|
|
1398
|
+
event: this.rowToEvent(row)
|
|
1399
|
+
}));
|
|
1400
|
+
}
|
|
1401
|
+
/**
|
|
1402
|
+
* Import events with fixed IDs (used for cross-machine replication).
|
|
1403
|
+
* Idempotent: skips if event id or dedupeKey already exists.
|
|
1404
|
+
*
|
|
1405
|
+
* NOTE: This bypasses the append() id generation to preserve stable IDs.
|
|
1406
|
+
*/
|
|
1407
|
+
async importEvents(events) {
|
|
1408
|
+
if (events.length === 0)
|
|
1409
|
+
return { inserted: 0, skipped: 0 };
|
|
1410
|
+
if (this.readOnly)
|
|
1411
|
+
return { inserted: 0, skipped: events.length };
|
|
1412
|
+
await this.initialize();
|
|
1413
|
+
const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
|
|
1414
|
+
const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
|
|
1415
|
+
const insertEvent = this.db.prepare(`
|
|
1416
|
+
INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
|
|
1417
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1418
|
+
`);
|
|
1419
|
+
const insertDedup = this.db.prepare(`
|
|
1420
|
+
INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
|
|
1421
|
+
`);
|
|
1422
|
+
const insertLevel = this.db.prepare(`
|
|
1423
|
+
INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
|
|
1424
|
+
`);
|
|
1425
|
+
let inserted = 0;
|
|
1426
|
+
let skipped = 0;
|
|
1427
|
+
const insertedEvents = [];
|
|
1428
|
+
const tx = this.db.transaction((batch) => {
|
|
1429
|
+
for (const ev of batch) {
|
|
1430
|
+
const existingById = getById.get(ev.id);
|
|
1431
|
+
if (existingById) {
|
|
1432
|
+
skipped++;
|
|
1433
|
+
continue;
|
|
1434
|
+
}
|
|
1435
|
+
const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
|
|
1436
|
+
const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
|
|
1437
|
+
const existingByDedupe = getByDedupe.get(dedupeKey);
|
|
1438
|
+
if (existingByDedupe) {
|
|
1439
|
+
skipped++;
|
|
1440
|
+
continue;
|
|
1441
|
+
}
|
|
1442
|
+
const metadata = ev.metadata || {};
|
|
1443
|
+
const turnId = metadata.turnId;
|
|
1444
|
+
insertEvent.run(
|
|
1445
|
+
ev.id,
|
|
1446
|
+
ev.eventType,
|
|
1447
|
+
ev.sessionId,
|
|
1448
|
+
toSQLiteTimestamp(ev.timestamp),
|
|
1449
|
+
ev.content,
|
|
1450
|
+
canonicalKey,
|
|
1451
|
+
dedupeKey,
|
|
1452
|
+
JSON.stringify(metadata),
|
|
1453
|
+
turnId ?? null
|
|
1454
|
+
);
|
|
1455
|
+
insertDedup.run(dedupeKey, ev.id);
|
|
1456
|
+
insertLevel.run(ev.id);
|
|
1457
|
+
inserted++;
|
|
1458
|
+
insertedEvents.push(ev);
|
|
1459
|
+
}
|
|
1460
|
+
});
|
|
1461
|
+
tx(events);
|
|
1462
|
+
if (this.markdownMirror && insertedEvents.length > 0) {
|
|
1463
|
+
for (const ev of insertedEvents) {
|
|
1464
|
+
this.markdownMirror.append(ev).catch((err) => {
|
|
1465
|
+
console.warn("[SQLiteEventStore] markdown mirror append failed:", err);
|
|
1466
|
+
});
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
return { inserted, skipped };
|
|
1470
|
+
}
|
|
1257
1471
|
/**
|
|
1258
1472
|
* Create or update session
|
|
1259
1473
|
*/
|
|
@@ -1416,6 +1630,35 @@ var SQLiteEventStore = class {
|
|
|
1416
1630
|
[error, ...ids]
|
|
1417
1631
|
);
|
|
1418
1632
|
}
|
|
1633
|
+
/**
|
|
1634
|
+
* Get embedding/vector outbox health statistics
|
|
1635
|
+
*/
|
|
1636
|
+
async getOutboxStats() {
|
|
1637
|
+
await this.initialize();
|
|
1638
|
+
const embeddingRows = sqliteAll(
|
|
1639
|
+
this.db,
|
|
1640
|
+
`SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
|
|
1641
|
+
);
|
|
1642
|
+
const vectorRows = sqliteAll(
|
|
1643
|
+
this.db,
|
|
1644
|
+
`SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
|
|
1645
|
+
);
|
|
1646
|
+
const fromRows = (rows) => {
|
|
1647
|
+
const out = { pending: 0, processing: 0, failed: 0, total: 0 };
|
|
1648
|
+
for (const row of rows) {
|
|
1649
|
+
const key = row.status;
|
|
1650
|
+
if (key === "pending" || key === "processing" || key === "failed") {
|
|
1651
|
+
out[key] += row.count;
|
|
1652
|
+
}
|
|
1653
|
+
out.total += row.count;
|
|
1654
|
+
}
|
|
1655
|
+
return out;
|
|
1656
|
+
};
|
|
1657
|
+
return {
|
|
1658
|
+
embedding: fromRows(embeddingRows),
|
|
1659
|
+
vector: fromRows(vectorRows)
|
|
1660
|
+
};
|
|
1661
|
+
}
|
|
1419
1662
|
/**
|
|
1420
1663
|
* Update memory level
|
|
1421
1664
|
*/
|
|
@@ -1774,6 +2017,79 @@ var SQLiteEventStore = class {
|
|
|
1774
2017
|
getDatabase() {
|
|
1775
2018
|
return this.db;
|
|
1776
2019
|
}
|
|
2020
|
+
async recordRetrievalTrace(input) {
|
|
2021
|
+
await this.initialize();
|
|
2022
|
+
const traceId = randomUUID2();
|
|
2023
|
+
sqliteRun(
|
|
2024
|
+
this.db,
|
|
2025
|
+
`INSERT INTO retrieval_traces (
|
|
2026
|
+
trace_id, session_id, project_hash, query_text, strategy,
|
|
2027
|
+
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
2028
|
+
candidate_count, selected_count, confidence, fallback_trace
|
|
2029
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2030
|
+
[
|
|
2031
|
+
traceId,
|
|
2032
|
+
input.sessionId || null,
|
|
2033
|
+
input.projectHash || null,
|
|
2034
|
+
input.queryText,
|
|
2035
|
+
input.strategy || null,
|
|
2036
|
+
JSON.stringify(input.candidateEventIds || []),
|
|
2037
|
+
JSON.stringify(input.selectedEventIds || []),
|
|
2038
|
+
JSON.stringify(input.candidateDetails || []),
|
|
2039
|
+
JSON.stringify(input.selectedDetails || []),
|
|
2040
|
+
(input.candidateEventIds || []).length,
|
|
2041
|
+
(input.selectedEventIds || []).length,
|
|
2042
|
+
input.confidence || null,
|
|
2043
|
+
JSON.stringify(input.fallbackTrace || [])
|
|
2044
|
+
]
|
|
2045
|
+
);
|
|
2046
|
+
}
|
|
2047
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
2048
|
+
await this.initialize();
|
|
2049
|
+
const rows = sqliteAll(
|
|
2050
|
+
this.db,
|
|
2051
|
+
`SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
|
|
2052
|
+
[limit]
|
|
2053
|
+
);
|
|
2054
|
+
return rows.map((row) => ({
|
|
2055
|
+
traceId: row.trace_id,
|
|
2056
|
+
sessionId: row.session_id || void 0,
|
|
2057
|
+
projectHash: row.project_hash || void 0,
|
|
2058
|
+
queryText: row.query_text,
|
|
2059
|
+
strategy: row.strategy || void 0,
|
|
2060
|
+
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
2061
|
+
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
2062
|
+
candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json) : [],
|
|
2063
|
+
selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json) : [],
|
|
2064
|
+
candidateCount: Number(row.candidate_count || 0),
|
|
2065
|
+
selectedCount: Number(row.selected_count || 0),
|
|
2066
|
+
confidence: row.confidence || void 0,
|
|
2067
|
+
fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace) : [],
|
|
2068
|
+
createdAt: toDateFromSQLite(row.created_at)
|
|
2069
|
+
}));
|
|
2070
|
+
}
|
|
2071
|
+
async getRetrievalTraceStats() {
|
|
2072
|
+
await this.initialize();
|
|
2073
|
+
const row = sqliteGet(
|
|
2074
|
+
this.db,
|
|
2075
|
+
`SELECT
|
|
2076
|
+
COUNT(*) as total_queries,
|
|
2077
|
+
AVG(candidate_count) as avg_candidate_count,
|
|
2078
|
+
AVG(selected_count) as avg_selected_count,
|
|
2079
|
+
CASE
|
|
2080
|
+
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
2081
|
+
ELSE 0
|
|
2082
|
+
END as selection_rate
|
|
2083
|
+
FROM retrieval_traces`,
|
|
2084
|
+
[]
|
|
2085
|
+
);
|
|
2086
|
+
return {
|
|
2087
|
+
totalQueries: Number(row?.total_queries || 0),
|
|
2088
|
+
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
2089
|
+
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
2090
|
+
selectionRate: Number(row?.selection_rate || 0)
|
|
2091
|
+
};
|
|
2092
|
+
}
|
|
1777
2093
|
/**
|
|
1778
2094
|
* Close database connection
|
|
1779
2095
|
*/
|
|
@@ -2091,7 +2407,7 @@ var SyncWorker = class {
|
|
|
2091
2407
|
* Sleep utility
|
|
2092
2408
|
*/
|
|
2093
2409
|
sleep(ms) {
|
|
2094
|
-
return new Promise((
|
|
2410
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
2095
2411
|
}
|
|
2096
2412
|
/**
|
|
2097
2413
|
* Get sync statistics
|
|
@@ -2635,7 +2951,20 @@ var DEFAULT_OPTIONS = {
|
|
|
2635
2951
|
topK: 5,
|
|
2636
2952
|
minScore: 0.7,
|
|
2637
2953
|
maxTokens: 2e3,
|
|
2638
|
-
includeSessionContext: true
|
|
2954
|
+
includeSessionContext: true,
|
|
2955
|
+
strategy: "auto",
|
|
2956
|
+
rerankWithKeyword: true,
|
|
2957
|
+
decayPolicy: {
|
|
2958
|
+
enabled: true,
|
|
2959
|
+
windowDays: 30,
|
|
2960
|
+
maxPenalty: 0.15
|
|
2961
|
+
},
|
|
2962
|
+
graphHop: {
|
|
2963
|
+
enabled: true,
|
|
2964
|
+
maxHops: 1,
|
|
2965
|
+
hopPenalty: 0.08
|
|
2966
|
+
},
|
|
2967
|
+
projectScopeMode: "global"
|
|
2639
2968
|
};
|
|
2640
2969
|
var Retriever = class {
|
|
2641
2970
|
eventStore;
|
|
@@ -2645,6 +2974,7 @@ var Retriever = class {
|
|
|
2645
2974
|
sharedStore;
|
|
2646
2975
|
sharedVectorStore;
|
|
2647
2976
|
graduation;
|
|
2977
|
+
queryRewriter;
|
|
2648
2978
|
constructor(eventStore, vectorStore, embedder, matcher, sharedOptions) {
|
|
2649
2979
|
this.eventStore = eventStore;
|
|
2650
2980
|
this.vectorStore = vectorStore;
|
|
@@ -2653,47 +2983,105 @@ var Retriever = class {
|
|
|
2653
2983
|
this.sharedStore = sharedOptions?.sharedStore;
|
|
2654
2984
|
this.sharedVectorStore = sharedOptions?.sharedVectorStore;
|
|
2655
2985
|
}
|
|
2656
|
-
/**
|
|
2657
|
-
* Set graduation pipeline for access tracking
|
|
2658
|
-
*/
|
|
2659
2986
|
setGraduationPipeline(graduation) {
|
|
2660
2987
|
this.graduation = graduation;
|
|
2661
2988
|
}
|
|
2662
|
-
/**
|
|
2663
|
-
* Set shared stores after construction
|
|
2664
|
-
*/
|
|
2665
2989
|
setSharedStores(sharedStore, sharedVectorStore) {
|
|
2666
2990
|
this.sharedStore = sharedStore;
|
|
2667
2991
|
this.sharedVectorStore = sharedVectorStore;
|
|
2668
2992
|
}
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2993
|
+
setQueryRewriter(rewriter) {
|
|
2994
|
+
this.queryRewriter = rewriter;
|
|
2995
|
+
}
|
|
2672
2996
|
async retrieve(query, options = {}) {
|
|
2673
2997
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
2674
|
-
const
|
|
2675
|
-
const
|
|
2676
|
-
|
|
2677
|
-
|
|
2998
|
+
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
2999
|
+
const fallbackTrace = [];
|
|
3000
|
+
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
3001
|
+
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
3002
|
+
let current = await this.runStage(query, {
|
|
3003
|
+
strategy: primaryStrategy,
|
|
3004
|
+
topK: opts.topK,
|
|
2678
3005
|
minScore: opts.minScore,
|
|
2679
|
-
sessionId:
|
|
3006
|
+
sessionId: sessionFilter,
|
|
3007
|
+
scope: opts.scope,
|
|
3008
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3009
|
+
rerankWeights: opts.rerankWeights,
|
|
3010
|
+
decayPolicy: opts.decayPolicy,
|
|
3011
|
+
intentRewrite: opts.intentRewrite === true,
|
|
3012
|
+
graphHop: opts.graphHop,
|
|
3013
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3014
|
+
projectHash: opts.projectHash,
|
|
3015
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
2680
3016
|
});
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
3017
|
+
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
3018
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
3019
|
+
current = await this.runStage(query, {
|
|
3020
|
+
strategy: "deep",
|
|
3021
|
+
topK: opts.topK,
|
|
3022
|
+
minScore: opts.minScore,
|
|
3023
|
+
sessionId: sessionFilter,
|
|
3024
|
+
scope: opts.scope,
|
|
3025
|
+
rerankWithKeyword: opts.rerankWithKeyword !== false,
|
|
3026
|
+
rerankWeights: opts.rerankWeights,
|
|
3027
|
+
decayPolicy: opts.decayPolicy,
|
|
3028
|
+
graphHop: opts.graphHop,
|
|
3029
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3030
|
+
projectHash: opts.projectHash,
|
|
3031
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3032
|
+
});
|
|
3033
|
+
fallbackTrace.push("fallback:deep");
|
|
3034
|
+
}
|
|
3035
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3036
|
+
current = await this.runStage(query, {
|
|
3037
|
+
strategy: "deep",
|
|
3038
|
+
topK: opts.topK,
|
|
3039
|
+
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
3040
|
+
sessionId: void 0,
|
|
3041
|
+
scope: void 0,
|
|
3042
|
+
rerankWithKeyword: true,
|
|
3043
|
+
rerankWeights: opts.rerankWeights,
|
|
3044
|
+
decayPolicy: opts.decayPolicy,
|
|
3045
|
+
graphHop: opts.graphHop,
|
|
3046
|
+
projectScopeMode: opts.projectScopeMode,
|
|
3047
|
+
projectHash: opts.projectHash,
|
|
3048
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
3049
|
+
});
|
|
3050
|
+
fallbackTrace.push("fallback:scope-expanded");
|
|
3051
|
+
}
|
|
3052
|
+
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
3053
|
+
const summary = await this.buildSummaryFallback(query, opts.topK);
|
|
3054
|
+
current = {
|
|
3055
|
+
results: summary,
|
|
3056
|
+
candidateResults: summary,
|
|
3057
|
+
matchResult: this.matcher.matchSearchResults(summary, () => 0)
|
|
3058
|
+
};
|
|
3059
|
+
fallbackTrace.push("fallback:summary");
|
|
3060
|
+
}
|
|
3061
|
+
const memories = await this.enrichResults(current.results.slice(0, opts.topK), opts);
|
|
2686
3062
|
const context = this.buildContext(memories, opts.maxTokens);
|
|
2687
3063
|
return {
|
|
2688
3064
|
memories,
|
|
2689
|
-
matchResult,
|
|
3065
|
+
matchResult: current.matchResult,
|
|
2690
3066
|
totalTokens: this.estimateTokens(context),
|
|
2691
|
-
context
|
|
3067
|
+
context,
|
|
3068
|
+
fallbackTrace,
|
|
3069
|
+
selectedDebug: current.results.slice(0, opts.topK).map((r) => ({
|
|
3070
|
+
eventId: r.eventId,
|
|
3071
|
+
score: r.score,
|
|
3072
|
+
semanticScore: r.semanticScore,
|
|
3073
|
+
lexicalScore: r.lexicalScore,
|
|
3074
|
+
recencyScore: r.recencyScore
|
|
3075
|
+
})),
|
|
3076
|
+
candidateDebug: (current.candidateResults || []).slice(0, Math.max(opts.topK * 3, 20)).map((r) => ({
|
|
3077
|
+
eventId: r.eventId,
|
|
3078
|
+
score: r.score,
|
|
3079
|
+
semanticScore: r.semanticScore,
|
|
3080
|
+
lexicalScore: r.lexicalScore,
|
|
3081
|
+
recencyScore: r.recencyScore
|
|
3082
|
+
}))
|
|
2692
3083
|
};
|
|
2693
3084
|
}
|
|
2694
|
-
/**
|
|
2695
|
-
* Retrieve with unified search (project + shared)
|
|
2696
|
-
*/
|
|
2697
3085
|
async retrieveUnified(query, options = {}) {
|
|
2698
3086
|
const projectResult = await this.retrieve(query, options);
|
|
2699
3087
|
if (!options.includeShared || !this.sharedStore || !this.sharedVectorStore) {
|
|
@@ -2701,22 +3089,19 @@ var Retriever = class {
|
|
|
2701
3089
|
}
|
|
2702
3090
|
try {
|
|
2703
3091
|
const queryEmbedding = await this.embedder.embed(query);
|
|
2704
|
-
const sharedVectorResults = await this.sharedVectorStore.search(
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
excludeProjectHash: options.projectHash
|
|
2710
|
-
}
|
|
2711
|
-
);
|
|
3092
|
+
const sharedVectorResults = await this.sharedVectorStore.search(queryEmbedding.vector, {
|
|
3093
|
+
limit: options.topK || 5,
|
|
3094
|
+
minScore: options.minScore || 0.7,
|
|
3095
|
+
excludeProjectHash: options.projectHash
|
|
3096
|
+
});
|
|
2712
3097
|
const sharedMemories = [];
|
|
2713
3098
|
for (const result of sharedVectorResults) {
|
|
2714
3099
|
const entry = await this.sharedStore.get(result.entryId);
|
|
2715
|
-
if (entry)
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
3100
|
+
if (!entry)
|
|
3101
|
+
continue;
|
|
3102
|
+
if (!options.projectHash || entry.sourceProjectHash !== options.projectHash) {
|
|
3103
|
+
sharedMemories.push(entry);
|
|
3104
|
+
await this.sharedStore.recordUsage(entry.entryId);
|
|
2720
3105
|
}
|
|
2721
3106
|
}
|
|
2722
3107
|
const unifiedContext = this.buildUnifiedContext(projectResult, sharedMemories);
|
|
@@ -2731,50 +3116,243 @@ var Retriever = class {
|
|
|
2731
3116
|
return projectResult;
|
|
2732
3117
|
}
|
|
2733
3118
|
}
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
3119
|
+
async runStage(query, input) {
|
|
3120
|
+
let initialResults = await this.searchByStrategy(query, {
|
|
3121
|
+
strategy: input.strategy,
|
|
3122
|
+
topK: input.topK,
|
|
3123
|
+
minScore: input.minScore,
|
|
3124
|
+
sessionId: input.sessionId
|
|
3125
|
+
});
|
|
3126
|
+
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
3127
|
+
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
3128
|
+
if (rewritten && rewritten !== query) {
|
|
3129
|
+
const rewrittenResults = await this.searchByStrategy(rewritten, {
|
|
3130
|
+
strategy: "deep",
|
|
3131
|
+
topK: input.topK,
|
|
3132
|
+
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
3133
|
+
sessionId: input.sessionId
|
|
3134
|
+
});
|
|
3135
|
+
initialResults = this.mergeResults(initialResults, rewrittenResults, input.topK * 3);
|
|
3136
|
+
}
|
|
3137
|
+
}
|
|
3138
|
+
const expandedResults = input.graphHop?.enabled === false ? initialResults : await this.expandGraphHops(initialResults, {
|
|
3139
|
+
maxHops: Math.max(1, input.graphHop?.maxHops ?? 1),
|
|
3140
|
+
hopPenalty: Math.max(0, input.graphHop?.hopPenalty ?? 0.08),
|
|
3141
|
+
limit: input.topK * 4
|
|
3142
|
+
});
|
|
3143
|
+
const rerankedResults = input.rerankWithKeyword ? this.rerankByKeywordOverlap(expandedResults, query, input.rerankWeights, input.decayPolicy) : expandedResults;
|
|
3144
|
+
const filtered = await this.applyScopeFilters(rerankedResults, {
|
|
3145
|
+
scope: input.scope,
|
|
3146
|
+
projectScopeMode: input.projectScopeMode,
|
|
3147
|
+
projectHash: input.projectHash,
|
|
3148
|
+
allowedProjectHashes: input.allowedProjectHashes
|
|
3149
|
+
});
|
|
3150
|
+
const top = filtered.slice(0, input.topK);
|
|
3151
|
+
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
3152
|
+
return { results: top, candidateResults: filtered, matchResult };
|
|
3153
|
+
}
|
|
3154
|
+
mergeResults(primary, secondary, limit) {
|
|
3155
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3156
|
+
for (const row of primary)
|
|
3157
|
+
byId.set(row.eventId, row);
|
|
3158
|
+
for (const row of secondary) {
|
|
3159
|
+
const prev = byId.get(row.eventId);
|
|
3160
|
+
if (!prev || row.score > prev.score) {
|
|
3161
|
+
byId.set(row.eventId, row);
|
|
3162
|
+
}
|
|
3163
|
+
}
|
|
3164
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, limit);
|
|
3165
|
+
}
|
|
3166
|
+
async expandGraphHops(seeds, opts) {
|
|
3167
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3168
|
+
for (const s of seeds)
|
|
3169
|
+
byId.set(s.eventId, s);
|
|
3170
|
+
let frontier = seeds.map((s) => ({ row: s, hop: 0 }));
|
|
3171
|
+
for (let hop = 1; hop <= opts.maxHops; hop += 1) {
|
|
3172
|
+
const next = [];
|
|
3173
|
+
for (const f of frontier) {
|
|
3174
|
+
const ev = await this.eventStore.getEvent(f.row.eventId);
|
|
3175
|
+
if (!ev)
|
|
3176
|
+
continue;
|
|
3177
|
+
const rel = ev.metadata?.relatedEventIds ?? [];
|
|
3178
|
+
const relatedIds = Array.isArray(rel) ? rel.filter((x) => typeof x === "string") : [];
|
|
3179
|
+
for (const rid of relatedIds) {
|
|
3180
|
+
if (byId.has(rid))
|
|
3181
|
+
continue;
|
|
3182
|
+
const target = await this.eventStore.getEvent(rid);
|
|
3183
|
+
if (!target)
|
|
3184
|
+
continue;
|
|
3185
|
+
const score = Math.max(0, f.row.score - opts.hopPenalty * hop);
|
|
3186
|
+
const row = {
|
|
3187
|
+
id: `hop-${hop}-${rid}`,
|
|
3188
|
+
eventId: target.id,
|
|
3189
|
+
content: target.content,
|
|
3190
|
+
score,
|
|
3191
|
+
sessionId: target.sessionId,
|
|
3192
|
+
eventType: target.eventType,
|
|
3193
|
+
timestamp: target.timestamp.toISOString()
|
|
3194
|
+
};
|
|
3195
|
+
byId.set(row.eventId, row);
|
|
3196
|
+
next.push({ row, hop });
|
|
3197
|
+
if (byId.size >= opts.limit)
|
|
3198
|
+
break;
|
|
2755
3199
|
}
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
`;
|
|
3200
|
+
if (byId.size >= opts.limit)
|
|
3201
|
+
break;
|
|
2759
3202
|
}
|
|
3203
|
+
frontier = next;
|
|
3204
|
+
if (frontier.length === 0 || byId.size >= opts.limit)
|
|
3205
|
+
break;
|
|
2760
3206
|
}
|
|
2761
|
-
return
|
|
3207
|
+
return [...byId.values()].sort((a, b) => b.score - a.score).slice(0, opts.limit);
|
|
3208
|
+
}
|
|
3209
|
+
shouldFallback(matchResult, results) {
|
|
3210
|
+
if (results.length === 0)
|
|
3211
|
+
return true;
|
|
3212
|
+
if (matchResult.confidence === "none")
|
|
3213
|
+
return true;
|
|
3214
|
+
return false;
|
|
3215
|
+
}
|
|
3216
|
+
async buildSummaryFallback(query, topK) {
|
|
3217
|
+
const recent = await this.eventStore.getRecentEvents(Math.max(topK * 6, 20));
|
|
3218
|
+
const q = this.tokenize(query);
|
|
3219
|
+
const ranked = recent.map((e) => ({ e, overlap: this.keywordOverlap(q, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, topK).map((row, idx) => ({
|
|
3220
|
+
id: `summary-${row.e.id}`,
|
|
3221
|
+
eventId: row.e.id,
|
|
3222
|
+
content: row.e.content,
|
|
3223
|
+
score: Math.max(0.25, 0.6 - idx * 0.05),
|
|
3224
|
+
sessionId: row.e.sessionId,
|
|
3225
|
+
eventType: row.e.eventType,
|
|
3226
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3227
|
+
}));
|
|
3228
|
+
return ranked;
|
|
3229
|
+
}
|
|
3230
|
+
async searchByStrategy(query, input) {
|
|
3231
|
+
const strategy = input.strategy === "auto" ? "deep" : input.strategy;
|
|
3232
|
+
if (strategy === "fast") {
|
|
3233
|
+
const keyword = await this.searchByKeyword(query, {
|
|
3234
|
+
limit: Math.max(5, input.topK * 3),
|
|
3235
|
+
sessionId: input.sessionId
|
|
3236
|
+
});
|
|
3237
|
+
return keyword;
|
|
3238
|
+
}
|
|
3239
|
+
const queryEmbedding = await this.embedder.embed(query);
|
|
3240
|
+
return this.vectorStore.search(queryEmbedding.vector, {
|
|
3241
|
+
limit: Math.max(5, input.topK * 3),
|
|
3242
|
+
minScore: input.minScore,
|
|
3243
|
+
sessionId: input.sessionId
|
|
3244
|
+
});
|
|
3245
|
+
}
|
|
3246
|
+
async searchByKeyword(query, input) {
|
|
3247
|
+
if (this.eventStore.keywordSearch) {
|
|
3248
|
+
const rows = await this.eventStore.keywordSearch(query, input.limit);
|
|
3249
|
+
const filtered2 = input.sessionId ? rows.filter((r) => r.event.sessionId === input.sessionId) : rows;
|
|
3250
|
+
return filtered2.map((row, idx) => ({
|
|
3251
|
+
id: `kw-${row.event.id}`,
|
|
3252
|
+
eventId: row.event.id,
|
|
3253
|
+
content: row.event.content,
|
|
3254
|
+
score: Math.max(0.4, 1 - idx * 0.04),
|
|
3255
|
+
sessionId: row.event.sessionId,
|
|
3256
|
+
eventType: row.event.eventType,
|
|
3257
|
+
timestamp: row.event.timestamp.toISOString()
|
|
3258
|
+
}));
|
|
3259
|
+
}
|
|
3260
|
+
const recent = await this.eventStore.getRecentEvents(input.limit * 4);
|
|
3261
|
+
const tokens = this.tokenize(query);
|
|
3262
|
+
const filtered = recent.filter((e) => input.sessionId ? e.sessionId === input.sessionId : true).map((e) => ({ e, overlap: this.keywordOverlap(tokens, this.tokenize(e.content)) })).filter((r) => r.overlap > 0).sort((a, b) => b.overlap - a.overlap).slice(0, input.limit);
|
|
3263
|
+
return filtered.map((row, idx) => ({
|
|
3264
|
+
id: `kw-fallback-${row.e.id}`,
|
|
3265
|
+
eventId: row.e.id,
|
|
3266
|
+
content: row.e.content,
|
|
3267
|
+
score: Math.max(0.3, 0.9 - idx * 0.05),
|
|
3268
|
+
sessionId: row.e.sessionId,
|
|
3269
|
+
eventType: row.e.eventType,
|
|
3270
|
+
timestamp: row.e.timestamp.toISOString()
|
|
3271
|
+
}));
|
|
3272
|
+
}
|
|
3273
|
+
rerankByKeywordOverlap(results, query, weights, decayPolicy) {
|
|
3274
|
+
const q = this.tokenize(query);
|
|
3275
|
+
const now = Date.now();
|
|
3276
|
+
const sw = Math.max(0, weights?.semantic ?? 0.7);
|
|
3277
|
+
const lw = Math.max(0, weights?.lexical ?? 0.2);
|
|
3278
|
+
const rw = Math.max(0, weights?.recency ?? 0.1);
|
|
3279
|
+
const total = sw + lw + rw || 1;
|
|
3280
|
+
const decayEnabled = decayPolicy?.enabled !== false;
|
|
3281
|
+
const decayWindow = Math.max(1, decayPolicy?.windowDays ?? 30);
|
|
3282
|
+
const decayMaxPenalty = Math.max(0, decayPolicy?.maxPenalty ?? 0.15);
|
|
3283
|
+
return [...results].map((r) => {
|
|
3284
|
+
const overlap = this.keywordOverlap(q, this.tokenize(r.content));
|
|
3285
|
+
const recencyDays = Math.max(0, (now - new Date(r.timestamp).getTime()) / (1e3 * 60 * 60 * 24));
|
|
3286
|
+
const recency = Math.max(0, 1 - recencyDays / decayWindow);
|
|
3287
|
+
let blended = (r.score * sw + overlap * lw + recency * rw) / total;
|
|
3288
|
+
if (decayEnabled && recencyDays > decayWindow && overlap < 0.5) {
|
|
3289
|
+
const ageFactor = Math.min(1, (recencyDays - decayWindow) / decayWindow);
|
|
3290
|
+
blended -= decayMaxPenalty * ageFactor;
|
|
3291
|
+
}
|
|
3292
|
+
return { ...r, score: Math.max(0, blended), semanticScore: r.score, lexicalScore: overlap, recencyScore: recency };
|
|
3293
|
+
}).sort((a, b) => b.score - a.score);
|
|
3294
|
+
}
|
|
3295
|
+
async applyScopeFilters(results, options) {
|
|
3296
|
+
const scope = options?.scope;
|
|
3297
|
+
const projectScopeMode = options?.projectScopeMode ?? "global";
|
|
3298
|
+
const allowedProjectHashes = new Set(
|
|
3299
|
+
[options?.projectHash, ...options?.allowedProjectHashes || []].filter(
|
|
3300
|
+
(value) => typeof value === "string" && value.length > 0
|
|
3301
|
+
)
|
|
3302
|
+
);
|
|
3303
|
+
if (!scope && projectScopeMode === "global")
|
|
3304
|
+
return results;
|
|
3305
|
+
const normalizedIncludes = (scope?.contentIncludes || []).map((s) => s.toLowerCase());
|
|
3306
|
+
const filtered = [];
|
|
3307
|
+
for (const result of results) {
|
|
3308
|
+
if (scope?.sessionId && result.sessionId !== scope.sessionId)
|
|
3309
|
+
continue;
|
|
3310
|
+
if (scope?.sessionIdPrefix && !result.sessionId.startsWith(scope.sessionIdPrefix))
|
|
3311
|
+
continue;
|
|
3312
|
+
if (scope?.eventTypes && scope.eventTypes.length > 0 && !scope.eventTypes.includes(result.eventType))
|
|
3313
|
+
continue;
|
|
3314
|
+
const event = await this.eventStore.getEvent(result.eventId);
|
|
3315
|
+
if (!event)
|
|
3316
|
+
continue;
|
|
3317
|
+
if (scope?.canonicalKeyPrefix && !event.canonicalKey.startsWith(scope.canonicalKeyPrefix))
|
|
3318
|
+
continue;
|
|
3319
|
+
if (normalizedIncludes.length > 0) {
|
|
3320
|
+
const lc = event.content.toLowerCase();
|
|
3321
|
+
if (!normalizedIncludes.some((needle) => lc.includes(needle)))
|
|
3322
|
+
continue;
|
|
3323
|
+
}
|
|
3324
|
+
if (scope?.metadata && !this.matchesMetadataScope(event.metadata, scope.metadata))
|
|
3325
|
+
continue;
|
|
3326
|
+
const projectHash = this.extractProjectHash(event.metadata);
|
|
3327
|
+
filtered.push({ result, projectHash });
|
|
3328
|
+
}
|
|
3329
|
+
if (projectScopeMode === "global" || allowedProjectHashes.size === 0) {
|
|
3330
|
+
return filtered.map((x) => x.result);
|
|
3331
|
+
}
|
|
3332
|
+
const projectMatched = filtered.filter((x) => x.projectHash && allowedProjectHashes.has(x.projectHash));
|
|
3333
|
+
if (projectScopeMode === "strict") {
|
|
3334
|
+
return projectMatched.map((x) => x.result);
|
|
3335
|
+
}
|
|
3336
|
+
return (projectMatched.length > 0 ? projectMatched : filtered).map((x) => x.result);
|
|
3337
|
+
}
|
|
3338
|
+
extractProjectHash(metadata) {
|
|
3339
|
+
if (!metadata || typeof metadata !== "object")
|
|
3340
|
+
return void 0;
|
|
3341
|
+
const scope = metadata.scope;
|
|
3342
|
+
if (!scope || typeof scope !== "object")
|
|
3343
|
+
return void 0;
|
|
3344
|
+
const project = scope.project;
|
|
3345
|
+
if (!project || typeof project !== "object")
|
|
3346
|
+
return void 0;
|
|
3347
|
+
const hash = project.hash;
|
|
3348
|
+
return typeof hash === "string" && hash.length > 0 ? hash : void 0;
|
|
2762
3349
|
}
|
|
2763
|
-
/**
|
|
2764
|
-
* Retrieve memories from a specific session
|
|
2765
|
-
*/
|
|
2766
3350
|
async retrieveFromSession(sessionId) {
|
|
2767
3351
|
return this.eventStore.getSessionEvents(sessionId);
|
|
2768
3352
|
}
|
|
2769
|
-
/**
|
|
2770
|
-
* Get recent memories across all sessions
|
|
2771
|
-
*/
|
|
2772
3353
|
async retrieveRecent(limit = 100) {
|
|
2773
3354
|
return this.eventStore.getRecentEvents(limit);
|
|
2774
3355
|
}
|
|
2775
|
-
/**
|
|
2776
|
-
* Enrich search results with full event data
|
|
2777
|
-
*/
|
|
2778
3356
|
async enrichResults(results, options) {
|
|
2779
3357
|
const memories = [];
|
|
2780
3358
|
for (const result of results) {
|
|
@@ -2782,27 +3360,16 @@ var Retriever = class {
|
|
|
2782
3360
|
if (!event)
|
|
2783
3361
|
continue;
|
|
2784
3362
|
if (this.graduation) {
|
|
2785
|
-
this.graduation.recordAccess(
|
|
2786
|
-
event.id,
|
|
2787
|
-
options.sessionId || "unknown",
|
|
2788
|
-
result.score
|
|
2789
|
-
);
|
|
3363
|
+
this.graduation.recordAccess(event.id, options.sessionId || "unknown", result.score);
|
|
2790
3364
|
}
|
|
2791
3365
|
let sessionContext;
|
|
2792
3366
|
if (options.includeSessionContext) {
|
|
2793
3367
|
sessionContext = await this.getSessionContext(event.sessionId, event.id);
|
|
2794
3368
|
}
|
|
2795
|
-
memories.push({
|
|
2796
|
-
event,
|
|
2797
|
-
score: result.score,
|
|
2798
|
-
sessionContext
|
|
2799
|
-
});
|
|
3369
|
+
memories.push({ event, score: result.score, sessionContext });
|
|
2800
3370
|
}
|
|
2801
3371
|
return memories;
|
|
2802
3372
|
}
|
|
2803
|
-
/**
|
|
2804
|
-
* Get surrounding context from the same session
|
|
2805
|
-
*/
|
|
2806
3373
|
async getSessionContext(sessionId, eventId) {
|
|
2807
3374
|
const sessionEvents = await this.eventStore.getSessionEvents(sessionId);
|
|
2808
3375
|
const eventIndex = sessionEvents.findIndex((e) => e.id === eventId);
|
|
@@ -2815,55 +3382,86 @@ var Retriever = class {
|
|
|
2815
3382
|
return void 0;
|
|
2816
3383
|
return contextEvents.filter((e) => e.id !== eventId).map((e) => `[${e.eventType}]: ${e.content.slice(0, 200)}...`).join("\n");
|
|
2817
3384
|
}
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
3385
|
+
buildUnifiedContext(projectResult, sharedMemories) {
|
|
3386
|
+
let context = projectResult.context;
|
|
3387
|
+
if (sharedMemories.length === 0)
|
|
3388
|
+
return context;
|
|
3389
|
+
context += "\n\n## Cross-Project Knowledge\n\n";
|
|
3390
|
+
for (const memory of sharedMemories.slice(0, 3)) {
|
|
3391
|
+
context += `### ${memory.title}
|
|
3392
|
+
`;
|
|
3393
|
+
if (memory.symptoms.length > 0)
|
|
3394
|
+
context += `**Symptoms:** ${memory.symptoms.join(", ")}
|
|
3395
|
+
`;
|
|
3396
|
+
context += `**Root Cause:** ${memory.rootCause}
|
|
3397
|
+
`;
|
|
3398
|
+
context += `**Solution:** ${memory.solution}
|
|
3399
|
+
`;
|
|
3400
|
+
if (memory.technologies && memory.technologies.length > 0)
|
|
3401
|
+
context += `**Technologies:** ${memory.technologies.join(", ")}
|
|
3402
|
+
`;
|
|
3403
|
+
context += `_Confidence: ${(memory.confidence * 100).toFixed(0)}%_
|
|
3404
|
+
|
|
3405
|
+
`;
|
|
3406
|
+
}
|
|
3407
|
+
return context;
|
|
3408
|
+
}
|
|
2821
3409
|
buildContext(memories, maxTokens) {
|
|
2822
3410
|
const parts = [];
|
|
2823
3411
|
let currentTokens = 0;
|
|
2824
3412
|
for (const memory of memories) {
|
|
2825
3413
|
const memoryText = this.formatMemory(memory);
|
|
2826
3414
|
const memoryTokens = this.estimateTokens(memoryText);
|
|
2827
|
-
if (currentTokens + memoryTokens > maxTokens)
|
|
3415
|
+
if (currentTokens + memoryTokens > maxTokens)
|
|
2828
3416
|
break;
|
|
2829
|
-
}
|
|
2830
3417
|
parts.push(memoryText);
|
|
2831
3418
|
currentTokens += memoryTokens;
|
|
2832
3419
|
}
|
|
2833
|
-
if (parts.length === 0)
|
|
3420
|
+
if (parts.length === 0)
|
|
2834
3421
|
return "";
|
|
2835
|
-
}
|
|
2836
3422
|
return `## Relevant Memories
|
|
2837
3423
|
|
|
2838
3424
|
${parts.join("\n\n---\n\n")}`;
|
|
2839
3425
|
}
|
|
2840
|
-
/**
|
|
2841
|
-
* Format a single memory for context
|
|
2842
|
-
*/
|
|
2843
3426
|
formatMemory(memory) {
|
|
2844
3427
|
const { event, score, sessionContext } = memory;
|
|
2845
3428
|
const date = event.timestamp.toISOString().split("T")[0];
|
|
2846
3429
|
let text = `**${event.eventType}** (${date}, score: ${score.toFixed(2)})
|
|
2847
3430
|
${event.content}`;
|
|
2848
|
-
if (sessionContext)
|
|
3431
|
+
if (sessionContext)
|
|
2849
3432
|
text += `
|
|
2850
3433
|
|
|
2851
3434
|
_Context:_ ${sessionContext}`;
|
|
2852
|
-
}
|
|
2853
3435
|
return text;
|
|
2854
3436
|
}
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
|
|
3437
|
+
matchesMetadataScope(metadata, expected) {
|
|
3438
|
+
if (!metadata)
|
|
3439
|
+
return false;
|
|
3440
|
+
return Object.entries(expected).every(([path10, value]) => {
|
|
3441
|
+
const actual = path10.split(".").reduce((acc, key) => {
|
|
3442
|
+
if (typeof acc !== "object" || acc === null)
|
|
3443
|
+
return void 0;
|
|
3444
|
+
return acc[key];
|
|
3445
|
+
}, metadata);
|
|
3446
|
+
return actual === value;
|
|
3447
|
+
});
|
|
3448
|
+
}
|
|
3449
|
+
tokenize(text) {
|
|
3450
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
3451
|
+
}
|
|
3452
|
+
keywordOverlap(a, b) {
|
|
3453
|
+
if (a.length === 0 || b.length === 0)
|
|
3454
|
+
return 0;
|
|
3455
|
+
const bs = new Set(b);
|
|
3456
|
+
let hit = 0;
|
|
3457
|
+
for (const t of a)
|
|
3458
|
+
if (bs.has(t))
|
|
3459
|
+
hit += 1;
|
|
3460
|
+
return hit / a.length;
|
|
3461
|
+
}
|
|
2858
3462
|
estimateTokens(text) {
|
|
2859
3463
|
return Math.ceil(text.length / 4);
|
|
2860
3464
|
}
|
|
2861
|
-
/**
|
|
2862
|
-
* Get event age in days (for recency scoring)
|
|
2863
|
-
*/
|
|
2864
|
-
getEventAgeDays(eventId) {
|
|
2865
|
-
return 0;
|
|
2866
|
-
}
|
|
2867
3465
|
};
|
|
2868
3466
|
function createRetriever(eventStore, vectorStore, embedder, matcher) {
|
|
2869
3467
|
return new Retriever(eventStore, vectorStore, embedder, matcher);
|
|
@@ -4154,14 +4752,67 @@ var ConsolidatedStore = class {
|
|
|
4154
4752
|
);
|
|
4155
4753
|
}
|
|
4156
4754
|
/**
|
|
4157
|
-
*
|
|
4755
|
+
* Create a long-term rule promoted from stable summaries
|
|
4158
4756
|
*/
|
|
4159
|
-
async
|
|
4160
|
-
const
|
|
4757
|
+
async createRule(input) {
|
|
4758
|
+
const ruleId = randomUUID6();
|
|
4759
|
+
await dbRun(
|
|
4161
4760
|
this.db,
|
|
4162
|
-
`
|
|
4761
|
+
`INSERT INTO consolidated_rules
|
|
4762
|
+
(rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
|
|
4763
|
+
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
|
4764
|
+
[
|
|
4765
|
+
ruleId,
|
|
4766
|
+
input.rule,
|
|
4767
|
+
JSON.stringify(input.topics),
|
|
4768
|
+
JSON.stringify(input.sourceMemoryIds),
|
|
4769
|
+
JSON.stringify(input.sourceEvents),
|
|
4770
|
+
input.confidence
|
|
4771
|
+
]
|
|
4163
4772
|
);
|
|
4164
|
-
return
|
|
4773
|
+
return ruleId;
|
|
4774
|
+
}
|
|
4775
|
+
async getRules(options) {
|
|
4776
|
+
const limit = options?.limit || 100;
|
|
4777
|
+
const rows = await dbAll(
|
|
4778
|
+
this.db,
|
|
4779
|
+
`SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
|
|
4780
|
+
[limit]
|
|
4781
|
+
);
|
|
4782
|
+
return rows.map((row) => ({
|
|
4783
|
+
ruleId: row.rule_id,
|
|
4784
|
+
rule: row.rule,
|
|
4785
|
+
topics: JSON.parse(row.topics || "[]"),
|
|
4786
|
+
sourceMemoryIds: JSON.parse(row.source_memory_ids || "[]"),
|
|
4787
|
+
sourceEvents: JSON.parse(row.source_events || "[]"),
|
|
4788
|
+
confidence: Number(row.confidence ?? 0.5),
|
|
4789
|
+
createdAt: toDate(row.created_at) || /* @__PURE__ */ new Date()
|
|
4790
|
+
}));
|
|
4791
|
+
}
|
|
4792
|
+
async countRules() {
|
|
4793
|
+
const result = await dbAll(
|
|
4794
|
+
this.db,
|
|
4795
|
+
`SELECT COUNT(*) as count FROM consolidated_rules`
|
|
4796
|
+
);
|
|
4797
|
+
return result[0]?.count || 0;
|
|
4798
|
+
}
|
|
4799
|
+
async hasRuleForSourceMemory(memoryId) {
|
|
4800
|
+
const rows = await dbAll(
|
|
4801
|
+
this.db,
|
|
4802
|
+
`SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
|
|
4803
|
+
[`%"${memoryId}"%`]
|
|
4804
|
+
);
|
|
4805
|
+
return (rows[0]?.count || 0) > 0;
|
|
4806
|
+
}
|
|
4807
|
+
/**
|
|
4808
|
+
* Get count of consolidated memories
|
|
4809
|
+
*/
|
|
4810
|
+
async count() {
|
|
4811
|
+
const result = await dbAll(
|
|
4812
|
+
this.db,
|
|
4813
|
+
`SELECT COUNT(*) as count FROM consolidated_memories`
|
|
4814
|
+
);
|
|
4815
|
+
return result[0]?.count || 0;
|
|
4165
4816
|
}
|
|
4166
4817
|
/**
|
|
4167
4818
|
* Get most accessed memories (for importance scoring)
|
|
@@ -4302,7 +4953,14 @@ var ConsolidationWorker = class {
|
|
|
4302
4953
|
* Force a consolidation run (manual trigger)
|
|
4303
4954
|
*/
|
|
4304
4955
|
async forceRun() {
|
|
4305
|
-
|
|
4956
|
+
const out = await this.consolidateWithReport();
|
|
4957
|
+
return out.consolidatedCount;
|
|
4958
|
+
}
|
|
4959
|
+
/**
|
|
4960
|
+
* Force a consolidation run and return metrics report
|
|
4961
|
+
*/
|
|
4962
|
+
async forceRunWithReport() {
|
|
4963
|
+
return this.consolidateWithReport();
|
|
4306
4964
|
}
|
|
4307
4965
|
/**
|
|
4308
4966
|
* Schedule the next consolidation check
|
|
@@ -4342,12 +5000,21 @@ var ConsolidationWorker = class {
|
|
|
4342
5000
|
* Perform consolidation
|
|
4343
5001
|
*/
|
|
4344
5002
|
async consolidate() {
|
|
5003
|
+
const out = await this.consolidateWithReport();
|
|
5004
|
+
return out.consolidatedCount;
|
|
5005
|
+
}
|
|
5006
|
+
async consolidateWithReport() {
|
|
4345
5007
|
const workingSet = await this.workingSetStore.get();
|
|
4346
5008
|
if (workingSet.recentEvents.length < 3) {
|
|
4347
|
-
return
|
|
5009
|
+
return {
|
|
5010
|
+
consolidatedCount: 0,
|
|
5011
|
+
promotedRuleCount: 0,
|
|
5012
|
+
report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
|
|
5013
|
+
};
|
|
4348
5014
|
}
|
|
4349
5015
|
const groups = this.groupByTopic(workingSet.recentEvents);
|
|
4350
5016
|
let consolidatedCount = 0;
|
|
5017
|
+
const createdMemoryIds = [];
|
|
4351
5018
|
for (const group of groups) {
|
|
4352
5019
|
if (group.events.length < 3)
|
|
4353
5020
|
continue;
|
|
@@ -4356,14 +5023,16 @@ var ConsolidationWorker = class {
|
|
|
4356
5023
|
if (alreadyConsolidated)
|
|
4357
5024
|
continue;
|
|
4358
5025
|
const summary = await this.summarize(group);
|
|
4359
|
-
await this.consolidatedStore.create({
|
|
5026
|
+
const memoryId = await this.consolidatedStore.create({
|
|
4360
5027
|
summary,
|
|
4361
5028
|
topics: group.topics,
|
|
4362
5029
|
sourceEvents: eventIds,
|
|
4363
5030
|
confidence: this.calculateConfidence(group)
|
|
4364
5031
|
});
|
|
5032
|
+
createdMemoryIds.push(memoryId);
|
|
4365
5033
|
consolidatedCount++;
|
|
4366
5034
|
}
|
|
5035
|
+
const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
|
|
4367
5036
|
if (consolidatedCount > 0) {
|
|
4368
5037
|
const consolidatedEventIds = groups.filter((g) => g.events.length >= 3).flatMap((g) => g.events.map((e) => e.id));
|
|
4369
5038
|
const oldEventIds = consolidatedEventIds.filter((id) => {
|
|
@@ -4377,7 +5046,61 @@ var ConsolidationWorker = class {
|
|
|
4377
5046
|
await this.workingSetStore.prune(oldEventIds);
|
|
4378
5047
|
}
|
|
4379
5048
|
}
|
|
4380
|
-
|
|
5049
|
+
const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
|
|
5050
|
+
return { consolidatedCount, promotedRuleCount, report };
|
|
5051
|
+
}
|
|
5052
|
+
async promoteStableSummariesToRules(memoryIds) {
|
|
5053
|
+
let promoted = 0;
|
|
5054
|
+
for (const memoryId of memoryIds) {
|
|
5055
|
+
const memory = await this.consolidatedStore.get(memoryId);
|
|
5056
|
+
if (!memory)
|
|
5057
|
+
continue;
|
|
5058
|
+
if (memory.confidence < 0.55)
|
|
5059
|
+
continue;
|
|
5060
|
+
if (memory.sourceEvents.length < 4)
|
|
5061
|
+
continue;
|
|
5062
|
+
const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
|
|
5063
|
+
if (exists)
|
|
5064
|
+
continue;
|
|
5065
|
+
const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
|
|
5066
|
+
if (!rule)
|
|
5067
|
+
continue;
|
|
5068
|
+
await this.consolidatedStore.createRule({
|
|
5069
|
+
rule,
|
|
5070
|
+
topics: memory.topics,
|
|
5071
|
+
sourceMemoryIds: [memory.memoryId],
|
|
5072
|
+
sourceEvents: memory.sourceEvents,
|
|
5073
|
+
confidence: Math.min(1, memory.confidence + 0.08)
|
|
5074
|
+
});
|
|
5075
|
+
promoted++;
|
|
5076
|
+
}
|
|
5077
|
+
return promoted;
|
|
5078
|
+
}
|
|
5079
|
+
buildRuleFromSummary(summary, topics) {
|
|
5080
|
+
const lines = summary.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).filter((l) => !l.toLowerCase().startsWith("topics:"));
|
|
5081
|
+
const bullet = lines.find((l) => l.startsWith("- "))?.replace(/^-\s*/, "");
|
|
5082
|
+
const seed = bullet || lines[0];
|
|
5083
|
+
if (!seed || seed.length < 8)
|
|
5084
|
+
return null;
|
|
5085
|
+
const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(", ")}] ` : "";
|
|
5086
|
+
return `${topicPrefix}${seed}`;
|
|
5087
|
+
}
|
|
5088
|
+
buildCostQualityReport(events, groups, consolidatedCount) {
|
|
5089
|
+
const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
|
|
5090
|
+
const afterSummaries = groups.filter((g) => g.events.length >= 3).slice(0, Math.max(consolidatedCount, 1));
|
|
5091
|
+
const afterTokenEstimate = afterSummaries.length > 0 ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0) : beforeTokenEstimate;
|
|
5092
|
+
const reductionRatio = beforeTokenEstimate > 0 ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate) : 0;
|
|
5093
|
+
const qualityGuardPassed = consolidatedCount === 0 ? true : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
|
|
5094
|
+
return {
|
|
5095
|
+
beforeTokenEstimate,
|
|
5096
|
+
afterTokenEstimate,
|
|
5097
|
+
reductionRatio,
|
|
5098
|
+
qualityGuardPassed,
|
|
5099
|
+
details: `groups=${groups.length}, consolidated=${consolidatedCount}`
|
|
5100
|
+
};
|
|
5101
|
+
}
|
|
5102
|
+
estimateTokens(text) {
|
|
5103
|
+
return Math.ceil((text || "").length / 4);
|
|
4381
5104
|
}
|
|
4382
5105
|
/**
|
|
4383
5106
|
* Check if consolidation should run
|
|
@@ -4937,13 +5660,185 @@ function createGraduationWorker(eventStore, graduation, config) {
|
|
|
4937
5660
|
);
|
|
4938
5661
|
}
|
|
4939
5662
|
|
|
5663
|
+
// src/core/md-mirror.ts
|
|
5664
|
+
import * as fs3 from "node:fs";
|
|
5665
|
+
import * as path2 from "node:path";
|
|
5666
|
+
function sanitizeSegment2(input, fallback) {
|
|
5667
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
5668
|
+
return v || fallback;
|
|
5669
|
+
}
|
|
5670
|
+
function getAtPath(obj, dotted) {
|
|
5671
|
+
if (!obj)
|
|
5672
|
+
return void 0;
|
|
5673
|
+
return dotted.split(".").reduce((acc, key) => {
|
|
5674
|
+
if (!acc || typeof acc !== "object")
|
|
5675
|
+
return void 0;
|
|
5676
|
+
return acc[key];
|
|
5677
|
+
}, obj);
|
|
5678
|
+
}
|
|
5679
|
+
function buildMirrorPath2(rootDir, event) {
|
|
5680
|
+
const meta = event.metadata;
|
|
5681
|
+
const namespaceRaw = getAtPath(meta, "namespace") ?? getAtPath(meta, "scope.namespace") ?? event.eventType;
|
|
5682
|
+
const namespace = sanitizeSegment2(typeof namespaceRaw === "string" ? namespaceRaw : void 0, "general");
|
|
5683
|
+
const categoryRaw = getAtPath(meta, "categoryPath") ?? getAtPath(meta, "scope.categoryPath");
|
|
5684
|
+
const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0 ? categoryRaw.map((x) => sanitizeSegment2(typeof x === "string" ? x : void 0, "uncategorized")) : ["uncategorized"];
|
|
5685
|
+
const d = event.timestamp;
|
|
5686
|
+
const yyyy = d.getFullYear();
|
|
5687
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
5688
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
5689
|
+
return path2.join(rootDir, "memory", namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
|
|
5690
|
+
}
|
|
5691
|
+
var MarkdownMirror2 = class {
|
|
5692
|
+
constructor(rootDir) {
|
|
5693
|
+
this.rootDir = rootDir;
|
|
5694
|
+
}
|
|
5695
|
+
async append(event, eventId) {
|
|
5696
|
+
const out = buildMirrorPath2(this.rootDir, event);
|
|
5697
|
+
fs3.mkdirSync(path2.dirname(out), { recursive: true });
|
|
5698
|
+
const lines = [
|
|
5699
|
+
"",
|
|
5700
|
+
`## ${event.timestamp.toISOString()} | ${eventId ?? "pending-id"}`,
|
|
5701
|
+
`- type: ${event.eventType}`,
|
|
5702
|
+
`- session: ${event.sessionId}`,
|
|
5703
|
+
event.content
|
|
5704
|
+
];
|
|
5705
|
+
await fs3.promises.appendFile(out, lines.join("\n"), "utf8");
|
|
5706
|
+
await this.refreshIndex();
|
|
5707
|
+
}
|
|
5708
|
+
async refreshIndex() {
|
|
5709
|
+
const memoryRoot = path2.join(this.rootDir, "memory");
|
|
5710
|
+
await fs3.promises.mkdir(memoryRoot, { recursive: true });
|
|
5711
|
+
const files = [];
|
|
5712
|
+
await this.walk(memoryRoot, files);
|
|
5713
|
+
const mdFiles = files.filter((f) => f.endsWith(".md")).map((f) => path2.relative(this.rootDir, f)).filter((rel) => rel !== path2.join("memory", "_index.md")).sort();
|
|
5714
|
+
const index = [
|
|
5715
|
+
"# Memory Index",
|
|
5716
|
+
"",
|
|
5717
|
+
"Generated automatically by MarkdownMirror.",
|
|
5718
|
+
"",
|
|
5719
|
+
...mdFiles.map((rel) => `- ${rel}`),
|
|
5720
|
+
""
|
|
5721
|
+
].join("\n");
|
|
5722
|
+
await fs3.promises.writeFile(path2.join(memoryRoot, "_index.md"), index, "utf8");
|
|
5723
|
+
}
|
|
5724
|
+
async walk(dir, out) {
|
|
5725
|
+
const entries = await fs3.promises.readdir(dir, { withFileTypes: true });
|
|
5726
|
+
for (const e of entries) {
|
|
5727
|
+
const full = path2.join(dir, e.name);
|
|
5728
|
+
if (e.isDirectory()) {
|
|
5729
|
+
await this.walk(full, out);
|
|
5730
|
+
} else {
|
|
5731
|
+
out.push(full);
|
|
5732
|
+
}
|
|
5733
|
+
}
|
|
5734
|
+
}
|
|
5735
|
+
};
|
|
5736
|
+
|
|
5737
|
+
// src/core/ingest-interceptor.ts
|
|
5738
|
+
var IngestInterceptorRegistry = class {
|
|
5739
|
+
before = [];
|
|
5740
|
+
after = [];
|
|
5741
|
+
onError = [];
|
|
5742
|
+
registerBefore(interceptor) {
|
|
5743
|
+
this.before.push(interceptor);
|
|
5744
|
+
return () => {
|
|
5745
|
+
this.before = this.before.filter((i) => i !== interceptor);
|
|
5746
|
+
};
|
|
5747
|
+
}
|
|
5748
|
+
registerAfter(interceptor) {
|
|
5749
|
+
this.after.push(interceptor);
|
|
5750
|
+
return () => {
|
|
5751
|
+
this.after = this.after.filter((i) => i !== interceptor);
|
|
5752
|
+
};
|
|
5753
|
+
}
|
|
5754
|
+
registerOnError(interceptor) {
|
|
5755
|
+
this.onError.push(interceptor);
|
|
5756
|
+
return () => {
|
|
5757
|
+
this.onError = this.onError.filter((i) => i !== interceptor);
|
|
5758
|
+
};
|
|
5759
|
+
}
|
|
5760
|
+
async run(stage, context) {
|
|
5761
|
+
const interceptors = stage === "before" ? this.before : stage === "after" ? this.after : this.onError;
|
|
5762
|
+
for (const interceptor of interceptors) {
|
|
5763
|
+
await interceptor({ ...context, stage });
|
|
5764
|
+
}
|
|
5765
|
+
}
|
|
5766
|
+
};
|
|
5767
|
+
function mergeHierarchicalMetadata(base, patch) {
|
|
5768
|
+
if (!base && !patch)
|
|
5769
|
+
return void 0;
|
|
5770
|
+
if (!base)
|
|
5771
|
+
return patch;
|
|
5772
|
+
if (!patch)
|
|
5773
|
+
return base;
|
|
5774
|
+
const result = { ...base };
|
|
5775
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
5776
|
+
const current = result[key];
|
|
5777
|
+
if (typeof current === "object" && current !== null && !Array.isArray(current) && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
5778
|
+
result[key] = mergeHierarchicalMetadata(
|
|
5779
|
+
current,
|
|
5780
|
+
value
|
|
5781
|
+
);
|
|
5782
|
+
} else {
|
|
5783
|
+
result[key] = value;
|
|
5784
|
+
}
|
|
5785
|
+
}
|
|
5786
|
+
return result;
|
|
5787
|
+
}
|
|
5788
|
+
|
|
5789
|
+
// src/core/tag-taxonomy.ts
|
|
5790
|
+
var TAG_NAMESPACES = {
|
|
5791
|
+
SYSTEM: "sys:",
|
|
5792
|
+
QUALITY: "q:",
|
|
5793
|
+
PROJECT: "proj:",
|
|
5794
|
+
TOPIC: "topic:",
|
|
5795
|
+
TEMPORAL: "t:",
|
|
5796
|
+
USER: "user:",
|
|
5797
|
+
AGENT: "agent:"
|
|
5798
|
+
};
|
|
5799
|
+
var VALID_TAG_NAMESPACES = new Set(Object.values(TAG_NAMESPACES));
|
|
5800
|
+
function parseTag(tag) {
|
|
5801
|
+
const value = (tag || "").trim();
|
|
5802
|
+
const idx = value.indexOf(":");
|
|
5803
|
+
if (idx <= 0)
|
|
5804
|
+
return { value };
|
|
5805
|
+
const namespace = `${value.slice(0, idx)}:`;
|
|
5806
|
+
const tagValue = value.slice(idx + 1);
|
|
5807
|
+
if (!tagValue)
|
|
5808
|
+
return { value };
|
|
5809
|
+
return { namespace, value: tagValue };
|
|
5810
|
+
}
|
|
5811
|
+
function validateTag(tag) {
|
|
5812
|
+
const normalized = (tag || "").trim();
|
|
5813
|
+
if (!normalized)
|
|
5814
|
+
return false;
|
|
5815
|
+
const { namespace } = parseTag(normalized);
|
|
5816
|
+
if (!namespace)
|
|
5817
|
+
return true;
|
|
5818
|
+
return VALID_TAG_NAMESPACES.has(namespace);
|
|
5819
|
+
}
|
|
5820
|
+
function normalizeTags(tags) {
|
|
5821
|
+
if (!Array.isArray(tags))
|
|
5822
|
+
return [];
|
|
5823
|
+
const dedup = /* @__PURE__ */ new Set();
|
|
5824
|
+
for (const item of tags) {
|
|
5825
|
+
if (typeof item !== "string")
|
|
5826
|
+
continue;
|
|
5827
|
+
const normalized = item.trim();
|
|
5828
|
+
if (!validateTag(normalized))
|
|
5829
|
+
continue;
|
|
5830
|
+
dedup.add(normalized);
|
|
5831
|
+
}
|
|
5832
|
+
return [...dedup];
|
|
5833
|
+
}
|
|
5834
|
+
|
|
4940
5835
|
// src/services/memory-service.ts
|
|
4941
5836
|
function normalizePath(projectPath) {
|
|
4942
|
-
const expanded = projectPath.startsWith("~") ?
|
|
5837
|
+
const expanded = projectPath.startsWith("~") ? path3.join(os.homedir(), projectPath.slice(1)) : projectPath;
|
|
4943
5838
|
try {
|
|
4944
|
-
return
|
|
5839
|
+
return fs4.realpathSync(expanded);
|
|
4945
5840
|
} catch {
|
|
4946
|
-
return
|
|
5841
|
+
return path3.resolve(expanded);
|
|
4947
5842
|
}
|
|
4948
5843
|
}
|
|
4949
5844
|
function hashProjectPath(projectPath) {
|
|
@@ -4952,14 +5847,14 @@ function hashProjectPath(projectPath) {
|
|
|
4952
5847
|
}
|
|
4953
5848
|
function getProjectStoragePath(projectPath) {
|
|
4954
5849
|
const hash = hashProjectPath(projectPath);
|
|
4955
|
-
return
|
|
5850
|
+
return path3.join(os.homedir(), ".claude-code", "memory", "projects", hash);
|
|
4956
5851
|
}
|
|
4957
|
-
var REGISTRY_PATH =
|
|
4958
|
-
var SHARED_STORAGE_PATH =
|
|
5852
|
+
var REGISTRY_PATH = path3.join(os.homedir(), ".claude-code", "memory", "session-registry.json");
|
|
5853
|
+
var SHARED_STORAGE_PATH = path3.join(os.homedir(), ".claude-code", "memory", "shared");
|
|
4959
5854
|
function loadSessionRegistry() {
|
|
4960
5855
|
try {
|
|
4961
|
-
if (
|
|
4962
|
-
const data =
|
|
5856
|
+
if (fs4.existsSync(REGISTRY_PATH)) {
|
|
5857
|
+
const data = fs4.readFileSync(REGISTRY_PATH, "utf-8");
|
|
4963
5858
|
return JSON.parse(data);
|
|
4964
5859
|
}
|
|
4965
5860
|
} catch (error) {
|
|
@@ -4968,13 +5863,13 @@ function loadSessionRegistry() {
|
|
|
4968
5863
|
return { version: 1, sessions: {} };
|
|
4969
5864
|
}
|
|
4970
5865
|
function saveSessionRegistry(registry) {
|
|
4971
|
-
const dir =
|
|
4972
|
-
if (!
|
|
4973
|
-
|
|
5866
|
+
const dir = path3.dirname(REGISTRY_PATH);
|
|
5867
|
+
if (!fs4.existsSync(dir)) {
|
|
5868
|
+
fs4.mkdirSync(dir, { recursive: true });
|
|
4974
5869
|
}
|
|
4975
5870
|
const tempPath = REGISTRY_PATH + ".tmp";
|
|
4976
|
-
|
|
4977
|
-
|
|
5871
|
+
fs4.writeFileSync(tempPath, JSON.stringify(registry, null, 2));
|
|
5872
|
+
fs4.renameSync(tempPath, REGISTRY_PATH);
|
|
4978
5873
|
}
|
|
4979
5874
|
function registerSession(sessionId, projectPath) {
|
|
4980
5875
|
const registry = loadSessionRegistry();
|
|
@@ -5006,6 +5901,7 @@ var MemoryService = class {
|
|
|
5006
5901
|
vectorWorker = null;
|
|
5007
5902
|
graduationWorker = null;
|
|
5008
5903
|
initialized = false;
|
|
5904
|
+
ingestInterceptors = new IngestInterceptorRegistry();
|
|
5009
5905
|
// Endless Mode components
|
|
5010
5906
|
workingSetStore = null;
|
|
5011
5907
|
consolidatedStore = null;
|
|
@@ -5019,20 +5915,27 @@ var MemoryService = class {
|
|
|
5019
5915
|
sharedPromoter = null;
|
|
5020
5916
|
sharedStoreConfig = null;
|
|
5021
5917
|
projectHash = null;
|
|
5918
|
+
projectPath = null;
|
|
5022
5919
|
readOnly;
|
|
5023
5920
|
lightweightMode;
|
|
5921
|
+
mdMirror;
|
|
5024
5922
|
constructor(config) {
|
|
5025
5923
|
const storagePath = this.expandPath(config.storagePath);
|
|
5026
5924
|
this.readOnly = config.readOnly ?? false;
|
|
5027
5925
|
this.lightweightMode = config.lightweightMode ?? false;
|
|
5028
|
-
|
|
5029
|
-
|
|
5926
|
+
this.mdMirror = new MarkdownMirror2(process.cwd());
|
|
5927
|
+
if (!this.readOnly && !fs4.existsSync(storagePath)) {
|
|
5928
|
+
fs4.mkdirSync(storagePath, { recursive: true });
|
|
5030
5929
|
}
|
|
5031
5930
|
this.projectHash = config.projectHash || null;
|
|
5931
|
+
this.projectPath = config.projectPath || null;
|
|
5032
5932
|
this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
|
|
5033
5933
|
this.sqliteStore = new SQLiteEventStore(
|
|
5034
|
-
|
|
5035
|
-
{
|
|
5934
|
+
path3.join(storagePath, "events.sqlite"),
|
|
5935
|
+
{
|
|
5936
|
+
readonly: this.readOnly,
|
|
5937
|
+
markdownMirrorRoot: storagePath
|
|
5938
|
+
}
|
|
5036
5939
|
);
|
|
5037
5940
|
const analyticsEnabled = config.analyticsEnabled ?? this.readOnly;
|
|
5038
5941
|
if (!analyticsEnabled) {
|
|
@@ -5040,7 +5943,7 @@ var MemoryService = class {
|
|
|
5040
5943
|
} else if (this.readOnly) {
|
|
5041
5944
|
try {
|
|
5042
5945
|
this.analyticsStore = new EventStore(
|
|
5043
|
-
|
|
5946
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5044
5947
|
{ readOnly: true }
|
|
5045
5948
|
);
|
|
5046
5949
|
} catch {
|
|
@@ -5048,11 +5951,11 @@ var MemoryService = class {
|
|
|
5048
5951
|
}
|
|
5049
5952
|
} else {
|
|
5050
5953
|
this.analyticsStore = new EventStore(
|
|
5051
|
-
|
|
5954
|
+
path3.join(storagePath, "analytics.duckdb"),
|
|
5052
5955
|
{ readOnly: false }
|
|
5053
5956
|
);
|
|
5054
5957
|
}
|
|
5055
|
-
this.vectorStore = new VectorStore(
|
|
5958
|
+
this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
|
|
5056
5959
|
this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
|
|
5057
5960
|
this.matcher = getDefaultMatcher();
|
|
5058
5961
|
this.retriever = createRetriever(
|
|
@@ -5062,6 +5965,7 @@ var MemoryService = class {
|
|
|
5062
5965
|
this.embedder,
|
|
5063
5966
|
this.matcher
|
|
5064
5967
|
);
|
|
5968
|
+
this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
|
|
5065
5969
|
this.graduation = createGraduationPipeline(this.sqliteStore);
|
|
5066
5970
|
}
|
|
5067
5971
|
/**
|
|
@@ -5121,16 +6025,16 @@ var MemoryService = class {
|
|
|
5121
6025
|
*/
|
|
5122
6026
|
async initializeSharedStore() {
|
|
5123
6027
|
const sharedPath = this.sharedStoreConfig?.sharedStoragePath ? this.expandPath(this.sharedStoreConfig.sharedStoragePath) : SHARED_STORAGE_PATH;
|
|
5124
|
-
if (!
|
|
5125
|
-
|
|
6028
|
+
if (!fs4.existsSync(sharedPath)) {
|
|
6029
|
+
fs4.mkdirSync(sharedPath, { recursive: true });
|
|
5126
6030
|
}
|
|
5127
6031
|
this.sharedEventStore = createSharedEventStore(
|
|
5128
|
-
|
|
6032
|
+
path3.join(sharedPath, "shared.duckdb")
|
|
5129
6033
|
);
|
|
5130
6034
|
await this.sharedEventStore.initialize();
|
|
5131
6035
|
this.sharedStore = createSharedStore(this.sharedEventStore);
|
|
5132
6036
|
this.sharedVectorStore = createSharedVectorStore(
|
|
5133
|
-
|
|
6037
|
+
path3.join(sharedPath, "vectors")
|
|
5134
6038
|
);
|
|
5135
6039
|
await this.sharedVectorStore.initialize();
|
|
5136
6040
|
this.sharedPromoter = createSharedPromoter(
|
|
@@ -5141,6 +6045,86 @@ var MemoryService = class {
|
|
|
5141
6045
|
);
|
|
5142
6046
|
this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
|
|
5143
6047
|
}
|
|
6048
|
+
registerIngestBefore(interceptor) {
|
|
6049
|
+
return this.ingestInterceptors.registerBefore(interceptor);
|
|
6050
|
+
}
|
|
6051
|
+
registerIngestAfter(interceptor) {
|
|
6052
|
+
return this.ingestInterceptors.registerAfter(interceptor);
|
|
6053
|
+
}
|
|
6054
|
+
registerIngestOnError(interceptor) {
|
|
6055
|
+
return this.ingestInterceptors.registerOnError(interceptor);
|
|
6056
|
+
}
|
|
6057
|
+
async ingestWithInterceptors(operation, input, onSuccess) {
|
|
6058
|
+
const normalizedInput = {
|
|
6059
|
+
...input,
|
|
6060
|
+
metadata: mergeHierarchicalMetadata(
|
|
6061
|
+
{
|
|
6062
|
+
ingest: {
|
|
6063
|
+
operation,
|
|
6064
|
+
pipeline: "default",
|
|
6065
|
+
ts: (/* @__PURE__ */ new Date()).toISOString()
|
|
6066
|
+
},
|
|
6067
|
+
...this.projectHash ? {
|
|
6068
|
+
scope: {
|
|
6069
|
+
project: {
|
|
6070
|
+
hash: this.projectHash,
|
|
6071
|
+
...this.projectPath ? { path: this.projectPath } : {}
|
|
6072
|
+
}
|
|
6073
|
+
},
|
|
6074
|
+
tags: [`proj:${this.projectHash}`]
|
|
6075
|
+
} : {}
|
|
6076
|
+
},
|
|
6077
|
+
input.metadata
|
|
6078
|
+
)
|
|
6079
|
+
};
|
|
6080
|
+
if (this.projectHash && normalizedInput.metadata) {
|
|
6081
|
+
const meta = normalizedInput.metadata;
|
|
6082
|
+
const currentTags = Array.isArray(meta.tags) ? meta.tags.filter((x) => typeof x === "string") : [];
|
|
6083
|
+
const projectTag = `proj:${this.projectHash}`;
|
|
6084
|
+
if (!currentTags.includes(projectTag)) {
|
|
6085
|
+
meta.tags = [...currentTags, projectTag];
|
|
6086
|
+
}
|
|
6087
|
+
}
|
|
6088
|
+
if (normalizedInput.metadata) {
|
|
6089
|
+
const meta = normalizedInput.metadata;
|
|
6090
|
+
const normalizedTags = normalizeTags(meta.tags);
|
|
6091
|
+
if (normalizedTags.length > 0) {
|
|
6092
|
+
meta.tags = normalizedTags;
|
|
6093
|
+
}
|
|
6094
|
+
}
|
|
6095
|
+
await this.ingestInterceptors.run("before", {
|
|
6096
|
+
operation,
|
|
6097
|
+
sessionId: normalizedInput.sessionId,
|
|
6098
|
+
event: normalizedInput
|
|
6099
|
+
});
|
|
6100
|
+
try {
|
|
6101
|
+
const result = await this.sqliteStore.append(normalizedInput);
|
|
6102
|
+
if (result.success && !result.isDuplicate) {
|
|
6103
|
+
if (onSuccess) {
|
|
6104
|
+
await onSuccess(result.eventId);
|
|
6105
|
+
}
|
|
6106
|
+
try {
|
|
6107
|
+
await this.mdMirror.append(normalizedInput, result.eventId);
|
|
6108
|
+
} catch {
|
|
6109
|
+
}
|
|
6110
|
+
}
|
|
6111
|
+
await this.ingestInterceptors.run("after", {
|
|
6112
|
+
operation,
|
|
6113
|
+
sessionId: normalizedInput.sessionId,
|
|
6114
|
+
event: normalizedInput
|
|
6115
|
+
});
|
|
6116
|
+
return result;
|
|
6117
|
+
} catch (error) {
|
|
6118
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
6119
|
+
await this.ingestInterceptors.run("error", {
|
|
6120
|
+
operation,
|
|
6121
|
+
sessionId: normalizedInput.sessionId,
|
|
6122
|
+
event: normalizedInput,
|
|
6123
|
+
error: normalizedError
|
|
6124
|
+
});
|
|
6125
|
+
throw error;
|
|
6126
|
+
}
|
|
6127
|
+
}
|
|
5144
6128
|
/**
|
|
5145
6129
|
* Start a new session
|
|
5146
6130
|
*/
|
|
@@ -5168,50 +6152,57 @@ var MemoryService = class {
|
|
|
5168
6152
|
*/
|
|
5169
6153
|
async storeUserPrompt(sessionId, content, metadata) {
|
|
5170
6154
|
await this.initialize();
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
6155
|
+
return this.ingestWithInterceptors(
|
|
6156
|
+
"user_prompt",
|
|
6157
|
+
{
|
|
6158
|
+
eventType: "user_prompt",
|
|
6159
|
+
sessionId,
|
|
6160
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6161
|
+
content,
|
|
6162
|
+
metadata
|
|
6163
|
+
},
|
|
6164
|
+
async (eventId) => {
|
|
6165
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6166
|
+
}
|
|
6167
|
+
);
|
|
5182
6168
|
}
|
|
5183
6169
|
/**
|
|
5184
6170
|
* Store an agent response
|
|
5185
6171
|
*/
|
|
5186
6172
|
async storeAgentResponse(sessionId, content, metadata) {
|
|
5187
6173
|
await this.initialize();
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
6174
|
+
return this.ingestWithInterceptors(
|
|
6175
|
+
"agent_response",
|
|
6176
|
+
{
|
|
6177
|
+
eventType: "agent_response",
|
|
6178
|
+
sessionId,
|
|
6179
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6180
|
+
content,
|
|
6181
|
+
metadata
|
|
6182
|
+
},
|
|
6183
|
+
async (eventId) => {
|
|
6184
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, content);
|
|
6185
|
+
}
|
|
6186
|
+
);
|
|
5199
6187
|
}
|
|
5200
6188
|
/**
|
|
5201
6189
|
* Store a session summary
|
|
5202
6190
|
*/
|
|
5203
|
-
async storeSessionSummary(sessionId, summary) {
|
|
6191
|
+
async storeSessionSummary(sessionId, summary, metadata) {
|
|
5204
6192
|
await this.initialize();
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
6193
|
+
return this.ingestWithInterceptors(
|
|
6194
|
+
"session_summary",
|
|
6195
|
+
{
|
|
6196
|
+
eventType: "session_summary",
|
|
6197
|
+
sessionId,
|
|
6198
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6199
|
+
content: summary,
|
|
6200
|
+
metadata
|
|
6201
|
+
},
|
|
6202
|
+
async (eventId) => {
|
|
6203
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, summary);
|
|
6204
|
+
}
|
|
6205
|
+
);
|
|
5215
6206
|
}
|
|
5216
6207
|
/**
|
|
5217
6208
|
* Store a tool observation
|
|
@@ -5220,40 +6211,181 @@ var MemoryService = class {
|
|
|
5220
6211
|
await this.initialize();
|
|
5221
6212
|
const content = JSON.stringify(payload);
|
|
5222
6213
|
const turnId = payload.metadata?.turnId;
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
6214
|
+
return this.ingestWithInterceptors(
|
|
6215
|
+
"tool_observation",
|
|
6216
|
+
{
|
|
6217
|
+
eventType: "tool_observation",
|
|
6218
|
+
sessionId,
|
|
6219
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
6220
|
+
content,
|
|
6221
|
+
metadata: {
|
|
6222
|
+
toolName: payload.toolName,
|
|
6223
|
+
success: payload.success,
|
|
6224
|
+
...turnId ? { turnId } : {}
|
|
6225
|
+
}
|
|
6226
|
+
},
|
|
6227
|
+
async (eventId) => {
|
|
6228
|
+
const embeddingContent = createToolObservationEmbedding(
|
|
6229
|
+
payload.toolName,
|
|
6230
|
+
payload.metadata || {},
|
|
6231
|
+
payload.success
|
|
6232
|
+
);
|
|
6233
|
+
await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
|
|
5232
6234
|
}
|
|
5233
|
-
|
|
5234
|
-
if (result.success && !result.isDuplicate) {
|
|
5235
|
-
const embeddingContent = createToolObservationEmbedding(
|
|
5236
|
-
payload.toolName,
|
|
5237
|
-
payload.metadata || {},
|
|
5238
|
-
payload.success
|
|
5239
|
-
);
|
|
5240
|
-
await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
|
|
5241
|
-
}
|
|
5242
|
-
return result;
|
|
6235
|
+
);
|
|
5243
6236
|
}
|
|
5244
6237
|
/**
|
|
5245
6238
|
* Retrieve relevant memories for a query
|
|
5246
6239
|
*/
|
|
5247
6240
|
async retrieveMemories(query, options) {
|
|
5248
6241
|
await this.initialize();
|
|
6242
|
+
const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
|
|
6243
|
+
let result;
|
|
5249
6244
|
if (options?.includeShared && this.sharedStore) {
|
|
5250
|
-
|
|
6245
|
+
result = await this.retriever.retrieveUnified(query, {
|
|
5251
6246
|
...options,
|
|
6247
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6248
|
+
rerankWeights,
|
|
5252
6249
|
includeShared: true,
|
|
5253
|
-
projectHash: this.projectHash || void 0
|
|
6250
|
+
projectHash: this.projectHash || void 0,
|
|
6251
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6252
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6253
|
+
});
|
|
6254
|
+
} else {
|
|
6255
|
+
result = await this.retriever.retrieve(query, {
|
|
6256
|
+
...options,
|
|
6257
|
+
intentRewrite: options?.intentRewrite === true,
|
|
6258
|
+
rerankWeights,
|
|
6259
|
+
projectHash: this.projectHash || void 0,
|
|
6260
|
+
projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? "strict" : "global"),
|
|
6261
|
+
allowedProjectHashes: options?.allowedProjectHashes
|
|
6262
|
+
});
|
|
6263
|
+
}
|
|
6264
|
+
try {
|
|
6265
|
+
const selectedEventIds = result.memories.map((m) => m.event.id);
|
|
6266
|
+
const selectedDetails = (result.selectedDebug || []).map((d) => ({
|
|
6267
|
+
eventId: d.eventId,
|
|
6268
|
+
score: d.score,
|
|
6269
|
+
semanticScore: d.semanticScore,
|
|
6270
|
+
lexicalScore: d.lexicalScore,
|
|
6271
|
+
recencyScore: d.recencyScore
|
|
6272
|
+
}));
|
|
6273
|
+
const candidateDetails = (result.candidateDebug || []).map((d) => ({
|
|
6274
|
+
eventId: d.eventId,
|
|
6275
|
+
score: d.score,
|
|
6276
|
+
semanticScore: d.semanticScore,
|
|
6277
|
+
lexicalScore: d.lexicalScore,
|
|
6278
|
+
recencyScore: d.recencyScore
|
|
6279
|
+
}));
|
|
6280
|
+
const candidateEventIds = candidateDetails.length > 0 ? candidateDetails.map((d) => d.eventId) : selectedEventIds;
|
|
6281
|
+
await this.sqliteStore.recordRetrievalTrace({
|
|
6282
|
+
sessionId: options?.sessionId,
|
|
6283
|
+
projectHash: this.projectHash || void 0,
|
|
6284
|
+
queryText: query,
|
|
6285
|
+
strategy: options?.strategy || "auto",
|
|
6286
|
+
candidateEventIds,
|
|
6287
|
+
selectedEventIds,
|
|
6288
|
+
candidateDetails,
|
|
6289
|
+
selectedDetails,
|
|
6290
|
+
confidence: result.matchResult.confidence,
|
|
6291
|
+
fallbackTrace: result.fallbackTrace || []
|
|
5254
6292
|
});
|
|
6293
|
+
} catch {
|
|
6294
|
+
}
|
|
6295
|
+
return result;
|
|
6296
|
+
}
|
|
6297
|
+
getConfiguredRerankWeights() {
|
|
6298
|
+
const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? "");
|
|
6299
|
+
const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? "");
|
|
6300
|
+
const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? "");
|
|
6301
|
+
const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
|
|
6302
|
+
if (!allFinite)
|
|
6303
|
+
return void 0;
|
|
6304
|
+
const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
|
|
6305
|
+
const total = semantic + lexical + recency;
|
|
6306
|
+
if (!nonNegative || total <= 0)
|
|
6307
|
+
return void 0;
|
|
6308
|
+
return {
|
|
6309
|
+
semantic: semantic / total,
|
|
6310
|
+
lexical: lexical / total,
|
|
6311
|
+
recency: recency / total
|
|
6312
|
+
};
|
|
6313
|
+
}
|
|
6314
|
+
async getRerankWeights(adaptive) {
|
|
6315
|
+
const configured = this.getConfiguredRerankWeights();
|
|
6316
|
+
if (configured)
|
|
6317
|
+
return configured;
|
|
6318
|
+
if (adaptive)
|
|
6319
|
+
return this.getAdaptiveRerankWeights();
|
|
6320
|
+
return void 0;
|
|
6321
|
+
}
|
|
6322
|
+
async rewriteQueryIntent(query) {
|
|
6323
|
+
if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== "1")
|
|
6324
|
+
return null;
|
|
6325
|
+
const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
|
|
6326
|
+
if (!apiUrl)
|
|
6327
|
+
return null;
|
|
6328
|
+
const controller = new AbortController();
|
|
6329
|
+
const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5e3);
|
|
6330
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
6331
|
+
try {
|
|
6332
|
+
const prompt = [
|
|
6333
|
+
"Rewrite user query for memory retrieval intent expansion.",
|
|
6334
|
+
"Return plain text only, one line, no markdown.",
|
|
6335
|
+
`Query: ${query}`
|
|
6336
|
+
].join("\n");
|
|
6337
|
+
const res = await fetch(apiUrl, {
|
|
6338
|
+
method: "POST",
|
|
6339
|
+
headers: {
|
|
6340
|
+
"Content-Type": "application/json",
|
|
6341
|
+
Accept: "*/*",
|
|
6342
|
+
Origin: process.env.COMPANY_INT_ORIGIN || "http://company-int.aplusai.ai",
|
|
6343
|
+
Referer: process.env.COMPANY_INT_REFERER || "http://company-int.aplusai.ai/"
|
|
6344
|
+
},
|
|
6345
|
+
body: JSON.stringify({
|
|
6346
|
+
question: prompt,
|
|
6347
|
+
company_name: null,
|
|
6348
|
+
conversation_id: null
|
|
6349
|
+
}),
|
|
6350
|
+
signal: controller.signal
|
|
6351
|
+
});
|
|
6352
|
+
const text = (await res.text()).trim();
|
|
6353
|
+
if (!text)
|
|
6354
|
+
return null;
|
|
6355
|
+
const oneLine = text.replace(/^data:\s*/gm, "").split(/\r?\n/).map((x) => x.trim()).filter(Boolean).join(" ").slice(0, 240);
|
|
6356
|
+
if (!oneLine || oneLine.toLowerCase() === query.toLowerCase())
|
|
6357
|
+
return null;
|
|
6358
|
+
return oneLine;
|
|
6359
|
+
} catch {
|
|
6360
|
+
return null;
|
|
6361
|
+
} finally {
|
|
6362
|
+
clearTimeout(timeout);
|
|
6363
|
+
}
|
|
6364
|
+
}
|
|
6365
|
+
async getAdaptiveRerankWeights() {
|
|
6366
|
+
try {
|
|
6367
|
+
const s = await this.sqliteStore.getHelpfulnessStats();
|
|
6368
|
+
if (s.totalEvaluated < 20)
|
|
6369
|
+
return void 0;
|
|
6370
|
+
let semantic = 0.7;
|
|
6371
|
+
let lexical = 0.2;
|
|
6372
|
+
let recency = 0.1;
|
|
6373
|
+
if (s.avgScore < 0.45) {
|
|
6374
|
+
semantic -= 0.1;
|
|
6375
|
+
lexical += 0.1;
|
|
6376
|
+
} else if (s.avgScore > 0.75) {
|
|
6377
|
+
semantic += 0.05;
|
|
6378
|
+
lexical -= 0.05;
|
|
6379
|
+
}
|
|
6380
|
+
if (s.unhelpful > s.helpful) {
|
|
6381
|
+
recency += 0.05;
|
|
6382
|
+
semantic -= 0.03;
|
|
6383
|
+
lexical -= 0.02;
|
|
6384
|
+
}
|
|
6385
|
+
return { semantic, lexical, recency };
|
|
6386
|
+
} catch {
|
|
6387
|
+
return void 0;
|
|
5255
6388
|
}
|
|
5256
|
-
return this.retriever.retrieve(query, options);
|
|
5257
6389
|
}
|
|
5258
6390
|
/**
|
|
5259
6391
|
* Fast keyword search using SQLite FTS5
|
|
@@ -5295,6 +6427,18 @@ var MemoryService = class {
|
|
|
5295
6427
|
/**
|
|
5296
6428
|
* Get memory statistics
|
|
5297
6429
|
*/
|
|
6430
|
+
async getOutboxStats() {
|
|
6431
|
+
await this.initialize();
|
|
6432
|
+
return this.sqliteStore.getOutboxStats();
|
|
6433
|
+
}
|
|
6434
|
+
async getRetrievalTraceStats() {
|
|
6435
|
+
await this.initialize();
|
|
6436
|
+
return this.sqliteStore.getRetrievalTraceStats();
|
|
6437
|
+
}
|
|
6438
|
+
async getRecentRetrievalTraces(limit = 50) {
|
|
6439
|
+
await this.initialize();
|
|
6440
|
+
return this.sqliteStore.getRecentRetrievalTraces(limit);
|
|
6441
|
+
}
|
|
5298
6442
|
async getStats() {
|
|
5299
6443
|
await this.initialize();
|
|
5300
6444
|
const recentEvents = await this.sqliteStore.getRecentEvents(1e4);
|
|
@@ -5785,7 +6929,7 @@ var MemoryService = class {
|
|
|
5785
6929
|
*/
|
|
5786
6930
|
expandPath(p) {
|
|
5787
6931
|
if (p.startsWith("~")) {
|
|
5788
|
-
return
|
|
6932
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
5789
6933
|
}
|
|
5790
6934
|
return p;
|
|
5791
6935
|
}
|
|
@@ -5821,6 +6965,7 @@ function getMemoryServiceForProject(projectPath, sharedStoreConfig) {
|
|
|
5821
6965
|
serviceCache.set(hash, new MemoryService({
|
|
5822
6966
|
storagePath,
|
|
5823
6967
|
projectHash: hash,
|
|
6968
|
+
projectPath,
|
|
5824
6969
|
// Override shared store config - hooks don't need DuckDB
|
|
5825
6970
|
sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
|
|
5826
6971
|
analyticsEnabled: false
|
|
@@ -5831,8 +6976,8 @@ function getMemoryServiceForProject(projectPath, sharedStoreConfig) {
|
|
|
5831
6976
|
}
|
|
5832
6977
|
|
|
5833
6978
|
// src/services/session-history-importer.ts
|
|
5834
|
-
import * as
|
|
5835
|
-
import * as
|
|
6979
|
+
import * as fs5 from "fs";
|
|
6980
|
+
import * as path4 from "path";
|
|
5836
6981
|
import * as os2 from "os";
|
|
5837
6982
|
import * as readline from "readline";
|
|
5838
6983
|
import { randomUUID as randomUUID9 } from "crypto";
|
|
@@ -5876,7 +7021,7 @@ var SessionHistoryImporter = class {
|
|
|
5876
7021
|
claudeDir;
|
|
5877
7022
|
constructor(memoryService) {
|
|
5878
7023
|
this.memoryService = memoryService;
|
|
5879
|
-
this.claudeDir =
|
|
7024
|
+
this.claudeDir = path4.join(os2.homedir(), ".claude");
|
|
5880
7025
|
}
|
|
5881
7026
|
/**
|
|
5882
7027
|
* Import all sessions from a project
|
|
@@ -5899,7 +7044,7 @@ var SessionHistoryImporter = class {
|
|
|
5899
7044
|
}
|
|
5900
7045
|
const sessionFiles = await this.findSessionFiles(projectDir);
|
|
5901
7046
|
result.totalSessions = sessionFiles.length;
|
|
5902
|
-
onProgress?.({ phase: "scan", message: `Found ${sessionFiles.length} sessions in ${
|
|
7047
|
+
onProgress?.({ phase: "scan", message: `Found ${sessionFiles.length} sessions in ${path4.basename(projectDir)}` });
|
|
5903
7048
|
if (options.verbose) {
|
|
5904
7049
|
console.log(`Found ${sessionFiles.length} session files in ${projectDir}`);
|
|
5905
7050
|
}
|
|
@@ -5940,11 +7085,11 @@ var SessionHistoryImporter = class {
|
|
|
5940
7085
|
skippedDuplicates: 0,
|
|
5941
7086
|
errors: []
|
|
5942
7087
|
};
|
|
5943
|
-
if (!
|
|
7088
|
+
if (!fs5.existsSync(filePath)) {
|
|
5944
7089
|
result.errors.push(`File not found: ${filePath}`);
|
|
5945
7090
|
return result;
|
|
5946
7091
|
}
|
|
5947
|
-
const sessionId =
|
|
7092
|
+
const sessionId = path4.basename(filePath, ".jsonl");
|
|
5948
7093
|
if (options.force) {
|
|
5949
7094
|
const deleted = await this.memoryService.deleteSessionEvents(sessionId);
|
|
5950
7095
|
if (options.verbose && deleted > 0) {
|
|
@@ -5952,7 +7097,7 @@ var SessionHistoryImporter = class {
|
|
|
5952
7097
|
}
|
|
5953
7098
|
}
|
|
5954
7099
|
await this.memoryService.startSession(sessionId, options.projectPath);
|
|
5955
|
-
const fileStream =
|
|
7100
|
+
const fileStream = fs5.createReadStream(filePath);
|
|
5956
7101
|
const rl = readline.createInterface({
|
|
5957
7102
|
input: fileStream,
|
|
5958
7103
|
crlfDelay: Infinity
|
|
@@ -6057,13 +7202,13 @@ var SessionHistoryImporter = class {
|
|
|
6057
7202
|
errors: []
|
|
6058
7203
|
};
|
|
6059
7204
|
const onProgress = options.onProgress;
|
|
6060
|
-
const projectsDir =
|
|
6061
|
-
if (!
|
|
7205
|
+
const projectsDir = path4.join(this.claudeDir, "projects");
|
|
7206
|
+
if (!fs5.existsSync(projectsDir)) {
|
|
6062
7207
|
result.errors.push(`Projects directory not found: ${projectsDir}`);
|
|
6063
7208
|
return result;
|
|
6064
7209
|
}
|
|
6065
7210
|
onProgress?.({ phase: "scan", message: "Scanning all projects..." });
|
|
6066
|
-
const projectDirs =
|
|
7211
|
+
const projectDirs = fs5.readdirSync(projectsDir).map((name) => path4.join(projectsDir, name)).filter((p) => fs5.statSync(p).isDirectory());
|
|
6067
7212
|
const allSessionFiles = [];
|
|
6068
7213
|
for (const projectDir of projectDirs) {
|
|
6069
7214
|
const sessionFiles = await this.findSessionFiles(projectDir);
|
|
@@ -6104,14 +7249,14 @@ var SessionHistoryImporter = class {
|
|
|
6104
7249
|
* Find project directory from project path
|
|
6105
7250
|
*/
|
|
6106
7251
|
async findProjectDir(projectPath) {
|
|
6107
|
-
const projectsDir =
|
|
6108
|
-
if (!
|
|
7252
|
+
const projectsDir = path4.join(this.claudeDir, "projects");
|
|
7253
|
+
if (!fs5.existsSync(projectsDir)) {
|
|
6109
7254
|
return null;
|
|
6110
7255
|
}
|
|
6111
|
-
const projectDirs =
|
|
7256
|
+
const projectDirs = fs5.readdirSync(projectsDir).map((name) => path4.join(projectsDir, name)).filter((p) => fs5.statSync(p).isDirectory());
|
|
6112
7257
|
const normalizedPath = projectPath.replace(/\//g, "-").replace(/^-/, "");
|
|
6113
7258
|
for (const dir of projectDirs) {
|
|
6114
|
-
const dirName =
|
|
7259
|
+
const dirName = path4.basename(dir);
|
|
6115
7260
|
if (dirName.includes(normalizedPath) || normalizedPath.includes(dirName)) {
|
|
6116
7261
|
return dir;
|
|
6117
7262
|
}
|
|
@@ -6122,10 +7267,10 @@ var SessionHistoryImporter = class {
|
|
|
6122
7267
|
* Find all JSONL session files in a directory
|
|
6123
7268
|
*/
|
|
6124
7269
|
async findSessionFiles(dir) {
|
|
6125
|
-
if (!
|
|
7270
|
+
if (!fs5.existsSync(dir)) {
|
|
6126
7271
|
return [];
|
|
6127
7272
|
}
|
|
6128
|
-
return
|
|
7273
|
+
return fs5.readdirSync(dir).filter((name) => name.endsWith(".jsonl")).map((name) => path4.join(dir, name)).filter((p) => fs5.statSync(p).isFile());
|
|
6129
7274
|
}
|
|
6130
7275
|
/**
|
|
6131
7276
|
* Extract text content from Claude message
|
|
@@ -6156,17 +7301,17 @@ var SessionHistoryImporter = class {
|
|
|
6156
7301
|
projectDirs = [projectDir];
|
|
6157
7302
|
}
|
|
6158
7303
|
} else {
|
|
6159
|
-
const projectsDir =
|
|
6160
|
-
if (
|
|
6161
|
-
projectDirs =
|
|
7304
|
+
const projectsDir = path4.join(this.claudeDir, "projects");
|
|
7305
|
+
if (fs5.existsSync(projectsDir)) {
|
|
7306
|
+
projectDirs = fs5.readdirSync(projectsDir).map((name) => path4.join(projectsDir, name)).filter((p) => fs5.statSync(p).isDirectory());
|
|
6162
7307
|
}
|
|
6163
7308
|
}
|
|
6164
7309
|
for (const projectDir of projectDirs) {
|
|
6165
7310
|
const sessionFiles = await this.findSessionFiles(projectDir);
|
|
6166
7311
|
for (const filePath of sessionFiles) {
|
|
6167
|
-
const stats =
|
|
7312
|
+
const stats = fs5.statSync(filePath);
|
|
6168
7313
|
sessions.push({
|
|
6169
|
-
sessionId:
|
|
7314
|
+
sessionId: path4.basename(filePath, ".jsonl"),
|
|
6170
7315
|
filePath,
|
|
6171
7316
|
size: stats.size,
|
|
6172
7317
|
modifiedAt: stats.mtime
|
|
@@ -6181,23 +7326,418 @@ function createSessionHistoryImporter(memoryService) {
|
|
|
6181
7326
|
return new SessionHistoryImporter(memoryService);
|
|
6182
7327
|
}
|
|
6183
7328
|
|
|
7329
|
+
// src/services/bootstrap-organizer.ts
|
|
7330
|
+
import * as fs6 from "node:fs";
|
|
7331
|
+
import * as path5 from "node:path";
|
|
7332
|
+
import { execSync } from "node:child_process";
|
|
7333
|
+
var EXCLUDED_DIRS = /* @__PURE__ */ new Set([".git", "node_modules", "dist", "build", "coverage", ".next", ".turbo", "memory"]);
|
|
7334
|
+
var CODE_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
7335
|
+
".ts",
|
|
7336
|
+
".tsx",
|
|
7337
|
+
".js",
|
|
7338
|
+
".jsx",
|
|
7339
|
+
".mjs",
|
|
7340
|
+
".cjs",
|
|
7341
|
+
".py",
|
|
7342
|
+
".go",
|
|
7343
|
+
".rs",
|
|
7344
|
+
".java",
|
|
7345
|
+
".kt",
|
|
7346
|
+
".swift",
|
|
7347
|
+
".rb",
|
|
7348
|
+
".php",
|
|
7349
|
+
".cs",
|
|
7350
|
+
".scala",
|
|
7351
|
+
".sh",
|
|
7352
|
+
".zsh",
|
|
7353
|
+
".yaml",
|
|
7354
|
+
".yml",
|
|
7355
|
+
".json",
|
|
7356
|
+
".sql",
|
|
7357
|
+
".md"
|
|
7358
|
+
]);
|
|
7359
|
+
function safeRel(base, target) {
|
|
7360
|
+
return path5.relative(base, target).replaceAll("\\", "/");
|
|
7361
|
+
}
|
|
7362
|
+
function mkdirp(dir) {
|
|
7363
|
+
fs6.mkdirSync(dir, { recursive: true });
|
|
7364
|
+
}
|
|
7365
|
+
function walkCodeFiles(root) {
|
|
7366
|
+
const out = [];
|
|
7367
|
+
function walk(dir) {
|
|
7368
|
+
const entries = fs6.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
|
|
7369
|
+
for (const e of entries) {
|
|
7370
|
+
const full = path5.join(dir, e.name);
|
|
7371
|
+
if (e.isDirectory()) {
|
|
7372
|
+
if (!EXCLUDED_DIRS.has(e.name))
|
|
7373
|
+
walk(full);
|
|
7374
|
+
} else if (e.isFile()) {
|
|
7375
|
+
const ext = path5.extname(e.name).toLowerCase();
|
|
7376
|
+
if (CODE_EXTENSIONS.has(ext))
|
|
7377
|
+
out.push(full);
|
|
7378
|
+
}
|
|
7379
|
+
}
|
|
7380
|
+
}
|
|
7381
|
+
walk(root);
|
|
7382
|
+
return out.sort();
|
|
7383
|
+
}
|
|
7384
|
+
function detectLanguage(file) {
|
|
7385
|
+
const ext = path5.extname(file).toLowerCase();
|
|
7386
|
+
const map = {
|
|
7387
|
+
".ts": "TypeScript",
|
|
7388
|
+
".tsx": "TypeScript",
|
|
7389
|
+
".js": "JavaScript",
|
|
7390
|
+
".jsx": "JavaScript",
|
|
7391
|
+
".mjs": "JavaScript",
|
|
7392
|
+
".cjs": "JavaScript",
|
|
7393
|
+
".py": "Python",
|
|
7394
|
+
".go": "Go",
|
|
7395
|
+
".rs": "Rust",
|
|
7396
|
+
".java": "Java",
|
|
7397
|
+
".kt": "Kotlin",
|
|
7398
|
+
".swift": "Swift",
|
|
7399
|
+
".rb": "Ruby",
|
|
7400
|
+
".php": "PHP",
|
|
7401
|
+
".cs": "C#",
|
|
7402
|
+
".scala": "Scala",
|
|
7403
|
+
".sh": "Shell",
|
|
7404
|
+
".zsh": "Shell",
|
|
7405
|
+
".yaml": "YAML",
|
|
7406
|
+
".yml": "YAML",
|
|
7407
|
+
".json": "JSON",
|
|
7408
|
+
".sql": "SQL",
|
|
7409
|
+
".md": "Markdown"
|
|
7410
|
+
};
|
|
7411
|
+
return map[ext] || "Other";
|
|
7412
|
+
}
|
|
7413
|
+
function summarizeModules(repoPath, files) {
|
|
7414
|
+
const modules = /* @__PURE__ */ new Map();
|
|
7415
|
+
for (const abs of files) {
|
|
7416
|
+
const rel = safeRel(repoPath, abs);
|
|
7417
|
+
const seg = rel.split("/").filter(Boolean);
|
|
7418
|
+
const top = seg[0] || "root";
|
|
7419
|
+
if (!modules.has(top))
|
|
7420
|
+
modules.set(top, { files: [], langs: /* @__PURE__ */ new Map() });
|
|
7421
|
+
const bucket = modules.get(top);
|
|
7422
|
+
bucket.files.push(rel);
|
|
7423
|
+
const lang = detectLanguage(abs);
|
|
7424
|
+
bucket.langs.set(lang, (bucket.langs.get(lang) || 0) + 1);
|
|
7425
|
+
}
|
|
7426
|
+
return [...modules.entries()].map(([name, data]) => ({
|
|
7427
|
+
name,
|
|
7428
|
+
root: name,
|
|
7429
|
+
fileCount: data.files.length,
|
|
7430
|
+
languages: [...data.langs.entries()].sort((a, b) => b[1] - a[1]).map(([l]) => l).slice(0, 5),
|
|
7431
|
+
entryCandidates: data.files.filter((f) => /(index|main|app|server|cli)\./i.test(path5.basename(f))).slice(0, 10)
|
|
7432
|
+
})).sort((a, b) => b.fileCount - a.fileCount || a.name.localeCompare(b.name));
|
|
7433
|
+
}
|
|
7434
|
+
function runGit(repoPath, command) {
|
|
7435
|
+
return execSync(`git -C ${JSON.stringify(repoPath)} ${command}`, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
7436
|
+
}
|
|
7437
|
+
function getGitCommits(repoPath, since = "180 days ago", maxCommits = 1e3) {
|
|
7438
|
+
try {
|
|
7439
|
+
const raw = runGit(
|
|
7440
|
+
repoPath,
|
|
7441
|
+
`log --since=${JSON.stringify(since)} -n ${Math.max(1, maxCommits)} --date=short --pretty=format:%H%x09%ad%x09%an%x09%s --name-only --reverse`
|
|
7442
|
+
);
|
|
7443
|
+
const lines = raw.split(/\r?\n/);
|
|
7444
|
+
const commits = [];
|
|
7445
|
+
let current = null;
|
|
7446
|
+
for (const line of lines) {
|
|
7447
|
+
if (!line.trim()) {
|
|
7448
|
+
if (current) {
|
|
7449
|
+
commits.push(current);
|
|
7450
|
+
current = null;
|
|
7451
|
+
}
|
|
7452
|
+
continue;
|
|
7453
|
+
}
|
|
7454
|
+
if (line.includes(" ") && line.split(" ").length >= 4) {
|
|
7455
|
+
if (current)
|
|
7456
|
+
commits.push(current);
|
|
7457
|
+
const [hash, date, author, ...subjectRest] = line.split(" ");
|
|
7458
|
+
current = { hash, date, author, subject: subjectRest.join(" ").trim(), files: [] };
|
|
7459
|
+
} else if (current) {
|
|
7460
|
+
current.files.push(line.trim());
|
|
7461
|
+
}
|
|
7462
|
+
}
|
|
7463
|
+
if (current)
|
|
7464
|
+
commits.push(current);
|
|
7465
|
+
return commits;
|
|
7466
|
+
} catch {
|
|
7467
|
+
return [];
|
|
7468
|
+
}
|
|
7469
|
+
}
|
|
7470
|
+
function extractDecisions(commits) {
|
|
7471
|
+
const decisionPattern = /(refactor|migrate|deprecat|remove|replace|introduce|adopt|switch|upgrade|breaking|architecture|feat|fix)/i;
|
|
7472
|
+
return commits.filter((c) => decisionPattern.test(c.subject));
|
|
7473
|
+
}
|
|
7474
|
+
function buildTimeline(commits) {
|
|
7475
|
+
const timeline = /* @__PURE__ */ new Map();
|
|
7476
|
+
for (const c of commits) {
|
|
7477
|
+
const key = (c.date || "").slice(0, 7) || "unknown";
|
|
7478
|
+
if (!timeline.has(key))
|
|
7479
|
+
timeline.set(key, []);
|
|
7480
|
+
timeline.get(key).push(c);
|
|
7481
|
+
}
|
|
7482
|
+
return new Map([...timeline.entries()].sort((a, b) => a[0].localeCompare(b[0])));
|
|
7483
|
+
}
|
|
7484
|
+
function buildGlossary(files) {
|
|
7485
|
+
const stop = /* @__PURE__ */ new Set(["src", "test", "dist", "lib", "core", "index", "main", "app", "server", "client", "utils"]);
|
|
7486
|
+
const freq = /* @__PURE__ */ new Map();
|
|
7487
|
+
for (const f of files) {
|
|
7488
|
+
const base = path5.basename(f, path5.extname(f));
|
|
7489
|
+
const tokens = base.split(/[^a-zA-Z0-9]+/).flatMap((t) => t.split(/(?=[A-Z])/)).map((t) => t.toLowerCase()).filter((t) => t.length >= 3 && !stop.has(t));
|
|
7490
|
+
for (const t of tokens)
|
|
7491
|
+
freq.set(t, (freq.get(t) || 0) + 1);
|
|
7492
|
+
}
|
|
7493
|
+
return [...freq.entries()].filter(([, count]) => count >= 2).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 80).map(([term]) => term);
|
|
7494
|
+
}
|
|
7495
|
+
function writeFile(filePath, content) {
|
|
7496
|
+
mkdirp(path5.dirname(filePath));
|
|
7497
|
+
fs6.writeFileSync(filePath, content, "utf8");
|
|
7498
|
+
}
|
|
7499
|
+
function confidenceByEvidence(sourceCount) {
|
|
7500
|
+
if (sourceCount >= 3)
|
|
7501
|
+
return "high";
|
|
7502
|
+
if (sourceCount >= 1)
|
|
7503
|
+
return "mid";
|
|
7504
|
+
return "low";
|
|
7505
|
+
}
|
|
7506
|
+
function sourceLine(source) {
|
|
7507
|
+
return `- source: ${source}`;
|
|
7508
|
+
}
|
|
7509
|
+
function loadExistingManifest(outDir) {
|
|
7510
|
+
try {
|
|
7511
|
+
const p = path5.join(outDir, "sources", "manifest.json");
|
|
7512
|
+
if (!fs6.existsSync(p))
|
|
7513
|
+
return null;
|
|
7514
|
+
const data = JSON.parse(fs6.readFileSync(p, "utf8"));
|
|
7515
|
+
return data;
|
|
7516
|
+
} catch {
|
|
7517
|
+
return null;
|
|
7518
|
+
}
|
|
7519
|
+
}
|
|
7520
|
+
function listMarkdownOutputs(outDir) {
|
|
7521
|
+
const out = [];
|
|
7522
|
+
const stack = [outDir];
|
|
7523
|
+
while (stack.length) {
|
|
7524
|
+
const dir = stack.pop();
|
|
7525
|
+
for (const entry of fs6.readdirSync(dir, { withFileTypes: true })) {
|
|
7526
|
+
const full = path5.join(dir, entry.name);
|
|
7527
|
+
if (entry.isDirectory())
|
|
7528
|
+
stack.push(full);
|
|
7529
|
+
else if (entry.isFile() && entry.name.endsWith(".md"))
|
|
7530
|
+
out.push(full);
|
|
7531
|
+
}
|
|
7532
|
+
}
|
|
7533
|
+
return out.sort((a, b) => a.localeCompare(b));
|
|
7534
|
+
}
|
|
7535
|
+
async function bootstrapKnowledgeBase(options) {
|
|
7536
|
+
const repoPath = path5.resolve(options.repoPath);
|
|
7537
|
+
const outDir = path5.resolve(options.outDir);
|
|
7538
|
+
const maxCommits = options.maxCommits ?? 1e3;
|
|
7539
|
+
const existingManifest = options.incremental ? loadExistingManifest(outDir) : null;
|
|
7540
|
+
const incrementalSince = existingManifest?.lastCommitDate || existingManifest?.generatedAt;
|
|
7541
|
+
const since = options.since || incrementalSince || "180 days ago";
|
|
7542
|
+
const codeFiles = walkCodeFiles(repoPath);
|
|
7543
|
+
const modules = summarizeModules(repoPath, codeFiles);
|
|
7544
|
+
const commits = getGitCommits(repoPath, since, maxCommits);
|
|
7545
|
+
const decisions = extractDecisions(commits);
|
|
7546
|
+
const timeline = buildTimeline(commits);
|
|
7547
|
+
const glossary = buildGlossary(codeFiles);
|
|
7548
|
+
const generatedFiles = [];
|
|
7549
|
+
const sections = {
|
|
7550
|
+
overview: path5.join(outDir, "overview"),
|
|
7551
|
+
modules: path5.join(outDir, "modules"),
|
|
7552
|
+
decisions: path5.join(outDir, "decisions"),
|
|
7553
|
+
timeline: path5.join(outDir, "timeline"),
|
|
7554
|
+
glossary: path5.join(outDir, "glossary"),
|
|
7555
|
+
sources: path5.join(outDir, "sources")
|
|
7556
|
+
};
|
|
7557
|
+
for (const sectionDir of Object.values(sections)) {
|
|
7558
|
+
mkdirp(sectionDir);
|
|
7559
|
+
}
|
|
7560
|
+
const overviewPath = path5.join(sections.overview, "overview.md");
|
|
7561
|
+
const overview = [
|
|
7562
|
+
"# Codebase Overview",
|
|
7563
|
+
"",
|
|
7564
|
+
`- generatedAt: ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
7565
|
+
"- deterministicPipeline: true",
|
|
7566
|
+
`- repo: ${repoPath}`,
|
|
7567
|
+
`- filesAnalyzed: ${codeFiles.length}`,
|
|
7568
|
+
`- commitsAnalyzed: ${commits.length}`,
|
|
7569
|
+
`- confidence: ${confidenceByEvidence(modules.length > 0 ? 3 : 0)}`,
|
|
7570
|
+
"",
|
|
7571
|
+
"## Directory / Module Map",
|
|
7572
|
+
...modules.slice(0, 50).map((m) => `- ${m.name}: ${m.fileCount} files (${m.languages.join(", ") || "n/a"})`),
|
|
7573
|
+
"",
|
|
7574
|
+
"## Fact",
|
|
7575
|
+
"- Generated from deterministic file scan and git history parsing.",
|
|
7576
|
+
"",
|
|
7577
|
+
"## Inference",
|
|
7578
|
+
"- Module responsibilities should be reviewed by maintainers for nuanced boundaries.",
|
|
7579
|
+
"",
|
|
7580
|
+
"## Sources",
|
|
7581
|
+
sourceLine(`repo-scan:${repoPath}`),
|
|
7582
|
+
sourceLine(`git-log:since=${since};max=${maxCommits}`),
|
|
7583
|
+
""
|
|
7584
|
+
].join("\n");
|
|
7585
|
+
writeFile(overviewPath, overview);
|
|
7586
|
+
generatedFiles.push(overviewPath);
|
|
7587
|
+
const touchedRoots = new Set(
|
|
7588
|
+
commits.flatMap((c) => c.files).map((f) => f.split("/").filter(Boolean)[0]).filter(Boolean)
|
|
7589
|
+
);
|
|
7590
|
+
const moduleTargets = options.incremental && touchedRoots.size > 0 ? modules.filter((m) => touchedRoots.has(m.root)).slice(0, 200) : modules.slice(0, 200);
|
|
7591
|
+
for (const m of moduleTargets) {
|
|
7592
|
+
const relatedCommits = commits.filter((c) => c.files.some((f) => f.startsWith(`${m.root}/`))).slice(0, 15);
|
|
7593
|
+
const content = [
|
|
7594
|
+
`# Module: ${m.name}`,
|
|
7595
|
+
"",
|
|
7596
|
+
`- responsibility: inferred from top-level path \`${m.root}/\``,
|
|
7597
|
+
`- files: ${m.fileCount}`,
|
|
7598
|
+
`- languages: ${m.languages.join(", ") || "n/a"}`,
|
|
7599
|
+
`- confidence: ${confidenceByEvidence(relatedCommits.length)}`,
|
|
7600
|
+
"",
|
|
7601
|
+
"## Entry Candidates",
|
|
7602
|
+
...m.entryCandidates.length > 0 ? m.entryCandidates.map((f) => `- ${f}`) : ["- none detected"],
|
|
7603
|
+
"",
|
|
7604
|
+
"## Related Commits (recent sample)",
|
|
7605
|
+
...relatedCommits.length > 0 ? relatedCommits.map((c) => `- ${c.date} ${c.hash.slice(0, 8)} ${c.subject}`) : ["- none in selected range"],
|
|
7606
|
+
"",
|
|
7607
|
+
"## Sources",
|
|
7608
|
+
sourceLine(`repo-path:${m.root}/**`),
|
|
7609
|
+
...relatedCommits.map((c) => sourceLine(`commit:${c.hash}`)),
|
|
7610
|
+
""
|
|
7611
|
+
].join("\n");
|
|
7612
|
+
const modulePath = path5.join(sections.modules, `${m.name.replace(/[^a-z0-9._-]+/gi, "-").toLowerCase()}.md`);
|
|
7613
|
+
writeFile(modulePath, content);
|
|
7614
|
+
generatedFiles.push(modulePath);
|
|
7615
|
+
}
|
|
7616
|
+
const decisionsPath = path5.join(sections.decisions, "decisions.md");
|
|
7617
|
+
const decisionsMd = [
|
|
7618
|
+
"# Decisions (extracted)",
|
|
7619
|
+
"",
|
|
7620
|
+
`- confidence: ${confidenceByEvidence(decisions.length)}`,
|
|
7621
|
+
"",
|
|
7622
|
+
...decisions.length > 0 ? decisions.slice(0, 500).map((d) => [
|
|
7623
|
+
`## ${d.date} | ${d.subject}`,
|
|
7624
|
+
"- status: active (inferred)",
|
|
7625
|
+
sourceLine(`commit:${d.hash}`),
|
|
7626
|
+
`- author: ${d.author}`,
|
|
7627
|
+
`- changedFiles: ${d.files.length}`,
|
|
7628
|
+
`- confidence: ${confidenceByEvidence(d.files.length > 0 ? 2 : 1)}`,
|
|
7629
|
+
""
|
|
7630
|
+
].join("\n")) : ["- No decision-like commits found in selected range.", ""],
|
|
7631
|
+
"## Sources",
|
|
7632
|
+
sourceLine(`git-log:since=${since};max=${maxCommits}`),
|
|
7633
|
+
""
|
|
7634
|
+
].join("\n");
|
|
7635
|
+
writeFile(decisionsPath, decisionsMd);
|
|
7636
|
+
generatedFiles.push(decisionsPath);
|
|
7637
|
+
const timelinePath = path5.join(sections.timeline, "timeline.md");
|
|
7638
|
+
const timelineMd = [
|
|
7639
|
+
"# Timeline",
|
|
7640
|
+
"",
|
|
7641
|
+
`- confidence: ${confidenceByEvidence(commits.length > 0 ? 2 : 0)}`,
|
|
7642
|
+
"",
|
|
7643
|
+
...[...timeline.entries()].flatMap(([month, list]) => [
|
|
7644
|
+
`## ${month}`,
|
|
7645
|
+
...list.slice(0, 40).map((c) => `- ${c.date} ${c.hash.slice(0, 8)} ${c.subject}`),
|
|
7646
|
+
""
|
|
7647
|
+
]),
|
|
7648
|
+
"## Sources",
|
|
7649
|
+
sourceLine(`git-log:since=${since};max=${maxCommits}`),
|
|
7650
|
+
""
|
|
7651
|
+
].join("\n");
|
|
7652
|
+
writeFile(timelinePath, timelineMd);
|
|
7653
|
+
generatedFiles.push(timelinePath);
|
|
7654
|
+
const glossaryPath = path5.join(sections.glossary, "glossary.md");
|
|
7655
|
+
const glossaryMd = [
|
|
7656
|
+
"# Glossary (auto-extracted)",
|
|
7657
|
+
"",
|
|
7658
|
+
`- confidence: ${confidenceByEvidence(glossary.length > 0 ? 1 : 0)}`,
|
|
7659
|
+
"",
|
|
7660
|
+
...glossary.map((t) => `- ${t}`),
|
|
7661
|
+
"",
|
|
7662
|
+
"## Sources",
|
|
7663
|
+
sourceLine(`repo-scan:${repoPath}`),
|
|
7664
|
+
""
|
|
7665
|
+
].join("\n");
|
|
7666
|
+
writeFile(glossaryPath, glossaryMd);
|
|
7667
|
+
generatedFiles.push(glossaryPath);
|
|
7668
|
+
const outputs = generatedFiles.map((f) => safeRel(outDir, f)).sort((a, b) => a.localeCompare(b));
|
|
7669
|
+
const allOutputs = listMarkdownOutputs(outDir).map((f) => safeRel(outDir, f));
|
|
7670
|
+
const sourceItems = [
|
|
7671
|
+
...codeFiles.slice(0, 200).map((f) => ({ type: "file", ref: safeRel(repoPath, f) })),
|
|
7672
|
+
...commits.slice(0, 400).map((c) => ({ type: "commit", ref: c.hash, date: c.date, subject: c.subject }))
|
|
7673
|
+
];
|
|
7674
|
+
const latestCommitDate = commits.length > 0 ? commits[commits.length - 1].date : void 0;
|
|
7675
|
+
const manifest = {
|
|
7676
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7677
|
+
deterministicPipeline: true,
|
|
7678
|
+
mode: options.incremental ? "incremental" : "full",
|
|
7679
|
+
repoPath,
|
|
7680
|
+
options: { since, maxCommits, incremental: Boolean(options.incremental) },
|
|
7681
|
+
stats: {
|
|
7682
|
+
filesAnalyzed: codeFiles.length,
|
|
7683
|
+
modules: modules.length,
|
|
7684
|
+
modulesGenerated: moduleTargets.length,
|
|
7685
|
+
commits: commits.length,
|
|
7686
|
+
decisions: decisions.length,
|
|
7687
|
+
glossaryTerms: glossary.length
|
|
7688
|
+
},
|
|
7689
|
+
lastCommitDate: latestCommitDate,
|
|
7690
|
+
outputs,
|
|
7691
|
+
allOutputs,
|
|
7692
|
+
sources: sourceItems
|
|
7693
|
+
};
|
|
7694
|
+
const manifestJsonPath = path5.join(sections.sources, "manifest.json");
|
|
7695
|
+
writeFile(manifestJsonPath, `${JSON.stringify(manifest, null, 2)}
|
|
7696
|
+
`);
|
|
7697
|
+
generatedFiles.push(manifestJsonPath);
|
|
7698
|
+
const manifestMdPath = path5.join(sections.sources, "manifest.md");
|
|
7699
|
+
const manifestMd = [
|
|
7700
|
+
"# Sources Manifest",
|
|
7701
|
+
"",
|
|
7702
|
+
"- deterministicPipeline: true",
|
|
7703
|
+
`- mode: ${options.incremental ? "incremental" : "full"}`,
|
|
7704
|
+
`- sourceCount: ${sourceItems.length}`,
|
|
7705
|
+
"",
|
|
7706
|
+
"## Outputs",
|
|
7707
|
+
...outputs.map((o) => `- ${o}`),
|
|
7708
|
+
"",
|
|
7709
|
+
"## Sources (sample)",
|
|
7710
|
+
...sourceItems.slice(0, 300).map((s) => `- ${s.type}:${s.ref}`),
|
|
7711
|
+
""
|
|
7712
|
+
].join("\n");
|
|
7713
|
+
writeFile(manifestMdPath, manifestMd);
|
|
7714
|
+
generatedFiles.push(manifestMdPath);
|
|
7715
|
+
return {
|
|
7716
|
+
outDir,
|
|
7717
|
+
fileCount: codeFiles.length,
|
|
7718
|
+
moduleCount: modules.length,
|
|
7719
|
+
commitCount: commits.length,
|
|
7720
|
+
generatedFiles: generatedFiles.sort((a, b) => a.localeCompare(b))
|
|
7721
|
+
};
|
|
7722
|
+
}
|
|
7723
|
+
|
|
6184
7724
|
// src/server/index.ts
|
|
6185
|
-
import { Hono as
|
|
7725
|
+
import { Hono as Hono11 } from "hono";
|
|
6186
7726
|
import { cors } from "hono/cors";
|
|
6187
7727
|
import { logger } from "hono/logger";
|
|
6188
7728
|
import { serve } from "@hono/node-server";
|
|
6189
7729
|
import { serveStatic } from "@hono/node-server/serve-static";
|
|
6190
|
-
import * as
|
|
6191
|
-
import * as
|
|
7730
|
+
import * as path8 from "path";
|
|
7731
|
+
import * as fs8 from "fs";
|
|
6192
7732
|
|
|
6193
7733
|
// src/server/api/index.ts
|
|
6194
|
-
import { Hono as
|
|
7734
|
+
import { Hono as Hono10 } from "hono";
|
|
6195
7735
|
|
|
6196
7736
|
// src/server/api/sessions.ts
|
|
6197
7737
|
import { Hono } from "hono";
|
|
6198
7738
|
|
|
6199
7739
|
// src/server/api/utils.ts
|
|
6200
|
-
import * as
|
|
7740
|
+
import * as path6 from "path";
|
|
6201
7741
|
import * as os3 from "os";
|
|
6202
7742
|
function getServiceFromQuery(c) {
|
|
6203
7743
|
const project = c.req.query("project");
|
|
@@ -6205,12 +7745,12 @@ function getServiceFromQuery(c) {
|
|
|
6205
7745
|
const isHash = /^[a-f0-9]{8}$/.test(project);
|
|
6206
7746
|
let storagePath;
|
|
6207
7747
|
if (isHash) {
|
|
6208
|
-
storagePath =
|
|
7748
|
+
storagePath = path6.join(os3.homedir(), ".claude-code", "memory", "projects", project);
|
|
6209
7749
|
} else {
|
|
6210
7750
|
const crypto3 = __require("crypto");
|
|
6211
7751
|
const normalized = project.replace(/\/+$/, "") || "/";
|
|
6212
7752
|
const hash = crypto3.createHash("sha256").update(normalized).digest("hex").slice(0, 8);
|
|
6213
|
-
storagePath =
|
|
7753
|
+
storagePath = path6.join(os3.homedir(), ".claude-code", "memory", "projects", hash);
|
|
6214
7754
|
}
|
|
6215
7755
|
return new MemoryService({
|
|
6216
7756
|
storagePath,
|
|
@@ -6601,6 +8141,7 @@ statsRouter.get("/", async (c) => {
|
|
|
6601
8141
|
acc[day] = (acc[day] || 0) + 1;
|
|
6602
8142
|
return acc;
|
|
6603
8143
|
}, {});
|
|
8144
|
+
const retrievalTrace = await memoryService.getRetrievalTraceStats();
|
|
6604
8145
|
return c.json({
|
|
6605
8146
|
storage: {
|
|
6606
8147
|
eventCount: stats.totalEvents,
|
|
@@ -6618,7 +8159,8 @@ statsRouter.get("/", async (c) => {
|
|
|
6618
8159
|
heapUsed: Math.round(process.memoryUsage().heapUsed / 1024 / 1024),
|
|
6619
8160
|
heapTotal: Math.round(process.memoryUsage().heapTotal / 1024 / 1024)
|
|
6620
8161
|
},
|
|
6621
|
-
levelStats: stats.levelStats
|
|
8162
|
+
levelStats: stats.levelStats,
|
|
8163
|
+
retrievalTrace
|
|
6622
8164
|
});
|
|
6623
8165
|
} catch (error) {
|
|
6624
8166
|
return c.json({ error: error.message }, 500);
|
|
@@ -6720,6 +8262,42 @@ statsRouter.get("/helpfulness", async (c) => {
|
|
|
6720
8262
|
await memoryService.shutdown();
|
|
6721
8263
|
}
|
|
6722
8264
|
});
|
|
8265
|
+
statsRouter.get("/retrieval-traces", async (c) => {
|
|
8266
|
+
const limit = parseInt(c.req.query("limit") || "50", 10);
|
|
8267
|
+
const memoryService = getServiceFromQuery(c);
|
|
8268
|
+
try {
|
|
8269
|
+
await memoryService.initialize();
|
|
8270
|
+
const traces = await memoryService.getRecentRetrievalTraces(limit);
|
|
8271
|
+
const traceStats = await memoryService.getRetrievalTraceStats();
|
|
8272
|
+
return c.json({
|
|
8273
|
+
stats: traceStats,
|
|
8274
|
+
traces: traces.map((t) => ({
|
|
8275
|
+
traceId: t.traceId,
|
|
8276
|
+
sessionId: t.sessionId || null,
|
|
8277
|
+
projectHash: t.projectHash || null,
|
|
8278
|
+
queryText: t.queryText,
|
|
8279
|
+
strategy: t.strategy || null,
|
|
8280
|
+
candidateEventIds: t.candidateEventIds,
|
|
8281
|
+
selectedEventIds: t.selectedEventIds,
|
|
8282
|
+
candidateDetails: t.candidateDetails || [],
|
|
8283
|
+
selectedDetails: t.selectedDetails || [],
|
|
8284
|
+
candidateCount: t.candidateCount,
|
|
8285
|
+
selectedCount: t.selectedCount,
|
|
8286
|
+
confidence: t.confidence || null,
|
|
8287
|
+
fallbackTrace: t.fallbackTrace,
|
|
8288
|
+
createdAt: t.createdAt.toISOString()
|
|
8289
|
+
}))
|
|
8290
|
+
});
|
|
8291
|
+
} catch (error) {
|
|
8292
|
+
return c.json({
|
|
8293
|
+
stats: { totalQueries: 0, avgCandidateCount: 0, avgSelectedCount: 0, selectionRate: 0 },
|
|
8294
|
+
traces: [],
|
|
8295
|
+
error: error.message
|
|
8296
|
+
}, 500);
|
|
8297
|
+
} finally {
|
|
8298
|
+
await memoryService.shutdown();
|
|
8299
|
+
}
|
|
8300
|
+
});
|
|
6723
8301
|
statsRouter.post("/graduation/run", async (c) => {
|
|
6724
8302
|
const memoryService = getServiceFromQuery(c);
|
|
6725
8303
|
try {
|
|
@@ -6967,19 +8545,19 @@ turnsRouter.post("/backfill", async (c) => {
|
|
|
6967
8545
|
|
|
6968
8546
|
// src/server/api/projects.ts
|
|
6969
8547
|
import { Hono as Hono7 } from "hono";
|
|
6970
|
-
import * as
|
|
6971
|
-
import * as
|
|
8548
|
+
import * as fs7 from "fs";
|
|
8549
|
+
import * as path7 from "path";
|
|
6972
8550
|
import * as os4 from "os";
|
|
6973
8551
|
var projectsRouter = new Hono7();
|
|
6974
8552
|
projectsRouter.get("/", async (c) => {
|
|
6975
8553
|
try {
|
|
6976
|
-
const projectsDir =
|
|
6977
|
-
if (!
|
|
8554
|
+
const projectsDir = path7.join(os4.homedir(), ".claude-code", "memory", "projects");
|
|
8555
|
+
if (!fs7.existsSync(projectsDir)) {
|
|
6978
8556
|
return c.json({ projects: [] });
|
|
6979
8557
|
}
|
|
6980
|
-
const projectHashes =
|
|
6981
|
-
const fullPath =
|
|
6982
|
-
return
|
|
8558
|
+
const projectHashes = fs7.readdirSync(projectsDir).filter((name) => {
|
|
8559
|
+
const fullPath = path7.join(projectsDir, name);
|
|
8560
|
+
return fs7.statSync(fullPath).isDirectory();
|
|
6983
8561
|
});
|
|
6984
8562
|
const registry = loadSessionRegistry();
|
|
6985
8563
|
const hashToPath = /* @__PURE__ */ new Map();
|
|
@@ -6989,17 +8567,17 @@ projectsRouter.get("/", async (c) => {
|
|
|
6989
8567
|
}
|
|
6990
8568
|
}
|
|
6991
8569
|
const projects = projectHashes.map((hash) => {
|
|
6992
|
-
const dirPath =
|
|
6993
|
-
const dbPath =
|
|
8570
|
+
const dirPath = path7.join(projectsDir, hash);
|
|
8571
|
+
const dbPath = path7.join(dirPath, "events.sqlite");
|
|
6994
8572
|
let dbSize = 0;
|
|
6995
|
-
if (
|
|
6996
|
-
dbSize =
|
|
8573
|
+
if (fs7.existsSync(dbPath)) {
|
|
8574
|
+
dbSize = fs7.statSync(dbPath).size;
|
|
6997
8575
|
}
|
|
6998
8576
|
const projectPath = hashToPath.get(hash) || `unknown (${hash})`;
|
|
6999
8577
|
return {
|
|
7000
8578
|
hash,
|
|
7001
8579
|
projectPath,
|
|
7002
|
-
projectName:
|
|
8580
|
+
projectName: path7.basename(projectPath),
|
|
7003
8581
|
dbSize,
|
|
7004
8582
|
dbSizeHuman: formatBytes(dbSize)
|
|
7005
8583
|
};
|
|
@@ -7124,7 +8702,7 @@ function buildPrompt(statsContext, memoryContext, history, currentMessage) {
|
|
|
7124
8702
|
return parts.join("\n");
|
|
7125
8703
|
}
|
|
7126
8704
|
function streamClaudeResponse(prompt, stream) {
|
|
7127
|
-
return new Promise((
|
|
8705
|
+
return new Promise((resolve4, reject) => {
|
|
7128
8706
|
const proc = spawn("claude", [
|
|
7129
8707
|
"-p",
|
|
7130
8708
|
"--output-format",
|
|
@@ -7196,29 +8774,71 @@ function streamClaudeResponse(prompt, stream) {
|
|
|
7196
8774
|
if (code !== 0 && code !== null) {
|
|
7197
8775
|
reject(new Error(`Claude CLI exited with code ${code}`));
|
|
7198
8776
|
} else {
|
|
7199
|
-
|
|
8777
|
+
resolve4();
|
|
7200
8778
|
}
|
|
7201
8779
|
});
|
|
7202
8780
|
});
|
|
7203
8781
|
}
|
|
7204
8782
|
|
|
8783
|
+
// src/server/api/health.ts
|
|
8784
|
+
import { Hono as Hono9 } from "hono";
|
|
8785
|
+
var healthRouter = new Hono9();
|
|
8786
|
+
healthRouter.get("/", async (c) => {
|
|
8787
|
+
const memoryService = getServiceFromQuery(c);
|
|
8788
|
+
try {
|
|
8789
|
+
await memoryService.initialize();
|
|
8790
|
+
const [stats, outbox] = await Promise.all([
|
|
8791
|
+
memoryService.getStats(),
|
|
8792
|
+
memoryService.getOutboxStats()
|
|
8793
|
+
]);
|
|
8794
|
+
const outboxPending = outbox.embedding.pending + outbox.vector.pending;
|
|
8795
|
+
const outboxFailed = outbox.embedding.failed + outbox.vector.failed;
|
|
8796
|
+
const status = outboxFailed > 0 ? "needs-attention" : "ok";
|
|
8797
|
+
return c.json({
|
|
8798
|
+
status,
|
|
8799
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8800
|
+
storage: {
|
|
8801
|
+
totalEvents: stats.totalEvents,
|
|
8802
|
+
vectorCount: stats.vectorCount
|
|
8803
|
+
},
|
|
8804
|
+
outbox: {
|
|
8805
|
+
embedding: outbox.embedding,
|
|
8806
|
+
vector: outbox.vector,
|
|
8807
|
+
totals: {
|
|
8808
|
+
pending: outboxPending,
|
|
8809
|
+
failed: outboxFailed
|
|
8810
|
+
}
|
|
8811
|
+
},
|
|
8812
|
+
levelStats: stats.levelStats
|
|
8813
|
+
});
|
|
8814
|
+
} catch (error) {
|
|
8815
|
+
return c.json({
|
|
8816
|
+
status: "error",
|
|
8817
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8818
|
+
error: error.message
|
|
8819
|
+
}, 500);
|
|
8820
|
+
} finally {
|
|
8821
|
+
await memoryService.shutdown();
|
|
8822
|
+
}
|
|
8823
|
+
});
|
|
8824
|
+
|
|
7205
8825
|
// src/server/api/index.ts
|
|
7206
|
-
var apiRouter = new
|
|
8826
|
+
var apiRouter = new Hono10().route("/sessions", sessionsRouter).route("/events", eventsRouter).route("/search", searchRouter).route("/stats", statsRouter).route("/citations", citationsRouter).route("/turns", turnsRouter).route("/projects", projectsRouter).route("/chat", chatRouter).route("/health", healthRouter);
|
|
7207
8827
|
|
|
7208
8828
|
// src/server/index.ts
|
|
7209
|
-
var app = new
|
|
8829
|
+
var app = new Hono11();
|
|
7210
8830
|
app.use("/*", cors());
|
|
7211
8831
|
app.use("/*", logger());
|
|
7212
8832
|
app.route("/api", apiRouter);
|
|
7213
8833
|
app.get("/health", (c) => c.json({ status: "ok", timestamp: (/* @__PURE__ */ new Date()).toISOString() }));
|
|
7214
|
-
var uiPath =
|
|
7215
|
-
if (
|
|
8834
|
+
var uiPath = path8.join(__dirname, "../../dist/ui");
|
|
8835
|
+
if (fs8.existsSync(uiPath)) {
|
|
7216
8836
|
app.use("/*", serveStatic({ root: uiPath }));
|
|
7217
8837
|
}
|
|
7218
8838
|
app.get("*", (c) => {
|
|
7219
|
-
const indexPath =
|
|
7220
|
-
if (
|
|
7221
|
-
return c.html(
|
|
8839
|
+
const indexPath = path8.join(uiPath, "index.html");
|
|
8840
|
+
if (fs8.existsSync(indexPath)) {
|
|
8841
|
+
return c.html(fs8.readFileSync(indexPath, "utf-8"));
|
|
7222
8842
|
}
|
|
7223
8843
|
return c.text('UI not built. Run "npm run build:ui" first.', 404);
|
|
7224
8844
|
});
|
|
@@ -7255,29 +8875,303 @@ if (isMainModule) {
|
|
|
7255
8875
|
startServer(port);
|
|
7256
8876
|
}
|
|
7257
8877
|
|
|
8878
|
+
// src/core/mongo-sync-worker.ts
|
|
8879
|
+
import { randomUUID as randomUUID10 } from "crypto";
|
|
8880
|
+
import * as os5 from "os";
|
|
8881
|
+
import { MongoClient } from "mongodb";
|
|
8882
|
+
function redactMongoUri(uri) {
|
|
8883
|
+
const schemeIdx = uri.indexOf("://");
|
|
8884
|
+
if (schemeIdx === -1)
|
|
8885
|
+
return uri;
|
|
8886
|
+
const atIdx = uri.indexOf("@", schemeIdx + 3);
|
|
8887
|
+
if (atIdx === -1)
|
|
8888
|
+
return uri;
|
|
8889
|
+
const creds = uri.slice(schemeIdx + 3, atIdx);
|
|
8890
|
+
const colonIdx = creds.indexOf(":");
|
|
8891
|
+
if (colonIdx === -1)
|
|
8892
|
+
return uri;
|
|
8893
|
+
const prefix = uri.slice(0, schemeIdx + 3 + colonIdx + 1);
|
|
8894
|
+
const suffix = uri.slice(atIdx);
|
|
8895
|
+
return `${prefix}***${suffix}`;
|
|
8896
|
+
}
|
|
8897
|
+
function parseIntOrZero(value) {
|
|
8898
|
+
if (!value)
|
|
8899
|
+
return 0;
|
|
8900
|
+
const n = parseInt(value, 10);
|
|
8901
|
+
return Number.isFinite(n) ? n : 0;
|
|
8902
|
+
}
|
|
8903
|
+
var MongoSyncWorker = class {
|
|
8904
|
+
constructor(sqliteStore, config) {
|
|
8905
|
+
this.sqliteStore = sqliteStore;
|
|
8906
|
+
this.config = {
|
|
8907
|
+
uri: config.uri,
|
|
8908
|
+
dbName: config.dbName,
|
|
8909
|
+
projectKey: config.projectKey,
|
|
8910
|
+
direction: config.direction ?? "both",
|
|
8911
|
+
intervalMs: config.intervalMs ?? 3e4,
|
|
8912
|
+
batchSize: config.batchSize ?? 500,
|
|
8913
|
+
instanceId: config.instanceId ?? randomUUID10()
|
|
8914
|
+
};
|
|
8915
|
+
}
|
|
8916
|
+
config;
|
|
8917
|
+
intervalHandle = null;
|
|
8918
|
+
running = false;
|
|
8919
|
+
client = null;
|
|
8920
|
+
db = null;
|
|
8921
|
+
counters = null;
|
|
8922
|
+
events = null;
|
|
8923
|
+
indexesEnsured = false;
|
|
8924
|
+
stats = {
|
|
8925
|
+
lastSyncAt: null,
|
|
8926
|
+
pushedEvents: 0,
|
|
8927
|
+
pulledEvents: 0,
|
|
8928
|
+
errors: 0,
|
|
8929
|
+
status: "idle"
|
|
8930
|
+
};
|
|
8931
|
+
start() {
|
|
8932
|
+
if (this.running)
|
|
8933
|
+
return;
|
|
8934
|
+
this.running = true;
|
|
8935
|
+
this.stats.status = "idle";
|
|
8936
|
+
this.syncNow().catch((err) => {
|
|
8937
|
+
console.error("[MongoSyncWorker] Initial sync failed:", err);
|
|
8938
|
+
});
|
|
8939
|
+
this.intervalHandle = setInterval(() => {
|
|
8940
|
+
this.syncNow().catch((err) => {
|
|
8941
|
+
console.error("[MongoSyncWorker] Periodic sync failed:", err);
|
|
8942
|
+
});
|
|
8943
|
+
}, this.config.intervalMs);
|
|
8944
|
+
}
|
|
8945
|
+
stop() {
|
|
8946
|
+
this.running = false;
|
|
8947
|
+
this.stats.status = "stopped";
|
|
8948
|
+
if (this.intervalHandle) {
|
|
8949
|
+
clearInterval(this.intervalHandle);
|
|
8950
|
+
this.intervalHandle = null;
|
|
8951
|
+
}
|
|
8952
|
+
}
|
|
8953
|
+
async shutdown() {
|
|
8954
|
+
this.stop();
|
|
8955
|
+
await this.disconnect();
|
|
8956
|
+
}
|
|
8957
|
+
getStats() {
|
|
8958
|
+
return { ...this.stats };
|
|
8959
|
+
}
|
|
8960
|
+
isRunning() {
|
|
8961
|
+
return this.running;
|
|
8962
|
+
}
|
|
8963
|
+
async syncNow() {
|
|
8964
|
+
if (this.stats.status === "syncing")
|
|
8965
|
+
return { pushed: 0, pulled: 0 };
|
|
8966
|
+
this.stats.status = "syncing";
|
|
8967
|
+
let pushed = 0;
|
|
8968
|
+
let pulled = 0;
|
|
8969
|
+
try {
|
|
8970
|
+
await this.sqliteStore.initialize();
|
|
8971
|
+
await this.ensureConnected();
|
|
8972
|
+
await this.ensureIndexes();
|
|
8973
|
+
if (this.config.direction === "push" || this.config.direction === "both") {
|
|
8974
|
+
pushed = await this.pushEvents();
|
|
8975
|
+
this.stats.pushedEvents += pushed;
|
|
8976
|
+
}
|
|
8977
|
+
if (this.config.direction === "pull" || this.config.direction === "both") {
|
|
8978
|
+
pulled = await this.pullEvents();
|
|
8979
|
+
this.stats.pulledEvents += pulled;
|
|
8980
|
+
}
|
|
8981
|
+
this.stats.lastSyncAt = /* @__PURE__ */ new Date();
|
|
8982
|
+
this.stats.status = "idle";
|
|
8983
|
+
return { pushed, pulled };
|
|
8984
|
+
} catch (error) {
|
|
8985
|
+
this.stats.errors++;
|
|
8986
|
+
this.stats.status = "error";
|
|
8987
|
+
throw error;
|
|
8988
|
+
}
|
|
8989
|
+
}
|
|
8990
|
+
async ensureConnected() {
|
|
8991
|
+
if (this.client && this.db && this.counters && this.events)
|
|
8992
|
+
return;
|
|
8993
|
+
try {
|
|
8994
|
+
this.client = new MongoClient(this.config.uri, {
|
|
8995
|
+
appName: "claude-memory-layer",
|
|
8996
|
+
serverSelectionTimeoutMS: 5e3
|
|
8997
|
+
});
|
|
8998
|
+
await this.client.connect();
|
|
8999
|
+
this.db = this.client.db(this.config.dbName);
|
|
9000
|
+
this.counters = this.db.collection("cml_counters");
|
|
9001
|
+
this.events = this.db.collection("cml_events");
|
|
9002
|
+
} catch (err) {
|
|
9003
|
+
const safeUri = redactMongoUri(this.config.uri);
|
|
9004
|
+
throw new Error(`MongoDB connection failed (${safeUri}, db=${this.config.dbName}): ${String(err)}`);
|
|
9005
|
+
}
|
|
9006
|
+
}
|
|
9007
|
+
async disconnect() {
|
|
9008
|
+
try {
|
|
9009
|
+
await this.client?.close();
|
|
9010
|
+
} finally {
|
|
9011
|
+
this.client = null;
|
|
9012
|
+
this.db = null;
|
|
9013
|
+
this.counters = null;
|
|
9014
|
+
this.events = null;
|
|
9015
|
+
this.indexesEnsured = false;
|
|
9016
|
+
}
|
|
9017
|
+
}
|
|
9018
|
+
async ensureIndexes() {
|
|
9019
|
+
if (this.indexesEnsured)
|
|
9020
|
+
return;
|
|
9021
|
+
if (!this.events || !this.counters)
|
|
9022
|
+
throw new Error("Mongo not connected");
|
|
9023
|
+
try {
|
|
9024
|
+
await this.events.createIndex({ projectKey: 1, seq: 1 }, { unique: true });
|
|
9025
|
+
await this.events.createIndex({ projectKey: 1, eventId: 1 }, { unique: true });
|
|
9026
|
+
await this.events.createIndex({ projectKey: 1, dedupeKey: 1 });
|
|
9027
|
+
} catch (err) {
|
|
9028
|
+
console.warn("[MongoSyncWorker] Failed to ensure indexes (continuing):", err);
|
|
9029
|
+
}
|
|
9030
|
+
this.indexesEnsured = true;
|
|
9031
|
+
}
|
|
9032
|
+
counterKey(kind) {
|
|
9033
|
+
return `${kind}:${this.config.projectKey}`;
|
|
9034
|
+
}
|
|
9035
|
+
async allocateSeqRange(kind, count) {
|
|
9036
|
+
if (!this.counters)
|
|
9037
|
+
throw new Error("Mongo not connected");
|
|
9038
|
+
if (count <= 0)
|
|
9039
|
+
return 1;
|
|
9040
|
+
const key = this.counterKey(kind);
|
|
9041
|
+
const doc = await this.counters.findOneAndUpdate(
|
|
9042
|
+
{ _id: key },
|
|
9043
|
+
{ $inc: { seq: count } },
|
|
9044
|
+
{ upsert: true, returnDocument: "after" }
|
|
9045
|
+
);
|
|
9046
|
+
const endSeq = doc?.seq;
|
|
9047
|
+
if (typeof endSeq !== "number") {
|
|
9048
|
+
throw new Error(`Failed to allocate seq range for ${key}`);
|
|
9049
|
+
}
|
|
9050
|
+
return endSeq - count + 1;
|
|
9051
|
+
}
|
|
9052
|
+
pushTargetName() {
|
|
9053
|
+
return `mongo_push_events_rowid:${this.config.projectKey}`;
|
|
9054
|
+
}
|
|
9055
|
+
pullTargetName() {
|
|
9056
|
+
return `mongo_pull_events_seq:${this.config.projectKey}`;
|
|
9057
|
+
}
|
|
9058
|
+
async pushEvents() {
|
|
9059
|
+
if (!this.events)
|
|
9060
|
+
throw new Error("Mongo not connected");
|
|
9061
|
+
const position = await this.sqliteStore.getSyncPosition(this.pushTargetName());
|
|
9062
|
+
let lastRowid = parseIntOrZero(position.lastEventId);
|
|
9063
|
+
let pushed = 0;
|
|
9064
|
+
while (true) {
|
|
9065
|
+
const batch = await this.sqliteStore.getEventsSinceRowid(lastRowid, this.config.batchSize);
|
|
9066
|
+
if (batch.length === 0)
|
|
9067
|
+
break;
|
|
9068
|
+
const startSeq = await this.allocateSeqRange("events", batch.length);
|
|
9069
|
+
const now = /* @__PURE__ */ new Date();
|
|
9070
|
+
const hostname2 = os5.hostname();
|
|
9071
|
+
const ops = batch.map((item, idx) => {
|
|
9072
|
+
const event = item.event;
|
|
9073
|
+
const seq = startSeq + idx;
|
|
9074
|
+
const docId = `${this.config.projectKey}:${event.id}`;
|
|
9075
|
+
return {
|
|
9076
|
+
updateOne: {
|
|
9077
|
+
filter: { _id: docId },
|
|
9078
|
+
update: {
|
|
9079
|
+
$setOnInsert: {
|
|
9080
|
+
_id: docId,
|
|
9081
|
+
projectKey: this.config.projectKey,
|
|
9082
|
+
seq,
|
|
9083
|
+
eventId: event.id,
|
|
9084
|
+
eventType: event.eventType,
|
|
9085
|
+
sessionId: event.sessionId,
|
|
9086
|
+
timestamp: event.timestamp,
|
|
9087
|
+
content: event.content,
|
|
9088
|
+
canonicalKey: event.canonicalKey,
|
|
9089
|
+
dedupeKey: event.dedupeKey,
|
|
9090
|
+
metadata: event.metadata ?? null,
|
|
9091
|
+
insertedAt: now,
|
|
9092
|
+
updatedAt: now,
|
|
9093
|
+
source: { hostname: hostname2, instanceId: this.config.instanceId }
|
|
9094
|
+
}
|
|
9095
|
+
},
|
|
9096
|
+
upsert: true
|
|
9097
|
+
}
|
|
9098
|
+
};
|
|
9099
|
+
});
|
|
9100
|
+
await this.events.bulkWrite(ops, { ordered: false });
|
|
9101
|
+
const last = batch[batch.length - 1];
|
|
9102
|
+
lastRowid = last.rowid;
|
|
9103
|
+
await this.sqliteStore.updateSyncPosition(
|
|
9104
|
+
this.pushTargetName(),
|
|
9105
|
+
String(lastRowid),
|
|
9106
|
+
last.event.timestamp.toISOString()
|
|
9107
|
+
);
|
|
9108
|
+
pushed += batch.length;
|
|
9109
|
+
if (batch.length < this.config.batchSize)
|
|
9110
|
+
break;
|
|
9111
|
+
}
|
|
9112
|
+
return pushed;
|
|
9113
|
+
}
|
|
9114
|
+
async pullEvents() {
|
|
9115
|
+
if (!this.events)
|
|
9116
|
+
throw new Error("Mongo not connected");
|
|
9117
|
+
const position = await this.sqliteStore.getSyncPosition(this.pullTargetName());
|
|
9118
|
+
let lastSeq = parseIntOrZero(position.lastEventId);
|
|
9119
|
+
let pulled = 0;
|
|
9120
|
+
while (true) {
|
|
9121
|
+
const docs = await this.events.find(
|
|
9122
|
+
{ projectKey: this.config.projectKey, seq: { $gt: lastSeq } },
|
|
9123
|
+
{ sort: { seq: 1 }, limit: this.config.batchSize }
|
|
9124
|
+
).toArray();
|
|
9125
|
+
if (docs.length === 0)
|
|
9126
|
+
break;
|
|
9127
|
+
const events = docs.map((d) => ({
|
|
9128
|
+
id: d.eventId,
|
|
9129
|
+
eventType: d.eventType,
|
|
9130
|
+
sessionId: d.sessionId,
|
|
9131
|
+
timestamp: d.timestamp instanceof Date ? d.timestamp : new Date(d.timestamp),
|
|
9132
|
+
content: d.content,
|
|
9133
|
+
canonicalKey: d.canonicalKey,
|
|
9134
|
+
dedupeKey: d.dedupeKey,
|
|
9135
|
+
metadata: d.metadata ?? void 0
|
|
9136
|
+
}));
|
|
9137
|
+
const result = await this.sqliteStore.importEvents(events);
|
|
9138
|
+
pulled += result.inserted;
|
|
9139
|
+
lastSeq = docs[docs.length - 1].seq;
|
|
9140
|
+
await this.sqliteStore.updateSyncPosition(
|
|
9141
|
+
this.pullTargetName(),
|
|
9142
|
+
String(lastSeq),
|
|
9143
|
+
(/* @__PURE__ */ new Date()).toISOString()
|
|
9144
|
+
);
|
|
9145
|
+
if (docs.length < this.config.batchSize)
|
|
9146
|
+
break;
|
|
9147
|
+
}
|
|
9148
|
+
return pulled;
|
|
9149
|
+
}
|
|
9150
|
+
};
|
|
9151
|
+
|
|
7258
9152
|
// src/cli/index.ts
|
|
7259
|
-
var CLAUDE_SETTINGS_PATH =
|
|
9153
|
+
var CLAUDE_SETTINGS_PATH = path9.join(os6.homedir(), ".claude", "settings.json");
|
|
7260
9154
|
function getPluginPath() {
|
|
7261
9155
|
const possiblePaths = [
|
|
7262
|
-
|
|
9156
|
+
path9.join(__dirname, ".."),
|
|
7263
9157
|
// When running from dist/cli
|
|
7264
|
-
|
|
9158
|
+
path9.join(__dirname, "../..", "dist"),
|
|
7265
9159
|
// When running from src
|
|
7266
|
-
|
|
9160
|
+
path9.join(process.cwd(), "dist")
|
|
7267
9161
|
// Current working directory
|
|
7268
9162
|
];
|
|
7269
9163
|
for (const p of possiblePaths) {
|
|
7270
|
-
const hooksPath =
|
|
7271
|
-
if (
|
|
9164
|
+
const hooksPath = path9.join(p, "hooks", "user-prompt-submit.js");
|
|
9165
|
+
if (fs9.existsSync(hooksPath)) {
|
|
7272
9166
|
return p;
|
|
7273
9167
|
}
|
|
7274
9168
|
}
|
|
7275
|
-
return
|
|
9169
|
+
return path9.join(os6.homedir(), ".npm-global", "lib", "node_modules", "claude-memory-layer", "dist");
|
|
7276
9170
|
}
|
|
7277
9171
|
function loadClaudeSettings() {
|
|
7278
9172
|
try {
|
|
7279
|
-
if (
|
|
7280
|
-
const content =
|
|
9173
|
+
if (fs9.existsSync(CLAUDE_SETTINGS_PATH)) {
|
|
9174
|
+
const content = fs9.readFileSync(CLAUDE_SETTINGS_PATH, "utf-8");
|
|
7281
9175
|
return JSON.parse(content);
|
|
7282
9176
|
}
|
|
7283
9177
|
} catch (error) {
|
|
@@ -7286,13 +9180,13 @@ function loadClaudeSettings() {
|
|
|
7286
9180
|
return {};
|
|
7287
9181
|
}
|
|
7288
9182
|
function saveClaudeSettings(settings) {
|
|
7289
|
-
const dir =
|
|
7290
|
-
if (!
|
|
7291
|
-
|
|
9183
|
+
const dir = path9.dirname(CLAUDE_SETTINGS_PATH);
|
|
9184
|
+
if (!fs9.existsSync(dir)) {
|
|
9185
|
+
fs9.mkdirSync(dir, { recursive: true });
|
|
7292
9186
|
}
|
|
7293
9187
|
const tempPath = CLAUDE_SETTINGS_PATH + ".tmp";
|
|
7294
|
-
|
|
7295
|
-
|
|
9188
|
+
fs9.writeFileSync(tempPath, JSON.stringify(settings, null, 2));
|
|
9189
|
+
fs9.renameSync(tempPath, CLAUDE_SETTINGS_PATH);
|
|
7296
9190
|
}
|
|
7297
9191
|
function getHooksConfig(pluginPath) {
|
|
7298
9192
|
return {
|
|
@@ -7302,7 +9196,7 @@ function getHooksConfig(pluginPath) {
|
|
|
7302
9196
|
hooks: [
|
|
7303
9197
|
{
|
|
7304
9198
|
type: "command",
|
|
7305
|
-
command: `node ${
|
|
9199
|
+
command: `node ${path9.join(pluginPath, "hooks", "user-prompt-submit.js")}`
|
|
7306
9200
|
}
|
|
7307
9201
|
]
|
|
7308
9202
|
}
|
|
@@ -7313,7 +9207,7 @@ function getHooksConfig(pluginPath) {
|
|
|
7313
9207
|
hooks: [
|
|
7314
9208
|
{
|
|
7315
9209
|
type: "command",
|
|
7316
|
-
command: `node ${
|
|
9210
|
+
command: `node ${path9.join(pluginPath, "hooks", "post-tool-use.js")}`
|
|
7317
9211
|
}
|
|
7318
9212
|
]
|
|
7319
9213
|
}
|
|
@@ -7321,12 +9215,12 @@ function getHooksConfig(pluginPath) {
|
|
|
7321
9215
|
};
|
|
7322
9216
|
}
|
|
7323
9217
|
var program = new Command();
|
|
7324
|
-
program.name("claude-memory-layer").description("Claude Code Memory Plugin CLI").version("1.0.
|
|
9218
|
+
program.name("claude-memory-layer").description("Claude Code Memory Plugin CLI").version("1.0.12");
|
|
7325
9219
|
program.command("install").description("Install hooks into Claude Code settings").option("--path <path>", "Custom plugin path (defaults to auto-detect)").action(async (options) => {
|
|
7326
9220
|
try {
|
|
7327
9221
|
const pluginPath = options.path || getPluginPath();
|
|
7328
|
-
const userPromptHook =
|
|
7329
|
-
if (!
|
|
9222
|
+
const userPromptHook = path9.join(pluginPath, "hooks", "user-prompt-submit.js");
|
|
9223
|
+
if (!fs9.existsSync(userPromptHook)) {
|
|
7330
9224
|
console.error(`
|
|
7331
9225
|
\u274C Hook files not found at: ${pluginPath}`);
|
|
7332
9226
|
console.error(' Make sure you have built the plugin with "npm run build"');
|
|
@@ -7392,7 +9286,7 @@ program.command("status").description("Check plugin installation status").action
|
|
|
7392
9286
|
console.log("Hooks:");
|
|
7393
9287
|
console.log(` UserPromptSubmit: ${hasUserPromptHook ? "\u2705 Installed" : "\u274C Not installed"}`);
|
|
7394
9288
|
console.log(` PostToolUse: ${hasPostToolHook ? "\u2705 Installed" : "\u274C Not installed"}`);
|
|
7395
|
-
const hooksExist =
|
|
9289
|
+
const hooksExist = fs9.existsSync(path9.join(pluginPath, "hooks", "user-prompt-submit.js"));
|
|
7396
9290
|
console.log(`
|
|
7397
9291
|
Plugin files: ${hooksExist ? "\u2705 Found" : "\u274C Not found"}`);
|
|
7398
9292
|
console.log(` Path: ${pluginPath}`);
|
|
@@ -7520,6 +9414,81 @@ program.command("process").description("Process pending embeddings").option("-p,
|
|
|
7520
9414
|
process.exit(1);
|
|
7521
9415
|
}
|
|
7522
9416
|
});
|
|
9417
|
+
program.command("mongo-sync").description("Sync events with MongoDB for multi-server collaboration (optional)").option("-p, --project <path>", "Project path (defaults to cwd)").option("--mongo-uri <uri>", "MongoDB connection URI (env: CLAUDE_MEMORY_MONGO_URI)").option("--mongo-db <name>", "MongoDB database name (env: CLAUDE_MEMORY_MONGO_DB)").option("--mongo-project <key>", "Remote project key (env: CLAUDE_MEMORY_MONGO_PROJECT, default: basename(projectPath))").option("--direction <dir>", "push|pull|both", "both").option("--batch-size <n>", "Batch size", "500").option("--interval <ms>", "Watch interval ms", "30000").option("--watch", "Run continuously").action(async (options) => {
|
|
9418
|
+
const projectPath = options.project || process.cwd();
|
|
9419
|
+
const mongoUri = options.mongoUri || process.env.CLAUDE_MEMORY_MONGO_URI;
|
|
9420
|
+
const mongoDb = options.mongoDb || process.env.CLAUDE_MEMORY_MONGO_DB;
|
|
9421
|
+
const projectKey = options.mongoProject || process.env.CLAUDE_MEMORY_MONGO_PROJECT || path9.basename(projectPath);
|
|
9422
|
+
const direction = String(options.direction || "both").toLowerCase();
|
|
9423
|
+
if (!mongoUri || !mongoDb) {
|
|
9424
|
+
console.error("\n\u274C MongoDB sync is not configured.");
|
|
9425
|
+
console.error(" Set --mongo-uri/--mongo-db or env CLAUDE_MEMORY_MONGO_URI/CLAUDE_MEMORY_MONGO_DB.\n");
|
|
9426
|
+
process.exit(1);
|
|
9427
|
+
}
|
|
9428
|
+
if (!["push", "pull", "both"].includes(direction)) {
|
|
9429
|
+
console.error("\n\u274C Invalid --direction. Use: push | pull | both\n");
|
|
9430
|
+
process.exit(1);
|
|
9431
|
+
}
|
|
9432
|
+
const storagePath = getProjectStoragePath(projectPath);
|
|
9433
|
+
if (!fs9.existsSync(storagePath)) {
|
|
9434
|
+
fs9.mkdirSync(storagePath, { recursive: true });
|
|
9435
|
+
}
|
|
9436
|
+
const batchSizeParsed = parseInt(options.batchSize, 10);
|
|
9437
|
+
const intervalParsed = parseInt(options.interval, 10);
|
|
9438
|
+
const batchSize = Number.isFinite(batchSizeParsed) && batchSizeParsed > 0 ? batchSizeParsed : 500;
|
|
9439
|
+
const intervalMs = Number.isFinite(intervalParsed) && intervalParsed > 0 ? intervalParsed : 3e4;
|
|
9440
|
+
const sqliteStore = new SQLiteEventStore(path9.join(storagePath, "events.sqlite"));
|
|
9441
|
+
const worker = new MongoSyncWorker(sqliteStore, {
|
|
9442
|
+
uri: mongoUri,
|
|
9443
|
+
dbName: mongoDb,
|
|
9444
|
+
projectKey,
|
|
9445
|
+
direction,
|
|
9446
|
+
batchSize,
|
|
9447
|
+
intervalMs
|
|
9448
|
+
});
|
|
9449
|
+
const runOnce = async () => {
|
|
9450
|
+
const { pushed, pulled } = await worker.syncNow();
|
|
9451
|
+
const ts = (/* @__PURE__ */ new Date()).toISOString();
|
|
9452
|
+
process.stdout.write(`[mongo-sync] ${ts} project=${projectKey} pushed=${pushed} pulled=${pulled}
|
|
9453
|
+
`);
|
|
9454
|
+
};
|
|
9455
|
+
try {
|
|
9456
|
+
if (!options.watch) {
|
|
9457
|
+
await runOnce();
|
|
9458
|
+
await worker.shutdown();
|
|
9459
|
+
sqliteStore.close();
|
|
9460
|
+
return;
|
|
9461
|
+
}
|
|
9462
|
+
console.log(`[mongo-sync] Watch mode started (interval=${intervalMs}ms, project=${projectKey})`);
|
|
9463
|
+
const handle = setInterval(() => {
|
|
9464
|
+
runOnce().catch((err) => {
|
|
9465
|
+
console.error("[mongo-sync] Sync failed:", err);
|
|
9466
|
+
});
|
|
9467
|
+
}, intervalMs);
|
|
9468
|
+
const shutdown = async () => {
|
|
9469
|
+
clearInterval(handle);
|
|
9470
|
+
console.log("\n[mongo-sync] Shutting down...");
|
|
9471
|
+
try {
|
|
9472
|
+
await worker.shutdown();
|
|
9473
|
+
} finally {
|
|
9474
|
+
sqliteStore.close();
|
|
9475
|
+
}
|
|
9476
|
+
process.exit(0);
|
|
9477
|
+
};
|
|
9478
|
+
process.on("SIGINT", () => {
|
|
9479
|
+
void shutdown();
|
|
9480
|
+
});
|
|
9481
|
+
process.on("SIGTERM", () => {
|
|
9482
|
+
void shutdown();
|
|
9483
|
+
});
|
|
9484
|
+
await runOnce();
|
|
9485
|
+
await new Promise(() => {
|
|
9486
|
+
});
|
|
9487
|
+
} catch (error) {
|
|
9488
|
+
console.error("[mongo-sync] Failed:", error);
|
|
9489
|
+
process.exit(1);
|
|
9490
|
+
}
|
|
9491
|
+
});
|
|
7523
9492
|
function renderProgress(event) {
|
|
7524
9493
|
switch (event.phase) {
|
|
7525
9494
|
case "scan":
|
|
@@ -7527,7 +9496,7 @@ function renderProgress(event) {
|
|
|
7527
9496
|
break;
|
|
7528
9497
|
case "session-start": {
|
|
7529
9498
|
const pct = Math.round(event.sessionIndex / event.totalSessions * 100);
|
|
7530
|
-
const sessionName =
|
|
9499
|
+
const sessionName = path9.basename(event.filePath, ".jsonl").slice(0, 8);
|
|
7531
9500
|
process.stdout.write(
|
|
7532
9501
|
`\r \u{1F4C4} [${event.sessionIndex + 1}/${event.totalSessions}] ${pct}% | Session ${sessionName}... `
|
|
7533
9502
|
);
|
|
@@ -7593,6 +9562,140 @@ function printImportSummary(result, embedCount) {
|
|
|
7593
9562
|
}
|
|
7594
9563
|
}
|
|
7595
9564
|
}
|
|
9565
|
+
function sanitizeSegment3(input, fallback) {
|
|
9566
|
+
const v = (input || "").trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
9567
|
+
return v || fallback;
|
|
9568
|
+
}
|
|
9569
|
+
async function listMarkdownFiles(root) {
|
|
9570
|
+
const out = [];
|
|
9571
|
+
const stack = [root];
|
|
9572
|
+
while (stack.length > 0) {
|
|
9573
|
+
const dir = stack.pop();
|
|
9574
|
+
const entries = await fs9.promises.readdir(dir, { withFileTypes: true });
|
|
9575
|
+
for (const e of entries) {
|
|
9576
|
+
const full = path9.join(dir, e.name);
|
|
9577
|
+
if (e.isDirectory())
|
|
9578
|
+
stack.push(full);
|
|
9579
|
+
else if (e.isFile() && e.name.endsWith(".md") && e.name !== "_index.md")
|
|
9580
|
+
out.push(full);
|
|
9581
|
+
}
|
|
9582
|
+
}
|
|
9583
|
+
return out.sort();
|
|
9584
|
+
}
|
|
9585
|
+
function deriveNamespaceCategory(sourceRoot, filePath) {
|
|
9586
|
+
const rel = path9.relative(sourceRoot, filePath);
|
|
9587
|
+
const dirSeg = path9.dirname(rel).split(path9.sep).filter(Boolean);
|
|
9588
|
+
if (dirSeg.length >= 2) {
|
|
9589
|
+
const namespace = sanitizeSegment3(dirSeg[0], "default");
|
|
9590
|
+
const categoryPath = dirSeg.slice(1).map((s) => sanitizeSegment3(s, "uncategorized"));
|
|
9591
|
+
return { namespace, categoryPath: categoryPath.length > 0 ? categoryPath : ["uncategorized"] };
|
|
9592
|
+
}
|
|
9593
|
+
return { namespace: "default", categoryPath: ["uncategorized"] };
|
|
9594
|
+
}
|
|
9595
|
+
function extractImportEvidence(markdown) {
|
|
9596
|
+
const confidenceMatch = markdown.match(/^-\s*confidence:\s*([^\n]+)/m);
|
|
9597
|
+
const sources = markdown.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.startsWith("- source:")).map((line) => line.replace(/^-\s*source:\s*/i, "").trim()).filter(Boolean).slice(0, 30);
|
|
9598
|
+
return {
|
|
9599
|
+
confidence: confidenceMatch ? confidenceMatch[1].trim() : void 0,
|
|
9600
|
+
sources
|
|
9601
|
+
};
|
|
9602
|
+
}
|
|
9603
|
+
program.command("organize-import [sourceDir]").description("Import existing markdown memory files, or bootstrap knowledge docs from codebase/git when markdown is missing").option("-p, --project <path>", "Project path (defaults to cwd)").option("--session <id>", "Session id for imported events (default: import:organized)").option("--limit <n>", "Limit number of files to import").option("--dry-run", "Preview mapping without writing").option("--bootstrap", "Force-generate structured markdown from codebase + git history before import").option("--bootstrap-if-empty", "Auto-bootstrap when source has no markdown files (default: true)", true).option("--no-bootstrap-if-empty", "Disable auto-bootstrap when source has no markdown files").option("--force-bootstrap", "Run bootstrap even when markdown files exist").option("--repo <path>", "Repository root for bootstrap analysis (default: project path)").option("--out <path>", "Output directory for generated bootstrap markdown (default: <sourceDir>/bootstrap-kb)").option("--since <range>", 'Git history range for bootstrap (default: "180 days ago")').option("--max-commits <n>", "Max commits to analyze for bootstrap (default: 1000)").option("--incremental", "Use previous bootstrap manifest as baseline for incremental updates (default: true)", true).option("--no-incremental", "Disable incremental bootstrap; regenerate full snapshot").action(async (sourceDir, options) => {
|
|
9604
|
+
const projectPath = options.project || process.cwd();
|
|
9605
|
+
const sessionId = options.session || "import:organized";
|
|
9606
|
+
const sourceRoot = path9.resolve(sourceDir || options.out || projectPath);
|
|
9607
|
+
const repoPath = path9.resolve(options.repo || projectPath);
|
|
9608
|
+
if (!fs9.existsSync(sourceRoot)) {
|
|
9609
|
+
fs9.mkdirSync(sourceRoot, { recursive: true });
|
|
9610
|
+
}
|
|
9611
|
+
const service = getMemoryServiceForProject(projectPath);
|
|
9612
|
+
try {
|
|
9613
|
+
let activeSourceRoot = sourceRoot;
|
|
9614
|
+
let importRoot = sourceRoot;
|
|
9615
|
+
let files = await listMarkdownFiles(importRoot);
|
|
9616
|
+
const hasMarkdown = files.length > 0;
|
|
9617
|
+
const shouldBootstrap = Boolean(options.forceBootstrap || options.bootstrap || !hasMarkdown && options.bootstrapIfEmpty);
|
|
9618
|
+
if (shouldBootstrap) {
|
|
9619
|
+
const outDir = path9.resolve(options.out || path9.join(sourceRoot, "bootstrap-kb"));
|
|
9620
|
+
const since = options.since || "180 days ago";
|
|
9621
|
+
const maxCommits = options.maxCommits ? Math.max(1, parseInt(options.maxCommits, 10)) : 1e3;
|
|
9622
|
+
console.log("\n\u{1F9E0} Bootstrapping markdown knowledge base...");
|
|
9623
|
+
const bootstrap = await bootstrapKnowledgeBase({
|
|
9624
|
+
repoPath,
|
|
9625
|
+
outDir,
|
|
9626
|
+
since,
|
|
9627
|
+
maxCommits,
|
|
9628
|
+
incremental: options.incremental
|
|
9629
|
+
});
|
|
9630
|
+
console.log(` Repo: ${repoPath}`);
|
|
9631
|
+
console.log(` Output: ${bootstrap.outDir}`);
|
|
9632
|
+
console.log(` Files analyzed: ${bootstrap.fileCount}`);
|
|
9633
|
+
console.log(` Commits analyzed: ${bootstrap.commitCount}`);
|
|
9634
|
+
console.log(` Modules: ${bootstrap.moduleCount}`);
|
|
9635
|
+
activeSourceRoot = outDir;
|
|
9636
|
+
importRoot = outDir;
|
|
9637
|
+
files = await listMarkdownFiles(importRoot);
|
|
9638
|
+
}
|
|
9639
|
+
if (files.length === 0) {
|
|
9640
|
+
console.error("\n\u274C organize-import found no markdown files to import.\n");
|
|
9641
|
+
process.exit(1);
|
|
9642
|
+
}
|
|
9643
|
+
const limit = options.limit ? Math.max(1, parseInt(options.limit, 10)) : files.length;
|
|
9644
|
+
const targets = files.slice(0, limit);
|
|
9645
|
+
console.log(`
|
|
9646
|
+
\u{1F4E6} organize-import`);
|
|
9647
|
+
console.log(` Source: ${activeSourceRoot}`);
|
|
9648
|
+
console.log(` Project: ${projectPath}`);
|
|
9649
|
+
console.log(` Files: ${targets.length}${targets.length < files.length ? `/${files.length}` : ""}`);
|
|
9650
|
+
console.log(` Dry-run: ${options.dryRun ? "yes" : "no"}
|
|
9651
|
+
`);
|
|
9652
|
+
if (!options.dryRun) {
|
|
9653
|
+
await service.initialize();
|
|
9654
|
+
}
|
|
9655
|
+
let imported = 0;
|
|
9656
|
+
let skipped = 0;
|
|
9657
|
+
for (const file of targets) {
|
|
9658
|
+
const text = await fs9.promises.readFile(file, "utf8");
|
|
9659
|
+
if (!text.trim()) {
|
|
9660
|
+
skipped += 1;
|
|
9661
|
+
continue;
|
|
9662
|
+
}
|
|
9663
|
+
const { namespace, categoryPath } = deriveNamespaceCategory(activeSourceRoot, file);
|
|
9664
|
+
const rel = path9.relative(activeSourceRoot, file);
|
|
9665
|
+
const evidence = extractImportEvidence(text);
|
|
9666
|
+
if (options.dryRun) {
|
|
9667
|
+
console.log(`- ${rel} -> namespace=${namespace} category=${categoryPath.join("/")} confidence=${evidence.confidence || "n/a"} sources=${evidence.sources.length}`);
|
|
9668
|
+
continue;
|
|
9669
|
+
}
|
|
9670
|
+
await service.storeSessionSummary(sessionId, text, {
|
|
9671
|
+
namespace,
|
|
9672
|
+
categoryPath,
|
|
9673
|
+
confidence: evidence.confidence,
|
|
9674
|
+
sources: evidence.sources,
|
|
9675
|
+
import: {
|
|
9676
|
+
sourceFile: rel,
|
|
9677
|
+
importedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9678
|
+
bootstrap: shouldBootstrap === true
|
|
9679
|
+
}
|
|
9680
|
+
});
|
|
9681
|
+
imported += 1;
|
|
9682
|
+
}
|
|
9683
|
+
if (!options.dryRun) {
|
|
9684
|
+
const embed = await service.processPendingEmbeddings();
|
|
9685
|
+
await service.shutdown();
|
|
9686
|
+
console.log(`
|
|
9687
|
+
\u2705 Imported: ${imported}, skipped-empty: ${skipped}, embeddings: ${embed}
|
|
9688
|
+
`);
|
|
9689
|
+
} else {
|
|
9690
|
+
console.log(`
|
|
9691
|
+
\u2705 Dry-run complete (planned imports: ${targets.length - skipped}, skipped-empty: ${skipped})
|
|
9692
|
+
`);
|
|
9693
|
+
}
|
|
9694
|
+
} catch (error) {
|
|
9695
|
+
console.error("\n\u274C organize-import failed:", error);
|
|
9696
|
+
process.exit(1);
|
|
9697
|
+
}
|
|
9698
|
+
});
|
|
7596
9699
|
program.command("import").description("Import existing Claude Code conversation history").option("-p, --project <path>", "Import from specific project path").option("-s, --session <file>", "Import specific session file (JSONL)").option("-a, --all", "Import all sessions from all projects").option("-l, --limit <number>", "Limit messages per session").option("-f, --force", "Force reimport: delete existing events and reimport with turn_id grouping").option("-v, --verbose", "Show detailed progress").action(async (options) => {
|
|
7597
9700
|
const startTime = Date.now();
|
|
7598
9701
|
const targetProjectPath = options.project || process.cwd();
|