nexusmem 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +218 -126
- package/README.md +117 -39
- package/dist/cli/index.js +818 -97
- package/dist/cli/index.js.map +1 -1
- package/package.json +7 -4
package/dist/cli/index.js
CHANGED
|
@@ -2,13 +2,68 @@
|
|
|
2
2
|
|
|
3
3
|
// src/cli/index.ts
|
|
4
4
|
import { Command } from "commander";
|
|
5
|
-
import
|
|
5
|
+
import pc14 from "picocolors";
|
|
6
6
|
|
|
7
7
|
// src/config/workspace.ts
|
|
8
8
|
import { existsSync } from "fs";
|
|
9
9
|
import { mkdir, readFile, writeFile } from "fs/promises";
|
|
10
10
|
import { join } from "path";
|
|
11
11
|
import { z } from "zod";
|
|
12
|
+
|
|
13
|
+
// src/slm/provider.ts
|
|
14
|
+
var DEFAULT_BASE_URL = "http://127.0.0.1:11434";
|
|
15
|
+
var DEFAULT_SLM_MODEL = "qwen2.5:3b";
|
|
16
|
+
var DEFAULT_TIMEOUT_MS = 12e4;
|
|
17
|
+
var DEFAULT_MAX_TOKENS = 400;
|
|
18
|
+
var OllamaChatProvider = class {
|
|
19
|
+
identity;
|
|
20
|
+
baseUrl;
|
|
21
|
+
model;
|
|
22
|
+
timeoutMs;
|
|
23
|
+
maxTokens;
|
|
24
|
+
constructor(opts = {}) {
|
|
25
|
+
this.baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
|
|
26
|
+
this.model = opts.model ?? DEFAULT_SLM_MODEL;
|
|
27
|
+
this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
28
|
+
this.maxTokens = opts.maxTokens ?? DEFAULT_MAX_TOKENS;
|
|
29
|
+
this.identity = `ollama:${this.model}`;
|
|
30
|
+
}
|
|
31
|
+
async complete(prompt) {
|
|
32
|
+
const controller = new AbortController();
|
|
33
|
+
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
34
|
+
try {
|
|
35
|
+
const res = await fetch(`${this.baseUrl}/api/generate`, {
|
|
36
|
+
method: "POST",
|
|
37
|
+
headers: { "content-type": "application/json" },
|
|
38
|
+
body: JSON.stringify({
|
|
39
|
+
model: this.model,
|
|
40
|
+
prompt,
|
|
41
|
+
stream: false,
|
|
42
|
+
options: {
|
|
43
|
+
// Deterministic: the same session must summarize to the same text
|
|
44
|
+
// across syncs, or the content hash that suppresses re-work would
|
|
45
|
+
// never match and every sync would rewrite every summary node.
|
|
46
|
+
temperature: 0,
|
|
47
|
+
seed: 1,
|
|
48
|
+
num_predict: this.maxTokens
|
|
49
|
+
}
|
|
50
|
+
}),
|
|
51
|
+
signal: controller.signal
|
|
52
|
+
});
|
|
53
|
+
if (!res.ok) return null;
|
|
54
|
+
const data = await res.json();
|
|
55
|
+
if (typeof data.response !== "string") return null;
|
|
56
|
+
const text = data.response.trim();
|
|
57
|
+
return text.length > 0 ? text : null;
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
} finally {
|
|
61
|
+
clearTimeout(timeout);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
// src/config/workspace.ts
|
|
12
67
|
var WORKSPACE_DIR = ".nexusmem";
|
|
13
68
|
function resolveWorkspace(repoRoot) {
|
|
14
69
|
const dir = join(repoRoot, WORKSPACE_DIR);
|
|
@@ -46,6 +101,32 @@ var ConfigSchema = z.object({
|
|
|
46
101
|
conversation: z.object({
|
|
47
102
|
enabled: z.boolean().default(false)
|
|
48
103
|
}).default({ enabled: false }),
|
|
104
|
+
/**
|
|
105
|
+
* One distilled node per finished working session, written by a local
|
|
106
|
+
* small language model.
|
|
107
|
+
*
|
|
108
|
+
* Opt-in for the same reason as `conversation` -- it reads the same
|
|
109
|
+
* transcripts -- and additionally because it is the only source that
|
|
110
|
+
* costs real compute. Independent of `conversation.enabled`: summaries
|
|
111
|
+
* without the raw exchanges is a legitimate, and much smaller, way to
|
|
112
|
+
* remember a session.
|
|
113
|
+
*/
|
|
114
|
+
session: z.object({
|
|
115
|
+
enabled: z.boolean().default(false),
|
|
116
|
+
/** Ollama model tag. Must be pulled locally; nothing is downloaded automatically. */
|
|
117
|
+
model: z.string().default(DEFAULT_SLM_MODEL),
|
|
118
|
+
/** Minutes of quiet before a session counts as finished and can be summarized. */
|
|
119
|
+
settleMinutes: z.number().int().nonnegative().default(30),
|
|
120
|
+
/** Sessions summarized per sync. Each is a model call measured in seconds. */
|
|
121
|
+
maxSessions: z.number().int().positive().default(10),
|
|
122
|
+
maxPromptChars: z.number().int().positive().default(12e3)
|
|
123
|
+
}).default({
|
|
124
|
+
enabled: false,
|
|
125
|
+
model: DEFAULT_SLM_MODEL,
|
|
126
|
+
settleMinutes: 30,
|
|
127
|
+
maxSessions: 10,
|
|
128
|
+
maxPromptChars: 12e3
|
|
129
|
+
}),
|
|
49
130
|
/** Tracked `.md` files -- README, architecture docs. On by default like git/shell: no secrets risk, just project prose. */
|
|
50
131
|
docs: z.object({
|
|
51
132
|
enabled: z.boolean().default(true),
|
|
@@ -71,6 +152,13 @@ var ConfigSchema = z.object({
|
|
|
71
152
|
git: { enabled: true, since: null, includeMerges: true },
|
|
72
153
|
shell: { enabled: true, tailLines: 300 },
|
|
73
154
|
conversation: { enabled: false },
|
|
155
|
+
session: {
|
|
156
|
+
enabled: false,
|
|
157
|
+
model: DEFAULT_SLM_MODEL,
|
|
158
|
+
settleMinutes: 30,
|
|
159
|
+
maxSessions: 10,
|
|
160
|
+
maxPromptChars: 12e3
|
|
161
|
+
},
|
|
74
162
|
docs: { enabled: true, include: ["*.md"] },
|
|
75
163
|
diff: { enabled: true, maxCommits: 200, maxFilesPerCommit: 20, contextLines: 3 }
|
|
76
164
|
}),
|
|
@@ -209,9 +297,9 @@ async function* gitStream(cwd, args, opts = {}) {
|
|
|
209
297
|
for (let attempt = 0; ; attempt += 1) {
|
|
210
298
|
let produced = false;
|
|
211
299
|
try {
|
|
212
|
-
for await (const
|
|
300
|
+
for await (const chunk2 of runGitOnce(cwd, args, opts)) {
|
|
213
301
|
produced = true;
|
|
214
|
-
yield
|
|
302
|
+
yield chunk2;
|
|
215
303
|
}
|
|
216
304
|
return;
|
|
217
305
|
} catch (err) {
|
|
@@ -227,8 +315,8 @@ async function* runGitOnce(cwd, args, opts) {
|
|
|
227
315
|
child.stdout.setEncoding("utf8");
|
|
228
316
|
child.stderr.setEncoding("utf8");
|
|
229
317
|
let stderr = "";
|
|
230
|
-
child.stderr.on("data", (
|
|
231
|
-
if (stderr.length < 64 * 1024) stderr +=
|
|
318
|
+
child.stderr.on("data", (chunk2) => {
|
|
319
|
+
if (stderr.length < 64 * 1024) stderr += chunk2;
|
|
232
320
|
});
|
|
233
321
|
const exited = new Promise((resolve2, reject) => {
|
|
234
322
|
child.once("error", (err) => reject(toSpawnError(err, cwd, fullArgs)));
|
|
@@ -237,8 +325,8 @@ async function* runGitOnce(cwd, args, opts) {
|
|
|
237
325
|
exited.catch(() => {
|
|
238
326
|
});
|
|
239
327
|
try {
|
|
240
|
-
for await (const
|
|
241
|
-
yield
|
|
328
|
+
for await (const chunk2 of child.stdout) {
|
|
329
|
+
yield chunk2;
|
|
242
330
|
}
|
|
243
331
|
} finally {
|
|
244
332
|
if (child.exitCode === null) child.kill();
|
|
@@ -267,7 +355,7 @@ async function* runGitOnce(cwd, args, opts) {
|
|
|
267
355
|
}
|
|
268
356
|
async function git(cwd, args, opts = {}) {
|
|
269
357
|
let out = "";
|
|
270
|
-
for await (const
|
|
358
|
+
for await (const chunk2 of gitStream(cwd, args, opts)) out += chunk2;
|
|
271
359
|
return out;
|
|
272
360
|
}
|
|
273
361
|
async function gitOrNull(cwd, args, opts = {}) {
|
|
@@ -610,10 +698,13 @@ import * as sqliteVec from "sqlite-vec";
|
|
|
610
698
|
|
|
611
699
|
// src/store/fts.ts
|
|
612
700
|
var FTS_SYNTAX = /["'()*:^{}[\]-]/g;
|
|
701
|
+
var LOW_SIGNAL_TOKENS = /* @__PURE__ */ new Set(["id"]);
|
|
613
702
|
function toMatchQuery(input) {
|
|
614
703
|
const tokens = input.replace(FTS_SYNTAX, " ").split(/\s+/).map((t) => t.trim()).filter((t) => t.length > 0);
|
|
615
704
|
if (tokens.length === 0) return null;
|
|
616
|
-
|
|
705
|
+
const signal = tokens.filter((t) => !LOW_SIGNAL_TOKENS.has(t.toLowerCase()));
|
|
706
|
+
const kept = signal.length > 0 ? signal : tokens;
|
|
707
|
+
return kept.map((t) => `"${t}"*`).join(" OR ");
|
|
617
708
|
}
|
|
618
709
|
|
|
619
710
|
// src/store/schema.ts
|
|
@@ -760,6 +851,20 @@ var MemoryStore = class _MemoryStore {
|
|
|
760
851
|
markSynced(projectId) {
|
|
761
852
|
this.db.prepare("UPDATE projects SET last_synced_at = ? WHERE id = ?").run(Date.now(), projectId);
|
|
762
853
|
}
|
|
854
|
+
/**
|
|
855
|
+
* Every other project id ever recorded in THIS repo's own database.
|
|
856
|
+
*
|
|
857
|
+
* A repo's `.nexusmem/memory.db` is never shared with another repo (each
|
|
858
|
+
* gets its own, gitignored), so any id here besides `currentProjectId` is
|
|
859
|
+
* evidence of a prior identity for this same repo -- typically its git
|
|
860
|
+
* remote URL changed since the last sync. See `reconcileProjectId` in
|
|
861
|
+
* `store/reconcile.ts`.
|
|
862
|
+
*/
|
|
863
|
+
listOtherProjectIds(currentProjectId) {
|
|
864
|
+
return this.db.prepare("SELECT id FROM projects WHERE id != ?").all(currentProjectId).map(
|
|
865
|
+
(r) => r.id
|
|
866
|
+
);
|
|
867
|
+
}
|
|
763
868
|
/**
|
|
764
869
|
* Write a batch of nodes in one transaction.
|
|
765
870
|
*
|
|
@@ -831,6 +936,20 @@ var MemoryStore = class _MemoryStore {
|
|
|
831
936
|
run(nodes);
|
|
832
937
|
return stats;
|
|
833
938
|
}
|
|
939
|
+
/**
|
|
940
|
+
* The stored `meta` blob for one node, or null if it has never been
|
|
941
|
+
* written. Used by the session summarizer to recognise work it has
|
|
942
|
+
* already done without re-reading the node's whole body.
|
|
943
|
+
*/
|
|
944
|
+
getNodeMeta(id) {
|
|
945
|
+
const row = this.db.prepare("SELECT meta FROM nodes WHERE id = ?").get(id);
|
|
946
|
+
if (!row) return null;
|
|
947
|
+
try {
|
|
948
|
+
return JSON.parse(row.meta);
|
|
949
|
+
} catch {
|
|
950
|
+
return null;
|
|
951
|
+
}
|
|
952
|
+
}
|
|
834
953
|
getSyncCursor(projectId, source) {
|
|
835
954
|
const row = this.db.prepare("SELECT cursor FROM sync_state WHERE project_id = ? AND source = ?").get(projectId, source);
|
|
836
955
|
return row?.cursor ?? null;
|
|
@@ -893,19 +1012,57 @@ var MemoryStore = class _MemoryStore {
|
|
|
893
1012
|
return this.db.prepare(`DELETE FROM nodes WHERE ${scope}`).run(params).changes;
|
|
894
1013
|
})();
|
|
895
1014
|
}
|
|
896
|
-
/**
|
|
897
|
-
|
|
1015
|
+
/**
|
|
1016
|
+
* Nodes for this project that have no embedding yet (new, or invalidated
|
|
1017
|
+
* by a content change).
|
|
1018
|
+
*
|
|
1019
|
+
* `afterRowid` makes paging monotonic: the pass walks rowids strictly
|
|
1020
|
+
* upward instead of re-reading "the first N still pending". That matters
|
|
1021
|
+
* because a node the provider *failed* on stays pending -- an offset-free
|
|
1022
|
+
* loop would fetch the same failures forever, which is exactly the shape
|
|
1023
|
+
* of an infinite sync.
|
|
1024
|
+
*/
|
|
1025
|
+
findNodesNeedingEmbedding(projectId, limit = 200, afterRowid = 0) {
|
|
898
1026
|
return this.db.prepare(
|
|
899
1027
|
`SELECT n.rowid AS rowid, n.id AS id, n.title AS title, n.body AS body
|
|
900
1028
|
FROM nodes n
|
|
901
1029
|
LEFT JOIN nodes_vec v ON v.rowid = n.rowid
|
|
902
|
-
WHERE n.project_id = ? AND v.rowid IS NULL
|
|
1030
|
+
WHERE n.project_id = ? AND v.rowid IS NULL AND n.rowid > ?
|
|
1031
|
+
ORDER BY n.rowid
|
|
903
1032
|
LIMIT ?`
|
|
904
|
-
).all(projectId, limit);
|
|
1033
|
+
).all(projectId, afterRowid, limit);
|
|
1034
|
+
}
|
|
1035
|
+
/** How many of this project's nodes still need a vector. For progress reporting. */
|
|
1036
|
+
countNodesNeedingEmbedding(projectId) {
|
|
1037
|
+
const row = this.db.prepare(
|
|
1038
|
+
`SELECT COUNT(*) AS n
|
|
1039
|
+
FROM nodes n
|
|
1040
|
+
LEFT JOIN nodes_vec v ON v.rowid = n.rowid
|
|
1041
|
+
WHERE n.project_id = ? AND v.rowid IS NULL`
|
|
1042
|
+
).get(projectId);
|
|
1043
|
+
return row.n;
|
|
905
1044
|
}
|
|
906
1045
|
upsertEmbedding(rowid, embedding) {
|
|
907
1046
|
this.db.prepare("INSERT OR REPLACE INTO nodes_vec (rowid, embedding) VALUES (?, ?)").run(BigInt(rowid), embedding);
|
|
908
1047
|
}
|
|
1048
|
+
/**
|
|
1049
|
+
* Drop every vector in this database, across all projects.
|
|
1050
|
+
*
|
|
1051
|
+
* Whole-database on purpose: `nodes_vec` is shared and holds no
|
|
1052
|
+
* provenance, so once the vectors in it stopped being comparable there is
|
|
1053
|
+
* no subset that is still trustworthy. Nodes are untouched, so the next
|
|
1054
|
+
* embedding pass simply rebuilds them.
|
|
1055
|
+
*/
|
|
1056
|
+
dropAllEmbeddings() {
|
|
1057
|
+
return this.db.prepare("DELETE FROM nodes_vec").run().changes;
|
|
1058
|
+
}
|
|
1059
|
+
getMeta(key) {
|
|
1060
|
+
const row = this.db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
|
|
1061
|
+
return row?.value ?? null;
|
|
1062
|
+
}
|
|
1063
|
+
setMeta(key, value) {
|
|
1064
|
+
this.db.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
|
|
1065
|
+
}
|
|
909
1066
|
/**
|
|
910
1067
|
* Nearest-neighbour search over the corpus.
|
|
911
1068
|
*
|
|
@@ -970,7 +1127,7 @@ var MemoryStore = class _MemoryStore {
|
|
|
970
1127
|
|
|
971
1128
|
// src/cli/commands/init.ts
|
|
972
1129
|
async function runInit(opts) {
|
|
973
|
-
const out = opts.out ?? ((
|
|
1130
|
+
const out = opts.out ?? ((chunk2) => void process.stdout.write(chunk2));
|
|
974
1131
|
const repo = await readRepoInfo(opts.cwd);
|
|
975
1132
|
const ws = resolveWorkspace(repo.root);
|
|
976
1133
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
@@ -1091,7 +1248,7 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
|
|
|
1091
1248
|
import { z as z3 } from "zod";
|
|
1092
1249
|
|
|
1093
1250
|
// src/mcp/tools.ts
|
|
1094
|
-
import { basename as
|
|
1251
|
+
import { basename as basename3 } from "path";
|
|
1095
1252
|
|
|
1096
1253
|
// src/core/text.ts
|
|
1097
1254
|
function truncate(s, max) {
|
|
@@ -1105,6 +1262,8 @@ function approxTokens(text) {
|
|
|
1105
1262
|
var DEFAULT_SUMMARY_CHARS = 320;
|
|
1106
1263
|
var NODE_OVERHEAD_TOKENS = 8;
|
|
1107
1264
|
var CONVERSATION_ANSWER_MARKER = "\n\nA: ";
|
|
1265
|
+
var MAX_PER_FAMILY = 2;
|
|
1266
|
+
var CHUNKED_KINDS = /* @__PURE__ */ new Set(["conversation_turn", "doc_section"]);
|
|
1108
1267
|
var HUNK_BOUNDARY = "\n@@ ";
|
|
1109
1268
|
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
1110
1269
|
"the",
|
|
@@ -1223,7 +1382,14 @@ function packContext(ranked, tokensBudget, opts = {}) {
|
|
|
1223
1382
|
const nodes = [];
|
|
1224
1383
|
let tokensUsed = 0;
|
|
1225
1384
|
let droppedForBudget = 0;
|
|
1385
|
+
let droppedForDiversity = 0;
|
|
1386
|
+
const familyCounts = /* @__PURE__ */ new Map();
|
|
1226
1387
|
for (const hit of ranked) {
|
|
1388
|
+
const familyKey = CHUNKED_KINDS.has(hit.kind) ? `${hit.kind}:${hit.ts}` : null;
|
|
1389
|
+
if (familyKey && (familyCounts.get(familyKey) ?? 0) >= MAX_PER_FAMILY) {
|
|
1390
|
+
droppedForDiversity += 1;
|
|
1391
|
+
continue;
|
|
1392
|
+
}
|
|
1227
1393
|
const summary = summarize(hit, summaryChars, query);
|
|
1228
1394
|
const tokens = approxTokens(hit.title) + approxTokens(summary) + NODE_OVERHEAD_TOKENS;
|
|
1229
1395
|
if (tokensUsed + tokens > tokensBudget) {
|
|
@@ -1242,8 +1408,9 @@ function packContext(ranked, tokensBudget, opts = {}) {
|
|
|
1242
1408
|
...hit.project ? { project: hit.project } : {}
|
|
1243
1409
|
});
|
|
1244
1410
|
tokensUsed += tokens;
|
|
1411
|
+
if (familyKey) familyCounts.set(familyKey, (familyCounts.get(familyKey) ?? 0) + 1);
|
|
1245
1412
|
}
|
|
1246
|
-
return { nodes, tokensUsed, tokensBudget, consideredNodes: ranked.length, droppedForBudget };
|
|
1413
|
+
return { nodes, tokensUsed, tokensBudget, consideredNodes: ranked.length, droppedForBudget, droppedForDiversity };
|
|
1247
1414
|
}
|
|
1248
1415
|
function renderContextBlock(query, result) {
|
|
1249
1416
|
if (result.nodes.length === 0) return `No remembered context matched "${query}".`;
|
|
@@ -1291,8 +1458,10 @@ var RECENCY_FLOOR = 0.3;
|
|
|
1291
1458
|
var DEFAULT_HALF_LIFE_DAYS = 30;
|
|
1292
1459
|
var MS_PER_DAY = 864e5;
|
|
1293
1460
|
var MAX_PRIOR_OVERTURN = 2;
|
|
1294
|
-
var
|
|
1295
|
-
var
|
|
1461
|
+
var PRIOR_COUNT = 2;
|
|
1462
|
+
var PER_PRIOR_OVERTURN = MAX_PRIOR_OVERTURN ** (1 / PRIOR_COUNT);
|
|
1463
|
+
var SIGNAL_EXPONENT = Math.log(PER_PRIOR_OVERTURN) / Math.log(1 / SIGNAL_FLOOR);
|
|
1464
|
+
var RECENCY_EXPONENT = Math.log(PER_PRIOR_OVERTURN) / Math.log(1 / RECENCY_FLOOR);
|
|
1296
1465
|
function normalizeRelevance(hits) {
|
|
1297
1466
|
const costs = hits.map((h) => h.rank);
|
|
1298
1467
|
const min = Math.min(...costs);
|
|
@@ -1420,37 +1589,52 @@ function labelProjects(projects) {
|
|
|
1420
1589
|
}
|
|
1421
1590
|
|
|
1422
1591
|
// src/vector/embed.ts
|
|
1423
|
-
var
|
|
1592
|
+
var DEFAULT_BASE_URL2 = "http://127.0.0.1:11434";
|
|
1424
1593
|
var DEFAULT_MODEL = "nomic-embed-text";
|
|
1425
1594
|
var DEFAULT_DIMENSION = 768;
|
|
1426
|
-
var
|
|
1595
|
+
var DEFAULT_TIMEOUT_MS2 = 1e4;
|
|
1596
|
+
var DEFAULT_MAX_TIMEOUT_MS = 12e4;
|
|
1597
|
+
var EMBED_PATH = "/api/embed";
|
|
1427
1598
|
var OllamaEmbeddingProvider = class {
|
|
1428
1599
|
dimension;
|
|
1600
|
+
identity;
|
|
1429
1601
|
baseUrl;
|
|
1430
1602
|
model;
|
|
1431
1603
|
timeoutMs;
|
|
1604
|
+
maxTimeoutMs;
|
|
1432
1605
|
constructor(opts = {}) {
|
|
1433
|
-
this.baseUrl = opts.baseUrl ??
|
|
1606
|
+
this.baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL2;
|
|
1434
1607
|
this.model = opts.model ?? DEFAULT_MODEL;
|
|
1435
1608
|
this.dimension = opts.dimension ?? DEFAULT_DIMENSION;
|
|
1436
|
-
this.timeoutMs = opts.timeoutMs ??
|
|
1609
|
+
this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS2;
|
|
1610
|
+
this.maxTimeoutMs = opts.maxTimeoutMs ?? DEFAULT_MAX_TIMEOUT_MS;
|
|
1611
|
+
this.identity = `ollama${EMBED_PATH}:${this.model}:${this.dimension}`;
|
|
1437
1612
|
}
|
|
1438
1613
|
async embed(text) {
|
|
1614
|
+
const [only] = await this.embedBatch([text]);
|
|
1615
|
+
return only ?? null;
|
|
1616
|
+
}
|
|
1617
|
+
async embedBatch(texts) {
|
|
1618
|
+
if (texts.length === 0) return [];
|
|
1439
1619
|
const controller = new AbortController();
|
|
1440
|
-
const
|
|
1620
|
+
const budget = Math.min(this.timeoutMs * texts.length, this.maxTimeoutMs);
|
|
1621
|
+
const timeout = setTimeout(() => controller.abort(), budget);
|
|
1441
1622
|
try {
|
|
1442
|
-
const res = await fetch(`${this.baseUrl}
|
|
1623
|
+
const res = await fetch(`${this.baseUrl}${EMBED_PATH}`, {
|
|
1443
1624
|
method: "POST",
|
|
1444
1625
|
headers: { "content-type": "application/json" },
|
|
1445
|
-
body: JSON.stringify({ model: this.model,
|
|
1626
|
+
body: JSON.stringify({ model: this.model, input: texts }),
|
|
1446
1627
|
signal: controller.signal
|
|
1447
1628
|
});
|
|
1448
|
-
if (!res.ok) return null;
|
|
1629
|
+
if (!res.ok) return texts.map(() => null);
|
|
1449
1630
|
const data = await res.json();
|
|
1450
|
-
if (!Array.isArray(data.
|
|
1451
|
-
|
|
1631
|
+
if (!Array.isArray(data.embeddings)) return texts.map(() => null);
|
|
1632
|
+
if (data.embeddings.length !== texts.length) return texts.map(() => null);
|
|
1633
|
+
return data.embeddings.map(
|
|
1634
|
+
(row) => Array.isArray(row) && row.length === this.dimension ? new Float32Array(row) : null
|
|
1635
|
+
);
|
|
1452
1636
|
} catch {
|
|
1453
|
-
return null;
|
|
1637
|
+
return texts.map(() => null);
|
|
1454
1638
|
} finally {
|
|
1455
1639
|
clearTimeout(timeout);
|
|
1456
1640
|
}
|
|
@@ -1580,25 +1764,25 @@ function toMemoryNodes(turn, projectId, opts = {}) {
|
|
|
1580
1764
|
const assistantRedacted = redact(turn.assistantText);
|
|
1581
1765
|
const chunks = chunkAssistantText(assistantRedacted.text, maxChunk);
|
|
1582
1766
|
if (chunks.length === 0) return [];
|
|
1583
|
-
return chunks.map((
|
|
1584
|
-
const body = [`Q: ${userRedacted.text}`, "", `A: ${
|
|
1767
|
+
return chunks.map((chunk2, index) => {
|
|
1768
|
+
const body = [`Q: ${userRedacted.text}`, "", `A: ${chunk2.text}`].join("\n");
|
|
1585
1769
|
return {
|
|
1586
1770
|
id: makeNodeId(projectId, "conversation_turn", `${turn.naturalKey}:${index}`),
|
|
1587
1771
|
kind: "conversation_turn",
|
|
1588
1772
|
projectId,
|
|
1589
1773
|
ts: turn.ts,
|
|
1590
1774
|
source: `conversation:${turn.source}`,
|
|
1591
|
-
title: chunkTitle(userFirstLine,
|
|
1775
|
+
title: chunkTitle(userFirstLine, chunk2.heading, index, chunks.length),
|
|
1592
1776
|
body: truncate(body, maxBody),
|
|
1593
1777
|
files: extractMentionedFiles(`${userRedacted.text}
|
|
1594
|
-
${
|
|
1595
|
-
signal: scoreConversationTurn(userRedacted.text,
|
|
1778
|
+
${chunk2.text}`),
|
|
1779
|
+
signal: scoreConversationTurn(userRedacted.text, chunk2.text),
|
|
1596
1780
|
meta: {
|
|
1597
1781
|
cwd: turn.cwd,
|
|
1598
1782
|
source: turn.source,
|
|
1599
1783
|
chunkIndex: index,
|
|
1600
1784
|
chunkCount: chunks.length,
|
|
1601
|
-
heading:
|
|
1785
|
+
heading: chunk2.heading,
|
|
1602
1786
|
// Redaction runs once over the whole reply before chunking (so a
|
|
1603
1787
|
// secret can never straddle a chunk boundary and slip through) --
|
|
1604
1788
|
// this is the turn's total, repeated on every chunk it produced,
|
|
@@ -1754,8 +1938,8 @@ async function* readCommitDiffs(cwd, opts = {}) {
|
|
|
1754
1938
|
const args = buildDiffLogArgs(opts);
|
|
1755
1939
|
let buffer = "";
|
|
1756
1940
|
try {
|
|
1757
|
-
for await (const
|
|
1758
|
-
buffer +=
|
|
1941
|
+
for await (const chunk2 of gitStream(cwd, args)) {
|
|
1942
|
+
buffer += chunk2;
|
|
1759
1943
|
const { records, rest } = splitRecords(buffer);
|
|
1760
1944
|
buffer = rest;
|
|
1761
1945
|
for (const record of records) {
|
|
@@ -1911,8 +2095,8 @@ async function* readCommits(cwd, opts = {}) {
|
|
|
1911
2095
|
const args = buildLogArgs(opts);
|
|
1912
2096
|
let buffer = "";
|
|
1913
2097
|
try {
|
|
1914
|
-
for await (const
|
|
1915
|
-
buffer +=
|
|
2098
|
+
for await (const chunk2 of gitStream(cwd, args)) {
|
|
2099
|
+
buffer += chunk2;
|
|
1916
2100
|
const { records, rest } = splitRecords(buffer);
|
|
1917
2101
|
buffer = rest;
|
|
1918
2102
|
for (const record of records) {
|
|
@@ -2147,8 +2331,8 @@ function toMemoryNodes3(file, projectId, opts = {}) {
|
|
|
2147
2331
|
const chunks = chunkAssistantText(file.content, maxChunk);
|
|
2148
2332
|
if (chunks.length === 0) return [];
|
|
2149
2333
|
const seenSlugs = /* @__PURE__ */ new Map();
|
|
2150
|
-
return chunks.map((
|
|
2151
|
-
const baseSlug = slugify(
|
|
2334
|
+
return chunks.map((chunk2, index) => {
|
|
2335
|
+
const baseSlug = slugify(chunk2.heading, index);
|
|
2152
2336
|
const occurrence = seenSlugs.get(baseSlug) ?? 0;
|
|
2153
2337
|
seenSlugs.set(baseSlug, occurrence + 1);
|
|
2154
2338
|
const naturalKey = occurrence === 0 ? `${file.path}#${baseSlug}` : `${file.path}#${baseSlug}:${occurrence}`;
|
|
@@ -2158,13 +2342,13 @@ function toMemoryNodes3(file, projectId, opts = {}) {
|
|
|
2158
2342
|
projectId,
|
|
2159
2343
|
ts: file.ts,
|
|
2160
2344
|
source: "docs",
|
|
2161
|
-
title: sectionTitle(file.path,
|
|
2162
|
-
body: truncate(
|
|
2345
|
+
title: sectionTitle(file.path, chunk2.heading, index, chunks.length),
|
|
2346
|
+
body: truncate(chunk2.text, maxBody),
|
|
2163
2347
|
files: [{ path: file.path, insertions: null, deletions: null, binary: false }],
|
|
2164
|
-
signal: scoreDocSection(file.path,
|
|
2348
|
+
signal: scoreDocSection(file.path, chunk2.heading, chunk2.text),
|
|
2165
2349
|
meta: {
|
|
2166
2350
|
path: file.path,
|
|
2167
|
-
heading:
|
|
2351
|
+
heading: chunk2.heading,
|
|
2168
2352
|
chunkIndex: index,
|
|
2169
2353
|
chunkCount: chunks.length
|
|
2170
2354
|
}
|
|
@@ -2175,9 +2359,218 @@ function collectDocFiles(files, projectId, opts = {}) {
|
|
|
2175
2359
|
return files.flatMap((file) => toMemoryNodes3(file, projectId, opts));
|
|
2176
2360
|
}
|
|
2177
2361
|
|
|
2178
|
-
// src/
|
|
2179
|
-
var
|
|
2362
|
+
// src/slm/summarize.ts
|
|
2363
|
+
var MAX_USER_CHARS = 400;
|
|
2364
|
+
var MAX_REPLY_CHARS = 700;
|
|
2365
|
+
var DEFAULT_MAX_PROMPT_CHARS = 12e3;
|
|
2366
|
+
function groupTurnsIntoSessions(turns) {
|
|
2367
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2368
|
+
for (const turn of turns) {
|
|
2369
|
+
const existing = groups.get(turn.sessionKey);
|
|
2370
|
+
if (existing) {
|
|
2371
|
+
existing.turns.push(turn);
|
|
2372
|
+
if (turn.ts < existing.startedAt) existing.startedAt = turn.ts;
|
|
2373
|
+
if (turn.ts > existing.endedAt) existing.endedAt = turn.ts;
|
|
2374
|
+
existing.cwd ??= turn.cwd;
|
|
2375
|
+
continue;
|
|
2376
|
+
}
|
|
2377
|
+
groups.set(turn.sessionKey, {
|
|
2378
|
+
sessionKey: turn.sessionKey,
|
|
2379
|
+
source: turn.source,
|
|
2380
|
+
cwd: turn.cwd,
|
|
2381
|
+
turns: [turn],
|
|
2382
|
+
startedAt: turn.ts,
|
|
2383
|
+
endedAt: turn.ts
|
|
2384
|
+
});
|
|
2385
|
+
}
|
|
2386
|
+
for (const group of groups.values()) {
|
|
2387
|
+
group.turns.sort((a, b) => a.ts.localeCompare(b.ts));
|
|
2388
|
+
}
|
|
2389
|
+
return [...groups.values()].sort((a, b) => a.startedAt.localeCompare(b.startedAt));
|
|
2390
|
+
}
|
|
2391
|
+
function selectSettledSessions(sessions, settleMinutes, now = /* @__PURE__ */ new Date()) {
|
|
2392
|
+
const cutoff = now.getTime() - settleMinutes * 6e4;
|
|
2393
|
+
return sessions.filter((s) => {
|
|
2394
|
+
const ended = Date.parse(s.endedAt);
|
|
2395
|
+
return Number.isNaN(ended) || ended <= cutoff;
|
|
2396
|
+
});
|
|
2397
|
+
}
|
|
2398
|
+
var SESSION_INSTRUCTIONS = `You are summarizing one working session between a developer and an AI coding assistant, so that a future assistant can recall what happened without re-reading the transcript.
|
|
2399
|
+
|
|
2400
|
+
Your first line MUST begin with "TITLE: " and nothing else. Start your reply with those six characters.
|
|
2401
|
+
|
|
2402
|
+
Write your answer in exactly this shape:
|
|
2403
|
+
TITLE: <under 15 words, naming the specific work, e.g. "Raised the Node floor to 22 after a CI failure">
|
|
2404
|
+
- <a decision that was made, and why>
|
|
2405
|
+
- <a problem that was diagnosed, and its cause>
|
|
2406
|
+
- <what was left unfinished or explicitly deferred>
|
|
2407
|
+
|
|
2408
|
+
Rules:
|
|
2409
|
+
- Answer in English, whatever language the transcript is in. This summary is stored in a keyword index that cannot segment languages without spaces between words.
|
|
2410
|
+
- The title must name this session specifically. "Summary of the session" or "Project update" are wrong.
|
|
2411
|
+
- Prefer reasons over narration. "Chose X over Y because Z" is worth more than "worked on X".
|
|
2412
|
+
- Only state what the transcript supports. Do not guess or invent.
|
|
2413
|
+
- Between 3 and 6 bullets. No preamble, no closing remarks, no markdown bold.`;
|
|
2414
|
+
function buildSessionPrompt(session, maxPromptChars = DEFAULT_MAX_PROMPT_CHARS) {
|
|
2415
|
+
const rendered = session.turns.map((turn) => {
|
|
2416
|
+
const user = redact(turn.userText).text;
|
|
2417
|
+
const reply = redact(turn.assistantText).text;
|
|
2418
|
+
return {
|
|
2419
|
+
ts: turn.ts,
|
|
2420
|
+
text: [`[${turn.ts}] developer: ${truncate(user, MAX_USER_CHARS)}`, `assistant: ${truncate(reply, MAX_REPLY_CHARS)}`].join(
|
|
2421
|
+
"\n"
|
|
2422
|
+
),
|
|
2423
|
+
signal: scoreConversationTurn(user, reply)
|
|
2424
|
+
};
|
|
2425
|
+
});
|
|
2426
|
+
const budget = maxPromptChars - SESSION_INSTRUCTIONS.length;
|
|
2427
|
+
const kept = [];
|
|
2428
|
+
let used = 0;
|
|
2429
|
+
for (const turn of [...rendered].sort((a, b) => b.signal - a.signal)) {
|
|
2430
|
+
if (used + turn.text.length > budget) continue;
|
|
2431
|
+
kept.push(turn);
|
|
2432
|
+
used += turn.text.length;
|
|
2433
|
+
}
|
|
2434
|
+
kept.sort((a, b) => a.ts.localeCompare(b.ts));
|
|
2435
|
+
const body = kept.map((t) => t.text).join("\n\n");
|
|
2436
|
+
const prompt = `${SESSION_INSTRUCTIONS}
|
|
2437
|
+
|
|
2438
|
+
---
|
|
2439
|
+
|
|
2440
|
+
${body}
|
|
2441
|
+
|
|
2442
|
+
---
|
|
2443
|
+
|
|
2444
|
+
Summary:`;
|
|
2445
|
+
return { prompt, hash: sha256Hex(prompt), includedTurns: kept.length };
|
|
2446
|
+
}
|
|
2447
|
+
function sessionFallbackTitle(session) {
|
|
2448
|
+
const opening = session.turns[0];
|
|
2449
|
+
if (!opening) return "Working session";
|
|
2450
|
+
const line = redact(opening.userText).text.split(/\r?\n/)[0]?.trim();
|
|
2451
|
+
return line && line.length > 0 ? line : "Working session";
|
|
2452
|
+
}
|
|
2180
2453
|
var MAX_TITLE_CHARS5 = 200;
|
|
2454
|
+
var GENERIC_TITLE = /^(a |the )?(session |conversation |project |work )?(summary|update|overview|recap|status)\b/i;
|
|
2455
|
+
var ROLE_PREAMBLE_TITLE = /^(role\s*[::]|you\s*(?:'re|are)\s+(acting as|serving as|playing the role of|a\b)|i\s*(?:'m|am)\s+(acting as|a\b)|acting as\b|บทบาท\s*[::]|ในฐานะ|คุณ(กำลัง)?(ทำหน้าที่เป็น|เป็น|รับบทบาทเป็น)|(ผม|ฉัน)(กำลัง)?(ทำหน้าที่เป็น|เป็น))/i;
|
|
2456
|
+
function cleanTitle(line) {
|
|
2457
|
+
return line.replace(/^[-*#>\s]+/, "").replace(/\*+/g, "").replace(/\s*:\s*$/, "").trim();
|
|
2458
|
+
}
|
|
2459
|
+
function parseSummary(raw, fallbackTitle) {
|
|
2460
|
+
const text = redact(raw).text.trim();
|
|
2461
|
+
if (text.length === 0) return null;
|
|
2462
|
+
const lines = text.split(/\r?\n/);
|
|
2463
|
+
const firstIndex = lines.findIndex((line) => line.trim().length > 0);
|
|
2464
|
+
if (firstIndex === -1) return null;
|
|
2465
|
+
const first = lines[firstIndex].trim();
|
|
2466
|
+
const labelled = /^TITLE:\s*(.+)$/i.exec(first);
|
|
2467
|
+
const fallback = truncate(cleanTitle(fallbackTitle) || "Working session", MAX_TITLE_CHARS5);
|
|
2468
|
+
if (!labelled) return { title: fallback, body: text };
|
|
2469
|
+
const candidate = cleanTitle(labelled[1]);
|
|
2470
|
+
const rest = lines.slice(firstIndex + 1).join("\n").trim();
|
|
2471
|
+
return {
|
|
2472
|
+
title: candidate.length > 0 && !GENERIC_TITLE.test(candidate) && !ROLE_PREAMBLE_TITLE.test(candidate) ? truncate(candidate, MAX_TITLE_CHARS5) : fallback,
|
|
2473
|
+
// A model that emitted only a title still gets a usable node: the title
|
|
2474
|
+
// doubles as the body rather than storing an empty one.
|
|
2475
|
+
body: rest.length > 0 ? rest : candidate
|
|
2476
|
+
};
|
|
2477
|
+
}
|
|
2478
|
+
|
|
2479
|
+
// src/collectors/sessions.ts
|
|
2480
|
+
var SESSION_SOURCE_PREFIX = "session";
|
|
2481
|
+
var DEFAULT_SETTLE_MINUTES = 30;
|
|
2482
|
+
var DEFAULT_MAX_BODY_CHARS3 = 2500;
|
|
2483
|
+
var DEFAULT_MAX_SESSIONS = 10;
|
|
2484
|
+
function scoreSession(turnCount) {
|
|
2485
|
+
const score = 0.6 + Math.min(0.25, turnCount / 100);
|
|
2486
|
+
return Number(score.toFixed(3));
|
|
2487
|
+
}
|
|
2488
|
+
function toNode(session, projectId, summary, meta, maxBodyChars) {
|
|
2489
|
+
const header = `Session of ${session.startedAt.slice(0, 10)} \u2014 ${session.turns.length} exchange(s)`;
|
|
2490
|
+
const body = `${header}
|
|
2491
|
+
|
|
2492
|
+
${summary.body}`;
|
|
2493
|
+
return {
|
|
2494
|
+
id: makeNodeId(projectId, "session_summary", session.sessionKey),
|
|
2495
|
+
kind: "session_summary",
|
|
2496
|
+
projectId,
|
|
2497
|
+
// The session's end, not its start: a summary describes a finished piece
|
|
2498
|
+
// of work, and recency ranking should treat it as being as fresh as the
|
|
2499
|
+
// last thing that happened in it.
|
|
2500
|
+
ts: session.endedAt,
|
|
2501
|
+
source: `${SESSION_SOURCE_PREFIX}:${session.source}`,
|
|
2502
|
+
title: truncate(summary.title, 200),
|
|
2503
|
+
body: truncate(body, maxBodyChars),
|
|
2504
|
+
// Drawn from the whole session rather than only the summary: the model
|
|
2505
|
+
// mentions few paths, but `node_files` is what lets "why is this file
|
|
2506
|
+
// like this" reach the session that explains it.
|
|
2507
|
+
files: extractMentionedFiles(session.turns.map((t) => `${t.userText}
|
|
2508
|
+
${t.assistantText}`).join("\n")),
|
|
2509
|
+
signal: scoreSession(session.turns.length),
|
|
2510
|
+
meta: {
|
|
2511
|
+
sessionKey: session.sessionKey,
|
|
2512
|
+
source: session.source,
|
|
2513
|
+
turnCount: session.turns.length,
|
|
2514
|
+
summarizedTurns: meta.includedTurns,
|
|
2515
|
+
startedAt: session.startedAt,
|
|
2516
|
+
endedAt: session.endedAt,
|
|
2517
|
+
cwd: session.cwd,
|
|
2518
|
+
model: meta.model,
|
|
2519
|
+
contentHash: meta.hash
|
|
2520
|
+
}
|
|
2521
|
+
};
|
|
2522
|
+
}
|
|
2523
|
+
async function collectSessionSummaries(turns, projectId, provider, opts = {}) {
|
|
2524
|
+
const maxBodyChars = opts.maxBodyChars ?? DEFAULT_MAX_BODY_CHARS3;
|
|
2525
|
+
const maxSessions = opts.maxSessions ?? DEFAULT_MAX_SESSIONS;
|
|
2526
|
+
const all = groupTurnsIntoSessions(turns);
|
|
2527
|
+
const settled = selectSettledSessions(all, opts.settleMinutes ?? DEFAULT_SETTLE_MINUTES, opts.now);
|
|
2528
|
+
const nodes = [];
|
|
2529
|
+
let cached = 0;
|
|
2530
|
+
let failed = 0;
|
|
2531
|
+
let attempted = 0;
|
|
2532
|
+
const candidates = [...settled].sort((a, b) => b.endedAt.localeCompare(a.endedAt));
|
|
2533
|
+
const pending = [];
|
|
2534
|
+
for (const session of candidates) {
|
|
2535
|
+
const prompt = buildSessionPrompt(session, opts.maxPromptChars);
|
|
2536
|
+
if (opts.knownHash?.(session.sessionKey) === prompt.hash) {
|
|
2537
|
+
cached += 1;
|
|
2538
|
+
continue;
|
|
2539
|
+
}
|
|
2540
|
+
pending.push({ session, prompt });
|
|
2541
|
+
}
|
|
2542
|
+
for (const { session, prompt } of pending.slice(0, maxSessions)) {
|
|
2543
|
+
const raw = await provider.complete(prompt.prompt);
|
|
2544
|
+
attempted += 1;
|
|
2545
|
+
const summary = raw === null ? null : parseSummary(raw, sessionFallbackTitle(session));
|
|
2546
|
+
if (!summary) {
|
|
2547
|
+
failed += 1;
|
|
2548
|
+
continue;
|
|
2549
|
+
}
|
|
2550
|
+
nodes.push(
|
|
2551
|
+
toNode(
|
|
2552
|
+
session,
|
|
2553
|
+
projectId,
|
|
2554
|
+
summary,
|
|
2555
|
+
{ hash: prompt.hash, model: provider.identity, includedTurns: prompt.includedTurns },
|
|
2556
|
+
maxBodyChars
|
|
2557
|
+
)
|
|
2558
|
+
);
|
|
2559
|
+
opts.onProgress?.(nodes.length, Math.min(pending.length, maxSessions));
|
|
2560
|
+
}
|
|
2561
|
+
return {
|
|
2562
|
+
nodes,
|
|
2563
|
+
cached,
|
|
2564
|
+
unsettled: all.length - settled.length,
|
|
2565
|
+
deferred: Math.max(0, pending.length - maxSessions),
|
|
2566
|
+
failed,
|
|
2567
|
+
providerUnavailable: attempted > 0 && nodes.length === 0
|
|
2568
|
+
};
|
|
2569
|
+
}
|
|
2570
|
+
|
|
2571
|
+
// src/collectors/shell-history.ts
|
|
2572
|
+
var DEFAULT_MAX_BODY_CHARS4 = 1e3;
|
|
2573
|
+
var MAX_TITLE_CHARS6 = 200;
|
|
2181
2574
|
var NOISE = /^(cd|ls|dir|pwd|clear|cls|exit|history|whoami|date|type|cat|more|less|ll|la)\b/i;
|
|
2182
2575
|
var BUILD_TEST = /^(npm|pnpm|yarn)\s+(run\s+)?(test|build|lint|typecheck|tsc)\b|^(pytest|go\s+test|cargo\s+(test|build)|mvn\s+test|gradle\s+test|dotnet\s+(test|build))\b/i;
|
|
2183
2576
|
var INSTALL = /^(npm|pnpm|yarn)\s+(install|add|remove|uninstall|ci)\b|^pip\s+install\b|^(cargo\s+add|go\s+get|composer\s+require)\b/i;
|
|
@@ -2210,7 +2603,7 @@ function renderBody(entry, maxChars) {
|
|
|
2210
2603
|
return truncate(parts.join("\n"), maxChars);
|
|
2211
2604
|
}
|
|
2212
2605
|
function toMemoryNode2(entry, projectId, opts = {}) {
|
|
2213
|
-
const maxBody = opts.maxBodyChars ??
|
|
2606
|
+
const maxBody = opts.maxBodyChars ?? DEFAULT_MAX_BODY_CHARS4;
|
|
2214
2607
|
const titleLine = entry.command.split(/\r?\n/)[0] ?? entry.command;
|
|
2215
2608
|
return {
|
|
2216
2609
|
id: makeNodeId(projectId, "shell_command", entry.naturalKey),
|
|
@@ -2218,7 +2611,7 @@ function toMemoryNode2(entry, projectId, opts = {}) {
|
|
|
2218
2611
|
projectId,
|
|
2219
2612
|
ts: entry.ts,
|
|
2220
2613
|
source: `shell:${entry.shell}`,
|
|
2221
|
-
title: truncate(titleLine,
|
|
2614
|
+
title: truncate(titleLine, MAX_TITLE_CHARS6),
|
|
2222
2615
|
body: renderBody(entry, maxBody),
|
|
2223
2616
|
files: [],
|
|
2224
2617
|
signal: scoreShellCommand(entry),
|
|
@@ -2238,6 +2631,7 @@ function collectShellHistory(entries, projectId, opts = {}) {
|
|
|
2238
2631
|
|
|
2239
2632
|
// src/conversation/claude-code-reader.ts
|
|
2240
2633
|
import { readFile as readFile4 } from "fs/promises";
|
|
2634
|
+
import { basename as basename2 } from "path";
|
|
2241
2635
|
|
|
2242
2636
|
// src/conversation/paths.ts
|
|
2243
2637
|
import { existsSync as existsSync3 } from "fs";
|
|
@@ -2284,6 +2678,7 @@ function extractAssistantText(line) {
|
|
|
2284
2678
|
}
|
|
2285
2679
|
function parseClaudeCodeTranscript(raw, opts = {}) {
|
|
2286
2680
|
const source = opts.source ?? "claude-code";
|
|
2681
|
+
const sessionKey = `${source}:${opts.sessionId ?? "unknown"}`;
|
|
2287
2682
|
const turns = [];
|
|
2288
2683
|
let current = null;
|
|
2289
2684
|
const flush = () => {
|
|
@@ -2295,7 +2690,8 @@ function parseClaudeCodeTranscript(raw, opts = {}) {
|
|
|
2295
2690
|
assistantText,
|
|
2296
2691
|
ts: current.ts,
|
|
2297
2692
|
cwd: current.cwd,
|
|
2298
|
-
source
|
|
2693
|
+
source,
|
|
2694
|
+
sessionKey
|
|
2299
2695
|
});
|
|
2300
2696
|
current = null;
|
|
2301
2697
|
};
|
|
@@ -2328,7 +2724,7 @@ async function collectClaudeCodeTranscripts(repoRoot) {
|
|
|
2328
2724
|
const turns = [];
|
|
2329
2725
|
for (const file of files) {
|
|
2330
2726
|
const raw = await readFile4(file, "utf8");
|
|
2331
|
-
turns.push(...parseClaudeCodeTranscript(raw));
|
|
2727
|
+
turns.push(...parseClaudeCodeTranscript(raw, { sessionId: basename2(file, ".jsonl") }));
|
|
2332
2728
|
}
|
|
2333
2729
|
return turns;
|
|
2334
2730
|
}
|
|
@@ -2553,22 +2949,171 @@ async function collectAvailableShellHistory(opts = {}) {
|
|
|
2553
2949
|
return results;
|
|
2554
2950
|
}
|
|
2555
2951
|
|
|
2952
|
+
// src/store/reconcile.ts
|
|
2953
|
+
function recomputeByNaturalKey(db, oldProjectId, newProjectId, kind, source, computeNaturalKey) {
|
|
2954
|
+
const rows = source ? db.prepare("SELECT * FROM nodes WHERE project_id = ? AND kind = ? AND source = ?").all(oldProjectId, kind, source) : db.prepare("SELECT * FROM nodes WHERE project_id = ? AND kind = ?").all(oldProjectId, kind);
|
|
2955
|
+
const nodeExists = db.prepare("SELECT 1 FROM nodes WHERE id = ?");
|
|
2956
|
+
const insertNode = db.prepare(
|
|
2957
|
+
`INSERT INTO nodes (id, kind, project_id, ts, ts_epoch, source, title, body, signal, meta, created_at)
|
|
2958
|
+
VALUES (@id, @kind, @projectId, @ts, @tsEpoch, @source, @title, @body, @signal, @meta, @createdAt)`
|
|
2959
|
+
);
|
|
2960
|
+
const readFiles = db.prepare("SELECT path, previous_path, insertions, deletions, is_binary FROM node_files WHERE node_id = ?");
|
|
2961
|
+
const insertFile = db.prepare(
|
|
2962
|
+
`INSERT INTO node_files (node_id, path, previous_path, insertions, deletions, is_binary)
|
|
2963
|
+
VALUES (@nodeId, @path, @previousPath, @insertions, @deletions, @isBinary)`
|
|
2964
|
+
);
|
|
2965
|
+
const dropEmbedding = db.prepare("DELETE FROM nodes_vec WHERE rowid = (SELECT rowid FROM nodes WHERE id = ?)");
|
|
2966
|
+
const deleteNode = db.prepare("DELETE FROM nodes WHERE id = ?");
|
|
2967
|
+
let migrated = 0;
|
|
2968
|
+
let deduped = 0;
|
|
2969
|
+
let skipped = 0;
|
|
2970
|
+
for (const row of rows) {
|
|
2971
|
+
let meta;
|
|
2972
|
+
try {
|
|
2973
|
+
meta = JSON.parse(row.meta);
|
|
2974
|
+
} catch {
|
|
2975
|
+
skipped += 1;
|
|
2976
|
+
continue;
|
|
2977
|
+
}
|
|
2978
|
+
const naturalKey = computeNaturalKey(row, meta);
|
|
2979
|
+
if (naturalKey === null) {
|
|
2980
|
+
skipped += 1;
|
|
2981
|
+
continue;
|
|
2982
|
+
}
|
|
2983
|
+
const newId = makeNodeId(newProjectId, kind, naturalKey);
|
|
2984
|
+
if (nodeExists.get(newId)) {
|
|
2985
|
+
deduped += 1;
|
|
2986
|
+
} else {
|
|
2987
|
+
insertNode.run({
|
|
2988
|
+
id: newId,
|
|
2989
|
+
kind: row.kind,
|
|
2990
|
+
projectId: newProjectId,
|
|
2991
|
+
ts: row.ts,
|
|
2992
|
+
tsEpoch: row.ts_epoch,
|
|
2993
|
+
source: row.source,
|
|
2994
|
+
title: row.title,
|
|
2995
|
+
body: row.body,
|
|
2996
|
+
signal: row.signal,
|
|
2997
|
+
meta: row.meta,
|
|
2998
|
+
createdAt: row.created_at
|
|
2999
|
+
});
|
|
3000
|
+
for (const file of readFiles.all(row.id)) {
|
|
3001
|
+
insertFile.run({
|
|
3002
|
+
nodeId: newId,
|
|
3003
|
+
path: file.path,
|
|
3004
|
+
previousPath: file.previous_path,
|
|
3005
|
+
insertions: file.insertions,
|
|
3006
|
+
deletions: file.deletions,
|
|
3007
|
+
isBinary: file.is_binary
|
|
3008
|
+
});
|
|
3009
|
+
}
|
|
3010
|
+
migrated += 1;
|
|
3011
|
+
}
|
|
3012
|
+
dropEmbedding.run(row.id);
|
|
3013
|
+
deleteNode.run(row.id);
|
|
3014
|
+
}
|
|
3015
|
+
return { migrated, deduped, skipped };
|
|
3016
|
+
}
|
|
3017
|
+
function reconcileProjectId(db, oldProjectId, newProjectId) {
|
|
3018
|
+
return db.transaction(() => {
|
|
3019
|
+
const sessions = recomputeByNaturalKey(
|
|
3020
|
+
db,
|
|
3021
|
+
oldProjectId,
|
|
3022
|
+
newProjectId,
|
|
3023
|
+
"session_summary",
|
|
3024
|
+
null,
|
|
3025
|
+
(_row, meta) => typeof meta.sessionKey === "string" ? meta.sessionKey : null
|
|
3026
|
+
);
|
|
3027
|
+
const hookShell = recomputeByNaturalKey(
|
|
3028
|
+
db,
|
|
3029
|
+
oldProjectId,
|
|
3030
|
+
newProjectId,
|
|
3031
|
+
"shell_command",
|
|
3032
|
+
"shell:pwsh-hook",
|
|
3033
|
+
(row, meta) => typeof meta.command === "string" ? `pwsh-hook:${row.ts}:${sha256Hex(meta.command).slice(0, 12)}` : null
|
|
3034
|
+
);
|
|
3035
|
+
const reassigned = db.prepare(`UPDATE nodes SET project_id = ? WHERE project_id = ? AND kind = 'conversation_turn'`).run(newProjectId, oldProjectId).changes;
|
|
3036
|
+
return {
|
|
3037
|
+
oldProjectId,
|
|
3038
|
+
migrated: sessions.migrated + hookShell.migrated,
|
|
3039
|
+
reassigned,
|
|
3040
|
+
deduped: sessions.deduped + hookShell.deduped,
|
|
3041
|
+
skipped: sessions.skipped + hookShell.skipped
|
|
3042
|
+
};
|
|
3043
|
+
})();
|
|
3044
|
+
}
|
|
3045
|
+
|
|
2556
3046
|
// src/vector/sync.ts
|
|
3047
|
+
var EMBEDDING_IDENTITY_KEY = "embedding.identity";
|
|
3048
|
+
var DEFAULT_BATCH_SIZE = 32;
|
|
3049
|
+
var DEFAULT_PAGE_SIZE = 500;
|
|
3050
|
+
var DEFAULT_FAILURE_TOLERANCE = 3;
|
|
3051
|
+
function reconcileProviderIdentity(store, provider) {
|
|
3052
|
+
if (store.getMeta(EMBEDDING_IDENTITY_KEY) === provider.identity) return 0;
|
|
3053
|
+
const invalidated = store.dropAllEmbeddings();
|
|
3054
|
+
store.setMeta(EMBEDDING_IDENTITY_KEY, provider.identity);
|
|
3055
|
+
return invalidated;
|
|
3056
|
+
}
|
|
3057
|
+
async function embedTexts(provider, texts) {
|
|
3058
|
+
if (provider.embedBatch) return provider.embedBatch(texts);
|
|
3059
|
+
const out = [];
|
|
3060
|
+
for (const text of texts) out.push(await provider.embed(text));
|
|
3061
|
+
return out;
|
|
3062
|
+
}
|
|
3063
|
+
function chunk(items, size) {
|
|
3064
|
+
const out = [];
|
|
3065
|
+
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
|
3066
|
+
return out;
|
|
3067
|
+
}
|
|
2557
3068
|
async function embedPendingNodes(store, provider, projectId, opts = {}) {
|
|
2558
|
-
const
|
|
3069
|
+
const batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
3070
|
+
const pageSize = opts.pageSize ?? DEFAULT_PAGE_SIZE;
|
|
3071
|
+
const failureTolerance = opts.failureTolerance ?? DEFAULT_FAILURE_TOLERANCE;
|
|
3072
|
+
const maxNodes = opts.maxNodes ?? Number.POSITIVE_INFINITY;
|
|
3073
|
+
const invalidated = reconcileProviderIdentity(store, provider);
|
|
3074
|
+
if (invalidated > 0) opts.onInvalidated?.(invalidated);
|
|
3075
|
+
const total = Math.min(store.countNodesNeedingEmbedding(projectId), maxNodes);
|
|
2559
3076
|
let embedded = 0;
|
|
2560
3077
|
let skipped = 0;
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
3078
|
+
let attempted = 0;
|
|
3079
|
+
let consecutiveFailedRequests = 0;
|
|
3080
|
+
let cursor = 0;
|
|
3081
|
+
outer: while (attempted < maxNodes) {
|
|
3082
|
+
const page = store.findNodesNeedingEmbedding(projectId, Math.min(pageSize, maxNodes - attempted), cursor);
|
|
3083
|
+
if (page.length === 0) break;
|
|
3084
|
+
cursor = page[page.length - 1].rowid;
|
|
3085
|
+
for (const group of chunk(page, batchSize)) {
|
|
3086
|
+
const vectors = await embedTexts(
|
|
3087
|
+
provider,
|
|
3088
|
+
group.map((node) => `${node.title}
|
|
3089
|
+
${node.body}`)
|
|
3090
|
+
);
|
|
3091
|
+
let embeddedHere = 0;
|
|
3092
|
+
for (const [index, node] of group.entries()) {
|
|
3093
|
+
const vector = vectors[index];
|
|
3094
|
+
if (vector && vector.length === provider.dimension) {
|
|
3095
|
+
store.upsertEmbedding(node.rowid, vector);
|
|
3096
|
+
embedded += 1;
|
|
3097
|
+
embeddedHere += 1;
|
|
3098
|
+
} else {
|
|
3099
|
+
skipped += 1;
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
attempted += group.length;
|
|
3103
|
+
consecutiveFailedRequests = embeddedHere === 0 ? consecutiveFailedRequests + 1 : 0;
|
|
3104
|
+
opts.onProgress?.(attempted, total);
|
|
3105
|
+
if (consecutiveFailedRequests >= failureTolerance) break outer;
|
|
2569
3106
|
}
|
|
2570
3107
|
}
|
|
2571
|
-
return {
|
|
3108
|
+
return {
|
|
3109
|
+
embedded,
|
|
3110
|
+
skipped,
|
|
3111
|
+
// Unchanged meaning: nothing came back at all. Reached far sooner now --
|
|
3112
|
+
// `failureTolerance` requests instead of the whole first page.
|
|
3113
|
+
providerUnavailable: attempted > 0 && embedded === 0,
|
|
3114
|
+
invalidated,
|
|
3115
|
+
remaining: store.countNodesNeedingEmbedding(projectId)
|
|
3116
|
+
};
|
|
2572
3117
|
}
|
|
2573
3118
|
|
|
2574
3119
|
// src/cli/context.ts
|
|
@@ -2581,6 +3126,8 @@ async function loadContext(cwd) {
|
|
|
2581
3126
|
|
|
2582
3127
|
// src/cli/commands/sync.ts
|
|
2583
3128
|
var BATCH_SIZE = 500;
|
|
3129
|
+
var PROGRESS_THRESHOLD = 200;
|
|
3130
|
+
var PROGRESS_EVERY = 100;
|
|
2584
3131
|
var GIT_SOURCE = "git";
|
|
2585
3132
|
function addStats(into, from) {
|
|
2586
3133
|
into.inserted += from.inserted;
|
|
@@ -2704,13 +3251,12 @@ async function syncShell(store, projectId, opts, repoRoot, config, log) {
|
|
|
2704
3251
|
return { totals, seen };
|
|
2705
3252
|
}
|
|
2706
3253
|
var CONVERSATION_SOURCE = "conversation:claude-code";
|
|
2707
|
-
|
|
3254
|
+
function syncConversation(store, projectId, turns, config, log, forceEnabled) {
|
|
2708
3255
|
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
2709
3256
|
const enabled = forceEnabled ?? config.sources.conversation.enabled;
|
|
2710
3257
|
if (!enabled) {
|
|
2711
3258
|
return { totals, seen: 0 };
|
|
2712
3259
|
}
|
|
2713
|
-
const turns = await collectClaudeCodeTranscripts(repoRoot);
|
|
2714
3260
|
if (turns.length === 0) {
|
|
2715
3261
|
log(`${pc4.dim("conversation")} no transcripts found`);
|
|
2716
3262
|
return { totals, seen: 0 };
|
|
@@ -2721,6 +3267,42 @@ async function syncConversation(store, projectId, repoRoot, config, log, forceEn
|
|
|
2721
3267
|
log(` ${pc4.dim(`${CONVERSATION_SOURCE}: ${nodes.length} of ${turns.length} exchange(s) kept`)}`);
|
|
2722
3268
|
return { totals, seen: nodes.length };
|
|
2723
3269
|
}
|
|
3270
|
+
var SESSION_SOURCE = "session:claude-code";
|
|
3271
|
+
async function syncSessions(store, projectId, turns, config, log) {
|
|
3272
|
+
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
3273
|
+
const settings = config.sources.session;
|
|
3274
|
+
if (!settings.enabled) return { totals, seen: 0 };
|
|
3275
|
+
if (turns.length === 0) {
|
|
3276
|
+
log(`${pc4.dim("session")} no transcripts found`);
|
|
3277
|
+
return { totals, seen: 0 };
|
|
3278
|
+
}
|
|
3279
|
+
const result = await collectSessionSummaries(turns, projectId, new OllamaChatProvider({ model: settings.model }), {
|
|
3280
|
+
settleMinutes: settings.settleMinutes,
|
|
3281
|
+
maxSessions: settings.maxSessions,
|
|
3282
|
+
maxPromptChars: settings.maxPromptChars,
|
|
3283
|
+
maxBodyChars: config.limits.maxBodyChars,
|
|
3284
|
+
knownHash: (sessionKey) => {
|
|
3285
|
+
const meta = store.getNodeMeta(makeNodeId(projectId, "session_summary", sessionKey));
|
|
3286
|
+
return typeof meta?.contentHash === "string" ? meta.contentHash : null;
|
|
3287
|
+
},
|
|
3288
|
+
onProgress: (done, total) => log(` ${pc4.dim(`session: summarizing ${done}/${total}`)}`)
|
|
3289
|
+
});
|
|
3290
|
+
if (result.nodes.length > 0) addStats(totals, store.upsertNodes(result.nodes));
|
|
3291
|
+
if (result.providerUnavailable) {
|
|
3292
|
+
log(
|
|
3293
|
+
`${pc4.dim("session")} summarization model unavailable (is Ollama running with \`${settings.model}\` pulled?) -- skipped`
|
|
3294
|
+
);
|
|
3295
|
+
} else {
|
|
3296
|
+
const parts = [`${result.nodes.length} summarized`];
|
|
3297
|
+
if (result.cached > 0) parts.push(`${result.cached} unchanged`);
|
|
3298
|
+
if (result.deferred > 0) parts.push(`${result.deferred} queued for the next sync`);
|
|
3299
|
+
if (result.unsettled > 0) parts.push(`${result.unsettled} still active`);
|
|
3300
|
+
if (result.failed > 0) parts.push(`${result.failed} failed`);
|
|
3301
|
+
log(` ${pc4.dim(`${SESSION_SOURCE}: ${parts.join(", ")}`)}`);
|
|
3302
|
+
}
|
|
3303
|
+
store.setSyncCursor(projectId, SESSION_SOURCE, `scanned:${result.nodes.length}`);
|
|
3304
|
+
return { totals, seen: result.nodes.length };
|
|
3305
|
+
}
|
|
2724
3306
|
var DOCS_SOURCE = "docs";
|
|
2725
3307
|
async function syncDocs(store, projectId, repoRoot, config, log) {
|
|
2726
3308
|
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
@@ -2753,26 +3335,62 @@ async function runSync(opts) {
|
|
|
2753
3335
|
if (!opts.quiet) process.stderr.write(`${line}
|
|
2754
3336
|
`);
|
|
2755
3337
|
};
|
|
2756
|
-
const out = opts.out ?? ((
|
|
3338
|
+
const out = opts.out ?? ((chunk2) => void process.stdout.write(chunk2));
|
|
2757
3339
|
const store = MemoryStore.open(ws.dbPath);
|
|
2758
3340
|
const started = Date.now();
|
|
2759
3341
|
try {
|
|
2760
3342
|
store.upsertProject({ id: projectId, root: repo.root, originUrl: repo.originUrl });
|
|
2761
|
-
await recordProject({ projectId, root: repo.root, dbPath: ws.dbPath, originUrl: repo.originUrl });
|
|
2762
3343
|
if (opts.rebuild) {
|
|
2763
3344
|
const removed = store.clearProject(projectId);
|
|
2764
3345
|
log(`${pc4.dim("rebuild")} dropped ${removed} existing node(s)`);
|
|
2765
3346
|
}
|
|
3347
|
+
const staleProjectIds = store.listOtherProjectIds(projectId);
|
|
3348
|
+
for (const staleId of staleProjectIds) {
|
|
3349
|
+
const result = reconcileProjectId(store.raw, staleId, projectId);
|
|
3350
|
+
const parts = [
|
|
3351
|
+
result.migrated > 0 ? `${result.migrated} migrated` : null,
|
|
3352
|
+
result.reassigned > 0 ? `${result.reassigned} reassigned` : null,
|
|
3353
|
+
result.deduped > 0 ? `${result.deduped} already up to date` : null,
|
|
3354
|
+
result.skipped > 0 ? `${result.skipped} left behind (not reconstructable)` : null
|
|
3355
|
+
].filter((part) => part !== null);
|
|
3356
|
+
if (parts.length > 0) {
|
|
3357
|
+
log(
|
|
3358
|
+
`${pc4.yellow("reconciled")} previous project identity ${pc4.dim(staleId)} (remote URL likely changed): ${parts.join(", ")}`
|
|
3359
|
+
);
|
|
3360
|
+
}
|
|
3361
|
+
}
|
|
3362
|
+
if (staleProjectIds.length > 0) {
|
|
3363
|
+
if (opts.rebuild) {
|
|
3364
|
+
for (const staleId of staleProjectIds) store.clearProject(staleId);
|
|
3365
|
+
}
|
|
3366
|
+
await forgetProjects(staleProjectIds);
|
|
3367
|
+
if (config.projectId !== projectId) await writeConfig(ws, { ...config, projectId });
|
|
3368
|
+
}
|
|
3369
|
+
await recordProject({ projectId, root: repo.root, dbPath: ws.dbPath, originUrl: repo.originUrl });
|
|
2766
3370
|
const git2 = await syncGit(store, projectId, opts, repo, config, log);
|
|
2767
3371
|
const diffs = await syncDiffs(store, projectId, opts, repo, config, log);
|
|
2768
3372
|
const shell = await syncShell(store, projectId, opts, repo.root, config, log);
|
|
2769
|
-
const
|
|
3373
|
+
const conversationEnabled = opts.conversationOverride ?? config.sources.conversation.enabled;
|
|
3374
|
+
const turns = conversationEnabled || config.sources.session.enabled ? await collectClaudeCodeTranscripts(repo.root) : [];
|
|
3375
|
+
const conversation = syncConversation(store, projectId, turns, config, log, opts.conversationOverride);
|
|
3376
|
+
const sessions = await syncSessions(store, projectId, turns, config, log);
|
|
2770
3377
|
const docs = await syncDocs(store, projectId, repo.root, config, log);
|
|
2771
3378
|
let embedLine = "";
|
|
2772
3379
|
if (!opts.noEmbed) {
|
|
2773
|
-
|
|
3380
|
+
let lastLogged = 0;
|
|
3381
|
+
const result = await embedPendingNodes(store, new OllamaEmbeddingProvider(), projectId, {
|
|
3382
|
+
maxNodes: opts.embedLimit,
|
|
3383
|
+
onInvalidated: (count) => log(`${pc4.yellow("vector")} embedding model changed -- dropped ${count} vector(s), re-embedding from scratch`),
|
|
3384
|
+
onProgress: (attempted, total) => {
|
|
3385
|
+
if (total < PROGRESS_THRESHOLD || attempted - lastLogged < PROGRESS_EVERY) return;
|
|
3386
|
+
lastLogged = attempted;
|
|
3387
|
+
log(` ${pc4.dim(`vector: ${attempted}/${total} embedded`)}`);
|
|
3388
|
+
}
|
|
3389
|
+
});
|
|
2774
3390
|
if (result.embedded > 0) {
|
|
2775
|
-
|
|
3391
|
+
const skippedPart = result.skipped > 0 ? pc4.dim(`, ${result.skipped} skipped`) : "";
|
|
3392
|
+
const remainingPart = result.remaining > 0 ? pc4.yellow(`, ${result.remaining} still pending`) : "";
|
|
3393
|
+
embedLine = ` ${pc4.dim(`vector: ${result.embedded} node(s) embedded`)}${skippedPart}${remainingPart}
|
|
2776
3394
|
`;
|
|
2777
3395
|
} else if (result.providerUnavailable) {
|
|
2778
3396
|
log(`${pc4.dim("vector")} embedding provider unavailable (is Ollama running with nomic-embed-text pulled?) -- BM25-only for now`);
|
|
@@ -2784,16 +3402,17 @@ async function runSync(opts) {
|
|
|
2784
3402
|
addStats(totals, diffs.totals);
|
|
2785
3403
|
addStats(totals, shell.totals);
|
|
2786
3404
|
addStats(totals, conversation.totals);
|
|
3405
|
+
addStats(totals, sessions.totals);
|
|
2787
3406
|
addStats(totals, docs.totals);
|
|
2788
3407
|
const stats = store.stats(projectId);
|
|
2789
3408
|
const elapsed = ((Date.now() - started) / 1e3).toFixed(2);
|
|
2790
|
-
const conversationEnabled = opts.conversationOverride ?? config.sources.conversation.enabled;
|
|
2791
3409
|
const conversationPart = conversationEnabled ? `, ${conversation.seen} conversation exchange(s)` : "";
|
|
3410
|
+
const sessionPart = config.sources.session.enabled ? `, ${sessions.seen} session summar${sessions.seen === 1 ? "y" : "ies"}` : "";
|
|
2792
3411
|
const docsPart = config.sources.docs.enabled ? `, ${docs.seen} doc section(s)` : "";
|
|
2793
3412
|
const diffPart = config.sources.diff.enabled ? `, ${diffs.seen} file diff(s)` : "";
|
|
2794
3413
|
out(
|
|
2795
3414
|
[
|
|
2796
|
-
`${pc4.green("synced")} ${git2.seen} commit(s)${diffPart}, ${shell.seen} shell entr${shell.seen === 1 ? "y" : "ies"}${conversationPart}${docsPart} in ${elapsed}s`,
|
|
3415
|
+
`${pc4.green("synced")} ${git2.seen} commit(s)${diffPart}, ${shell.seen} shell entr${shell.seen === 1 ? "y" : "ies"}${conversationPart}${sessionPart}${docsPart} in ${elapsed}s`,
|
|
2797
3416
|
` ${pc4.green(`+${totals.inserted} new`)} ${pc4.yellow(`~${totals.updated} updated`)} ${pc4.dim(`=${totals.unchanged} unchanged`)}`,
|
|
2798
3417
|
` ${pc4.dim(`${stats.total} node(s) total across ${stats.distinctFiles} file path(s)`)}`,
|
|
2799
3418
|
""
|
|
@@ -2844,7 +3463,7 @@ async function searchMemory(input) {
|
|
|
2844
3463
|
vectorMatched: vectorCount,
|
|
2845
3464
|
tokensUsed: packed.tokensUsed,
|
|
2846
3465
|
tokensBudget: packed.tokensBudget,
|
|
2847
|
-
projectsSearched: [
|
|
3466
|
+
projectsSearched: [basename3(repo.root) || repo.root]
|
|
2848
3467
|
};
|
|
2849
3468
|
} finally {
|
|
2850
3469
|
store.close();
|
|
@@ -2852,8 +3471,8 @@ async function searchMemory(input) {
|
|
|
2852
3471
|
}
|
|
2853
3472
|
async function syncProject(input) {
|
|
2854
3473
|
const chunks = [];
|
|
2855
|
-
const out = (
|
|
2856
|
-
chunks.push(
|
|
3474
|
+
const out = (chunk2) => {
|
|
3475
|
+
chunks.push(chunk2);
|
|
2857
3476
|
};
|
|
2858
3477
|
await runInit({ cwd: input.projectRoot, force: false, hook: false, enableConversation: false, out });
|
|
2859
3478
|
const opts = {
|
|
@@ -2887,7 +3506,7 @@ function createServer() {
|
|
|
2887
3506
|
"search_memory",
|
|
2888
3507
|
{
|
|
2889
3508
|
title: "Search remembered project history",
|
|
2890
|
-
description: "Search a NexusMem-tracked repository's remembered history: git commits, code diffs (the patch of each changed file), shell commands, tracked markdown docs, and (if enabled) conversation transcripts. Returns a token-budgeted, ranked context block -- not raw search results.",
|
|
3509
|
+
description: "Search a NexusMem-tracked repository's remembered history: git commits, code diffs (the patch of each changed file), shell commands, tracked markdown docs, and (if enabled) conversation transcripts and per-session summaries. Returns a token-budgeted, ranked context block -- not raw search results.",
|
|
2891
3510
|
inputSchema: {
|
|
2892
3511
|
projectRoot: z3.string().describe("Absolute path to the repository root"),
|
|
2893
3512
|
query: z3.string().describe("Free-text question or search terms"),
|
|
@@ -3002,7 +3621,8 @@ async function runQuery(opts) {
|
|
|
3002
3621
|
packed: packed.nodes,
|
|
3003
3622
|
tokensUsed: packed.tokensUsed,
|
|
3004
3623
|
tokensBudget: packed.tokensBudget,
|
|
3005
|
-
droppedForBudget: packed.droppedForBudget
|
|
3624
|
+
droppedForBudget: packed.droppedForBudget,
|
|
3625
|
+
droppedForDiversity: packed.droppedForDiversity
|
|
3006
3626
|
},
|
|
3007
3627
|
null,
|
|
3008
3628
|
2
|
|
@@ -3019,7 +3639,7 @@ async function runQuery(opts) {
|
|
|
3019
3639
|
process.stderr.write(
|
|
3020
3640
|
[
|
|
3021
3641
|
`${pc5.dim("matched")} ${bm25Count} bm25${vectorCount > 0 ? ` + ${vectorCount} vector` : ""}, packed ${pc5.bold(String(packed.nodes.length))} into budget`,
|
|
3022
|
-
`${pc5.dim("tokens ")} ${packed.tokensUsed}/${packed.tokensBudget}` + (packed.droppedForBudget ? pc5.dim(` (${packed.droppedForBudget} dropped for budget)`) : ""),
|
|
3642
|
+
`${pc5.dim("tokens ")} ${packed.tokensUsed}/${packed.tokensBudget}` + (packed.droppedForBudget ? pc5.dim(` (${packed.droppedForBudget} dropped for budget)`) : "") + (packed.droppedForDiversity ? pc5.dim(` (${packed.droppedForDiversity} dropped for diversity)`) : ""),
|
|
3023
3643
|
rawTokens > 0 ? `${pc5.dim("vs raw ")} ${rawTokens} tokens if these same matches were sent unpacked ${packerEfficiency >= 0 ? pc5.green(`(${(packerEfficiency * 100).toFixed(0)}% packer efficiency)`) : pc5.yellow(`(${(-packerEfficiency * 100).toFixed(0)}% larger -- overhead dominates on small result sets)`)}` : "",
|
|
3024
3644
|
""
|
|
3025
3645
|
].filter(Boolean).join("\n")
|
|
@@ -3249,17 +3869,101 @@ function formatNode4(node) {
|
|
|
3249
3869
|
return [formatSignal(node.signal, DOCS_SIGNAL_BANDS), node.title].join(" ");
|
|
3250
3870
|
}
|
|
3251
3871
|
|
|
3252
|
-
// src/cli/commands/scan-
|
|
3872
|
+
// src/cli/commands/scan-session.ts
|
|
3253
3873
|
import pc11 from "picocolors";
|
|
3874
|
+
async function runScanSession(opts) {
|
|
3875
|
+
const repo = await readRepoInfo(opts.cwd);
|
|
3876
|
+
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
3877
|
+
const turns = await collectClaudeCodeTranscripts(repo.root);
|
|
3878
|
+
if (turns.length === 0) {
|
|
3879
|
+
process.stderr.write(`${pc11.yellow("no transcripts found")} at ${claudeProjectTranscriptDir(repo.root)}
|
|
3880
|
+
`);
|
|
3881
|
+
return 0;
|
|
3882
|
+
}
|
|
3883
|
+
const sessions = groupTurnsIntoSessions(turns);
|
|
3884
|
+
const settled = selectSettledSessions(sessions, opts.settleMinutes);
|
|
3885
|
+
if (!opts.json) {
|
|
3886
|
+
process.stderr.write(
|
|
3887
|
+
`${pc11.dim("sessions")} ${sessions.length} found, ${settled.length} settled (quiet ${opts.settleMinutes}m+)
|
|
3888
|
+
|
|
3889
|
+
`
|
|
3890
|
+
);
|
|
3891
|
+
}
|
|
3892
|
+
if (opts.dryRun) {
|
|
3893
|
+
const previews = settled.slice(0, opts.maxSessions).map((session) => {
|
|
3894
|
+
const { prompt, hash, includedTurns } = buildSessionPrompt(session);
|
|
3895
|
+
return {
|
|
3896
|
+
sessionKey: session.sessionKey,
|
|
3897
|
+
startedAt: session.startedAt,
|
|
3898
|
+
endedAt: session.endedAt,
|
|
3899
|
+
turns: session.turns.length,
|
|
3900
|
+
includedTurns,
|
|
3901
|
+
hash,
|
|
3902
|
+
promptChars: prompt.length,
|
|
3903
|
+
prompt
|
|
3904
|
+
};
|
|
3905
|
+
});
|
|
3906
|
+
if (opts.json) {
|
|
3907
|
+
process.stdout.write(`${JSON.stringify(previews, null, 2)}
|
|
3908
|
+
`);
|
|
3909
|
+
return 0;
|
|
3910
|
+
}
|
|
3911
|
+
for (const preview of previews) {
|
|
3912
|
+
process.stdout.write(
|
|
3913
|
+
`${pc11.bold(preview.sessionKey)} ${preview.startedAt.slice(0, 16).replace("T", " ")} ${preview.includedTurns}/${preview.turns} turn(s), ${preview.promptChars} chars
|
|
3914
|
+
${preview.prompt}
|
|
3915
|
+
|
|
3916
|
+
`
|
|
3917
|
+
);
|
|
3918
|
+
}
|
|
3919
|
+
return 0;
|
|
3920
|
+
}
|
|
3921
|
+
const result = await collectSessionSummaries(turns, projectId, new OllamaChatProvider({ model: opts.model }), {
|
|
3922
|
+
settleMinutes: opts.settleMinutes,
|
|
3923
|
+
maxSessions: opts.maxSessions,
|
|
3924
|
+
onProgress: (done, total) => {
|
|
3925
|
+
if (!opts.json) process.stderr.write(` ${pc11.dim(`summarizing ${done}/${total}`)}
|
|
3926
|
+
`);
|
|
3927
|
+
}
|
|
3928
|
+
});
|
|
3929
|
+
if (opts.json) {
|
|
3930
|
+
process.stdout.write(`${JSON.stringify(result.nodes, null, 2)}
|
|
3931
|
+
`);
|
|
3932
|
+
return 0;
|
|
3933
|
+
}
|
|
3934
|
+
for (const node of result.nodes) {
|
|
3935
|
+
process.stdout.write(`${pc11.bold(node.title)}
|
|
3936
|
+
${pc11.dim(node.ts.slice(0, 16).replace("T", " "))}
|
|
3937
|
+
${node.body}
|
|
3938
|
+
|
|
3939
|
+
`);
|
|
3940
|
+
}
|
|
3941
|
+
if (result.providerUnavailable) {
|
|
3942
|
+
process.stderr.write(
|
|
3943
|
+
`${pc11.yellow("model unavailable")} -- is Ollama running with \`${opts.model}\` pulled? (\`ollama pull ${opts.model}\`)
|
|
3944
|
+
`
|
|
3945
|
+
);
|
|
3946
|
+
return 0;
|
|
3947
|
+
}
|
|
3948
|
+
process.stderr.write(
|
|
3949
|
+
`${pc11.bold(String(result.nodes.length))} summarized` + (result.failed > 0 ? `, ${pc11.yellow(`${result.failed} failed`)}` : "") + ` ${pc11.dim(`(model ${opts.model})`)}
|
|
3950
|
+
`
|
|
3951
|
+
);
|
|
3952
|
+
return 0;
|
|
3953
|
+
}
|
|
3954
|
+
var SCAN_SESSION_DEFAULT_MODEL = DEFAULT_SLM_MODEL;
|
|
3955
|
+
|
|
3956
|
+
// src/cli/commands/scan-shell.ts
|
|
3957
|
+
import pc12 from "picocolors";
|
|
3254
3958
|
async function runScanShell(opts) {
|
|
3255
3959
|
const repo = await readRepoInfo(opts.cwd);
|
|
3256
3960
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
3257
3961
|
const results = await collectAvailableShellHistory({ tailLines: opts.tailLines, repoRoot: repo.root });
|
|
3258
3962
|
if (!opts.json) {
|
|
3259
3963
|
process.stderr.write(
|
|
3260
|
-
results.length ? `${
|
|
3964
|
+
results.length ? `${pc12.dim("sources found")} ${results.map((r) => r.name).join(", ")}
|
|
3261
3965
|
|
|
3262
|
-
` : `${
|
|
3966
|
+
` : `${pc12.yellow("no shell history source found on this machine")}
|
|
3263
3967
|
`
|
|
3264
3968
|
);
|
|
3265
3969
|
}
|
|
@@ -3268,7 +3972,7 @@ async function runScanShell(opts) {
|
|
|
3268
3972
|
const nodes = collectShellHistory(result.entries, projectId).filter((n) => n.signal >= opts.minSignal);
|
|
3269
3973
|
allNodes.push(...nodes);
|
|
3270
3974
|
if (!opts.json) {
|
|
3271
|
-
process.stdout.write(`${
|
|
3975
|
+
process.stdout.write(`${pc12.bold(`shell:${result.name}`)} ${pc12.dim(`(${nodes.length} of ${result.entries.length} above threshold)`)}
|
|
3272
3976
|
`);
|
|
3273
3977
|
for (const node of nodes) process.stdout.write(`${formatNode5(node)}
|
|
3274
3978
|
`);
|
|
@@ -3281,20 +3985,20 @@ async function runScanShell(opts) {
|
|
|
3281
3985
|
return 0;
|
|
3282
3986
|
}
|
|
3283
3987
|
const approxTotal = allNodes.reduce((n, x) => n + approxTokens(x.body), 0);
|
|
3284
|
-
process.stderr.write(`${
|
|
3988
|
+
process.stderr.write(`${pc12.bold(String(allNodes.length))} node(s) total ${pc12.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}
|
|
3285
3989
|
`);
|
|
3286
3990
|
return 0;
|
|
3287
3991
|
}
|
|
3288
3992
|
function formatNode5(node) {
|
|
3289
|
-
const approx = node.meta.tsApprox ?
|
|
3993
|
+
const approx = node.meta.tsApprox ? pc12.dim("~") : " ";
|
|
3290
3994
|
const exit = node.meta.exitCode;
|
|
3291
|
-
const exitLabel = typeof exit === "number" && exit !== 0 ?
|
|
3995
|
+
const exitLabel = typeof exit === "number" && exit !== 0 ? pc12.red(`exit ${exit}`) : "";
|
|
3292
3996
|
return [formatSignal(node.signal, SHELL_SIGNAL_BANDS), approx + node.ts.slice(0, 16).replace("T", " "), node.title, exitLabel].filter(Boolean).join(" ");
|
|
3293
3997
|
}
|
|
3294
3998
|
|
|
3295
3999
|
// src/cli/commands/status.ts
|
|
3296
4000
|
import { statSync } from "fs";
|
|
3297
|
-
import
|
|
4001
|
+
import pc13 from "picocolors";
|
|
3298
4002
|
function humanBytes(bytes) {
|
|
3299
4003
|
if (bytes < 1024) return `${bytes} B`;
|
|
3300
4004
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
@@ -3319,23 +4023,23 @@ async function runStatus(opts) {
|
|
|
3319
4023
|
const kinds = Object.entries(stats.byKind).sort((a, b) => b[1] - a[1]).map(([kind, n]) => ` ${String(n).padStart(6)} ${kind}`);
|
|
3320
4024
|
process.stdout.write(
|
|
3321
4025
|
[
|
|
3322
|
-
`${
|
|
3323
|
-
`${
|
|
3324
|
-
`${
|
|
3325
|
-
`${
|
|
3326
|
-
`${
|
|
4026
|
+
`${pc13.dim("repo ")} ${repo.root}`,
|
|
4027
|
+
`${pc13.dim("branch ")} ${repo.branch ?? pc13.yellow("(detached)")}`,
|
|
4028
|
+
`${pc13.dim("project ")} ${pc13.cyan(projectId)}`,
|
|
4029
|
+
`${pc13.dim("schema ")} v${schema}${schema === LATEST_SCHEMA_VERSION ? "" : pc13.yellow(` (latest is v${LATEST_SCHEMA_VERSION})`)}`,
|
|
4030
|
+
`${pc13.dim("database")} ${ws.dbPath} ${pc13.dim(`(${humanBytes(dbBytes)})`)}`,
|
|
3327
4031
|
"",
|
|
3328
|
-
`${
|
|
4032
|
+
`${pc13.bold(String(stats.total))} node(s)${stats.total ? ` ${pc13.dim(`${stats.oldest?.slice(0, 10)} .. ${stats.newest?.slice(0, 10)}`)}` : ""}`,
|
|
3329
4033
|
...kinds,
|
|
3330
|
-
stats.total ? ` ${
|
|
4034
|
+
stats.total ? ` ${pc13.dim(`${stats.distinctFiles} distinct file path(s)`)}` : "",
|
|
3331
4035
|
"",
|
|
3332
|
-
sources.length ?
|
|
4036
|
+
sources.length ? pc13.dim("sources") : pc13.yellow("no sources synced yet"),
|
|
3333
4037
|
...sources.map((s) => {
|
|
3334
4038
|
const when = s.lastRunAt ? new Date(s.lastRunAt).toISOString().slice(0, 16).replace("T", " ") : "never";
|
|
3335
4039
|
const cursorLabel = s.source === "git" ? s.cursor?.slice(0, 7) ?? "-" : s.cursor ?? "-";
|
|
3336
|
-
return ` ${s.source.padEnd(14)} ${
|
|
4040
|
+
return ` ${s.source.padEnd(14)} ${pc13.dim(`last run ${when}`)} ${pc13.dim(`cursor ${cursorLabel}`)}`;
|
|
3337
4041
|
}),
|
|
3338
|
-
gitCursor && gitCursor !== repo.head ? `${
|
|
4042
|
+
gitCursor && gitCursor !== repo.head ? `${pc13.yellow("git behind HEAD")} \u2014 run ${pc13.bold("nexusmem sync")}` : "",
|
|
3339
4043
|
""
|
|
3340
4044
|
].filter((line) => line !== "").join("\n").concat("\n")
|
|
3341
4045
|
);
|
|
@@ -3359,7 +4063,7 @@ function guard(run) {
|
|
|
3359
4063
|
process.exitCode = await run();
|
|
3360
4064
|
} catch (err) {
|
|
3361
4065
|
if (isExpected(err)) {
|
|
3362
|
-
process.stderr.write(`${
|
|
4066
|
+
process.stderr.write(`${pc14.red("error")} ${err.message}
|
|
3363
4067
|
`);
|
|
3364
4068
|
process.exitCode = 1;
|
|
3365
4069
|
return;
|
|
@@ -3375,7 +4079,11 @@ program.command("init").description("Create the .nexusmem workspace and database
|
|
|
3375
4079
|
() => runInit({ cwd: options.cwd, force: options.force, hook: options.hook, enableConversation: options.enableConversation })
|
|
3376
4080
|
)()
|
|
3377
4081
|
);
|
|
3378
|
-
program.command("sync").description("Ingest new history into the local database").option("-C, --cwd <path>", "repository path", process.cwd()).option("--full", "ignore the stored cursor and re-walk all history", false).option("--rebuild", "drop this project's nodes and re-ingest from scratch", false).option("--since <date>", "override the configured git cutoff, e.g. 1.year.ago").option("--shell-lines <count>", "override the configured shell tail-window size", (v) => Number.parseInt(v, 10)).option("--conversation", "force the conversation source on for this run, without persisting it to config", false).option("--no-embed", "skip the vector-embedding pass for this run").option(
|
|
4082
|
+
program.command("sync").description("Ingest new history into the local database").option("-C, --cwd <path>", "repository path", process.cwd()).option("--full", "ignore the stored cursor and re-walk all history", false).option("--rebuild", "drop this project's nodes and re-ingest from scratch", false).option("--since <date>", "override the configured git cutoff, e.g. 1.year.ago").option("--shell-lines <count>", "override the configured shell tail-window size", (v) => Number.parseInt(v, 10)).option("--conversation", "force the conversation source on for this run, without persisting it to config", false).option("--no-embed", "skip the vector-embedding pass for this run").option(
|
|
4083
|
+
"--embed-limit <count>",
|
|
4084
|
+
"stop embedding after this many nodes (default: embed everything pending)",
|
|
4085
|
+
(v) => Number.parseInt(v, 10)
|
|
4086
|
+
).option("-q, --quiet", "only print the final summary", false).action(
|
|
3379
4087
|
(options) => guard(
|
|
3380
4088
|
() => runSync({
|
|
3381
4089
|
cwd: options.cwd,
|
|
@@ -3385,6 +4093,7 @@ program.command("sync").description("Ingest new history into the local database"
|
|
|
3385
4093
|
shellTailLines: options.shellLines,
|
|
3386
4094
|
conversationOverride: options.conversation ? true : void 0,
|
|
3387
4095
|
noEmbed: !options.embed,
|
|
4096
|
+
embedLimit: options.embedLimit,
|
|
3388
4097
|
quiet: options.quiet
|
|
3389
4098
|
})
|
|
3390
4099
|
)()
|
|
@@ -3443,11 +4152,23 @@ program.command("scan-shell").description("Preview the MemoryNodes shell history
|
|
|
3443
4152
|
program.command("scan-conversation").description("Preview the MemoryNodes the conversation transcript would produce (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("--min-signal <score>", "drop nodes below this signal", (v) => Number.parseFloat(v), 0).option("--json", "emit MemoryNodes as JSON on stdout", false).action(
|
|
3444
4153
|
(options) => guard(() => runScanConversation({ cwd: options.cwd, minSignal: options.minSignal, json: options.json }))()
|
|
3445
4154
|
);
|
|
4155
|
+
program.command("scan-session").description("Preview the session summaries a local model would produce (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("--model <name>", "Ollama model to summarize with", SCAN_SESSION_DEFAULT_MODEL).option("--settle-minutes <count>", "minutes of quiet before a session counts as finished", (v) => Number.parseInt(v, 10), 30).option("-n, --max-sessions <count>", "how many sessions to summarize", (v) => Number.parseInt(v, 10), 3).option("--dry-run", "print the prompts instead of calling the model", false).option("--json", "emit as JSON on stdout", false).action(
|
|
4156
|
+
(options) => guard(
|
|
4157
|
+
() => runScanSession({
|
|
4158
|
+
cwd: options.cwd,
|
|
4159
|
+
model: options.model,
|
|
4160
|
+
settleMinutes: options.settleMinutes,
|
|
4161
|
+
maxSessions: options.maxSessions,
|
|
4162
|
+
dryRun: options.dryRun,
|
|
4163
|
+
json: options.json
|
|
4164
|
+
})
|
|
4165
|
+
)()
|
|
4166
|
+
);
|
|
3446
4167
|
program.command("scan-docs").description("Preview the MemoryNodes tracked .md files would produce (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("--min-signal <score>", "drop nodes below this signal", (v) => Number.parseFloat(v), 0).option("--json", "emit MemoryNodes as JSON on stdout", false).action((options) => guard(() => runScanDocs({ cwd: options.cwd, minSignal: options.minSignal, json: options.json }))());
|
|
3447
4168
|
program.command("mcp").description("Start the MCP server (stdio transport) for Claude Desktop, Cursor, Windsurf, etc.").action(() => guard(() => runMcpServer().then(() => 0))());
|
|
3448
4169
|
program.parseAsync(process.argv).catch((err) => {
|
|
3449
4170
|
const message = err instanceof Error ? err.message : String(err);
|
|
3450
|
-
process.stderr.write(`${
|
|
4171
|
+
process.stderr.write(`${pc14.red("error")} ${message}
|
|
3451
4172
|
`);
|
|
3452
4173
|
process.exitCode = 1;
|
|
3453
4174
|
});
|