nexusmem 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +347 -254
- package/dist/cli/index.js +1669 -333
- package/dist/cli/index.js.map +1 -1
- package/package.json +4 -2
package/dist/cli/index.js
CHANGED
|
@@ -2,13 +2,68 @@
|
|
|
2
2
|
|
|
3
3
|
// src/cli/index.ts
|
|
4
4
|
import { Command } from "commander";
|
|
5
|
-
import
|
|
5
|
+
import pc14 from "picocolors";
|
|
6
6
|
|
|
7
7
|
// src/config/workspace.ts
|
|
8
8
|
import { existsSync } from "fs";
|
|
9
9
|
import { mkdir, readFile, writeFile } from "fs/promises";
|
|
10
10
|
import { join } from "path";
|
|
11
11
|
import { z } from "zod";
|
|
12
|
+
|
|
13
|
+
// src/slm/provider.ts
|
|
14
|
+
var DEFAULT_BASE_URL = "http://127.0.0.1:11434";
|
|
15
|
+
var DEFAULT_SLM_MODEL = "qwen2.5:3b";
|
|
16
|
+
var DEFAULT_TIMEOUT_MS = 12e4;
|
|
17
|
+
var DEFAULT_MAX_TOKENS = 400;
|
|
18
|
+
var OllamaChatProvider = class {
|
|
19
|
+
identity;
|
|
20
|
+
baseUrl;
|
|
21
|
+
model;
|
|
22
|
+
timeoutMs;
|
|
23
|
+
maxTokens;
|
|
24
|
+
constructor(opts = {}) {
|
|
25
|
+
this.baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
|
|
26
|
+
this.model = opts.model ?? DEFAULT_SLM_MODEL;
|
|
27
|
+
this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
28
|
+
this.maxTokens = opts.maxTokens ?? DEFAULT_MAX_TOKENS;
|
|
29
|
+
this.identity = `ollama:${this.model}`;
|
|
30
|
+
}
|
|
31
|
+
async complete(prompt) {
|
|
32
|
+
const controller = new AbortController();
|
|
33
|
+
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
34
|
+
try {
|
|
35
|
+
const res = await fetch(`${this.baseUrl}/api/generate`, {
|
|
36
|
+
method: "POST",
|
|
37
|
+
headers: { "content-type": "application/json" },
|
|
38
|
+
body: JSON.stringify({
|
|
39
|
+
model: this.model,
|
|
40
|
+
prompt,
|
|
41
|
+
stream: false,
|
|
42
|
+
options: {
|
|
43
|
+
// Deterministic: the same session must summarize to the same text
|
|
44
|
+
// across syncs, or the content hash that suppresses re-work would
|
|
45
|
+
// never match and every sync would rewrite every summary node.
|
|
46
|
+
temperature: 0,
|
|
47
|
+
seed: 1,
|
|
48
|
+
num_predict: this.maxTokens
|
|
49
|
+
}
|
|
50
|
+
}),
|
|
51
|
+
signal: controller.signal
|
|
52
|
+
});
|
|
53
|
+
if (!res.ok) return null;
|
|
54
|
+
const data = await res.json();
|
|
55
|
+
if (typeof data.response !== "string") return null;
|
|
56
|
+
const text = data.response.trim();
|
|
57
|
+
return text.length > 0 ? text : null;
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
} finally {
|
|
61
|
+
clearTimeout(timeout);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
// src/config/workspace.ts
|
|
12
67
|
var WORKSPACE_DIR = ".nexusmem";
|
|
13
68
|
function resolveWorkspace(repoRoot) {
|
|
14
69
|
const dir = join(repoRoot, WORKSPACE_DIR);
|
|
@@ -46,17 +101,66 @@ var ConfigSchema = z.object({
|
|
|
46
101
|
conversation: z.object({
|
|
47
102
|
enabled: z.boolean().default(false)
|
|
48
103
|
}).default({ enabled: false }),
|
|
104
|
+
/**
|
|
105
|
+
* One distilled node per finished working session, written by a local
|
|
106
|
+
* small language model.
|
|
107
|
+
*
|
|
108
|
+
* Opt-in for the same reason as `conversation` -- it reads the same
|
|
109
|
+
* transcripts -- and additionally because it is the only source that
|
|
110
|
+
* costs real compute. Independent of `conversation.enabled`: summaries
|
|
111
|
+
* without the raw exchanges is a legitimate, and much smaller, way to
|
|
112
|
+
* remember a session.
|
|
113
|
+
*/
|
|
114
|
+
session: z.object({
|
|
115
|
+
enabled: z.boolean().default(false),
|
|
116
|
+
/** Ollama model tag. Must be pulled locally; nothing is downloaded automatically. */
|
|
117
|
+
model: z.string().default(DEFAULT_SLM_MODEL),
|
|
118
|
+
/** Minutes of quiet before a session counts as finished and can be summarized. */
|
|
119
|
+
settleMinutes: z.number().int().nonnegative().default(30),
|
|
120
|
+
/** Sessions summarized per sync. Each is a model call measured in seconds. */
|
|
121
|
+
maxSessions: z.number().int().positive().default(10),
|
|
122
|
+
maxPromptChars: z.number().int().positive().default(12e3)
|
|
123
|
+
}).default({
|
|
124
|
+
enabled: false,
|
|
125
|
+
model: DEFAULT_SLM_MODEL,
|
|
126
|
+
settleMinutes: 30,
|
|
127
|
+
maxSessions: 10,
|
|
128
|
+
maxPromptChars: 12e3
|
|
129
|
+
}),
|
|
49
130
|
/** Tracked `.md` files -- README, architecture docs. On by default like git/shell: no secrets risk, just project prose. */
|
|
50
131
|
docs: z.object({
|
|
51
132
|
enabled: z.boolean().default(true),
|
|
52
133
|
/** git pathspecs passed to `git ls-files`. */
|
|
53
134
|
include: z.array(z.string()).default(["*.md"])
|
|
54
|
-
}).default({ enabled: true, include: ["*.md"] })
|
|
135
|
+
}).default({ enabled: true, include: ["*.md"] }),
|
|
136
|
+
/**
|
|
137
|
+
* The patch text of each commit, one node per changed file.
|
|
138
|
+
*
|
|
139
|
+
* Bounded by `maxCommits` rather than by `git.since`, because patches
|
|
140
|
+
* are an order of magnitude bulkier than commit messages: an unbounded
|
|
141
|
+
* first sync of a long-lived repository would spend most of its time
|
|
142
|
+
* and database on code nobody will ask about. Later syncs walk only
|
|
143
|
+
* `cursor..HEAD`, so the cap effectively applies to the first run.
|
|
144
|
+
*/
|
|
145
|
+
diff: z.object({
|
|
146
|
+
enabled: z.boolean().default(true),
|
|
147
|
+
maxCommits: z.number().int().positive().default(200),
|
|
148
|
+
maxFilesPerCommit: z.number().int().positive().default(20),
|
|
149
|
+
contextLines: z.number().int().nonnegative().default(3)
|
|
150
|
+
}).default({ enabled: true, maxCommits: 200, maxFilesPerCommit: 20, contextLines: 3 })
|
|
55
151
|
}).default({
|
|
56
152
|
git: { enabled: true, since: null, includeMerges: true },
|
|
57
153
|
shell: { enabled: true, tailLines: 300 },
|
|
58
154
|
conversation: { enabled: false },
|
|
59
|
-
|
|
155
|
+
session: {
|
|
156
|
+
enabled: false,
|
|
157
|
+
model: DEFAULT_SLM_MODEL,
|
|
158
|
+
settleMinutes: 30,
|
|
159
|
+
maxSessions: 10,
|
|
160
|
+
maxPromptChars: 12e3
|
|
161
|
+
},
|
|
162
|
+
docs: { enabled: true, include: ["*.md"] },
|
|
163
|
+
diff: { enabled: true, maxCommits: 200, maxFilesPerCommit: 20, contextLines: 3 }
|
|
60
164
|
}),
|
|
61
165
|
limits: z.object({
|
|
62
166
|
maxFilesPerNode: z.number().int().positive().default(40),
|
|
@@ -193,9 +297,9 @@ async function* gitStream(cwd, args, opts = {}) {
|
|
|
193
297
|
for (let attempt = 0; ; attempt += 1) {
|
|
194
298
|
let produced = false;
|
|
195
299
|
try {
|
|
196
|
-
for await (const
|
|
300
|
+
for await (const chunk2 of runGitOnce(cwd, args, opts)) {
|
|
197
301
|
produced = true;
|
|
198
|
-
yield
|
|
302
|
+
yield chunk2;
|
|
199
303
|
}
|
|
200
304
|
return;
|
|
201
305
|
} catch (err) {
|
|
@@ -211,8 +315,8 @@ async function* runGitOnce(cwd, args, opts) {
|
|
|
211
315
|
child.stdout.setEncoding("utf8");
|
|
212
316
|
child.stderr.setEncoding("utf8");
|
|
213
317
|
let stderr = "";
|
|
214
|
-
child.stderr.on("data", (
|
|
215
|
-
if (stderr.length < 64 * 1024) stderr +=
|
|
318
|
+
child.stderr.on("data", (chunk2) => {
|
|
319
|
+
if (stderr.length < 64 * 1024) stderr += chunk2;
|
|
216
320
|
});
|
|
217
321
|
const exited = new Promise((resolve2, reject) => {
|
|
218
322
|
child.once("error", (err) => reject(toSpawnError(err, cwd, fullArgs)));
|
|
@@ -221,8 +325,8 @@ async function* runGitOnce(cwd, args, opts) {
|
|
|
221
325
|
exited.catch(() => {
|
|
222
326
|
});
|
|
223
327
|
try {
|
|
224
|
-
for await (const
|
|
225
|
-
yield
|
|
328
|
+
for await (const chunk2 of child.stdout) {
|
|
329
|
+
yield chunk2;
|
|
226
330
|
}
|
|
227
331
|
} finally {
|
|
228
332
|
if (child.exitCode === null) child.kill();
|
|
@@ -251,7 +355,7 @@ async function* runGitOnce(cwd, args, opts) {
|
|
|
251
355
|
}
|
|
252
356
|
async function git(cwd, args, opts = {}) {
|
|
253
357
|
let out = "";
|
|
254
|
-
for await (const
|
|
358
|
+
for await (const chunk2 of gitStream(cwd, args, opts)) out += chunk2;
|
|
255
359
|
return out;
|
|
256
360
|
}
|
|
257
361
|
async function gitOrNull(cwd, args, opts = {}) {
|
|
@@ -314,25 +418,31 @@ import { dirname } from "path";
|
|
|
314
418
|
|
|
315
419
|
// src/shell/paths.ts
|
|
316
420
|
import { execFile } from "child_process";
|
|
421
|
+
import { homedir as homedir2 } from "os";
|
|
422
|
+
import { join as join3 } from "path";
|
|
423
|
+
import { promisify } from "util";
|
|
424
|
+
|
|
425
|
+
// src/config/paths.ts
|
|
317
426
|
import { homedir } from "os";
|
|
318
427
|
import { join as join2 } from "path";
|
|
319
|
-
|
|
428
|
+
function globalWorkspaceDir() {
|
|
429
|
+
return process.env.NEXUSMEM_HOME ?? join2(homedir(), ".nexusmem");
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// src/shell/paths.ts
|
|
320
433
|
var execFileAsync = promisify(execFile);
|
|
321
434
|
function psReadLineHistoryPath() {
|
|
322
|
-
const appData = process.env.APPDATA ??
|
|
323
|
-
return
|
|
435
|
+
const appData = process.env.APPDATA ?? join3(homedir2(), "AppData", "Roaming");
|
|
436
|
+
return join3(appData, "Microsoft", "Windows", "PowerShell", "PSReadLine", "ConsoleHost_history.txt");
|
|
324
437
|
}
|
|
325
438
|
function bashHistoryPath() {
|
|
326
|
-
return process.env.HISTFILE_BASH ??
|
|
439
|
+
return process.env.HISTFILE_BASH ?? join3(homedir2(), ".bash_history");
|
|
327
440
|
}
|
|
328
441
|
function zshHistoryPath() {
|
|
329
|
-
return process.env.HISTFILE ??
|
|
330
|
-
}
|
|
331
|
-
function globalWorkspaceDir() {
|
|
332
|
-
return join2(homedir(), ".nexusmem");
|
|
442
|
+
return process.env.HISTFILE ?? join3(homedir2(), ".zsh_history");
|
|
333
443
|
}
|
|
334
444
|
function hookLogPath() {
|
|
335
|
-
return
|
|
445
|
+
return join3(globalWorkspaceDir(), "shell-history.jsonl");
|
|
336
446
|
}
|
|
337
447
|
async function resolvePowerShellProfilePath(exe = "powershell") {
|
|
338
448
|
try {
|
|
@@ -486,6 +596,74 @@ async function runHookStatus(opts) {
|
|
|
486
596
|
import { relative } from "path";
|
|
487
597
|
import pc2 from "picocolors";
|
|
488
598
|
|
|
599
|
+
// src/config/registry.ts
|
|
600
|
+
import { existsSync as existsSync2 } from "fs";
|
|
601
|
+
import { mkdir as mkdir3, readFile as readFile3, rename, writeFile as writeFile3 } from "fs/promises";
|
|
602
|
+
import { join as join4 } from "path";
|
|
603
|
+
import { z as z2 } from "zod";
|
|
604
|
+
var ENTRY_SCHEMA = z2.object({
|
|
605
|
+
projectId: z2.string().min(1),
|
|
606
|
+
root: z2.string().min(1),
|
|
607
|
+
dbPath: z2.string().min(1),
|
|
608
|
+
originUrl: z2.string().nullable().default(null),
|
|
609
|
+
/** Epoch ms of the last `init`/`sync` that recorded this entry. */
|
|
610
|
+
lastSeenAt: z2.number().int().nonnegative()
|
|
611
|
+
});
|
|
612
|
+
var REGISTRY_SCHEMA = z2.object({
|
|
613
|
+
version: z2.literal(1),
|
|
614
|
+
projects: z2.array(ENTRY_SCHEMA).default([])
|
|
615
|
+
});
|
|
616
|
+
function registryPath() {
|
|
617
|
+
return join4(globalWorkspaceDir(), "projects.json");
|
|
618
|
+
}
|
|
619
|
+
async function readRegistry() {
|
|
620
|
+
let raw;
|
|
621
|
+
try {
|
|
622
|
+
raw = await readFile3(registryPath(), "utf8");
|
|
623
|
+
} catch {
|
|
624
|
+
return [];
|
|
625
|
+
}
|
|
626
|
+
try {
|
|
627
|
+
const parsed = REGISTRY_SCHEMA.safeParse(JSON.parse(raw));
|
|
628
|
+
if (!parsed.success) return [];
|
|
629
|
+
return [...parsed.data.projects].sort((a, b) => b.lastSeenAt - a.lastSeenAt);
|
|
630
|
+
} catch {
|
|
631
|
+
return [];
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
async function readLiveRegistry() {
|
|
635
|
+
const all = await readRegistry();
|
|
636
|
+
const entries = [];
|
|
637
|
+
const missing = [];
|
|
638
|
+
for (const entry of all) {
|
|
639
|
+
(existsSync2(entry.dbPath) ? entries : missing).push(entry);
|
|
640
|
+
}
|
|
641
|
+
return { entries, missing };
|
|
642
|
+
}
|
|
643
|
+
async function recordProject(input) {
|
|
644
|
+
const existing = await readRegistry();
|
|
645
|
+
const entry = { ...input, lastSeenAt: Date.now() };
|
|
646
|
+
const projects = [entry, ...existing.filter((e) => e.projectId !== input.projectId)];
|
|
647
|
+
await writeRegistry(projects);
|
|
648
|
+
return projects;
|
|
649
|
+
}
|
|
650
|
+
async function forgetProjects(projectIds) {
|
|
651
|
+
const existing = await readRegistry();
|
|
652
|
+
const drop = new Set(projectIds);
|
|
653
|
+
const kept = existing.filter((e) => !drop.has(e.projectId));
|
|
654
|
+
if (kept.length === existing.length) return 0;
|
|
655
|
+
await writeRegistry(kept);
|
|
656
|
+
return existing.length - kept.length;
|
|
657
|
+
}
|
|
658
|
+
async function writeRegistry(projects) {
|
|
659
|
+
const path = registryPath();
|
|
660
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
661
|
+
await mkdir3(globalWorkspaceDir(), { recursive: true });
|
|
662
|
+
await writeFile3(tmp, `${JSON.stringify({ version: 1, projects }, null, 2)}
|
|
663
|
+
`, "utf8");
|
|
664
|
+
await rename(tmp, path);
|
|
665
|
+
}
|
|
666
|
+
|
|
489
667
|
// src/core/ids.ts
|
|
490
668
|
import { createHash } from "crypto";
|
|
491
669
|
var KEY_SEP = "\0";
|
|
@@ -741,6 +919,20 @@ var MemoryStore = class _MemoryStore {
|
|
|
741
919
|
run(nodes);
|
|
742
920
|
return stats;
|
|
743
921
|
}
|
|
922
|
+
/**
|
|
923
|
+
* The stored `meta` blob for one node, or null if it has never been
|
|
924
|
+
* written. Used by the session summarizer to recognise work it has
|
|
925
|
+
* already done without re-reading the node's whole body.
|
|
926
|
+
*/
|
|
927
|
+
getNodeMeta(id) {
|
|
928
|
+
const row = this.db.prepare("SELECT meta FROM nodes WHERE id = ?").get(id);
|
|
929
|
+
if (!row) return null;
|
|
930
|
+
try {
|
|
931
|
+
return JSON.parse(row.meta);
|
|
932
|
+
} catch {
|
|
933
|
+
return null;
|
|
934
|
+
}
|
|
935
|
+
}
|
|
744
936
|
getSyncCursor(projectId, source) {
|
|
745
937
|
const row = this.db.prepare("SELECT cursor FROM sync_state WHERE project_id = ? AND source = ?").get(projectId, source);
|
|
746
938
|
return row?.cursor ?? null;
|
|
@@ -803,19 +995,57 @@ var MemoryStore = class _MemoryStore {
|
|
|
803
995
|
return this.db.prepare(`DELETE FROM nodes WHERE ${scope}`).run(params).changes;
|
|
804
996
|
})();
|
|
805
997
|
}
|
|
806
|
-
/**
|
|
807
|
-
|
|
998
|
+
/**
|
|
999
|
+
* Nodes for this project that have no embedding yet (new, or invalidated
|
|
1000
|
+
* by a content change).
|
|
1001
|
+
*
|
|
1002
|
+
* `afterRowid` makes paging monotonic: the pass walks rowids strictly
|
|
1003
|
+
* upward instead of re-reading "the first N still pending". That matters
|
|
1004
|
+
* because a node the provider *failed* on stays pending -- an offset-free
|
|
1005
|
+
* loop would fetch the same failures forever, which is exactly the shape
|
|
1006
|
+
* of an infinite sync.
|
|
1007
|
+
*/
|
|
1008
|
+
findNodesNeedingEmbedding(projectId, limit = 200, afterRowid = 0) {
|
|
808
1009
|
return this.db.prepare(
|
|
809
1010
|
`SELECT n.rowid AS rowid, n.id AS id, n.title AS title, n.body AS body
|
|
810
1011
|
FROM nodes n
|
|
811
1012
|
LEFT JOIN nodes_vec v ON v.rowid = n.rowid
|
|
812
|
-
WHERE n.project_id = ? AND v.rowid IS NULL
|
|
1013
|
+
WHERE n.project_id = ? AND v.rowid IS NULL AND n.rowid > ?
|
|
1014
|
+
ORDER BY n.rowid
|
|
813
1015
|
LIMIT ?`
|
|
814
|
-
).all(projectId, limit);
|
|
1016
|
+
).all(projectId, afterRowid, limit);
|
|
1017
|
+
}
|
|
1018
|
+
/** How many of this project's nodes still need a vector. For progress reporting. */
|
|
1019
|
+
countNodesNeedingEmbedding(projectId) {
|
|
1020
|
+
const row = this.db.prepare(
|
|
1021
|
+
`SELECT COUNT(*) AS n
|
|
1022
|
+
FROM nodes n
|
|
1023
|
+
LEFT JOIN nodes_vec v ON v.rowid = n.rowid
|
|
1024
|
+
WHERE n.project_id = ? AND v.rowid IS NULL`
|
|
1025
|
+
).get(projectId);
|
|
1026
|
+
return row.n;
|
|
815
1027
|
}
|
|
816
1028
|
upsertEmbedding(rowid, embedding) {
|
|
817
1029
|
this.db.prepare("INSERT OR REPLACE INTO nodes_vec (rowid, embedding) VALUES (?, ?)").run(BigInt(rowid), embedding);
|
|
818
1030
|
}
|
|
1031
|
+
/**
|
|
1032
|
+
* Drop every vector in this database, across all projects.
|
|
1033
|
+
*
|
|
1034
|
+
* Whole-database on purpose: `nodes_vec` is shared and holds no
|
|
1035
|
+
* provenance, so once the vectors in it stopped being comparable there is
|
|
1036
|
+
* no subset that is still trustworthy. Nodes are untouched, so the next
|
|
1037
|
+
* embedding pass simply rebuilds them.
|
|
1038
|
+
*/
|
|
1039
|
+
dropAllEmbeddings() {
|
|
1040
|
+
return this.db.prepare("DELETE FROM nodes_vec").run().changes;
|
|
1041
|
+
}
|
|
1042
|
+
getMeta(key) {
|
|
1043
|
+
const row = this.db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
|
|
1044
|
+
return row?.value ?? null;
|
|
1045
|
+
}
|
|
1046
|
+
setMeta(key, value) {
|
|
1047
|
+
this.db.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
|
|
1048
|
+
}
|
|
819
1049
|
/**
|
|
820
1050
|
* Nearest-neighbour search over the corpus.
|
|
821
1051
|
*
|
|
@@ -880,7 +1110,7 @@ var MemoryStore = class _MemoryStore {
|
|
|
880
1110
|
|
|
881
1111
|
// src/cli/commands/init.ts
|
|
882
1112
|
async function runInit(opts) {
|
|
883
|
-
const out = opts.out ?? ((
|
|
1113
|
+
const out = opts.out ?? ((chunk2) => void process.stdout.write(chunk2));
|
|
884
1114
|
const repo = await readRepoInfo(opts.cwd);
|
|
885
1115
|
const ws = resolveWorkspace(repo.root);
|
|
886
1116
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
@@ -905,6 +1135,7 @@ async function runInit(opts) {
|
|
|
905
1135
|
} finally {
|
|
906
1136
|
store.close();
|
|
907
1137
|
}
|
|
1138
|
+
await recordProject({ projectId, root: repo.root, dbPath: ws.dbPath, originUrl: repo.originUrl });
|
|
908
1139
|
const lines = [
|
|
909
1140
|
`${pc2.green("initialized")} ${ws.dir}`,
|
|
910
1141
|
` project ${pc2.cyan(projectId)}`,
|
|
@@ -939,10 +1170,68 @@ async function runInit(opts) {
|
|
|
939
1170
|
return 0;
|
|
940
1171
|
}
|
|
941
1172
|
|
|
1173
|
+
// src/cli/commands/projects.ts
|
|
1174
|
+
import pc3 from "picocolors";
|
|
1175
|
+
async function runProjects(opts) {
|
|
1176
|
+
const { entries, missing } = await readLiveRegistry();
|
|
1177
|
+
const rows = entries.map((entry) => {
|
|
1178
|
+
let nodes = null;
|
|
1179
|
+
try {
|
|
1180
|
+
const store = MemoryStore.open(entry.dbPath);
|
|
1181
|
+
try {
|
|
1182
|
+
nodes = store.stats(entry.projectId).total;
|
|
1183
|
+
} finally {
|
|
1184
|
+
store.close();
|
|
1185
|
+
}
|
|
1186
|
+
} catch {
|
|
1187
|
+
nodes = null;
|
|
1188
|
+
}
|
|
1189
|
+
return { ...entry, nodes };
|
|
1190
|
+
});
|
|
1191
|
+
if (opts.prune) {
|
|
1192
|
+
const removed = await forgetProjects(missing.map((entry) => entry.projectId));
|
|
1193
|
+
process.stderr.write(`${pc3.yellow("pruned")} ${removed} project(s) whose database is gone
|
|
1194
|
+
`);
|
|
1195
|
+
}
|
|
1196
|
+
if (opts.json) {
|
|
1197
|
+
process.stdout.write(`${JSON.stringify({ registry: registryPath(), projects: rows, missing }, null, 2)}
|
|
1198
|
+
`);
|
|
1199
|
+
return 0;
|
|
1200
|
+
}
|
|
1201
|
+
process.stderr.write(`${pc3.dim("registry")} ${registryPath()}
|
|
1202
|
+
|
|
1203
|
+
`);
|
|
1204
|
+
if (rows.length === 0) {
|
|
1205
|
+
process.stderr.write(`${pc3.yellow("no projects registered")} -- run ${pc3.bold("nexusmem sync")} in a repository
|
|
1206
|
+
`);
|
|
1207
|
+
return 0;
|
|
1208
|
+
}
|
|
1209
|
+
for (const row of rows) {
|
|
1210
|
+
const seen = new Date(row.lastSeenAt).toISOString().slice(0, 16).replace("T", " ");
|
|
1211
|
+
const count = row.nodes === null ? pc3.yellow("unreadable") : `${row.nodes} node(s)`;
|
|
1212
|
+
process.stdout.write(`${pc3.cyan(row.projectId.slice(0, 8))} ${row.root}
|
|
1213
|
+
${pc3.dim(`${count}, last seen ${seen}`)}
|
|
1214
|
+
`);
|
|
1215
|
+
}
|
|
1216
|
+
if (!opts.prune && missing.length > 0) {
|
|
1217
|
+
process.stderr.write(
|
|
1218
|
+
`
|
|
1219
|
+
${pc3.yellow(`${missing.length} registered project(s) have no database on disk`)} ${pc3.dim("-- run with --prune to forget them")}
|
|
1220
|
+
`
|
|
1221
|
+
);
|
|
1222
|
+
for (const entry of missing) process.stderr.write(` ${pc3.dim(entry.root)}
|
|
1223
|
+
`);
|
|
1224
|
+
}
|
|
1225
|
+
return 0;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
942
1228
|
// src/mcp/server.ts
|
|
943
1229
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
944
1230
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
945
|
-
import { z as
|
|
1231
|
+
import { z as z3 } from "zod";
|
|
1232
|
+
|
|
1233
|
+
// src/mcp/tools.ts
|
|
1234
|
+
import { basename as basename3 } from "path";
|
|
946
1235
|
|
|
947
1236
|
// src/core/text.ts
|
|
948
1237
|
function truncate(s, max) {
|
|
@@ -956,22 +1245,126 @@ function approxTokens(text) {
|
|
|
956
1245
|
var DEFAULT_SUMMARY_CHARS = 320;
|
|
957
1246
|
var NODE_OVERHEAD_TOKENS = 8;
|
|
958
1247
|
var CONVERSATION_ANSWER_MARKER = "\n\nA: ";
|
|
959
|
-
|
|
1248
|
+
var HUNK_BOUNDARY = "\n@@ ";
|
|
1249
|
+
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
1250
|
+
"the",
|
|
1251
|
+
"and",
|
|
1252
|
+
"for",
|
|
1253
|
+
"are",
|
|
1254
|
+
"was",
|
|
1255
|
+
"were",
|
|
1256
|
+
"that",
|
|
1257
|
+
"this",
|
|
1258
|
+
"with",
|
|
1259
|
+
"from",
|
|
1260
|
+
"into",
|
|
1261
|
+
"not",
|
|
1262
|
+
"but",
|
|
1263
|
+
"what",
|
|
1264
|
+
"why",
|
|
1265
|
+
"how",
|
|
1266
|
+
"when",
|
|
1267
|
+
"where",
|
|
1268
|
+
"which",
|
|
1269
|
+
"who",
|
|
1270
|
+
"does",
|
|
1271
|
+
"did",
|
|
1272
|
+
"has",
|
|
1273
|
+
"have",
|
|
1274
|
+
"had",
|
|
1275
|
+
"can",
|
|
1276
|
+
"could",
|
|
1277
|
+
"would",
|
|
1278
|
+
"should",
|
|
1279
|
+
"all",
|
|
1280
|
+
"any",
|
|
1281
|
+
"every",
|
|
1282
|
+
"each",
|
|
1283
|
+
"its",
|
|
1284
|
+
"our",
|
|
1285
|
+
"you",
|
|
1286
|
+
"your",
|
|
1287
|
+
"about"
|
|
1288
|
+
]);
|
|
1289
|
+
function queryTerms(query) {
|
|
1290
|
+
const words = query.toLowerCase().match(/[a-z0-9_]{3,}/g) ?? [];
|
|
1291
|
+
return [...new Set(words.filter((w) => !STOPWORDS.has(w)).map(singularize))];
|
|
1292
|
+
}
|
|
1293
|
+
function codeTokens(text) {
|
|
1294
|
+
const spaced = text.replace(/([a-z0-9])([A-Z])/g, "$1 $2").toLowerCase();
|
|
1295
|
+
return new Set((spaced.match(/[a-z0-9]{2,}/g) ?? []).map(singularize));
|
|
1296
|
+
}
|
|
1297
|
+
function singularize(word) {
|
|
1298
|
+
return word.length > 3 && word.endsWith("s") && !word.endsWith("ss") ? word.slice(0, -1) : word;
|
|
1299
|
+
}
|
|
1300
|
+
function pickHunk(patch, query) {
|
|
1301
|
+
const first = patch.indexOf("@@ ");
|
|
1302
|
+
if (first === -1) return patch;
|
|
1303
|
+
const hunks = [];
|
|
1304
|
+
let rest = patch.slice(first);
|
|
1305
|
+
for (; ; ) {
|
|
1306
|
+
const next = rest.indexOf(HUNK_BOUNDARY, 1);
|
|
1307
|
+
if (next === -1) {
|
|
1308
|
+
hunks.push(rest);
|
|
1309
|
+
break;
|
|
1310
|
+
}
|
|
1311
|
+
hunks.push(rest.slice(0, next));
|
|
1312
|
+
rest = rest.slice(next + 1);
|
|
1313
|
+
}
|
|
1314
|
+
const terms = queryTerms(query);
|
|
1315
|
+
if (terms.length === 0) return hunks[0] ?? patch;
|
|
1316
|
+
let best = hunks[0] ?? patch;
|
|
1317
|
+
let bestScore = 0;
|
|
1318
|
+
for (const hunk of hunks) {
|
|
1319
|
+
const tokens = codeTokens(hunk);
|
|
1320
|
+
const score = terms.reduce((n, term) => n + (tokens.has(term) ? 1 : 0), 0);
|
|
1321
|
+
if (score > bestScore) {
|
|
1322
|
+
best = hunk;
|
|
1323
|
+
bestScore = score;
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
return focusHunk(best, terms);
|
|
1327
|
+
}
|
|
1328
|
+
function focusHunk(hunk, terms) {
|
|
1329
|
+
const lines = hunk.split("\n");
|
|
1330
|
+
const header = lines[0] ?? "";
|
|
1331
|
+
const body = lines.slice(1);
|
|
1332
|
+
const isChange = (line) => line.startsWith("+") || line.startsWith("-");
|
|
1333
|
+
let idx = terms.length ? body.findIndex((line) => {
|
|
1334
|
+
if (!isChange(line)) return false;
|
|
1335
|
+
const tokens = codeTokens(line);
|
|
1336
|
+
return terms.some((term) => tokens.has(term));
|
|
1337
|
+
}) : -1;
|
|
1338
|
+
if (idx === -1) idx = body.findIndex(isChange);
|
|
1339
|
+
if (idx <= 1) return hunk;
|
|
1340
|
+
return [header, ...body.slice(idx - 1)].join("\n");
|
|
1341
|
+
}
|
|
1342
|
+
function summarize(hit, maxChars, query) {
|
|
960
1343
|
const answerIdx = hit.body.indexOf(CONVERSATION_ANSWER_MARKER);
|
|
961
1344
|
if (answerIdx !== -1) {
|
|
962
1345
|
const answer = hit.body.slice(answerIdx + CONVERSATION_ANSWER_MARKER.length).trim();
|
|
963
1346
|
if (answer) return truncate(answer, maxChars);
|
|
964
1347
|
}
|
|
1348
|
+
if (hit.kind === "code_diff") {
|
|
1349
|
+
const patchStart = hit.body.indexOf(HUNK_BOUNDARY);
|
|
1350
|
+
if (patchStart !== -1) {
|
|
1351
|
+
const head = hit.body.slice(0, patchStart).trim();
|
|
1352
|
+
const hunk = pickHunk(hit.body.slice(patchStart + 1), query);
|
|
1353
|
+
return truncate(`${head}
|
|
1354
|
+
${hunk}`, maxChars);
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
965
1357
|
const rest = hit.body.startsWith(hit.title) ? hit.body.slice(hit.title.length).trim() : hit.body;
|
|
966
1358
|
return truncate(rest || hit.title, maxChars);
|
|
967
1359
|
}
|
|
968
1360
|
function packContext(ranked, tokensBudget, opts = {}) {
|
|
969
1361
|
const summaryChars = opts.summaryChars ?? DEFAULT_SUMMARY_CHARS;
|
|
1362
|
+
const query = opts.query ?? "";
|
|
970
1363
|
const nodes = [];
|
|
971
1364
|
let tokensUsed = 0;
|
|
972
1365
|
let droppedForBudget = 0;
|
|
973
1366
|
for (const hit of ranked) {
|
|
974
|
-
const summary = summarize(hit, summaryChars);
|
|
1367
|
+
const summary = summarize(hit, summaryChars, query);
|
|
975
1368
|
const tokens = approxTokens(hit.title) + approxTokens(summary) + NODE_OVERHEAD_TOKENS;
|
|
976
1369
|
if (tokensUsed + tokens > tokensBudget) {
|
|
977
1370
|
droppedForBudget += 1;
|
|
@@ -985,7 +1378,8 @@ function packContext(ranked, tokensBudget, opts = {}) {
|
|
|
985
1378
|
signal: hit.signal,
|
|
986
1379
|
score: hit.score,
|
|
987
1380
|
summary,
|
|
988
|
-
tokens
|
|
1381
|
+
tokens,
|
|
1382
|
+
...hit.project ? { project: hit.project } : {}
|
|
989
1383
|
});
|
|
990
1384
|
tokensUsed += tokens;
|
|
991
1385
|
}
|
|
@@ -995,9 +1389,14 @@ function renderContextBlock(query, result) {
|
|
|
995
1389
|
if (result.nodes.length === 0) return `No remembered context matched "${query}".`;
|
|
996
1390
|
const lines = [`Relevant history for: ${query}`, ""];
|
|
997
1391
|
for (const node of result.nodes) {
|
|
998
|
-
|
|
1392
|
+
const project = node.project ? `[${node.project}] ` : "";
|
|
1393
|
+
lines.push(`- ${node.ts.slice(0, 10)} ${project}${node.title}`);
|
|
999
1394
|
if (node.summary && node.summary !== node.title) {
|
|
1000
|
-
|
|
1395
|
+
if (node.kind === "code_diff") {
|
|
1396
|
+
for (const line of node.summary.split("\n")) lines.push(` ${line}`);
|
|
1397
|
+
} else {
|
|
1398
|
+
lines.push(` ${node.summary.replace(/\n+/g, " ")}`);
|
|
1399
|
+
}
|
|
1001
1400
|
}
|
|
1002
1401
|
}
|
|
1003
1402
|
return lines.join("\n");
|
|
@@ -1032,8 +1431,10 @@ var RECENCY_FLOOR = 0.3;
|
|
|
1032
1431
|
var DEFAULT_HALF_LIFE_DAYS = 30;
|
|
1033
1432
|
var MS_PER_DAY = 864e5;
|
|
1034
1433
|
var MAX_PRIOR_OVERTURN = 2;
|
|
1035
|
-
var
|
|
1036
|
-
var
|
|
1434
|
+
var PRIOR_COUNT = 2;
|
|
1435
|
+
var PER_PRIOR_OVERTURN = MAX_PRIOR_OVERTURN ** (1 / PRIOR_COUNT);
|
|
1436
|
+
var SIGNAL_EXPONENT = Math.log(PER_PRIOR_OVERTURN) / Math.log(1 / SIGNAL_FLOOR);
|
|
1437
|
+
var RECENCY_EXPONENT = Math.log(PER_PRIOR_OVERTURN) / Math.log(1 / RECENCY_FLOOR);
|
|
1037
1438
|
function normalizeRelevance(hits) {
|
|
1038
1439
|
const costs = hits.map((h) => h.rank);
|
|
1039
1440
|
const min = Math.min(...costs);
|
|
@@ -1076,6 +1477,31 @@ function rankHits(hits, opts = {}) {
|
|
|
1076
1477
|
}
|
|
1077
1478
|
|
|
1078
1479
|
// src/retrieval/query-pipeline.ts
|
|
1480
|
+
async function runCrossProjectQuery(sources, query, opts) {
|
|
1481
|
+
const queryVector = opts.embeddingProvider ? await opts.embeddingProvider.embed(query) : null;
|
|
1482
|
+
const lists = [];
|
|
1483
|
+
const hits = [];
|
|
1484
|
+
const perProject = [];
|
|
1485
|
+
let bm25Count = 0;
|
|
1486
|
+
let vectorCount = 0;
|
|
1487
|
+
for (const source of sources) {
|
|
1488
|
+
const label = (hit) => ({ ...hit, project: source.label });
|
|
1489
|
+
const bm25Hits = source.store.search(source.projectId, query, opts.candidates).map(label);
|
|
1490
|
+
const vectorHits = queryVector ? source.store.vectorSearch(source.projectId, queryVector, opts.candidates) : [];
|
|
1491
|
+
bm25Count += bm25Hits.length;
|
|
1492
|
+
vectorCount += vectorHits.length;
|
|
1493
|
+
perProject.push({ label: source.label, bm25: bm25Hits.length, vector: vectorHits.length });
|
|
1494
|
+
if (bm25Hits.length > 0) lists.push(bm25Hits);
|
|
1495
|
+
if (vectorHits.length > 0) {
|
|
1496
|
+
lists.push(vectorHits.map((hit) => ({ ...hit, rank: 0, project: source.label })));
|
|
1497
|
+
}
|
|
1498
|
+
hits.push(...mergeSearchAndVectorHits(bm25Hits, vectorHits).map(label));
|
|
1499
|
+
}
|
|
1500
|
+
const relevanceScores = reciprocalRankFusion(lists);
|
|
1501
|
+
const ranked = rankHits(hits, { halfLifeDays: opts.halfLifeDays, relevanceScores });
|
|
1502
|
+
const packed = packContext(ranked, opts.budget, { query });
|
|
1503
|
+
return { bm25Count, vectorCount, hits, packed, perProject };
|
|
1504
|
+
}
|
|
1079
1505
|
async function runHybridQuery(store, projectId, query, opts) {
|
|
1080
1506
|
const bm25Hits = store.search(projectId, query, opts.candidates);
|
|
1081
1507
|
let vectorHits = [];
|
|
@@ -1086,42 +1512,102 @@ async function runHybridQuery(store, projectId, query, opts) {
|
|
|
1086
1512
|
const hits = vectorHits.length > 0 ? mergeSearchAndVectorHits(bm25Hits, vectorHits) : bm25Hits;
|
|
1087
1513
|
const relevanceScores = vectorHits.length > 0 ? reciprocalRankFusion([bm25Hits, vectorHits]) : void 0;
|
|
1088
1514
|
const ranked = rankHits(hits, { halfLifeDays: opts.halfLifeDays, relevanceScores });
|
|
1089
|
-
const packed = packContext(ranked, opts.budget);
|
|
1515
|
+
const packed = packContext(ranked, opts.budget, { query });
|
|
1090
1516
|
return { bm25Count: bm25Hits.length, vectorCount: vectorHits.length, hits, packed };
|
|
1091
1517
|
}
|
|
1092
1518
|
|
|
1519
|
+
// src/retrieval/sources.ts
|
|
1520
|
+
import { basename } from "path";
|
|
1521
|
+
async function openAllProjectSources(current) {
|
|
1522
|
+
const { entries, missing } = await readLiveRegistry();
|
|
1523
|
+
const wanted = [current];
|
|
1524
|
+
for (const entry of entries) {
|
|
1525
|
+
if (entry.projectId === current.projectId) continue;
|
|
1526
|
+
wanted.push({ projectId: entry.projectId, root: entry.root, dbPath: entry.dbPath });
|
|
1527
|
+
}
|
|
1528
|
+
const labels = labelProjects(wanted);
|
|
1529
|
+
const sources = [];
|
|
1530
|
+
const unreadable = [];
|
|
1531
|
+
wanted.forEach((project, index) => {
|
|
1532
|
+
try {
|
|
1533
|
+
sources.push({
|
|
1534
|
+
store: MemoryStore.open(project.dbPath),
|
|
1535
|
+
projectId: project.projectId,
|
|
1536
|
+
label: labels[index] ?? project.projectId.slice(0, 8)
|
|
1537
|
+
});
|
|
1538
|
+
} catch (err) {
|
|
1539
|
+
const entry = entries.find((e) => e.projectId === project.projectId);
|
|
1540
|
+
if (entry) unreadable.push({ entry, reason: err.message });
|
|
1541
|
+
}
|
|
1542
|
+
});
|
|
1543
|
+
return {
|
|
1544
|
+
sources,
|
|
1545
|
+
missing,
|
|
1546
|
+
unreadable,
|
|
1547
|
+
close: () => {
|
|
1548
|
+
for (const source of sources) source.store.close();
|
|
1549
|
+
}
|
|
1550
|
+
};
|
|
1551
|
+
}
|
|
1552
|
+
function labelProjects(projects) {
|
|
1553
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1554
|
+
for (const project of projects) {
|
|
1555
|
+
const name = basename(project.root) || project.root;
|
|
1556
|
+
counts.set(name, (counts.get(name) ?? 0) + 1);
|
|
1557
|
+
}
|
|
1558
|
+
return projects.map((project) => {
|
|
1559
|
+
const name = basename(project.root) || project.root;
|
|
1560
|
+
return (counts.get(name) ?? 0) > 1 ? `${name}#${project.projectId.slice(0, 6)}` : name;
|
|
1561
|
+
});
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1093
1564
|
// src/vector/embed.ts
|
|
1094
|
-
var
|
|
1565
|
+
var DEFAULT_BASE_URL2 = "http://127.0.0.1:11434";
|
|
1095
1566
|
var DEFAULT_MODEL = "nomic-embed-text";
|
|
1096
1567
|
var DEFAULT_DIMENSION = 768;
|
|
1097
|
-
var
|
|
1568
|
+
var DEFAULT_TIMEOUT_MS2 = 1e4;
|
|
1569
|
+
var DEFAULT_MAX_TIMEOUT_MS = 12e4;
|
|
1570
|
+
var EMBED_PATH = "/api/embed";
|
|
1098
1571
|
var OllamaEmbeddingProvider = class {
|
|
1099
1572
|
dimension;
|
|
1573
|
+
identity;
|
|
1100
1574
|
baseUrl;
|
|
1101
1575
|
model;
|
|
1102
1576
|
timeoutMs;
|
|
1577
|
+
maxTimeoutMs;
|
|
1103
1578
|
constructor(opts = {}) {
|
|
1104
|
-
this.baseUrl = opts.baseUrl ??
|
|
1579
|
+
this.baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL2;
|
|
1105
1580
|
this.model = opts.model ?? DEFAULT_MODEL;
|
|
1106
1581
|
this.dimension = opts.dimension ?? DEFAULT_DIMENSION;
|
|
1107
|
-
this.timeoutMs = opts.timeoutMs ??
|
|
1582
|
+
this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS2;
|
|
1583
|
+
this.maxTimeoutMs = opts.maxTimeoutMs ?? DEFAULT_MAX_TIMEOUT_MS;
|
|
1584
|
+
this.identity = `ollama${EMBED_PATH}:${this.model}:${this.dimension}`;
|
|
1108
1585
|
}
|
|
1109
1586
|
async embed(text) {
|
|
1587
|
+
const [only] = await this.embedBatch([text]);
|
|
1588
|
+
return only ?? null;
|
|
1589
|
+
}
|
|
1590
|
+
async embedBatch(texts) {
|
|
1591
|
+
if (texts.length === 0) return [];
|
|
1110
1592
|
const controller = new AbortController();
|
|
1111
|
-
const
|
|
1593
|
+
const budget = Math.min(this.timeoutMs * texts.length, this.maxTimeoutMs);
|
|
1594
|
+
const timeout = setTimeout(() => controller.abort(), budget);
|
|
1112
1595
|
try {
|
|
1113
|
-
const res = await fetch(`${this.baseUrl}
|
|
1596
|
+
const res = await fetch(`${this.baseUrl}${EMBED_PATH}`, {
|
|
1114
1597
|
method: "POST",
|
|
1115
1598
|
headers: { "content-type": "application/json" },
|
|
1116
|
-
body: JSON.stringify({ model: this.model,
|
|
1599
|
+
body: JSON.stringify({ model: this.model, input: texts }),
|
|
1117
1600
|
signal: controller.signal
|
|
1118
1601
|
});
|
|
1119
|
-
if (!res.ok) return null;
|
|
1602
|
+
if (!res.ok) return texts.map(() => null);
|
|
1120
1603
|
const data = await res.json();
|
|
1121
|
-
if (!Array.isArray(data.
|
|
1122
|
-
|
|
1604
|
+
if (!Array.isArray(data.embeddings)) return texts.map(() => null);
|
|
1605
|
+
if (data.embeddings.length !== texts.length) return texts.map(() => null);
|
|
1606
|
+
return data.embeddings.map(
|
|
1607
|
+
(row) => Array.isArray(row) && row.length === this.dimension ? new Float32Array(row) : null
|
|
1608
|
+
);
|
|
1123
1609
|
} catch {
|
|
1124
|
-
return null;
|
|
1610
|
+
return texts.map(() => null);
|
|
1125
1611
|
} finally {
|
|
1126
1612
|
clearTimeout(timeout);
|
|
1127
1613
|
}
|
|
@@ -1129,7 +1615,7 @@ var OllamaEmbeddingProvider = class {
|
|
|
1129
1615
|
};
|
|
1130
1616
|
|
|
1131
1617
|
// src/cli/commands/sync.ts
|
|
1132
|
-
import
|
|
1618
|
+
import pc4 from "picocolors";
|
|
1133
1619
|
|
|
1134
1620
|
// src/conversation/chunk.ts
|
|
1135
1621
|
var HEADING_LINE = /^#{1,6}\s+(.+)$/;
|
|
@@ -1169,21 +1655,31 @@ function chunkAssistantText(text, maxChars) {
|
|
|
1169
1655
|
|
|
1170
1656
|
// src/conversation/redact.ts
|
|
1171
1657
|
var RULES = [
|
|
1172
|
-
{
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1658
|
+
{
|
|
1659
|
+
name: "private-key-block",
|
|
1660
|
+
pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
|
|
1661
|
+
highConfidence: true
|
|
1662
|
+
},
|
|
1663
|
+
{ name: "aws-access-key", pattern: /\bAKIA[0-9A-Z]{16}\b/g, highConfidence: true },
|
|
1664
|
+
{ name: "github-token", pattern: /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g, highConfidence: true },
|
|
1665
|
+
{ name: "slack-token", pattern: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g, highConfidence: true },
|
|
1666
|
+
{
|
|
1667
|
+
name: "jwt",
|
|
1668
|
+
pattern: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
|
|
1669
|
+
highConfidence: true
|
|
1670
|
+
},
|
|
1177
1671
|
// key/token/secret/password = "value" or : value, in code, JSON, env-file or prose form.
|
|
1178
1672
|
{
|
|
1179
1673
|
name: "key-value-secret",
|
|
1180
|
-
pattern: /\b((?:api[_-]?key|secret|password|passwd|token|access[_-]?key)s?)\s*[:=]\s*['"]?[A-Za-z0-9_\-./+=]{8,}['"]?/gi
|
|
1674
|
+
pattern: /\b((?:api[_-]?key|secret|password|passwd|token|access[_-]?key)s?)\s*[:=]\s*['"]?[A-Za-z0-9_\-./+=]{8,}['"]?/gi,
|
|
1675
|
+
highConfidence: false
|
|
1181
1676
|
}
|
|
1182
1677
|
];
|
|
1183
|
-
function redact(text) {
|
|
1678
|
+
function redact(text, profile = "all") {
|
|
1184
1679
|
let redactedCount = 0;
|
|
1185
1680
|
let out = text;
|
|
1186
1681
|
for (const rule of RULES) {
|
|
1682
|
+
if (profile === "high-confidence" && !rule.highConfidence) continue;
|
|
1187
1683
|
out = out.replace(rule.pattern, (_match, ...rest) => {
|
|
1188
1684
|
redactedCount += 1;
|
|
1189
1685
|
const key = typeof rest[0] === "string" ? rest[0] : null;
|
|
@@ -1198,8 +1694,8 @@ var DEFAULT_MAX_BODY_CHARS = 2500;
|
|
|
1198
1694
|
var DEFAULT_MAX_CHUNK_CHARS = 900;
|
|
1199
1695
|
var MAX_TITLE_CHARS = 200;
|
|
1200
1696
|
var MAX_FILES_PER_NODE = 20;
|
|
1201
|
-
var EXPLANATION_MARKERS = /\b(because|the reason|design decision|trade-?off|instead of|rationale|so that)\b
|
|
1202
|
-
var TRIVIAL_ACK = /^(ok|okay|thanks
|
|
1697
|
+
var EXPLANATION_MARKERS = /\b(because|the reason|design decision|trade-?off|instead of|rationale|so that)\b/i;
|
|
1698
|
+
var TRIVIAL_ACK = /^(ok|okay|thanks?|got it|sounds good|👍|done)\.?!?$/i;
|
|
1203
1699
|
function scoreConversationTurn(userText, replyText) {
|
|
1204
1700
|
const text = `${userText}
|
|
1205
1701
|
${replyText}`;
|
|
@@ -1241,25 +1737,25 @@ function toMemoryNodes(turn, projectId, opts = {}) {
|
|
|
1241
1737
|
const assistantRedacted = redact(turn.assistantText);
|
|
1242
1738
|
const chunks = chunkAssistantText(assistantRedacted.text, maxChunk);
|
|
1243
1739
|
if (chunks.length === 0) return [];
|
|
1244
|
-
return chunks.map((
|
|
1245
|
-
const body = [`Q: ${userRedacted.text}`, "", `A: ${
|
|
1740
|
+
return chunks.map((chunk2, index) => {
|
|
1741
|
+
const body = [`Q: ${userRedacted.text}`, "", `A: ${chunk2.text}`].join("\n");
|
|
1246
1742
|
return {
|
|
1247
1743
|
id: makeNodeId(projectId, "conversation_turn", `${turn.naturalKey}:${index}`),
|
|
1248
1744
|
kind: "conversation_turn",
|
|
1249
1745
|
projectId,
|
|
1250
1746
|
ts: turn.ts,
|
|
1251
1747
|
source: `conversation:${turn.source}`,
|
|
1252
|
-
title: chunkTitle(userFirstLine,
|
|
1748
|
+
title: chunkTitle(userFirstLine, chunk2.heading, index, chunks.length),
|
|
1253
1749
|
body: truncate(body, maxBody),
|
|
1254
1750
|
files: extractMentionedFiles(`${userRedacted.text}
|
|
1255
|
-
${
|
|
1256
|
-
signal: scoreConversationTurn(userRedacted.text,
|
|
1751
|
+
${chunk2.text}`),
|
|
1752
|
+
signal: scoreConversationTurn(userRedacted.text, chunk2.text),
|
|
1257
1753
|
meta: {
|
|
1258
1754
|
cwd: turn.cwd,
|
|
1259
1755
|
source: turn.source,
|
|
1260
1756
|
chunkIndex: index,
|
|
1261
1757
|
chunkCount: chunks.length,
|
|
1262
|
-
heading:
|
|
1758
|
+
heading: chunk2.heading,
|
|
1263
1759
|
// Redaction runs once over the whole reply before chunking (so a
|
|
1264
1760
|
// secret can never straddle a chunk boundary and slip through) --
|
|
1265
1761
|
// this is the turn's total, repeated on every chunk it produced,
|
|
@@ -1273,63 +1769,6 @@ function collectConversationTurns(turns, projectId, opts = {}) {
|
|
|
1273
1769
|
return turns.filter((t) => t.assistantText.length > 0).flatMap((turn) => toMemoryNodes(turn, projectId, opts));
|
|
1274
1770
|
}
|
|
1275
1771
|
|
|
1276
|
-
// src/collectors/docs.ts
|
|
1277
|
-
var DEFAULT_MAX_BODY_CHARS2 = 2e3;
|
|
1278
|
-
var DEFAULT_MAX_CHUNK_CHARS2 = 1200;
|
|
1279
|
-
var MAX_TITLE_CHARS2 = 200;
|
|
1280
|
-
var EXPLANATION_MARKERS2 = /\b(because|the reason|design decision|trade-?off|instead of|rationale|why)\b/i;
|
|
1281
|
-
function scoreDocSection(path, heading, text) {
|
|
1282
|
-
let score = 0.45;
|
|
1283
|
-
if (EXPLANATION_MARKERS2.test(text)) score += 0.25;
|
|
1284
|
-
if (/(^|\/)readme\.md$/i.test(path)) score += 0.1;
|
|
1285
|
-
if (heading === null) score -= 0.1;
|
|
1286
|
-
if (text.length < 80) score -= 0.15;
|
|
1287
|
-
return Number(Math.min(1, Math.max(0.05, score)).toFixed(3));
|
|
1288
|
-
}
|
|
1289
|
-
function slugify(heading, index) {
|
|
1290
|
-
if (heading === null) return `_preamble-${index}`;
|
|
1291
|
-
const slug = heading.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
1292
|
-
return slug || `_section-${index}`;
|
|
1293
|
-
}
|
|
1294
|
-
function sectionTitle(path, heading, index, count) {
|
|
1295
|
-
if (heading) return truncate(`${path} \u2014 ${heading}`, MAX_TITLE_CHARS2);
|
|
1296
|
-
if (count > 1) return truncate(`${path} (part ${index + 1}/${count})`, MAX_TITLE_CHARS2);
|
|
1297
|
-
return truncate(path, MAX_TITLE_CHARS2);
|
|
1298
|
-
}
|
|
1299
|
-
function toMemoryNodes2(file, projectId, opts = {}) {
|
|
1300
|
-
const maxBody = opts.maxBodyChars ?? DEFAULT_MAX_BODY_CHARS2;
|
|
1301
|
-
const maxChunk = opts.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS2;
|
|
1302
|
-
const chunks = chunkAssistantText(file.content, maxChunk);
|
|
1303
|
-
if (chunks.length === 0) return [];
|
|
1304
|
-
const seenSlugs = /* @__PURE__ */ new Map();
|
|
1305
|
-
return chunks.map((chunk, index) => {
|
|
1306
|
-
const baseSlug = slugify(chunk.heading, index);
|
|
1307
|
-
const occurrence = seenSlugs.get(baseSlug) ?? 0;
|
|
1308
|
-
seenSlugs.set(baseSlug, occurrence + 1);
|
|
1309
|
-
const naturalKey = occurrence === 0 ? `${file.path}#${baseSlug}` : `${file.path}#${baseSlug}:${occurrence}`;
|
|
1310
|
-
return {
|
|
1311
|
-
id: makeNodeId(projectId, "doc_section", naturalKey),
|
|
1312
|
-
kind: "doc_section",
|
|
1313
|
-
projectId,
|
|
1314
|
-
ts: file.ts,
|
|
1315
|
-
source: "docs",
|
|
1316
|
-
title: sectionTitle(file.path, chunk.heading, index, chunks.length),
|
|
1317
|
-
body: truncate(chunk.text, maxBody),
|
|
1318
|
-
files: [{ path: file.path, insertions: null, deletions: null, binary: false }],
|
|
1319
|
-
signal: scoreDocSection(file.path, chunk.heading, chunk.text),
|
|
1320
|
-
meta: {
|
|
1321
|
-
path: file.path,
|
|
1322
|
-
heading: chunk.heading,
|
|
1323
|
-
chunkIndex: index,
|
|
1324
|
-
chunkCount: chunks.length
|
|
1325
|
-
}
|
|
1326
|
-
};
|
|
1327
|
-
});
|
|
1328
|
-
}
|
|
1329
|
-
function collectDocFiles(files, projectId, opts = {}) {
|
|
1330
|
-
return files.flatMap((file) => toMemoryNodes2(file, projectId, opts));
|
|
1331
|
-
}
|
|
1332
|
-
|
|
1333
1772
|
// src/git/parse.ts
|
|
1334
1773
|
var RECORD_SEP = "";
|
|
1335
1774
|
var UNIT_SEP = "";
|
|
@@ -1439,28 +1878,45 @@ function unquoteGitPath(p) {
|
|
|
1439
1878
|
return Buffer.from(bytes).toString("utf8");
|
|
1440
1879
|
}
|
|
1441
1880
|
|
|
1442
|
-
// src/git/
|
|
1881
|
+
// src/git/diff.ts
|
|
1882
|
+
var GIT_DIFF_LOG_FORMAT = "%x1e%H%x1f%h%x1f%aI%x1f%s%x1f";
|
|
1883
|
+
var FIELD_COUNT2 = 5;
|
|
1443
1884
|
var EMPTY_HISTORY = /does not have any commits yet|unknown revision|bad revision|ambiguous argument/i;
|
|
1444
|
-
function
|
|
1445
|
-
const { rev = "HEAD", afterCommit, since, maxCount,
|
|
1446
|
-
const args = [
|
|
1447
|
-
|
|
1885
|
+
function buildDiffLogArgs(opts = {}) {
|
|
1886
|
+
const { rev = "HEAD", afterCommit, since, maxCount, contextLines = 3, paths } = opts;
|
|
1887
|
+
const args = [
|
|
1888
|
+
"log",
|
|
1889
|
+
`--format=${GIT_DIFF_LOG_FORMAT}`,
|
|
1890
|
+
"--patch",
|
|
1891
|
+
"--no-color",
|
|
1892
|
+
// A merge produces no patch at all unless `-m`/`--cc` is passed, and the
|
|
1893
|
+
// combined diff those print is a different format from the one parsed
|
|
1894
|
+
// here. Excluding merges up front keeps the parser honest about what it
|
|
1895
|
+
// supports; the merge itself is still remembered as a `git_commit` node.
|
|
1896
|
+
"--no-merges",
|
|
1897
|
+
"--find-renames",
|
|
1898
|
+
// Never run a user-configured textconv filter: it would execute an
|
|
1899
|
+
// arbitrary program from repo config during a sync, and its output is not
|
|
1900
|
+
// the diff we claim to be indexing.
|
|
1901
|
+
"--no-textconv",
|
|
1902
|
+
`--unified=${Math.max(0, contextLines)}`
|
|
1903
|
+
];
|
|
1448
1904
|
if (maxCount && maxCount > 0) args.push(`--max-count=${maxCount}`);
|
|
1449
1905
|
if (since) args.push(`--since=${since}`);
|
|
1450
1906
|
args.push(afterCommit ? `${afterCommit}..${rev}` : rev);
|
|
1451
1907
|
if (paths?.length) args.push("--", ...paths);
|
|
1452
1908
|
return args;
|
|
1453
1909
|
}
|
|
1454
|
-
async function*
|
|
1455
|
-
const args =
|
|
1910
|
+
async function* readCommitDiffs(cwd, opts = {}) {
|
|
1911
|
+
const args = buildDiffLogArgs(opts);
|
|
1456
1912
|
let buffer = "";
|
|
1457
1913
|
try {
|
|
1458
|
-
for await (const
|
|
1459
|
-
buffer +=
|
|
1914
|
+
for await (const chunk2 of gitStream(cwd, args)) {
|
|
1915
|
+
buffer += chunk2;
|
|
1460
1916
|
const { records, rest } = splitRecords(buffer);
|
|
1461
1917
|
buffer = rest;
|
|
1462
1918
|
for (const record of records) {
|
|
1463
|
-
const commit =
|
|
1919
|
+
const commit = parseCommitDiffRecord(record);
|
|
1464
1920
|
if (commit) yield commit;
|
|
1465
1921
|
}
|
|
1466
1922
|
}
|
|
@@ -1469,55 +1925,212 @@ async function* readCommits(cwd, opts = {}) {
|
|
|
1469
1925
|
throw err;
|
|
1470
1926
|
}
|
|
1471
1927
|
for (const record of splitRecords(buffer, true).records) {
|
|
1472
|
-
const commit =
|
|
1928
|
+
const commit = parseCommitDiffRecord(record);
|
|
1473
1929
|
if (commit) yield commit;
|
|
1474
1930
|
}
|
|
1475
1931
|
}
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1932
|
+
function parseCommitDiffRecord(record) {
|
|
1933
|
+
let payload = record;
|
|
1934
|
+
while (payload.startsWith(RECORD_SEP)) payload = payload.slice(RECORD_SEP.length);
|
|
1935
|
+
const parts = payload.split(UNIT_SEP);
|
|
1936
|
+
if (parts.length < FIELD_COUNT2) return null;
|
|
1937
|
+
const sha = parts[0] ?? "";
|
|
1938
|
+
if (!/^[0-9a-f]{7,64}$/i.test(sha)) return null;
|
|
1939
|
+
const patchBlock = parts[parts.length - 1] ?? "";
|
|
1940
|
+
const subject = parts.slice(3, parts.length - 1).join(UNIT_SEP).trim();
|
|
1484
1941
|
return {
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1942
|
+
sha,
|
|
1943
|
+
shortSha: parts[1] ?? "",
|
|
1944
|
+
authoredAt: parts[2] ?? "",
|
|
1945
|
+
subject,
|
|
1946
|
+
files: parseFileDiffs(patchBlock)
|
|
1489
1947
|
};
|
|
1490
1948
|
}
|
|
1491
|
-
var
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1949
|
+
var FILE_HEADER = "diff --git ";
|
|
1950
|
+
var HUNK_HEADER = /^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@/;
|
|
1951
|
+
function parseFileDiffs(block) {
|
|
1952
|
+
const files = [];
|
|
1953
|
+
let section = null;
|
|
1954
|
+
const flush = () => {
|
|
1955
|
+
if (!section) return;
|
|
1956
|
+
const parsed = parseFileSection(section);
|
|
1957
|
+
if (parsed) files.push(parsed);
|
|
1958
|
+
section = null;
|
|
1959
|
+
};
|
|
1960
|
+
for (const raw of block.split("\n")) {
|
|
1961
|
+
const line = raw.endsWith("\r") ? raw.slice(0, -1) : raw;
|
|
1962
|
+
if (line.startsWith(FILE_HEADER)) {
|
|
1963
|
+
flush();
|
|
1964
|
+
section = [line];
|
|
1965
|
+
} else if (section) {
|
|
1966
|
+
section.push(line);
|
|
1967
|
+
}
|
|
1968
|
+
}
|
|
1969
|
+
flush();
|
|
1970
|
+
return files;
|
|
1971
|
+
}
|
|
1972
|
+
function parseFileSection(lines) {
|
|
1973
|
+
let status = "modified";
|
|
1974
|
+
let binary = false;
|
|
1975
|
+
let fromPath = null;
|
|
1976
|
+
let toPath = null;
|
|
1977
|
+
let renamedFrom = null;
|
|
1978
|
+
let hunkStart = -1;
|
|
1979
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
1980
|
+
const line = lines[i] ?? "";
|
|
1981
|
+
if (HUNK_HEADER.test(line)) {
|
|
1982
|
+
hunkStart = i;
|
|
1983
|
+
break;
|
|
1984
|
+
}
|
|
1985
|
+
if (line.startsWith("new file mode")) status = "added";
|
|
1986
|
+
else if (line.startsWith("deleted file mode")) status = "deleted";
|
|
1987
|
+
else if (line.startsWith("rename from ")) renamedFrom = unquoteGitPath(line.slice("rename from ".length));
|
|
1988
|
+
else if (line.startsWith("rename to ")) status = "renamed";
|
|
1989
|
+
else if (line.startsWith("copy from ")) renamedFrom = unquoteGitPath(line.slice("copy from ".length));
|
|
1990
|
+
else if (line.startsWith("Binary files ") || line.startsWith("GIT binary patch")) binary = true;
|
|
1991
|
+
else if (line.startsWith("--- ")) fromPath = stripDiffPathPrefix(line.slice(4));
|
|
1992
|
+
else if (line.startsWith("+++ ")) toPath = stripDiffPathPrefix(line.slice(4));
|
|
1993
|
+
}
|
|
1994
|
+
const header = parseDiffGitPaths(lines[0] ?? "");
|
|
1995
|
+
const path = toPath ?? header.b ?? fromPath ?? header.a;
|
|
1996
|
+
if (!path) return null;
|
|
1997
|
+
const previousPath = renamedFrom ?? (status === "renamed" ? fromPath ?? header.a ?? void 0 : void 0);
|
|
1998
|
+
if (binary || hunkStart === -1) {
|
|
1999
|
+
return {
|
|
2000
|
+
path,
|
|
2001
|
+
...previousPath && previousPath !== path ? { previousPath } : {},
|
|
2002
|
+
status,
|
|
2003
|
+
binary,
|
|
2004
|
+
insertions: 0,
|
|
2005
|
+
deletions: 0,
|
|
2006
|
+
hunkCount: 0,
|
|
2007
|
+
patch: ""
|
|
2008
|
+
};
|
|
2009
|
+
}
|
|
2010
|
+
const hunkLines = lines.slice(hunkStart);
|
|
2011
|
+
let insertions = 0;
|
|
2012
|
+
let deletions = 0;
|
|
2013
|
+
let hunkCount = 0;
|
|
2014
|
+
for (const line of hunkLines) {
|
|
2015
|
+
if (HUNK_HEADER.test(line)) hunkCount += 1;
|
|
2016
|
+
else if (line.startsWith("+")) insertions += 1;
|
|
2017
|
+
else if (line.startsWith("-")) deletions += 1;
|
|
2018
|
+
}
|
|
2019
|
+
return {
|
|
2020
|
+
path,
|
|
2021
|
+
...previousPath && previousPath !== path ? { previousPath } : {},
|
|
2022
|
+
status,
|
|
2023
|
+
binary: false,
|
|
2024
|
+
insertions,
|
|
2025
|
+
deletions,
|
|
2026
|
+
hunkCount,
|
|
2027
|
+
patch: hunkLines.join("\n").trimEnd()
|
|
2028
|
+
};
|
|
2029
|
+
}
|
|
2030
|
+
function stripDiffPathPrefix(raw) {
|
|
2031
|
+
const cleaned = unquoteGitPath(raw.trim());
|
|
2032
|
+
if (cleaned === "/dev/null") return null;
|
|
2033
|
+
return cleaned.replace(/^[ab]\//, "");
|
|
2034
|
+
}
|
|
2035
|
+
function parseDiffGitPaths(headerLine) {
|
|
2036
|
+
const rest = headerLine.slice(FILE_HEADER.length);
|
|
2037
|
+
if (rest.startsWith('"')) {
|
|
2038
|
+
const match = /^("(?:[^"\\]|\\.)*")\s+("(?:[^"\\]|\\.)*"|\S+)$/.exec(rest);
|
|
2039
|
+
if (match) {
|
|
2040
|
+
return { a: stripDiffPathPrefix(match[1] ?? ""), b: stripDiffPathPrefix(match[2] ?? "") };
|
|
2041
|
+
}
|
|
2042
|
+
}
|
|
2043
|
+
const quotedSecond = /^(\S+)\s+("(?:[^"\\]|\\.)*")$/.exec(rest);
|
|
2044
|
+
if (quotedSecond) {
|
|
2045
|
+
return { a: stripDiffPathPrefix(quotedSecond[1] ?? ""), b: stripDiffPathPrefix(quotedSecond[2] ?? "") };
|
|
2046
|
+
}
|
|
2047
|
+
const split = / b\//.exec(rest);
|
|
2048
|
+
if (!split || split.index <= 0) return { a: null, b: null };
|
|
2049
|
+
return {
|
|
2050
|
+
a: stripDiffPathPrefix(rest.slice(0, split.index)),
|
|
2051
|
+
b: stripDiffPathPrefix(rest.slice(split.index + 1))
|
|
2052
|
+
};
|
|
2053
|
+
}
|
|
2054
|
+
|
|
2055
|
+
// src/git/log.ts
|
|
2056
|
+
var EMPTY_HISTORY2 = /does not have any commits yet|unknown revision|bad revision|ambiguous argument/i;
|
|
2057
|
+
function buildLogArgs(opts = {}) {
|
|
2058
|
+
const { rev = "HEAD", afterCommit, since, maxCount, includeMerges = true, paths } = opts;
|
|
2059
|
+
const args = ["log", `--format=${GIT_LOG_FORMAT}`, "--numstat", "--no-color"];
|
|
2060
|
+
if (!includeMerges) args.push("--no-merges");
|
|
2061
|
+
if (maxCount && maxCount > 0) args.push(`--max-count=${maxCount}`);
|
|
2062
|
+
if (since) args.push(`--since=${since}`);
|
|
2063
|
+
args.push(afterCommit ? `${afterCommit}..${rev}` : rev);
|
|
2064
|
+
if (paths?.length) args.push("--", ...paths);
|
|
2065
|
+
return args;
|
|
2066
|
+
}
|
|
2067
|
+
async function* readCommits(cwd, opts = {}) {
|
|
2068
|
+
const args = buildLogArgs(opts);
|
|
2069
|
+
let buffer = "";
|
|
2070
|
+
try {
|
|
2071
|
+
for await (const chunk2 of gitStream(cwd, args)) {
|
|
2072
|
+
buffer += chunk2;
|
|
2073
|
+
const { records, rest } = splitRecords(buffer);
|
|
2074
|
+
buffer = rest;
|
|
2075
|
+
for (const record of records) {
|
|
2076
|
+
const commit = parseCommitRecord(record);
|
|
2077
|
+
if (commit) yield commit;
|
|
2078
|
+
}
|
|
2079
|
+
}
|
|
2080
|
+
} catch (err) {
|
|
2081
|
+
if (err instanceof GitError && EMPTY_HISTORY2.test(err.stderr)) return;
|
|
2082
|
+
throw err;
|
|
2083
|
+
}
|
|
2084
|
+
for (const record of splitRecords(buffer, true).records) {
|
|
2085
|
+
const commit = parseCommitRecord(record);
|
|
2086
|
+
if (commit) yield commit;
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
|
|
2090
|
+
// src/collectors/git-commits.ts
|
|
2091
|
+
var DEFAULTS = { maxFilesPerNode: 40, maxBodyChars: 4e3 };
|
|
2092
|
+
var MAX_TITLE_CHARS2 = 200;
|
|
2093
|
+
var CONVENTIONAL = /^([a-z]+)(?:\(([^)]*)\))?(!)?:\s*(.+)$/i;
|
|
2094
|
+
function parseConventionalHeader(subject) {
|
|
2095
|
+
const m = CONVENTIONAL.exec(subject.trim());
|
|
2096
|
+
if (!m) return { type: null, scope: null, breaking: false, description: subject.trim() };
|
|
2097
|
+
return {
|
|
2098
|
+
type: (m[1] ?? "").toLowerCase(),
|
|
2099
|
+
scope: m[2] ?? null,
|
|
2100
|
+
breaking: Boolean(m[3]),
|
|
2101
|
+
description: (m[4] ?? "").trim()
|
|
2102
|
+
};
|
|
2103
|
+
}
|
|
2104
|
+
var TYPE_WEIGHTS = {
|
|
2105
|
+
fix: 0.8,
|
|
2106
|
+
feat: 0.8,
|
|
2107
|
+
revert: 0.78,
|
|
2108
|
+
perf: 0.7,
|
|
2109
|
+
refactor: 0.68,
|
|
2110
|
+
security: 0.85,
|
|
2111
|
+
test: 0.45,
|
|
2112
|
+
docs: 0.35,
|
|
2113
|
+
build: 0.32,
|
|
2114
|
+
ci: 0.28,
|
|
2115
|
+
chore: 0.25,
|
|
2116
|
+
style: 0.2
|
|
2117
|
+
};
|
|
2118
|
+
var AUTOMATED = /^(merge (branch|pull request|remote)|bump |update dependenc|\[bot\]|revert "merge)/i;
|
|
2119
|
+
function scoreCommit(commit) {
|
|
2120
|
+
const header = parseConventionalHeader(commit.subject);
|
|
2121
|
+
let score = header.type ? TYPE_WEIGHTS[header.type] ?? 0.5 : 0.5;
|
|
2122
|
+
if (header.breaking) score += 0.12;
|
|
2123
|
+
if (commit.messageBody.length > 120) score += 0.1;
|
|
2124
|
+
if (commit.isMerge) score = Math.min(score, 0.3);
|
|
2125
|
+
if (AUTOMATED.test(commit.subject)) score -= 0.15;
|
|
2126
|
+
const churn = commit.files.reduce((n, f) => n + (f.insertions ?? 0) + (f.deletions ?? 0), 0);
|
|
2127
|
+
if (commit.files.length > 100 || churn > 5e3) score *= 0.75;
|
|
2128
|
+
if (commit.files.length <= 1 && churn <= 3) score -= 0.05;
|
|
2129
|
+
return Number(Math.min(1, Math.max(0.05, score)).toFixed(3));
|
|
2130
|
+
}
|
|
2131
|
+
function byChurnDesc(a, b) {
|
|
2132
|
+
const ca = (a.insertions ?? 0) + (a.deletions ?? 0);
|
|
2133
|
+
const cb = (b.insertions ?? 0) + (b.deletions ?? 0);
|
|
1521
2134
|
return cb - ca;
|
|
1522
2135
|
}
|
|
1523
2136
|
function renderFileLine(f) {
|
|
@@ -1545,7 +2158,7 @@ function toMemoryNode(commit, projectId, opts = {}) {
|
|
|
1545
2158
|
projectId,
|
|
1546
2159
|
ts: commit.authoredAt,
|
|
1547
2160
|
source: "git",
|
|
1548
|
-
title: truncate(commit.subject || `(no subject) ${commit.shortSha}`,
|
|
2161
|
+
title: truncate(commit.subject || `(no subject) ${commit.shortSha}`, MAX_TITLE_CHARS2),
|
|
1549
2162
|
body: truncate(bodyParts.join("\n"), maxBody),
|
|
1550
2163
|
files: keptFiles,
|
|
1551
2164
|
signal: scoreCommit(commit),
|
|
@@ -1572,9 +2185,364 @@ async function* collectGitCommits(cwd, projectId, opts = {}) {
|
|
|
1572
2185
|
}
|
|
1573
2186
|
}
|
|
1574
2187
|
|
|
1575
|
-
// src/collectors/
|
|
1576
|
-
var
|
|
2188
|
+
// src/collectors/diffs.ts
|
|
2189
|
+
var DIFF_SOURCE = "diff";
|
|
2190
|
+
var DEFAULTS2 = { maxFilesPerCommit: 20, maxBodyChars: 2e3 };
|
|
2191
|
+
var MAX_TITLE_CHARS3 = 200;
|
|
2192
|
+
var GENERATED_PATHS = [
|
|
2193
|
+
/(^|\/)(node_modules|dist|build|out|coverage|vendor|third_party)\//,
|
|
2194
|
+
/(^|\/)(package-lock\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|composer\.lock|Cargo\.lock|poetry\.lock|Gemfile\.lock|go\.sum)$/,
|
|
2195
|
+
/\.(min\.js|min\.css|map|snap)$/
|
|
2196
|
+
];
|
|
2197
|
+
function isGeneratedPath(path) {
|
|
2198
|
+
return GENERATED_PATHS.some((re) => re.test(path));
|
|
2199
|
+
}
|
|
2200
|
+
var TEST_PATHS = /(^|\/)(tests?|__tests__|spec|e2e)\/|\.(test|spec)\.[cm]?[jt]sx?$/;
|
|
2201
|
+
function scoreFileDiff(subject, file) {
|
|
2202
|
+
const header = parseConventionalHeader(subject);
|
|
2203
|
+
let score = header.type ? TYPE_WEIGHTS[header.type] ?? 0.5 : 0.5;
|
|
2204
|
+
if (header.breaking) score += 0.1;
|
|
2205
|
+
if (TEST_PATHS.test(file.path)) score -= 0.1;
|
|
2206
|
+
if (file.status === "added") score += 0.05;
|
|
2207
|
+
if (file.status === "deleted") score -= 0.1;
|
|
2208
|
+
const churn = file.insertions + file.deletions;
|
|
2209
|
+
if (churn <= 2) score -= 0.05;
|
|
2210
|
+
if (churn > 400) score *= 0.75;
|
|
2211
|
+
return Number(Math.min(1, Math.max(0.05, score)).toFixed(3));
|
|
2212
|
+
}
|
|
2213
|
+
function byChurnDesc2(a, b) {
|
|
2214
|
+
return b.insertions + b.deletions - (a.insertions + a.deletions);
|
|
2215
|
+
}
|
|
2216
|
+
function indexableFiles(files) {
|
|
2217
|
+
return files.filter((f) => !f.binary && f.hunkCount > 0 && !isGeneratedPath(f.path));
|
|
2218
|
+
}
|
|
2219
|
+
var STATUS_LABEL = {
|
|
2220
|
+
added: "added",
|
|
2221
|
+
deleted: "deleted",
|
|
2222
|
+
renamed: "renamed",
|
|
2223
|
+
modified: "modified"
|
|
2224
|
+
};
|
|
2225
|
+
function fileTitle(shortSha, subject, file) {
|
|
2226
|
+
return truncate(`${file.path} @ ${shortSha} \u2014 ${subject}`, MAX_TITLE_CHARS3);
|
|
2227
|
+
}
|
|
2228
|
+
function toMemoryNodes2(commit, projectId, opts = {}) {
|
|
2229
|
+
const maxFiles = opts.maxFilesPerCommit ?? DEFAULTS2.maxFilesPerCommit;
|
|
2230
|
+
const maxBody = opts.maxBodyChars ?? DEFAULTS2.maxBodyChars;
|
|
2231
|
+
const kept = indexableFiles(commit.files).sort(byChurnDesc2).slice(0, maxFiles);
|
|
2232
|
+
return kept.map((file) => {
|
|
2233
|
+
const churn = `+${file.insertions}/-${file.deletions}`;
|
|
2234
|
+
const renamePart = file.previousPath ? `, renamed from ${file.previousPath}` : "";
|
|
2235
|
+
const head = [
|
|
2236
|
+
`${commit.subject} (${commit.shortSha})`,
|
|
2237
|
+
`${STATUS_LABEL[file.status]} ${file.path} (${churn}, ${file.hunkCount} hunk${file.hunkCount === 1 ? "" : "s"}${renamePart})`,
|
|
2238
|
+
""
|
|
2239
|
+
].join("\n");
|
|
2240
|
+
const { text: patch } = redact(file.patch, "high-confidence");
|
|
2241
|
+
return {
|
|
2242
|
+
id: makeNodeId(projectId, "code_diff", `${commit.sha}:${file.path}`),
|
|
2243
|
+
kind: "code_diff",
|
|
2244
|
+
projectId,
|
|
2245
|
+
ts: commit.authoredAt,
|
|
2246
|
+
source: DIFF_SOURCE,
|
|
2247
|
+
title: fileTitle(commit.shortSha, commit.subject, file),
|
|
2248
|
+
body: truncate(head + patch, maxBody),
|
|
2249
|
+
files: [
|
|
2250
|
+
{
|
|
2251
|
+
path: file.path,
|
|
2252
|
+
...file.previousPath ? { previousPath: file.previousPath } : {},
|
|
2253
|
+
insertions: file.insertions,
|
|
2254
|
+
deletions: file.deletions,
|
|
2255
|
+
binary: false
|
|
2256
|
+
}
|
|
2257
|
+
],
|
|
2258
|
+
signal: scoreFileDiff(commit.subject, file),
|
|
2259
|
+
meta: {
|
|
2260
|
+
sha: commit.sha,
|
|
2261
|
+
shortSha: commit.shortSha,
|
|
2262
|
+
path: file.path,
|
|
2263
|
+
status: file.status,
|
|
2264
|
+
hunkCount: file.hunkCount,
|
|
2265
|
+
insertions: file.insertions,
|
|
2266
|
+
deletions: file.deletions,
|
|
2267
|
+
subject: commit.subject
|
|
2268
|
+
}
|
|
2269
|
+
};
|
|
2270
|
+
});
|
|
2271
|
+
}
|
|
2272
|
+
async function* collectCommitDiffs(cwd, projectId, opts = {}) {
|
|
2273
|
+
for await (const commit of readCommitDiffs(cwd, opts)) {
|
|
2274
|
+
for (const node of toMemoryNodes2(commit, projectId, opts)) yield node;
|
|
2275
|
+
}
|
|
2276
|
+
}
|
|
2277
|
+
|
|
2278
|
+
// src/collectors/docs.ts
|
|
2279
|
+
var DEFAULT_MAX_BODY_CHARS2 = 2e3;
|
|
2280
|
+
var DEFAULT_MAX_CHUNK_CHARS2 = 1200;
|
|
1577
2281
|
var MAX_TITLE_CHARS4 = 200;
|
|
2282
|
+
var EXPLANATION_MARKERS2 = /\b(because|the reason|design decision|trade-?off|instead of|rationale|why)\b/i;
|
|
2283
|
+
function scoreDocSection(path, heading, text) {
|
|
2284
|
+
let score = 0.45;
|
|
2285
|
+
if (EXPLANATION_MARKERS2.test(text)) score += 0.25;
|
|
2286
|
+
if (/(^|\/)readme\.md$/i.test(path)) score += 0.1;
|
|
2287
|
+
if (heading === null) score -= 0.1;
|
|
2288
|
+
if (text.length < 80) score -= 0.15;
|
|
2289
|
+
return Number(Math.min(1, Math.max(0.05, score)).toFixed(3));
|
|
2290
|
+
}
|
|
2291
|
+
function slugify(heading, index) {
|
|
2292
|
+
if (heading === null) return `_preamble-${index}`;
|
|
2293
|
+
const slug = heading.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
2294
|
+
return slug || `_section-${index}`;
|
|
2295
|
+
}
|
|
2296
|
+
function sectionTitle(path, heading, index, count) {
|
|
2297
|
+
if (heading) return truncate(`${path} \u2014 ${heading}`, MAX_TITLE_CHARS4);
|
|
2298
|
+
if (count > 1) return truncate(`${path} (part ${index + 1}/${count})`, MAX_TITLE_CHARS4);
|
|
2299
|
+
return truncate(path, MAX_TITLE_CHARS4);
|
|
2300
|
+
}
|
|
2301
|
+
function toMemoryNodes3(file, projectId, opts = {}) {
|
|
2302
|
+
const maxBody = opts.maxBodyChars ?? DEFAULT_MAX_BODY_CHARS2;
|
|
2303
|
+
const maxChunk = opts.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS2;
|
|
2304
|
+
const chunks = chunkAssistantText(file.content, maxChunk);
|
|
2305
|
+
if (chunks.length === 0) return [];
|
|
2306
|
+
const seenSlugs = /* @__PURE__ */ new Map();
|
|
2307
|
+
return chunks.map((chunk2, index) => {
|
|
2308
|
+
const baseSlug = slugify(chunk2.heading, index);
|
|
2309
|
+
const occurrence = seenSlugs.get(baseSlug) ?? 0;
|
|
2310
|
+
seenSlugs.set(baseSlug, occurrence + 1);
|
|
2311
|
+
const naturalKey = occurrence === 0 ? `${file.path}#${baseSlug}` : `${file.path}#${baseSlug}:${occurrence}`;
|
|
2312
|
+
return {
|
|
2313
|
+
id: makeNodeId(projectId, "doc_section", naturalKey),
|
|
2314
|
+
kind: "doc_section",
|
|
2315
|
+
projectId,
|
|
2316
|
+
ts: file.ts,
|
|
2317
|
+
source: "docs",
|
|
2318
|
+
title: sectionTitle(file.path, chunk2.heading, index, chunks.length),
|
|
2319
|
+
body: truncate(chunk2.text, maxBody),
|
|
2320
|
+
files: [{ path: file.path, insertions: null, deletions: null, binary: false }],
|
|
2321
|
+
signal: scoreDocSection(file.path, chunk2.heading, chunk2.text),
|
|
2322
|
+
meta: {
|
|
2323
|
+
path: file.path,
|
|
2324
|
+
heading: chunk2.heading,
|
|
2325
|
+
chunkIndex: index,
|
|
2326
|
+
chunkCount: chunks.length
|
|
2327
|
+
}
|
|
2328
|
+
};
|
|
2329
|
+
});
|
|
2330
|
+
}
|
|
2331
|
+
function collectDocFiles(files, projectId, opts = {}) {
|
|
2332
|
+
return files.flatMap((file) => toMemoryNodes3(file, projectId, opts));
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
// src/slm/summarize.ts
|
|
2336
|
+
var MAX_USER_CHARS = 400;
|
|
2337
|
+
var MAX_REPLY_CHARS = 700;
|
|
2338
|
+
var DEFAULT_MAX_PROMPT_CHARS = 12e3;
|
|
2339
|
+
function groupTurnsIntoSessions(turns) {
|
|
2340
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2341
|
+
for (const turn of turns) {
|
|
2342
|
+
const existing = groups.get(turn.sessionKey);
|
|
2343
|
+
if (existing) {
|
|
2344
|
+
existing.turns.push(turn);
|
|
2345
|
+
if (turn.ts < existing.startedAt) existing.startedAt = turn.ts;
|
|
2346
|
+
if (turn.ts > existing.endedAt) existing.endedAt = turn.ts;
|
|
2347
|
+
existing.cwd ??= turn.cwd;
|
|
2348
|
+
continue;
|
|
2349
|
+
}
|
|
2350
|
+
groups.set(turn.sessionKey, {
|
|
2351
|
+
sessionKey: turn.sessionKey,
|
|
2352
|
+
source: turn.source,
|
|
2353
|
+
cwd: turn.cwd,
|
|
2354
|
+
turns: [turn],
|
|
2355
|
+
startedAt: turn.ts,
|
|
2356
|
+
endedAt: turn.ts
|
|
2357
|
+
});
|
|
2358
|
+
}
|
|
2359
|
+
for (const group of groups.values()) {
|
|
2360
|
+
group.turns.sort((a, b) => a.ts.localeCompare(b.ts));
|
|
2361
|
+
}
|
|
2362
|
+
return [...groups.values()].sort((a, b) => a.startedAt.localeCompare(b.startedAt));
|
|
2363
|
+
}
|
|
2364
|
+
function selectSettledSessions(sessions, settleMinutes, now = /* @__PURE__ */ new Date()) {
|
|
2365
|
+
const cutoff = now.getTime() - settleMinutes * 6e4;
|
|
2366
|
+
return sessions.filter((s) => {
|
|
2367
|
+
const ended = Date.parse(s.endedAt);
|
|
2368
|
+
return Number.isNaN(ended) || ended <= cutoff;
|
|
2369
|
+
});
|
|
2370
|
+
}
|
|
2371
|
+
var SESSION_INSTRUCTIONS = `You are summarizing one working session between a developer and an AI coding assistant, so that a future assistant can recall what happened without re-reading the transcript.
|
|
2372
|
+
|
|
2373
|
+
Your first line MUST begin with "TITLE: " and nothing else. Start your reply with those six characters.
|
|
2374
|
+
|
|
2375
|
+
Write your answer in exactly this shape:
|
|
2376
|
+
TITLE: <under 15 words, naming the specific work, e.g. "Raised the Node floor to 22 after a CI failure">
|
|
2377
|
+
- <a decision that was made, and why>
|
|
2378
|
+
- <a problem that was diagnosed, and its cause>
|
|
2379
|
+
- <what was left unfinished or explicitly deferred>
|
|
2380
|
+
|
|
2381
|
+
Rules:
|
|
2382
|
+
- Answer in English, whatever language the transcript is in. This summary is stored in a keyword index that cannot segment languages without spaces between words.
|
|
2383
|
+
- The title must name this session specifically. "Summary of the session" or "Project update" are wrong.
|
|
2384
|
+
- Prefer reasons over narration. "Chose X over Y because Z" is worth more than "worked on X".
|
|
2385
|
+
- Only state what the transcript supports. Do not guess or invent.
|
|
2386
|
+
- Between 3 and 6 bullets. No preamble, no closing remarks, no markdown bold.`;
|
|
2387
|
+
function buildSessionPrompt(session, maxPromptChars = DEFAULT_MAX_PROMPT_CHARS) {
|
|
2388
|
+
const rendered = session.turns.map((turn) => {
|
|
2389
|
+
const user = redact(turn.userText).text;
|
|
2390
|
+
const reply = redact(turn.assistantText).text;
|
|
2391
|
+
return {
|
|
2392
|
+
ts: turn.ts,
|
|
2393
|
+
text: [`[${turn.ts}] developer: ${truncate(user, MAX_USER_CHARS)}`, `assistant: ${truncate(reply, MAX_REPLY_CHARS)}`].join(
|
|
2394
|
+
"\n"
|
|
2395
|
+
),
|
|
2396
|
+
signal: scoreConversationTurn(user, reply)
|
|
2397
|
+
};
|
|
2398
|
+
});
|
|
2399
|
+
const budget = maxPromptChars - SESSION_INSTRUCTIONS.length;
|
|
2400
|
+
const kept = [];
|
|
2401
|
+
let used = 0;
|
|
2402
|
+
for (const turn of [...rendered].sort((a, b) => b.signal - a.signal)) {
|
|
2403
|
+
if (used + turn.text.length > budget) continue;
|
|
2404
|
+
kept.push(turn);
|
|
2405
|
+
used += turn.text.length;
|
|
2406
|
+
}
|
|
2407
|
+
kept.sort((a, b) => a.ts.localeCompare(b.ts));
|
|
2408
|
+
const body = kept.map((t) => t.text).join("\n\n");
|
|
2409
|
+
const prompt = `${SESSION_INSTRUCTIONS}
|
|
2410
|
+
|
|
2411
|
+
---
|
|
2412
|
+
|
|
2413
|
+
${body}
|
|
2414
|
+
|
|
2415
|
+
---
|
|
2416
|
+
|
|
2417
|
+
Summary:`;
|
|
2418
|
+
return { prompt, hash: sha256Hex(prompt), includedTurns: kept.length };
|
|
2419
|
+
}
|
|
2420
|
+
function sessionFallbackTitle(session) {
|
|
2421
|
+
const opening = session.turns[0];
|
|
2422
|
+
if (!opening) return "Working session";
|
|
2423
|
+
const line = redact(opening.userText).text.split(/\r?\n/)[0]?.trim();
|
|
2424
|
+
return line && line.length > 0 ? line : "Working session";
|
|
2425
|
+
}
|
|
2426
|
+
var MAX_TITLE_CHARS5 = 200;
|
|
2427
|
+
var GENERIC_TITLE = /^(a |the )?(session |conversation |project |work )?(summary|update|overview|recap|status)\b/i;
|
|
2428
|
+
function cleanTitle(line) {
|
|
2429
|
+
return line.replace(/^[-*#>\s]+/, "").replace(/\*+/g, "").replace(/\s*:\s*$/, "").trim();
|
|
2430
|
+
}
|
|
2431
|
+
function parseSummary(raw, fallbackTitle) {
|
|
2432
|
+
const text = redact(raw).text.trim();
|
|
2433
|
+
if (text.length === 0) return null;
|
|
2434
|
+
const lines = text.split(/\r?\n/);
|
|
2435
|
+
const firstIndex = lines.findIndex((line) => line.trim().length > 0);
|
|
2436
|
+
if (firstIndex === -1) return null;
|
|
2437
|
+
const first = lines[firstIndex].trim();
|
|
2438
|
+
const labelled = /^TITLE:\s*(.+)$/i.exec(first);
|
|
2439
|
+
const fallback = truncate(cleanTitle(fallbackTitle) || "Working session", MAX_TITLE_CHARS5);
|
|
2440
|
+
if (!labelled) return { title: fallback, body: text };
|
|
2441
|
+
const candidate = cleanTitle(labelled[1]);
|
|
2442
|
+
const rest = lines.slice(firstIndex + 1).join("\n").trim();
|
|
2443
|
+
return {
|
|
2444
|
+
title: candidate.length > 0 && !GENERIC_TITLE.test(candidate) ? truncate(candidate, MAX_TITLE_CHARS5) : fallback,
|
|
2445
|
+
// A model that emitted only a title still gets a usable node: the title
|
|
2446
|
+
// doubles as the body rather than storing an empty one.
|
|
2447
|
+
body: rest.length > 0 ? rest : candidate
|
|
2448
|
+
};
|
|
2449
|
+
}
|
|
2450
|
+
|
|
2451
|
+
// src/collectors/sessions.ts
|
|
2452
|
+
var SESSION_SOURCE_PREFIX = "session";
|
|
2453
|
+
var DEFAULT_SETTLE_MINUTES = 30;
|
|
2454
|
+
var DEFAULT_MAX_BODY_CHARS3 = 2500;
|
|
2455
|
+
var DEFAULT_MAX_SESSIONS = 10;
|
|
2456
|
+
function scoreSession(turnCount) {
|
|
2457
|
+
const score = 0.6 + Math.min(0.25, turnCount / 100);
|
|
2458
|
+
return Number(score.toFixed(3));
|
|
2459
|
+
}
|
|
2460
|
+
function toNode(session, projectId, summary, meta, maxBodyChars) {
|
|
2461
|
+
const header = `Session of ${session.startedAt.slice(0, 10)} \u2014 ${session.turns.length} exchange(s)`;
|
|
2462
|
+
const body = `${header}
|
|
2463
|
+
|
|
2464
|
+
${summary.body}`;
|
|
2465
|
+
return {
|
|
2466
|
+
id: makeNodeId(projectId, "session_summary", session.sessionKey),
|
|
2467
|
+
kind: "session_summary",
|
|
2468
|
+
projectId,
|
|
2469
|
+
// The session's end, not its start: a summary describes a finished piece
|
|
2470
|
+
// of work, and recency ranking should treat it as being as fresh as the
|
|
2471
|
+
// last thing that happened in it.
|
|
2472
|
+
ts: session.endedAt,
|
|
2473
|
+
source: `${SESSION_SOURCE_PREFIX}:${session.source}`,
|
|
2474
|
+
title: truncate(summary.title, 200),
|
|
2475
|
+
body: truncate(body, maxBodyChars),
|
|
2476
|
+
// Drawn from the whole session rather than only the summary: the model
|
|
2477
|
+
// mentions few paths, but `node_files` is what lets "why is this file
|
|
2478
|
+
// like this" reach the session that explains it.
|
|
2479
|
+
files: extractMentionedFiles(session.turns.map((t) => `${t.userText}
|
|
2480
|
+
${t.assistantText}`).join("\n")),
|
|
2481
|
+
signal: scoreSession(session.turns.length),
|
|
2482
|
+
meta: {
|
|
2483
|
+
sessionKey: session.sessionKey,
|
|
2484
|
+
source: session.source,
|
|
2485
|
+
turnCount: session.turns.length,
|
|
2486
|
+
summarizedTurns: meta.includedTurns,
|
|
2487
|
+
startedAt: session.startedAt,
|
|
2488
|
+
endedAt: session.endedAt,
|
|
2489
|
+
cwd: session.cwd,
|
|
2490
|
+
model: meta.model,
|
|
2491
|
+
contentHash: meta.hash
|
|
2492
|
+
}
|
|
2493
|
+
};
|
|
2494
|
+
}
|
|
2495
|
+
async function collectSessionSummaries(turns, projectId, provider, opts = {}) {
|
|
2496
|
+
const maxBodyChars = opts.maxBodyChars ?? DEFAULT_MAX_BODY_CHARS3;
|
|
2497
|
+
const maxSessions = opts.maxSessions ?? DEFAULT_MAX_SESSIONS;
|
|
2498
|
+
const all = groupTurnsIntoSessions(turns);
|
|
2499
|
+
const settled = selectSettledSessions(all, opts.settleMinutes ?? DEFAULT_SETTLE_MINUTES, opts.now);
|
|
2500
|
+
const nodes = [];
|
|
2501
|
+
let cached = 0;
|
|
2502
|
+
let failed = 0;
|
|
2503
|
+
let attempted = 0;
|
|
2504
|
+
const candidates = [...settled].sort((a, b) => b.endedAt.localeCompare(a.endedAt));
|
|
2505
|
+
const pending = [];
|
|
2506
|
+
for (const session of candidates) {
|
|
2507
|
+
const prompt = buildSessionPrompt(session, opts.maxPromptChars);
|
|
2508
|
+
if (opts.knownHash?.(session.sessionKey) === prompt.hash) {
|
|
2509
|
+
cached += 1;
|
|
2510
|
+
continue;
|
|
2511
|
+
}
|
|
2512
|
+
pending.push({ session, prompt });
|
|
2513
|
+
}
|
|
2514
|
+
for (const { session, prompt } of pending.slice(0, maxSessions)) {
|
|
2515
|
+
const raw = await provider.complete(prompt.prompt);
|
|
2516
|
+
attempted += 1;
|
|
2517
|
+
const summary = raw === null ? null : parseSummary(raw, sessionFallbackTitle(session));
|
|
2518
|
+
if (!summary) {
|
|
2519
|
+
failed += 1;
|
|
2520
|
+
continue;
|
|
2521
|
+
}
|
|
2522
|
+
nodes.push(
|
|
2523
|
+
toNode(
|
|
2524
|
+
session,
|
|
2525
|
+
projectId,
|
|
2526
|
+
summary,
|
|
2527
|
+
{ hash: prompt.hash, model: provider.identity, includedTurns: prompt.includedTurns },
|
|
2528
|
+
maxBodyChars
|
|
2529
|
+
)
|
|
2530
|
+
);
|
|
2531
|
+
opts.onProgress?.(nodes.length, Math.min(pending.length, maxSessions));
|
|
2532
|
+
}
|
|
2533
|
+
return {
|
|
2534
|
+
nodes,
|
|
2535
|
+
cached,
|
|
2536
|
+
unsettled: all.length - settled.length,
|
|
2537
|
+
deferred: Math.max(0, pending.length - maxSessions),
|
|
2538
|
+
failed,
|
|
2539
|
+
providerUnavailable: attempted > 0 && nodes.length === 0
|
|
2540
|
+
};
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
// src/collectors/shell-history.ts
|
|
2544
|
+
var DEFAULT_MAX_BODY_CHARS4 = 1e3;
|
|
2545
|
+
var MAX_TITLE_CHARS6 = 200;
|
|
1578
2546
|
var NOISE = /^(cd|ls|dir|pwd|clear|cls|exit|history|whoami|date|type|cat|more|less|ll|la)\b/i;
|
|
1579
2547
|
var BUILD_TEST = /^(npm|pnpm|yarn)\s+(run\s+)?(test|build|lint|typecheck|tsc)\b|^(pytest|go\s+test|cargo\s+(test|build)|mvn\s+test|gradle\s+test|dotnet\s+(test|build))\b/i;
|
|
1580
2548
|
var INSTALL = /^(npm|pnpm|yarn)\s+(install|add|remove|uninstall|ci)\b|^pip\s+install\b|^(cargo\s+add|go\s+get|composer\s+require)\b/i;
|
|
@@ -1607,7 +2575,7 @@ function renderBody(entry, maxChars) {
|
|
|
1607
2575
|
return truncate(parts.join("\n"), maxChars);
|
|
1608
2576
|
}
|
|
1609
2577
|
function toMemoryNode2(entry, projectId, opts = {}) {
|
|
1610
|
-
const maxBody = opts.maxBodyChars ??
|
|
2578
|
+
const maxBody = opts.maxBodyChars ?? DEFAULT_MAX_BODY_CHARS4;
|
|
1611
2579
|
const titleLine = entry.command.split(/\r?\n/)[0] ?? entry.command;
|
|
1612
2580
|
return {
|
|
1613
2581
|
id: makeNodeId(projectId, "shell_command", entry.naturalKey),
|
|
@@ -1615,7 +2583,7 @@ function toMemoryNode2(entry, projectId, opts = {}) {
|
|
|
1615
2583
|
projectId,
|
|
1616
2584
|
ts: entry.ts,
|
|
1617
2585
|
source: `shell:${entry.shell}`,
|
|
1618
|
-
title: truncate(titleLine,
|
|
2586
|
+
title: truncate(titleLine, MAX_TITLE_CHARS6),
|
|
1619
2587
|
body: renderBody(entry, maxBody),
|
|
1620
2588
|
files: [],
|
|
1621
2589
|
signal: scoreShellCommand(entry),
|
|
@@ -1634,24 +2602,25 @@ function collectShellHistory(entries, projectId, opts = {}) {
|
|
|
1634
2602
|
}
|
|
1635
2603
|
|
|
1636
2604
|
// src/conversation/claude-code-reader.ts
|
|
1637
|
-
import { readFile as
|
|
2605
|
+
import { readFile as readFile4 } from "fs/promises";
|
|
2606
|
+
import { basename as basename2 } from "path";
|
|
1638
2607
|
|
|
1639
2608
|
// src/conversation/paths.ts
|
|
1640
|
-
import { existsSync as
|
|
2609
|
+
import { existsSync as existsSync3 } from "fs";
|
|
1641
2610
|
import { readdir } from "fs/promises";
|
|
1642
|
-
import { homedir as
|
|
1643
|
-
import { join as
|
|
2611
|
+
import { homedir as homedir3 } from "os";
|
|
2612
|
+
import { join as join5 } from "path";
|
|
1644
2613
|
function claudeProjectSlug(repoRoot) {
|
|
1645
2614
|
return repoRoot.replace(/[\\/:]/g, "-");
|
|
1646
2615
|
}
|
|
1647
2616
|
function claudeProjectTranscriptDir(repoRoot) {
|
|
1648
|
-
return
|
|
2617
|
+
return join5(homedir3(), ".claude", "projects", claudeProjectSlug(repoRoot));
|
|
1649
2618
|
}
|
|
1650
2619
|
async function listTranscriptFiles(repoRoot) {
|
|
1651
2620
|
const dir = claudeProjectTranscriptDir(repoRoot);
|
|
1652
|
-
if (!
|
|
2621
|
+
if (!existsSync3(dir)) return [];
|
|
1653
2622
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
1654
|
-
return entries.filter((e) => e.isFile() && e.name.endsWith(".jsonl")).map((e) =>
|
|
2623
|
+
return entries.filter((e) => e.isFile() && e.name.endsWith(".jsonl")).map((e) => join5(dir, e.name));
|
|
1655
2624
|
}
|
|
1656
2625
|
|
|
1657
2626
|
// src/conversation/claude-code-reader.ts
|
|
@@ -1681,6 +2650,7 @@ function extractAssistantText(line) {
|
|
|
1681
2650
|
}
|
|
1682
2651
|
function parseClaudeCodeTranscript(raw, opts = {}) {
|
|
1683
2652
|
const source = opts.source ?? "claude-code";
|
|
2653
|
+
const sessionKey = `${source}:${opts.sessionId ?? "unknown"}`;
|
|
1684
2654
|
const turns = [];
|
|
1685
2655
|
let current = null;
|
|
1686
2656
|
const flush = () => {
|
|
@@ -1692,7 +2662,8 @@ function parseClaudeCodeTranscript(raw, opts = {}) {
|
|
|
1692
2662
|
assistantText,
|
|
1693
2663
|
ts: current.ts,
|
|
1694
2664
|
cwd: current.cwd,
|
|
1695
|
-
source
|
|
2665
|
+
source,
|
|
2666
|
+
sessionKey
|
|
1696
2667
|
});
|
|
1697
2668
|
current = null;
|
|
1698
2669
|
};
|
|
@@ -1724,15 +2695,15 @@ async function collectClaudeCodeTranscripts(repoRoot) {
|
|
|
1724
2695
|
const files = await listTranscriptFiles(repoRoot);
|
|
1725
2696
|
const turns = [];
|
|
1726
2697
|
for (const file of files) {
|
|
1727
|
-
const raw = await
|
|
1728
|
-
turns.push(...parseClaudeCodeTranscript(raw));
|
|
2698
|
+
const raw = await readFile4(file, "utf8");
|
|
2699
|
+
turns.push(...parseClaudeCodeTranscript(raw, { sessionId: basename2(file, ".jsonl") }));
|
|
1729
2700
|
}
|
|
1730
2701
|
return turns;
|
|
1731
2702
|
}
|
|
1732
2703
|
|
|
1733
2704
|
// src/docs/read.ts
|
|
1734
|
-
import { readFile as
|
|
1735
|
-
import { join as
|
|
2705
|
+
import { readFile as readFile5, stat } from "fs/promises";
|
|
2706
|
+
import { join as join6 } from "path";
|
|
1736
2707
|
var DEFAULT_PATHSPECS = ["*.md"];
|
|
1737
2708
|
async function listDocFiles(repoRoot, opts = {}) {
|
|
1738
2709
|
const pathspecs = opts.include ?? DEFAULT_PATHSPECS;
|
|
@@ -1745,11 +2716,11 @@ async function readDocFiles(repoRoot, opts = {}) {
|
|
|
1745
2716
|
const unreadable = [];
|
|
1746
2717
|
for (const relPath of paths) {
|
|
1747
2718
|
const path = relPath.replace(/\\/g, "/");
|
|
1748
|
-
const absPath =
|
|
2719
|
+
const absPath = join6(repoRoot, relPath);
|
|
1749
2720
|
let content;
|
|
1750
2721
|
let mtime;
|
|
1751
2722
|
try {
|
|
1752
|
-
[content, { mtime }] = await Promise.all([
|
|
2723
|
+
[content, { mtime }] = await Promise.all([readFile5(absPath, "utf8"), stat(absPath)]);
|
|
1753
2724
|
} catch {
|
|
1754
2725
|
unreadable.push(path);
|
|
1755
2726
|
continue;
|
|
@@ -1760,11 +2731,11 @@ async function readDocFiles(repoRoot, opts = {}) {
|
|
|
1760
2731
|
}
|
|
1761
2732
|
|
|
1762
2733
|
// src/shell/detect.ts
|
|
1763
|
-
import { existsSync as
|
|
1764
|
-
import { readFile as
|
|
2734
|
+
import { existsSync as existsSync4 } from "fs";
|
|
2735
|
+
import { readFile as readFile7, stat as stat2 } from "fs/promises";
|
|
1765
2736
|
|
|
1766
2737
|
// src/shell/hook-log.ts
|
|
1767
|
-
import { appendFile, mkdir as
|
|
2738
|
+
import { appendFile, mkdir as mkdir4, readFile as readFile6 } from "fs/promises";
|
|
1768
2739
|
import { dirname as dirname3 } from "path";
|
|
1769
2740
|
function parseHookLogLine(line) {
|
|
1770
2741
|
const trimmed = line.trim();
|
|
@@ -1789,7 +2760,7 @@ function parseHookLogLine(line) {
|
|
|
1789
2760
|
async function readHookLog(path, fromLine) {
|
|
1790
2761
|
let raw;
|
|
1791
2762
|
try {
|
|
1792
|
-
raw = await
|
|
2763
|
+
raw = await readFile6(path, "utf8");
|
|
1793
2764
|
} catch {
|
|
1794
2765
|
return { entries: [], totalLines: fromLine };
|
|
1795
2766
|
}
|
|
@@ -1922,8 +2893,8 @@ function hookEntryToRaw(e) {
|
|
|
1922
2893
|
};
|
|
1923
2894
|
}
|
|
1924
2895
|
async function tryReadScrapeSource(path, parse, tailLines) {
|
|
1925
|
-
if (!
|
|
1926
|
-
const [raw, stats] = await Promise.all([
|
|
2896
|
+
if (!existsSync4(path)) return null;
|
|
2897
|
+
const [raw, stats] = await Promise.all([readFile7(path, "utf8"), stat2(path)]);
|
|
1927
2898
|
return parse(raw, stats.mtimeMs, { tailLines });
|
|
1928
2899
|
}
|
|
1929
2900
|
async function collectAvailableShellHistory(opts = {}) {
|
|
@@ -1931,7 +2902,7 @@ async function collectAvailableShellHistory(opts = {}) {
|
|
|
1931
2902
|
const tailLines = opts.tailLines ?? 300;
|
|
1932
2903
|
const preferHook = opts.preferHook ?? true;
|
|
1933
2904
|
const hookPath = hookLogPath();
|
|
1934
|
-
const hookExists =
|
|
2905
|
+
const hookExists = existsSync4(hookPath);
|
|
1935
2906
|
if (hookExists) {
|
|
1936
2907
|
const fromLine = Number(opts.hookCursor ?? "0") || 0;
|
|
1937
2908
|
const { entries, totalLines } = await readHookLog(hookPath, fromLine);
|
|
@@ -1951,21 +2922,76 @@ async function collectAvailableShellHistory(opts = {}) {
|
|
|
1951
2922
|
}
|
|
1952
2923
|
|
|
1953
2924
|
// src/vector/sync.ts
|
|
2925
|
+
var EMBEDDING_IDENTITY_KEY = "embedding.identity";
|
|
2926
|
+
var DEFAULT_BATCH_SIZE = 32;
|
|
2927
|
+
var DEFAULT_PAGE_SIZE = 500;
|
|
2928
|
+
var DEFAULT_FAILURE_TOLERANCE = 3;
|
|
2929
|
+
function reconcileProviderIdentity(store, provider) {
|
|
2930
|
+
if (store.getMeta(EMBEDDING_IDENTITY_KEY) === provider.identity) return 0;
|
|
2931
|
+
const invalidated = store.dropAllEmbeddings();
|
|
2932
|
+
store.setMeta(EMBEDDING_IDENTITY_KEY, provider.identity);
|
|
2933
|
+
return invalidated;
|
|
2934
|
+
}
|
|
2935
|
+
async function embedTexts(provider, texts) {
|
|
2936
|
+
if (provider.embedBatch) return provider.embedBatch(texts);
|
|
2937
|
+
const out = [];
|
|
2938
|
+
for (const text of texts) out.push(await provider.embed(text));
|
|
2939
|
+
return out;
|
|
2940
|
+
}
|
|
2941
|
+
function chunk(items, size) {
|
|
2942
|
+
const out = [];
|
|
2943
|
+
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
|
2944
|
+
return out;
|
|
2945
|
+
}
|
|
1954
2946
|
async function embedPendingNodes(store, provider, projectId, opts = {}) {
|
|
1955
|
-
const
|
|
2947
|
+
const batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
2948
|
+
const pageSize = opts.pageSize ?? DEFAULT_PAGE_SIZE;
|
|
2949
|
+
const failureTolerance = opts.failureTolerance ?? DEFAULT_FAILURE_TOLERANCE;
|
|
2950
|
+
const maxNodes = opts.maxNodes ?? Number.POSITIVE_INFINITY;
|
|
2951
|
+
const invalidated = reconcileProviderIdentity(store, provider);
|
|
2952
|
+
if (invalidated > 0) opts.onInvalidated?.(invalidated);
|
|
2953
|
+
const total = Math.min(store.countNodesNeedingEmbedding(projectId), maxNodes);
|
|
1956
2954
|
let embedded = 0;
|
|
1957
2955
|
let skipped = 0;
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
2956
|
+
let attempted = 0;
|
|
2957
|
+
let consecutiveFailedRequests = 0;
|
|
2958
|
+
let cursor = 0;
|
|
2959
|
+
outer: while (attempted < maxNodes) {
|
|
2960
|
+
const page = store.findNodesNeedingEmbedding(projectId, Math.min(pageSize, maxNodes - attempted), cursor);
|
|
2961
|
+
if (page.length === 0) break;
|
|
2962
|
+
cursor = page[page.length - 1].rowid;
|
|
2963
|
+
for (const group of chunk(page, batchSize)) {
|
|
2964
|
+
const vectors = await embedTexts(
|
|
2965
|
+
provider,
|
|
2966
|
+
group.map((node) => `${node.title}
|
|
2967
|
+
${node.body}`)
|
|
2968
|
+
);
|
|
2969
|
+
let embeddedHere = 0;
|
|
2970
|
+
for (const [index, node] of group.entries()) {
|
|
2971
|
+
const vector = vectors[index];
|
|
2972
|
+
if (vector && vector.length === provider.dimension) {
|
|
2973
|
+
store.upsertEmbedding(node.rowid, vector);
|
|
2974
|
+
embedded += 1;
|
|
2975
|
+
embeddedHere += 1;
|
|
2976
|
+
} else {
|
|
2977
|
+
skipped += 1;
|
|
2978
|
+
}
|
|
2979
|
+
}
|
|
2980
|
+
attempted += group.length;
|
|
2981
|
+
consecutiveFailedRequests = embeddedHere === 0 ? consecutiveFailedRequests + 1 : 0;
|
|
2982
|
+
opts.onProgress?.(attempted, total);
|
|
2983
|
+
if (consecutiveFailedRequests >= failureTolerance) break outer;
|
|
1966
2984
|
}
|
|
1967
2985
|
}
|
|
1968
|
-
return {
|
|
2986
|
+
return {
|
|
2987
|
+
embedded,
|
|
2988
|
+
skipped,
|
|
2989
|
+
// Unchanged meaning: nothing came back at all. Reached far sooner now --
|
|
2990
|
+
// `failureTolerance` requests instead of the whole first page.
|
|
2991
|
+
providerUnavailable: attempted > 0 && embedded === 0,
|
|
2992
|
+
invalidated,
|
|
2993
|
+
remaining: store.countNodesNeedingEmbedding(projectId)
|
|
2994
|
+
};
|
|
1969
2995
|
}
|
|
1970
2996
|
|
|
1971
2997
|
// src/cli/context.ts
|
|
@@ -1978,6 +3004,8 @@ async function loadContext(cwd) {
|
|
|
1978
3004
|
|
|
1979
3005
|
// src/cli/commands/sync.ts
|
|
1980
3006
|
var BATCH_SIZE = 500;
|
|
3007
|
+
var PROGRESS_THRESHOLD = 200;
|
|
3008
|
+
var PROGRESS_EVERY = 100;
|
|
1981
3009
|
var GIT_SOURCE = "git";
|
|
1982
3010
|
function addStats(into, from) {
|
|
1983
3011
|
into.inserted += from.inserted;
|
|
@@ -1987,25 +3015,25 @@ function addStats(into, from) {
|
|
|
1987
3015
|
async function syncGit(store, projectId, opts, repo, config, log) {
|
|
1988
3016
|
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
1989
3017
|
if (!repo.head) {
|
|
1990
|
-
log(`${
|
|
3018
|
+
log(`${pc4.yellow("git")} skipped -- repository has no commits yet`);
|
|
1991
3019
|
return { totals, seen: 0 };
|
|
1992
3020
|
}
|
|
1993
3021
|
if (!config.sources.git.enabled) {
|
|
1994
|
-
log(`${
|
|
3022
|
+
log(`${pc4.dim("git")} disabled in config`);
|
|
1995
3023
|
return { totals, seen: 0 };
|
|
1996
3024
|
}
|
|
1997
3025
|
let cursor = opts.full || opts.rebuild ? null : store.getSyncCursor(projectId, GIT_SOURCE);
|
|
1998
3026
|
if (cursor && !await isAncestor(repo.root, cursor, repo.head)) {
|
|
1999
|
-
log(`${
|
|
3027
|
+
log(`${pc4.yellow("git cursor stale")} ${cursor.slice(0, 7)} is not an ancestor of HEAD \u2014 falling back to a full walk`);
|
|
2000
3028
|
cursor = null;
|
|
2001
3029
|
}
|
|
2002
3030
|
if (cursor === repo.head) {
|
|
2003
|
-
log(`${
|
|
3031
|
+
log(`${pc4.green("git up to date")} at ${repo.head.slice(0, 7)}`);
|
|
2004
3032
|
store.setSyncCursor(projectId, GIT_SOURCE, repo.head);
|
|
2005
3033
|
return { totals, seen: 0 };
|
|
2006
3034
|
}
|
|
2007
3035
|
log(
|
|
2008
|
-
`${
|
|
3036
|
+
`${pc4.dim("git syncing")} ${repo.branch ?? "HEAD"} ${cursor ? `${cursor.slice(0, 7)}..${repo.head.slice(0, 7)}` : "(full history)"}`
|
|
2009
3037
|
);
|
|
2010
3038
|
let batch = [];
|
|
2011
3039
|
let seen = 0;
|
|
@@ -2013,7 +3041,7 @@ async function syncGit(store, projectId, opts, repo, config, log) {
|
|
|
2013
3041
|
if (batch.length === 0) return;
|
|
2014
3042
|
addStats(totals, store.upsertNodes(batch));
|
|
2015
3043
|
batch = [];
|
|
2016
|
-
log(` ${
|
|
3044
|
+
log(` ${pc4.dim(`${seen} commits read, ${totals.inserted} new`)}`);
|
|
2017
3045
|
};
|
|
2018
3046
|
const nodes = collectGitCommits(repo.root, projectId, {
|
|
2019
3047
|
afterCommit: cursor,
|
|
@@ -2031,10 +3059,51 @@ async function syncGit(store, projectId, opts, repo, config, log) {
|
|
|
2031
3059
|
store.setSyncCursor(projectId, GIT_SOURCE, repo.head);
|
|
2032
3060
|
return { totals, seen };
|
|
2033
3061
|
}
|
|
3062
|
+
async function syncDiffs(store, projectId, opts, repo, config, log) {
|
|
3063
|
+
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
3064
|
+
if (!repo.head) return { totals, seen: 0 };
|
|
3065
|
+
if (!config.sources.diff.enabled) {
|
|
3066
|
+
log(`${pc4.dim("diff")} disabled in config`);
|
|
3067
|
+
return { totals, seen: 0 };
|
|
3068
|
+
}
|
|
3069
|
+
let cursor = opts.full || opts.rebuild ? null : store.getSyncCursor(projectId, DIFF_SOURCE);
|
|
3070
|
+
if (cursor && !await isAncestor(repo.root, cursor, repo.head)) {
|
|
3071
|
+
log(`${pc4.yellow("diff cursor stale")} ${cursor.slice(0, 7)} is not an ancestor of HEAD \u2014 falling back to a bounded walk`);
|
|
3072
|
+
cursor = null;
|
|
3073
|
+
}
|
|
3074
|
+
if (cursor === repo.head) {
|
|
3075
|
+
store.setSyncCursor(projectId, DIFF_SOURCE, repo.head);
|
|
3076
|
+
return { totals, seen: 0 };
|
|
3077
|
+
}
|
|
3078
|
+
let batch = [];
|
|
3079
|
+
let seen = 0;
|
|
3080
|
+
const flush = () => {
|
|
3081
|
+
if (batch.length === 0) return;
|
|
3082
|
+
addStats(totals, store.upsertNodes(batch));
|
|
3083
|
+
batch = [];
|
|
3084
|
+
};
|
|
3085
|
+
const nodes = collectCommitDiffs(repo.root, projectId, {
|
|
3086
|
+
afterCommit: cursor,
|
|
3087
|
+
since: opts.since ?? config.sources.git.since,
|
|
3088
|
+
maxCount: config.sources.diff.maxCommits,
|
|
3089
|
+
maxFilesPerCommit: config.sources.diff.maxFilesPerCommit,
|
|
3090
|
+
contextLines: config.sources.diff.contextLines,
|
|
3091
|
+
maxBodyChars: config.limits.maxBodyChars
|
|
3092
|
+
});
|
|
3093
|
+
for await (const node of nodes) {
|
|
3094
|
+
batch.push(node);
|
|
3095
|
+
seen += 1;
|
|
3096
|
+
if (batch.length >= BATCH_SIZE) flush();
|
|
3097
|
+
}
|
|
3098
|
+
flush();
|
|
3099
|
+
store.setSyncCursor(projectId, DIFF_SOURCE, repo.head);
|
|
3100
|
+
log(` ${pc4.dim(`${DIFF_SOURCE}: ${seen} file diff(s) read`)}`);
|
|
3101
|
+
return { totals, seen };
|
|
3102
|
+
}
|
|
2034
3103
|
async function syncShell(store, projectId, opts, repoRoot, config, log) {
|
|
2035
3104
|
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
2036
3105
|
if (!config.sources.shell.enabled) {
|
|
2037
|
-
log(`${
|
|
3106
|
+
log(`${pc4.dim("shell")} disabled in config`);
|
|
2038
3107
|
return { totals, seen: 0 };
|
|
2039
3108
|
}
|
|
2040
3109
|
const results = await collectAvailableShellHistory({
|
|
@@ -2043,7 +3112,7 @@ async function syncShell(store, projectId, opts, repoRoot, config, log) {
|
|
|
2043
3112
|
hookCursor: store.getSyncCursor(projectId, "shell:pwsh-hook")
|
|
2044
3113
|
});
|
|
2045
3114
|
if (results.length === 0) {
|
|
2046
|
-
log(`${
|
|
3115
|
+
log(`${pc4.dim("shell")} no history source found on this machine`);
|
|
2047
3116
|
return { totals, seen: 0 };
|
|
2048
3117
|
}
|
|
2049
3118
|
let seen = 0;
|
|
@@ -2055,33 +3124,68 @@ async function syncShell(store, projectId, opts, repoRoot, config, log) {
|
|
|
2055
3124
|
addStats(totals, store.upsertNodes(nodes));
|
|
2056
3125
|
}
|
|
2057
3126
|
store.setSyncCursor(projectId, sourceKey, result.cursorAfter ?? `scanned:${result.entries.length}`);
|
|
2058
|
-
log(` ${
|
|
3127
|
+
log(` ${pc4.dim(`${sourceKey}: ${nodes.length} entr${nodes.length === 1 ? "y" : "ies"} read`)}`);
|
|
2059
3128
|
}
|
|
2060
3129
|
return { totals, seen };
|
|
2061
3130
|
}
|
|
2062
3131
|
var CONVERSATION_SOURCE = "conversation:claude-code";
|
|
2063
|
-
|
|
3132
|
+
function syncConversation(store, projectId, turns, config, log, forceEnabled) {
|
|
2064
3133
|
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
2065
3134
|
const enabled = forceEnabled ?? config.sources.conversation.enabled;
|
|
2066
3135
|
if (!enabled) {
|
|
2067
3136
|
return { totals, seen: 0 };
|
|
2068
3137
|
}
|
|
2069
|
-
const turns = await collectClaudeCodeTranscripts(repoRoot);
|
|
2070
3138
|
if (turns.length === 0) {
|
|
2071
|
-
log(`${
|
|
3139
|
+
log(`${pc4.dim("conversation")} no transcripts found`);
|
|
2072
3140
|
return { totals, seen: 0 };
|
|
2073
3141
|
}
|
|
2074
3142
|
const nodes = collectConversationTurns(turns, projectId, { maxBodyChars: config.limits.maxBodyChars });
|
|
2075
3143
|
if (nodes.length > 0) addStats(totals, store.upsertNodes(nodes));
|
|
2076
3144
|
store.setSyncCursor(projectId, CONVERSATION_SOURCE, `scanned:${nodes.length}`);
|
|
2077
|
-
log(` ${
|
|
3145
|
+
log(` ${pc4.dim(`${CONVERSATION_SOURCE}: ${nodes.length} of ${turns.length} exchange(s) kept`)}`);
|
|
2078
3146
|
return { totals, seen: nodes.length };
|
|
2079
3147
|
}
|
|
3148
|
+
var SESSION_SOURCE = "session:claude-code";
|
|
3149
|
+
async function syncSessions(store, projectId, turns, config, log) {
|
|
3150
|
+
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
3151
|
+
const settings = config.sources.session;
|
|
3152
|
+
if (!settings.enabled) return { totals, seen: 0 };
|
|
3153
|
+
if (turns.length === 0) {
|
|
3154
|
+
log(`${pc4.dim("session")} no transcripts found`);
|
|
3155
|
+
return { totals, seen: 0 };
|
|
3156
|
+
}
|
|
3157
|
+
const result = await collectSessionSummaries(turns, projectId, new OllamaChatProvider({ model: settings.model }), {
|
|
3158
|
+
settleMinutes: settings.settleMinutes,
|
|
3159
|
+
maxSessions: settings.maxSessions,
|
|
3160
|
+
maxPromptChars: settings.maxPromptChars,
|
|
3161
|
+
maxBodyChars: config.limits.maxBodyChars,
|
|
3162
|
+
knownHash: (sessionKey) => {
|
|
3163
|
+
const meta = store.getNodeMeta(makeNodeId(projectId, "session_summary", sessionKey));
|
|
3164
|
+
return typeof meta?.contentHash === "string" ? meta.contentHash : null;
|
|
3165
|
+
},
|
|
3166
|
+
onProgress: (done, total) => log(` ${pc4.dim(`session: summarizing ${done}/${total}`)}`)
|
|
3167
|
+
});
|
|
3168
|
+
if (result.nodes.length > 0) addStats(totals, store.upsertNodes(result.nodes));
|
|
3169
|
+
if (result.providerUnavailable) {
|
|
3170
|
+
log(
|
|
3171
|
+
`${pc4.dim("session")} summarization model unavailable (is Ollama running with \`${settings.model}\` pulled?) -- skipped`
|
|
3172
|
+
);
|
|
3173
|
+
} else {
|
|
3174
|
+
const parts = [`${result.nodes.length} summarized`];
|
|
3175
|
+
if (result.cached > 0) parts.push(`${result.cached} unchanged`);
|
|
3176
|
+
if (result.deferred > 0) parts.push(`${result.deferred} queued for the next sync`);
|
|
3177
|
+
if (result.unsettled > 0) parts.push(`${result.unsettled} still active`);
|
|
3178
|
+
if (result.failed > 0) parts.push(`${result.failed} failed`);
|
|
3179
|
+
log(` ${pc4.dim(`${SESSION_SOURCE}: ${parts.join(", ")}`)}`);
|
|
3180
|
+
}
|
|
3181
|
+
store.setSyncCursor(projectId, SESSION_SOURCE, `scanned:${result.nodes.length}`);
|
|
3182
|
+
return { totals, seen: result.nodes.length };
|
|
3183
|
+
}
|
|
2080
3184
|
var DOCS_SOURCE = "docs";
|
|
2081
3185
|
async function syncDocs(store, projectId, repoRoot, config, log) {
|
|
2082
3186
|
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
2083
3187
|
if (!config.sources.docs.enabled) {
|
|
2084
|
-
log(`${
|
|
3188
|
+
log(`${pc4.dim("docs")} disabled in config`);
|
|
2085
3189
|
return { totals, seen: 0 };
|
|
2086
3190
|
}
|
|
2087
3191
|
const { files, unreadable } = await readDocFiles(repoRoot, { include: config.sources.docs.include });
|
|
@@ -2095,11 +3199,11 @@ async function syncDocs(store, projectId, repoRoot, config, log) {
|
|
|
2095
3199
|
);
|
|
2096
3200
|
store.setSyncCursor(projectId, DOCS_SOURCE, `scanned:${nodes.length}`);
|
|
2097
3201
|
if (files.length === 0 && unreadable.length === 0) {
|
|
2098
|
-
log(`${
|
|
3202
|
+
log(`${pc4.dim("docs")} no tracked .md files found`);
|
|
2099
3203
|
} else {
|
|
2100
|
-
const prunedPart = pruned > 0 ? `, ${
|
|
3204
|
+
const prunedPart = pruned > 0 ? `, ${pc4.yellow(`${pruned} stale removed`)}` : "";
|
|
2101
3205
|
const skippedPart = unreadable.length > 0 ? `, ${unreadable.length} unreadable (kept)` : "";
|
|
2102
|
-
log(` ${
|
|
3206
|
+
log(` ${pc4.dim(`${DOCS_SOURCE}: ${nodes.length} section(s) from ${files.length} file(s)`)}${prunedPart}${pc4.dim(skippedPart)}`);
|
|
2103
3207
|
}
|
|
2104
3208
|
return { totals, seen: nodes.length };
|
|
2105
3209
|
}
|
|
@@ -2109,45 +3213,64 @@ async function runSync(opts) {
|
|
|
2109
3213
|
if (!opts.quiet) process.stderr.write(`${line}
|
|
2110
3214
|
`);
|
|
2111
3215
|
};
|
|
2112
|
-
const out = opts.out ?? ((
|
|
3216
|
+
const out = opts.out ?? ((chunk2) => void process.stdout.write(chunk2));
|
|
2113
3217
|
const store = MemoryStore.open(ws.dbPath);
|
|
2114
3218
|
const started = Date.now();
|
|
2115
3219
|
try {
|
|
2116
3220
|
store.upsertProject({ id: projectId, root: repo.root, originUrl: repo.originUrl });
|
|
3221
|
+
await recordProject({ projectId, root: repo.root, dbPath: ws.dbPath, originUrl: repo.originUrl });
|
|
2117
3222
|
if (opts.rebuild) {
|
|
2118
3223
|
const removed = store.clearProject(projectId);
|
|
2119
|
-
log(`${
|
|
3224
|
+
log(`${pc4.dim("rebuild")} dropped ${removed} existing node(s)`);
|
|
2120
3225
|
}
|
|
2121
3226
|
const git2 = await syncGit(store, projectId, opts, repo, config, log);
|
|
3227
|
+
const diffs = await syncDiffs(store, projectId, opts, repo, config, log);
|
|
2122
3228
|
const shell = await syncShell(store, projectId, opts, repo.root, config, log);
|
|
2123
|
-
const
|
|
3229
|
+
const conversationEnabled = opts.conversationOverride ?? config.sources.conversation.enabled;
|
|
3230
|
+
const turns = conversationEnabled || config.sources.session.enabled ? await collectClaudeCodeTranscripts(repo.root) : [];
|
|
3231
|
+
const conversation = syncConversation(store, projectId, turns, config, log, opts.conversationOverride);
|
|
3232
|
+
const sessions = await syncSessions(store, projectId, turns, config, log);
|
|
2124
3233
|
const docs = await syncDocs(store, projectId, repo.root, config, log);
|
|
2125
3234
|
let embedLine = "";
|
|
2126
3235
|
if (!opts.noEmbed) {
|
|
2127
|
-
|
|
3236
|
+
let lastLogged = 0;
|
|
3237
|
+
const result = await embedPendingNodes(store, new OllamaEmbeddingProvider(), projectId, {
|
|
3238
|
+
maxNodes: opts.embedLimit,
|
|
3239
|
+
onInvalidated: (count) => log(`${pc4.yellow("vector")} embedding model changed -- dropped ${count} vector(s), re-embedding from scratch`),
|
|
3240
|
+
onProgress: (attempted, total) => {
|
|
3241
|
+
if (total < PROGRESS_THRESHOLD || attempted - lastLogged < PROGRESS_EVERY) return;
|
|
3242
|
+
lastLogged = attempted;
|
|
3243
|
+
log(` ${pc4.dim(`vector: ${attempted}/${total} embedded`)}`);
|
|
3244
|
+
}
|
|
3245
|
+
});
|
|
2128
3246
|
if (result.embedded > 0) {
|
|
2129
|
-
|
|
3247
|
+
const skippedPart = result.skipped > 0 ? pc4.dim(`, ${result.skipped} skipped`) : "";
|
|
3248
|
+
const remainingPart = result.remaining > 0 ? pc4.yellow(`, ${result.remaining} still pending`) : "";
|
|
3249
|
+
embedLine = ` ${pc4.dim(`vector: ${result.embedded} node(s) embedded`)}${skippedPart}${remainingPart}
|
|
2130
3250
|
`;
|
|
2131
3251
|
} else if (result.providerUnavailable) {
|
|
2132
|
-
log(`${
|
|
3252
|
+
log(`${pc4.dim("vector")} embedding provider unavailable (is Ollama running with nomic-embed-text pulled?) -- BM25-only for now`);
|
|
2133
3253
|
}
|
|
2134
3254
|
}
|
|
2135
3255
|
store.markSynced(projectId);
|
|
2136
3256
|
const totals = { inserted: 0, updated: 0, unchanged: 0 };
|
|
2137
3257
|
addStats(totals, git2.totals);
|
|
3258
|
+
addStats(totals, diffs.totals);
|
|
2138
3259
|
addStats(totals, shell.totals);
|
|
2139
3260
|
addStats(totals, conversation.totals);
|
|
3261
|
+
addStats(totals, sessions.totals);
|
|
2140
3262
|
addStats(totals, docs.totals);
|
|
2141
3263
|
const stats = store.stats(projectId);
|
|
2142
3264
|
const elapsed = ((Date.now() - started) / 1e3).toFixed(2);
|
|
2143
|
-
const conversationEnabled = opts.conversationOverride ?? config.sources.conversation.enabled;
|
|
2144
3265
|
const conversationPart = conversationEnabled ? `, ${conversation.seen} conversation exchange(s)` : "";
|
|
3266
|
+
const sessionPart = config.sources.session.enabled ? `, ${sessions.seen} session summar${sessions.seen === 1 ? "y" : "ies"}` : "";
|
|
2145
3267
|
const docsPart = config.sources.docs.enabled ? `, ${docs.seen} doc section(s)` : "";
|
|
3268
|
+
const diffPart = config.sources.diff.enabled ? `, ${diffs.seen} file diff(s)` : "";
|
|
2146
3269
|
out(
|
|
2147
3270
|
[
|
|
2148
|
-
`${
|
|
2149
|
-
` ${
|
|
2150
|
-
` ${
|
|
3271
|
+
`${pc4.green("synced")} ${git2.seen} commit(s)${diffPart}, ${shell.seen} shell entr${shell.seen === 1 ? "y" : "ies"}${conversationPart}${sessionPart}${docsPart} in ${elapsed}s`,
|
|
3272
|
+
` ${pc4.green(`+${totals.inserted} new`)} ${pc4.yellow(`~${totals.updated} updated`)} ${pc4.dim(`=${totals.unchanged} unchanged`)}`,
|
|
3273
|
+
` ${pc4.dim(`${stats.total} node(s) total across ${stats.distinctFiles} file path(s)`)}`,
|
|
2151
3274
|
""
|
|
2152
3275
|
].join("\n") + embedLine
|
|
2153
3276
|
);
|
|
@@ -2164,20 +3287,39 @@ async function searchMemory(input) {
|
|
|
2164
3287
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
2165
3288
|
const budget = input.budget ?? 2e3;
|
|
2166
3289
|
const candidates = input.candidates ?? 30;
|
|
3290
|
+
const queryOpts = {
|
|
3291
|
+
budget,
|
|
3292
|
+
candidates,
|
|
3293
|
+
embeddingProvider: input.noVector ? null : new OllamaEmbeddingProvider()
|
|
3294
|
+
};
|
|
3295
|
+
if (input.allProjects) {
|
|
3296
|
+
const opened = await openAllProjectSources({ projectId, root: repo.root, dbPath: ws.dbPath });
|
|
3297
|
+
try {
|
|
3298
|
+
const { bm25Count, vectorCount, hits, packed } = await runCrossProjectQuery(opened.sources, input.query, queryOpts);
|
|
3299
|
+
return {
|
|
3300
|
+
text: renderContextBlock(input.query, packed),
|
|
3301
|
+
matched: hits.length,
|
|
3302
|
+
bm25Matched: bm25Count,
|
|
3303
|
+
vectorMatched: vectorCount,
|
|
3304
|
+
tokensUsed: packed.tokensUsed,
|
|
3305
|
+
tokensBudget: packed.tokensBudget,
|
|
3306
|
+
projectsSearched: opened.sources.map((s) => s.label)
|
|
3307
|
+
};
|
|
3308
|
+
} finally {
|
|
3309
|
+
opened.close();
|
|
3310
|
+
}
|
|
3311
|
+
}
|
|
2167
3312
|
const store = MemoryStore.open(ws.dbPath);
|
|
2168
3313
|
try {
|
|
2169
|
-
const { bm25Count, vectorCount, hits, packed } = await runHybridQuery(store, projectId, input.query,
|
|
2170
|
-
budget,
|
|
2171
|
-
candidates,
|
|
2172
|
-
embeddingProvider: input.noVector ? null : new OllamaEmbeddingProvider()
|
|
2173
|
-
});
|
|
3314
|
+
const { bm25Count, vectorCount, hits, packed } = await runHybridQuery(store, projectId, input.query, queryOpts);
|
|
2174
3315
|
return {
|
|
2175
3316
|
text: renderContextBlock(input.query, packed),
|
|
2176
3317
|
matched: hits.length,
|
|
2177
3318
|
bm25Matched: bm25Count,
|
|
2178
3319
|
vectorMatched: vectorCount,
|
|
2179
3320
|
tokensUsed: packed.tokensUsed,
|
|
2180
|
-
tokensBudget: packed.tokensBudget
|
|
3321
|
+
tokensBudget: packed.tokensBudget,
|
|
3322
|
+
projectsSearched: [basename3(repo.root) || repo.root]
|
|
2181
3323
|
};
|
|
2182
3324
|
} finally {
|
|
2183
3325
|
store.close();
|
|
@@ -2185,8 +3327,8 @@ async function searchMemory(input) {
|
|
|
2185
3327
|
}
|
|
2186
3328
|
async function syncProject(input) {
|
|
2187
3329
|
const chunks = [];
|
|
2188
|
-
const out = (
|
|
2189
|
-
chunks.push(
|
|
3330
|
+
const out = (chunk2) => {
|
|
3331
|
+
chunks.push(chunk2);
|
|
2190
3332
|
};
|
|
2191
3333
|
await runInit({ cwd: input.projectRoot, force: false, hook: false, enableConversation: false, out });
|
|
2192
3334
|
const opts = {
|
|
@@ -2220,15 +3362,18 @@ function createServer() {
|
|
|
2220
3362
|
"search_memory",
|
|
2221
3363
|
{
|
|
2222
3364
|
title: "Search remembered project history",
|
|
2223
|
-
description: "Search a NexusMem-tracked repository's remembered history: git commits, shell commands, tracked markdown docs, and (if enabled) conversation transcripts. Returns a token-budgeted, ranked context block -- not raw search results.",
|
|
3365
|
+
description: "Search a NexusMem-tracked repository's remembered history: git commits, code diffs (the patch of each changed file), shell commands, tracked markdown docs, and (if enabled) conversation transcripts and per-session summaries. Returns a token-budgeted, ranked context block -- not raw search results.",
|
|
2224
3366
|
inputSchema: {
|
|
2225
|
-
projectRoot:
|
|
2226
|
-
query:
|
|
2227
|
-
budget:
|
|
3367
|
+
projectRoot: z3.string().describe("Absolute path to the repository root"),
|
|
3368
|
+
query: z3.string().describe("Free-text question or search terms"),
|
|
3369
|
+
budget: z3.number().int().positive().optional().describe("Max tokens in the returned context block. Default 2000."),
|
|
3370
|
+
allProjects: z3.boolean().optional().describe(
|
|
3371
|
+
"Search every repository NexusMem has been run in on this machine, not just projectRoot. Use when the answer may live in a different project (a pattern solved elsewhere, a tool that failed the same way before). Each result is tagged with its repository."
|
|
3372
|
+
)
|
|
2228
3373
|
}
|
|
2229
3374
|
},
|
|
2230
|
-
async ({ projectRoot, query, budget }) => {
|
|
2231
|
-
const result = await searchMemory({ projectRoot, query, budget });
|
|
3375
|
+
async ({ projectRoot, query, budget, allProjects }) => {
|
|
3376
|
+
const result = await searchMemory({ projectRoot, query, budget, allProjects });
|
|
2232
3377
|
return {
|
|
2233
3378
|
content: [{ type: "text", text: result.text }],
|
|
2234
3379
|
structuredContent: {
|
|
@@ -2237,7 +3382,8 @@ function createServer() {
|
|
|
2237
3382
|
bm25Matched: result.bm25Matched,
|
|
2238
3383
|
vectorMatched: result.vectorMatched,
|
|
2239
3384
|
tokensUsed: result.tokensUsed,
|
|
2240
|
-
tokensBudget: result.tokensBudget
|
|
3385
|
+
tokensBudget: result.tokensBudget,
|
|
3386
|
+
projectsSearched: result.projectsSearched
|
|
2241
3387
|
}
|
|
2242
3388
|
};
|
|
2243
3389
|
}
|
|
@@ -2246,9 +3392,9 @@ function createServer() {
|
|
|
2246
3392
|
"sync_project",
|
|
2247
3393
|
{
|
|
2248
3394
|
title: "Sync remembered history",
|
|
2249
|
-
description: "Ingest new git, shell, docs and (if enabled) conversation history for a NexusMem-tracked repository into its local database.",
|
|
3395
|
+
description: "Ingest new git, diff, shell, docs and (if enabled) conversation history for a NexusMem-tracked repository into its local database.",
|
|
2250
3396
|
inputSchema: {
|
|
2251
|
-
projectRoot:
|
|
3397
|
+
projectRoot: z3.string().describe("Absolute path to the repository root")
|
|
2252
3398
|
}
|
|
2253
3399
|
},
|
|
2254
3400
|
async ({ projectRoot }) => {
|
|
@@ -2262,7 +3408,7 @@ function createServer() {
|
|
|
2262
3408
|
title: "Show what is remembered",
|
|
2263
3409
|
description: "Report how many nodes NexusMem currently remembers for a repository, broken down by kind and source.",
|
|
2264
3410
|
inputSchema: {
|
|
2265
|
-
projectRoot:
|
|
3411
|
+
projectRoot: z3.string().describe("Absolute path to the repository root")
|
|
2266
3412
|
}
|
|
2267
3413
|
},
|
|
2268
3414
|
async ({ projectRoot }) => {
|
|
@@ -2282,17 +3428,41 @@ async function runMcpServer() {
|
|
|
2282
3428
|
}
|
|
2283
3429
|
|
|
2284
3430
|
// src/cli/commands/query.ts
|
|
2285
|
-
import
|
|
3431
|
+
import pc5 from "picocolors";
|
|
2286
3432
|
async function runQuery(opts) {
|
|
2287
|
-
const { ws, projectId } = await loadContext(opts.cwd);
|
|
2288
|
-
const
|
|
3433
|
+
const { repo, ws, projectId } = await loadContext(opts.cwd);
|
|
3434
|
+
const opened = opts.allProjects ? await openAllProjectSources({ projectId, root: repo.root, dbPath: ws.dbPath }) : null;
|
|
3435
|
+
let store = null;
|
|
2289
3436
|
try {
|
|
2290
|
-
const
|
|
3437
|
+
const queryOpts = {
|
|
2291
3438
|
budget: opts.budget,
|
|
2292
3439
|
candidates: opts.candidates,
|
|
2293
3440
|
halfLifeDays: opts.halfLifeDays,
|
|
2294
3441
|
embeddingProvider: opts.noVector ? null : new OllamaEmbeddingProvider()
|
|
2295
|
-
}
|
|
3442
|
+
};
|
|
3443
|
+
let result;
|
|
3444
|
+
if (opened) {
|
|
3445
|
+
result = await runCrossProjectQuery(opened.sources, opts.query, queryOpts);
|
|
3446
|
+
} else {
|
|
3447
|
+
store = MemoryStore.open(ws.dbPath);
|
|
3448
|
+
result = await runHybridQuery(store, projectId, opts.query, queryOpts);
|
|
3449
|
+
}
|
|
3450
|
+
const { bm25Count, vectorCount, hits, packed } = result;
|
|
3451
|
+
if (opened && !opts.json) {
|
|
3452
|
+
const searched = opened.sources.map((s) => s.label).join(", ");
|
|
3453
|
+
process.stderr.write(`${pc5.dim("scope ")} ${opened.sources.length} project(s): ${searched}
|
|
3454
|
+
`);
|
|
3455
|
+
for (const { entry } of opened.unreadable) {
|
|
3456
|
+
process.stderr.write(`${pc5.yellow("unreadable")} ${entry.root} -- skipped
|
|
3457
|
+
`);
|
|
3458
|
+
}
|
|
3459
|
+
if (opened.missing.length > 0) {
|
|
3460
|
+
process.stderr.write(
|
|
3461
|
+
`${pc5.dim("skipped")} ${opened.missing.length} registered project(s) whose database is not on disk ${pc5.dim("(nexusmem projects --prune to forget them)")}
|
|
3462
|
+
`
|
|
3463
|
+
);
|
|
3464
|
+
}
|
|
3465
|
+
}
|
|
2296
3466
|
const matched = hits.length;
|
|
2297
3467
|
const rawTokens = hits.reduce((n, h) => n + approxTokens(h.body), 0);
|
|
2298
3468
|
const packerEfficiency = rawTokens > 0 ? 1 - packed.tokensUsed / rawTokens : 0;
|
|
@@ -2317,15 +3487,15 @@ async function runQuery(opts) {
|
|
|
2317
3487
|
return 0;
|
|
2318
3488
|
}
|
|
2319
3489
|
if (matched === 0) {
|
|
2320
|
-
process.stderr.write(`${
|
|
3490
|
+
process.stderr.write(`${pc5.yellow("no matches")} for "${opts.query}"
|
|
2321
3491
|
`);
|
|
2322
3492
|
return 0;
|
|
2323
3493
|
}
|
|
2324
3494
|
process.stderr.write(
|
|
2325
3495
|
[
|
|
2326
|
-
`${
|
|
2327
|
-
`${
|
|
2328
|
-
rawTokens > 0 ? `${
|
|
3496
|
+
`${pc5.dim("matched")} ${bm25Count} bm25${vectorCount > 0 ? ` + ${vectorCount} vector` : ""}, packed ${pc5.bold(String(packed.nodes.length))} into budget`,
|
|
3497
|
+
`${pc5.dim("tokens ")} ${packed.tokensUsed}/${packed.tokensBudget}` + (packed.droppedForBudget ? pc5.dim(` (${packed.droppedForBudget} dropped for budget)`) : ""),
|
|
3498
|
+
rawTokens > 0 ? `${pc5.dim("vs raw ")} ${rawTokens} tokens if these same matches were sent unpacked ${packerEfficiency >= 0 ? pc5.green(`(${(packerEfficiency * 100).toFixed(0)}% packer efficiency)`) : pc5.yellow(`(${(-packerEfficiency * 100).toFixed(0)}% larger -- overhead dominates on small result sets)`)}` : "",
|
|
2329
3499
|
""
|
|
2330
3500
|
].filter(Boolean).join("\n")
|
|
2331
3501
|
);
|
|
@@ -2333,28 +3503,30 @@ async function runQuery(opts) {
|
|
|
2333
3503
|
`);
|
|
2334
3504
|
return 0;
|
|
2335
3505
|
} finally {
|
|
2336
|
-
|
|
3506
|
+
opened?.close();
|
|
3507
|
+
store?.close();
|
|
2337
3508
|
}
|
|
2338
3509
|
}
|
|
2339
3510
|
|
|
2340
3511
|
// src/cli/commands/scan-conversation.ts
|
|
2341
|
-
import
|
|
3512
|
+
import pc7 from "picocolors";
|
|
2342
3513
|
|
|
2343
3514
|
// src/cli/format.ts
|
|
2344
|
-
import
|
|
3515
|
+
import pc6 from "picocolors";
|
|
2345
3516
|
var GIT_SIGNAL_BANDS = { high: 0.7, medium: 0.45 };
|
|
2346
3517
|
var SHELL_SIGNAL_BANDS = { high: 0.6, medium: 0.4 };
|
|
2347
3518
|
var CONVERSATION_SIGNAL_BANDS = { high: 0.55, medium: 0.35 };
|
|
2348
3519
|
var DOCS_SIGNAL_BANDS = { high: 0.55, medium: 0.35 };
|
|
3520
|
+
var DIFF_SIGNAL_BANDS = { high: 0.7, medium: 0.45 };
|
|
2349
3521
|
function signalBand(signal, bands) {
|
|
2350
3522
|
if (signal >= bands.high) return "high";
|
|
2351
3523
|
if (signal >= bands.medium) return "medium";
|
|
2352
3524
|
return "low";
|
|
2353
3525
|
}
|
|
2354
3526
|
var BAND_COLOR = {
|
|
2355
|
-
high:
|
|
2356
|
-
medium:
|
|
2357
|
-
low:
|
|
3527
|
+
high: pc6.green,
|
|
3528
|
+
medium: pc6.yellow,
|
|
3529
|
+
low: pc6.dim
|
|
2358
3530
|
};
|
|
2359
3531
|
function formatSignal(signal, bands) {
|
|
2360
3532
|
return BAND_COLOR[signalBand(signal, bands)](signal.toFixed(2));
|
|
@@ -2367,9 +3539,9 @@ async function runScanConversation(opts) {
|
|
|
2367
3539
|
const files = await listTranscriptFiles(repo.root);
|
|
2368
3540
|
if (!opts.json) {
|
|
2369
3541
|
process.stderr.write(
|
|
2370
|
-
files.length ? `${
|
|
3542
|
+
files.length ? `${pc7.dim("transcripts")} ${files.length} file(s) in ${claudeProjectTranscriptDir(repo.root)}
|
|
2371
3543
|
|
|
2372
|
-
` : `${
|
|
3544
|
+
` : `${pc7.yellow("no transcripts found")} at ${claudeProjectTranscriptDir(repo.root)}
|
|
2373
3545
|
`
|
|
2374
3546
|
);
|
|
2375
3547
|
}
|
|
@@ -2386,7 +3558,7 @@ async function runScanConversation(opts) {
|
|
|
2386
3558
|
const approxTotal = nodes.reduce((n, x) => n + approxTokens(x.body), 0);
|
|
2387
3559
|
process.stderr.write(
|
|
2388
3560
|
`
|
|
2389
|
-
${
|
|
3561
|
+
${pc7.bold(String(nodes.length))} of ${turns.length} exchange(s) above threshold ${pc7.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}` + (redactedTotal > 0 ? ` ${pc7.yellow(`${redactedTotal} secret-like value(s) redacted`)}` : "") + "\n"
|
|
2390
3562
|
);
|
|
2391
3563
|
return 0;
|
|
2392
3564
|
}
|
|
@@ -2394,44 +3566,8 @@ function formatNode(node) {
|
|
|
2394
3566
|
return [formatSignal(node.signal, CONVERSATION_SIGNAL_BANDS), node.ts.slice(0, 16).replace("T", " "), node.title].join(" ");
|
|
2395
3567
|
}
|
|
2396
3568
|
|
|
2397
|
-
// src/cli/commands/scan-
|
|
2398
|
-
import
|
|
2399
|
-
async function runScanDocs(opts) {
|
|
2400
|
-
const repo = await readRepoInfo(opts.cwd);
|
|
2401
|
-
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
2402
|
-
const { files, unreadable } = await readDocFiles(repo.root);
|
|
2403
|
-
if (!opts.json) {
|
|
2404
|
-
process.stderr.write(
|
|
2405
|
-
files.length ? `${pc7.dim("tracked .md files")} ${files.map((f) => f.path).join(", ")}
|
|
2406
|
-
|
|
2407
|
-
` : `${pc7.yellow("no tracked .md files found")}
|
|
2408
|
-
`
|
|
2409
|
-
);
|
|
2410
|
-
if (unreadable.length > 0) {
|
|
2411
|
-
process.stderr.write(`${pc7.yellow("unreadable")} ${unreadable.join(", ")}
|
|
2412
|
-
|
|
2413
|
-
`);
|
|
2414
|
-
}
|
|
2415
|
-
}
|
|
2416
|
-
const nodes = collectDocFiles(files, projectId).filter((n) => n.signal >= opts.minSignal);
|
|
2417
|
-
if (opts.json) {
|
|
2418
|
-
process.stdout.write(`${JSON.stringify(nodes, null, 2)}
|
|
2419
|
-
`);
|
|
2420
|
-
return 0;
|
|
2421
|
-
}
|
|
2422
|
-
for (const node of nodes) process.stdout.write(`${formatNode2(node)}
|
|
2423
|
-
`);
|
|
2424
|
-
const approxTotal = nodes.reduce((n, x) => n + approxTokens(x.body), 0);
|
|
2425
|
-
process.stderr.write(
|
|
2426
|
-
`
|
|
2427
|
-
${pc7.bold(String(nodes.length))} section(s) from ${files.length} file(s) above threshold ${pc7.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}
|
|
2428
|
-
`
|
|
2429
|
-
);
|
|
2430
|
-
return 0;
|
|
2431
|
-
}
|
|
2432
|
-
function formatNode2(node) {
|
|
2433
|
-
return [formatSignal(node.signal, DOCS_SIGNAL_BANDS), node.title].join(" ");
|
|
2434
|
-
}
|
|
3569
|
+
// src/cli/commands/scan-diff.ts
|
|
3570
|
+
import pc9 from "picocolors";
|
|
2435
3571
|
|
|
2436
3572
|
// src/cli/commands/scan-git.ts
|
|
2437
3573
|
import pc8 from "picocolors";
|
|
@@ -2458,7 +3594,7 @@ async function runScanGit(opts) {
|
|
|
2458
3594
|
for await (const node of collectGitCommits(repo.root, projectId, collectOpts)) {
|
|
2459
3595
|
if (node.signal < opts.minSignal) continue;
|
|
2460
3596
|
nodes.push(node);
|
|
2461
|
-
if (!opts.json) process.stdout.write(`${
|
|
3597
|
+
if (!opts.json) process.stdout.write(`${formatNode2(node)}
|
|
2462
3598
|
`);
|
|
2463
3599
|
}
|
|
2464
3600
|
if (opts.json) {
|
|
@@ -2471,7 +3607,7 @@ ${summarize2(nodes)}
|
|
|
2471
3607
|
`);
|
|
2472
3608
|
return 0;
|
|
2473
3609
|
}
|
|
2474
|
-
function
|
|
3610
|
+
function formatNode2(node) {
|
|
2475
3611
|
const sha = String(node.meta.shortSha ?? "").padEnd(9);
|
|
2476
3612
|
const date = node.ts.slice(0, 10);
|
|
2477
3613
|
const files = Number(node.meta.filesChanged ?? 0);
|
|
@@ -2502,17 +3638,187 @@ ${hottest.join("\n")}` : ""
|
|
|
2502
3638
|
].filter(Boolean).join("\n");
|
|
2503
3639
|
}
|
|
2504
3640
|
|
|
3641
|
+
// src/cli/commands/scan-diff.ts
|
|
3642
|
+
var DEFAULT_SCAN_COMMITS = 50;
|
|
3643
|
+
async function runScanDiff(opts) {
|
|
3644
|
+
const repo = await readRepoInfo(opts.cwd);
|
|
3645
|
+
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
3646
|
+
if (!opts.json) {
|
|
3647
|
+
process.stderr.write(
|
|
3648
|
+
[
|
|
3649
|
+
`${pc9.dim("repo ")} ${repo.root}`,
|
|
3650
|
+
`${pc9.dim("branch ")} ${repo.branch ?? pc9.yellow("(detached)")}`,
|
|
3651
|
+
`${pc9.dim("project")} ${pc9.cyan(projectId)}`,
|
|
3652
|
+
""
|
|
3653
|
+
].join("\n")
|
|
3654
|
+
);
|
|
3655
|
+
}
|
|
3656
|
+
const nodes = [];
|
|
3657
|
+
for await (const node of collectCommitDiffs(repo.root, projectId, {
|
|
3658
|
+
since: opts.since ?? null,
|
|
3659
|
+
maxCount: opts.limit ?? DEFAULT_SCAN_COMMITS
|
|
3660
|
+
})) {
|
|
3661
|
+
if (node.signal < opts.minSignal) continue;
|
|
3662
|
+
nodes.push(node);
|
|
3663
|
+
if (!opts.json) process.stdout.write(`${formatNode3(node)}
|
|
3664
|
+
`);
|
|
3665
|
+
}
|
|
3666
|
+
if (opts.json) {
|
|
3667
|
+
process.stdout.write(`${JSON.stringify(nodes, null, 2)}
|
|
3668
|
+
`);
|
|
3669
|
+
return 0;
|
|
3670
|
+
}
|
|
3671
|
+
process.stderr.write(`
|
|
3672
|
+
${summarize2(nodes)}
|
|
3673
|
+
`);
|
|
3674
|
+
return 0;
|
|
3675
|
+
}
|
|
3676
|
+
function formatNode3(node) {
|
|
3677
|
+
const sha = String(node.meta.shortSha ?? "").padEnd(9);
|
|
3678
|
+
const churn = `+${node.meta.insertions ?? 0}/-${node.meta.deletions ?? 0}`;
|
|
3679
|
+
return [
|
|
3680
|
+
formatSignal(node.signal, DIFF_SIGNAL_BANDS),
|
|
3681
|
+
pc9.dim(node.ts.slice(0, 10)),
|
|
3682
|
+
pc9.magenta(sha),
|
|
3683
|
+
String(node.meta.path ?? ""),
|
|
3684
|
+
pc9.dim(`(${churn}, ${node.meta.hunkCount ?? 0} hunk(s))`)
|
|
3685
|
+
].join(" ");
|
|
3686
|
+
}
|
|
3687
|
+
|
|
3688
|
+
// src/cli/commands/scan-docs.ts
|
|
3689
|
+
import pc10 from "picocolors";
|
|
3690
|
+
async function runScanDocs(opts) {
|
|
3691
|
+
const repo = await readRepoInfo(opts.cwd);
|
|
3692
|
+
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
3693
|
+
const { files, unreadable } = await readDocFiles(repo.root);
|
|
3694
|
+
if (!opts.json) {
|
|
3695
|
+
process.stderr.write(
|
|
3696
|
+
files.length ? `${pc10.dim("tracked .md files")} ${files.map((f) => f.path).join(", ")}
|
|
3697
|
+
|
|
3698
|
+
` : `${pc10.yellow("no tracked .md files found")}
|
|
3699
|
+
`
|
|
3700
|
+
);
|
|
3701
|
+
if (unreadable.length > 0) {
|
|
3702
|
+
process.stderr.write(`${pc10.yellow("unreadable")} ${unreadable.join(", ")}
|
|
3703
|
+
|
|
3704
|
+
`);
|
|
3705
|
+
}
|
|
3706
|
+
}
|
|
3707
|
+
const nodes = collectDocFiles(files, projectId).filter((n) => n.signal >= opts.minSignal);
|
|
3708
|
+
if (opts.json) {
|
|
3709
|
+
process.stdout.write(`${JSON.stringify(nodes, null, 2)}
|
|
3710
|
+
`);
|
|
3711
|
+
return 0;
|
|
3712
|
+
}
|
|
3713
|
+
for (const node of nodes) process.stdout.write(`${formatNode4(node)}
|
|
3714
|
+
`);
|
|
3715
|
+
const approxTotal = nodes.reduce((n, x) => n + approxTokens(x.body), 0);
|
|
3716
|
+
process.stderr.write(
|
|
3717
|
+
`
|
|
3718
|
+
${pc10.bold(String(nodes.length))} section(s) from ${files.length} file(s) above threshold ${pc10.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}
|
|
3719
|
+
`
|
|
3720
|
+
);
|
|
3721
|
+
return 0;
|
|
3722
|
+
}
|
|
3723
|
+
function formatNode4(node) {
|
|
3724
|
+
return [formatSignal(node.signal, DOCS_SIGNAL_BANDS), node.title].join(" ");
|
|
3725
|
+
}
|
|
3726
|
+
|
|
3727
|
+
// src/cli/commands/scan-session.ts
|
|
3728
|
+
import pc11 from "picocolors";
|
|
3729
|
+
async function runScanSession(opts) {
|
|
3730
|
+
const repo = await readRepoInfo(opts.cwd);
|
|
3731
|
+
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
3732
|
+
const turns = await collectClaudeCodeTranscripts(repo.root);
|
|
3733
|
+
if (turns.length === 0) {
|
|
3734
|
+
process.stderr.write(`${pc11.yellow("no transcripts found")} at ${claudeProjectTranscriptDir(repo.root)}
|
|
3735
|
+
`);
|
|
3736
|
+
return 0;
|
|
3737
|
+
}
|
|
3738
|
+
const sessions = groupTurnsIntoSessions(turns);
|
|
3739
|
+
const settled = selectSettledSessions(sessions, opts.settleMinutes);
|
|
3740
|
+
if (!opts.json) {
|
|
3741
|
+
process.stderr.write(
|
|
3742
|
+
`${pc11.dim("sessions")} ${sessions.length} found, ${settled.length} settled (quiet ${opts.settleMinutes}m+)
|
|
3743
|
+
|
|
3744
|
+
`
|
|
3745
|
+
);
|
|
3746
|
+
}
|
|
3747
|
+
if (opts.dryRun) {
|
|
3748
|
+
const previews = settled.slice(0, opts.maxSessions).map((session) => {
|
|
3749
|
+
const { prompt, hash, includedTurns } = buildSessionPrompt(session);
|
|
3750
|
+
return {
|
|
3751
|
+
sessionKey: session.sessionKey,
|
|
3752
|
+
startedAt: session.startedAt,
|
|
3753
|
+
endedAt: session.endedAt,
|
|
3754
|
+
turns: session.turns.length,
|
|
3755
|
+
includedTurns,
|
|
3756
|
+
hash,
|
|
3757
|
+
promptChars: prompt.length,
|
|
3758
|
+
prompt
|
|
3759
|
+
};
|
|
3760
|
+
});
|
|
3761
|
+
if (opts.json) {
|
|
3762
|
+
process.stdout.write(`${JSON.stringify(previews, null, 2)}
|
|
3763
|
+
`);
|
|
3764
|
+
return 0;
|
|
3765
|
+
}
|
|
3766
|
+
for (const preview of previews) {
|
|
3767
|
+
process.stdout.write(
|
|
3768
|
+
`${pc11.bold(preview.sessionKey)} ${preview.startedAt.slice(0, 16).replace("T", " ")} ${preview.includedTurns}/${preview.turns} turn(s), ${preview.promptChars} chars
|
|
3769
|
+
${preview.prompt}
|
|
3770
|
+
|
|
3771
|
+
`
|
|
3772
|
+
);
|
|
3773
|
+
}
|
|
3774
|
+
return 0;
|
|
3775
|
+
}
|
|
3776
|
+
const result = await collectSessionSummaries(turns, projectId, new OllamaChatProvider({ model: opts.model }), {
|
|
3777
|
+
settleMinutes: opts.settleMinutes,
|
|
3778
|
+
maxSessions: opts.maxSessions,
|
|
3779
|
+
onProgress: (done, total) => {
|
|
3780
|
+
if (!opts.json) process.stderr.write(` ${pc11.dim(`summarizing ${done}/${total}`)}
|
|
3781
|
+
`);
|
|
3782
|
+
}
|
|
3783
|
+
});
|
|
3784
|
+
if (opts.json) {
|
|
3785
|
+
process.stdout.write(`${JSON.stringify(result.nodes, null, 2)}
|
|
3786
|
+
`);
|
|
3787
|
+
return 0;
|
|
3788
|
+
}
|
|
3789
|
+
for (const node of result.nodes) {
|
|
3790
|
+
process.stdout.write(`${pc11.bold(node.title)}
|
|
3791
|
+
${pc11.dim(node.ts.slice(0, 16).replace("T", " "))}
|
|
3792
|
+
${node.body}
|
|
3793
|
+
|
|
3794
|
+
`);
|
|
3795
|
+
}
|
|
3796
|
+
if (result.providerUnavailable) {
|
|
3797
|
+
process.stderr.write(
|
|
3798
|
+
`${pc11.yellow("model unavailable")} -- is Ollama running with \`${opts.model}\` pulled? (\`ollama pull ${opts.model}\`)
|
|
3799
|
+
`
|
|
3800
|
+
);
|
|
3801
|
+
return 0;
|
|
3802
|
+
}
|
|
3803
|
+
process.stderr.write(
|
|
3804
|
+
`${pc11.bold(String(result.nodes.length))} summarized` + (result.failed > 0 ? `, ${pc11.yellow(`${result.failed} failed`)}` : "") + ` ${pc11.dim(`(model ${opts.model})`)}
|
|
3805
|
+
`
|
|
3806
|
+
);
|
|
3807
|
+
return 0;
|
|
3808
|
+
}
|
|
3809
|
+
var SCAN_SESSION_DEFAULT_MODEL = DEFAULT_SLM_MODEL;
|
|
3810
|
+
|
|
2505
3811
|
// src/cli/commands/scan-shell.ts
|
|
2506
|
-
import
|
|
3812
|
+
import pc12 from "picocolors";
|
|
2507
3813
|
async function runScanShell(opts) {
|
|
2508
3814
|
const repo = await readRepoInfo(opts.cwd);
|
|
2509
3815
|
const projectId = makeProjectId({ root: repo.root, originUrl: repo.originUrl });
|
|
2510
3816
|
const results = await collectAvailableShellHistory({ tailLines: opts.tailLines, repoRoot: repo.root });
|
|
2511
3817
|
if (!opts.json) {
|
|
2512
3818
|
process.stderr.write(
|
|
2513
|
-
results.length ? `${
|
|
3819
|
+
results.length ? `${pc12.dim("sources found")} ${results.map((r) => r.name).join(", ")}
|
|
2514
3820
|
|
|
2515
|
-
` : `${
|
|
3821
|
+
` : `${pc12.yellow("no shell history source found on this machine")}
|
|
2516
3822
|
`
|
|
2517
3823
|
);
|
|
2518
3824
|
}
|
|
@@ -2521,9 +3827,9 @@ async function runScanShell(opts) {
|
|
|
2521
3827
|
const nodes = collectShellHistory(result.entries, projectId).filter((n) => n.signal >= opts.minSignal);
|
|
2522
3828
|
allNodes.push(...nodes);
|
|
2523
3829
|
if (!opts.json) {
|
|
2524
|
-
process.stdout.write(`${
|
|
3830
|
+
process.stdout.write(`${pc12.bold(`shell:${result.name}`)} ${pc12.dim(`(${nodes.length} of ${result.entries.length} above threshold)`)}
|
|
2525
3831
|
`);
|
|
2526
|
-
for (const node of nodes) process.stdout.write(`${
|
|
3832
|
+
for (const node of nodes) process.stdout.write(`${formatNode5(node)}
|
|
2527
3833
|
`);
|
|
2528
3834
|
process.stdout.write("\n");
|
|
2529
3835
|
}
|
|
@@ -2534,20 +3840,20 @@ async function runScanShell(opts) {
|
|
|
2534
3840
|
return 0;
|
|
2535
3841
|
}
|
|
2536
3842
|
const approxTotal = allNodes.reduce((n, x) => n + approxTokens(x.body), 0);
|
|
2537
|
-
process.stderr.write(`${
|
|
3843
|
+
process.stderr.write(`${pc12.bold(String(allNodes.length))} node(s) total ${pc12.dim(`~${approxTotal.toLocaleString()} tokens if sent raw`)}
|
|
2538
3844
|
`);
|
|
2539
3845
|
return 0;
|
|
2540
3846
|
}
|
|
2541
|
-
function
|
|
2542
|
-
const approx = node.meta.tsApprox ?
|
|
3847
|
+
function formatNode5(node) {
|
|
3848
|
+
const approx = node.meta.tsApprox ? pc12.dim("~") : " ";
|
|
2543
3849
|
const exit = node.meta.exitCode;
|
|
2544
|
-
const exitLabel = typeof exit === "number" && exit !== 0 ?
|
|
3850
|
+
const exitLabel = typeof exit === "number" && exit !== 0 ? pc12.red(`exit ${exit}`) : "";
|
|
2545
3851
|
return [formatSignal(node.signal, SHELL_SIGNAL_BANDS), approx + node.ts.slice(0, 16).replace("T", " "), node.title, exitLabel].filter(Boolean).join(" ");
|
|
2546
3852
|
}
|
|
2547
3853
|
|
|
2548
3854
|
// src/cli/commands/status.ts
|
|
2549
3855
|
import { statSync } from "fs";
|
|
2550
|
-
import
|
|
3856
|
+
import pc13 from "picocolors";
|
|
2551
3857
|
function humanBytes(bytes) {
|
|
2552
3858
|
if (bytes < 1024) return `${bytes} B`;
|
|
2553
3859
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
@@ -2572,23 +3878,23 @@ async function runStatus(opts) {
|
|
|
2572
3878
|
const kinds = Object.entries(stats.byKind).sort((a, b) => b[1] - a[1]).map(([kind, n]) => ` ${String(n).padStart(6)} ${kind}`);
|
|
2573
3879
|
process.stdout.write(
|
|
2574
3880
|
[
|
|
2575
|
-
`${
|
|
2576
|
-
`${
|
|
2577
|
-
`${
|
|
2578
|
-
`${
|
|
2579
|
-
`${
|
|
3881
|
+
`${pc13.dim("repo ")} ${repo.root}`,
|
|
3882
|
+
`${pc13.dim("branch ")} ${repo.branch ?? pc13.yellow("(detached)")}`,
|
|
3883
|
+
`${pc13.dim("project ")} ${pc13.cyan(projectId)}`,
|
|
3884
|
+
`${pc13.dim("schema ")} v${schema}${schema === LATEST_SCHEMA_VERSION ? "" : pc13.yellow(` (latest is v${LATEST_SCHEMA_VERSION})`)}`,
|
|
3885
|
+
`${pc13.dim("database")} ${ws.dbPath} ${pc13.dim(`(${humanBytes(dbBytes)})`)}`,
|
|
2580
3886
|
"",
|
|
2581
|
-
`${
|
|
3887
|
+
`${pc13.bold(String(stats.total))} node(s)${stats.total ? ` ${pc13.dim(`${stats.oldest?.slice(0, 10)} .. ${stats.newest?.slice(0, 10)}`)}` : ""}`,
|
|
2582
3888
|
...kinds,
|
|
2583
|
-
stats.total ? ` ${
|
|
3889
|
+
stats.total ? ` ${pc13.dim(`${stats.distinctFiles} distinct file path(s)`)}` : "",
|
|
2584
3890
|
"",
|
|
2585
|
-
sources.length ?
|
|
3891
|
+
sources.length ? pc13.dim("sources") : pc13.yellow("no sources synced yet"),
|
|
2586
3892
|
...sources.map((s) => {
|
|
2587
3893
|
const when = s.lastRunAt ? new Date(s.lastRunAt).toISOString().slice(0, 16).replace("T", " ") : "never";
|
|
2588
3894
|
const cursorLabel = s.source === "git" ? s.cursor?.slice(0, 7) ?? "-" : s.cursor ?? "-";
|
|
2589
|
-
return ` ${s.source.padEnd(14)} ${
|
|
3895
|
+
return ` ${s.source.padEnd(14)} ${pc13.dim(`last run ${when}`)} ${pc13.dim(`cursor ${cursorLabel}`)}`;
|
|
2590
3896
|
}),
|
|
2591
|
-
gitCursor && gitCursor !== repo.head ? `${
|
|
3897
|
+
gitCursor && gitCursor !== repo.head ? `${pc13.yellow("git behind HEAD")} \u2014 run ${pc13.bold("nexusmem sync")}` : "",
|
|
2592
3898
|
""
|
|
2593
3899
|
].filter((line) => line !== "").join("\n").concat("\n")
|
|
2594
3900
|
);
|
|
@@ -2612,7 +3918,7 @@ function guard(run) {
|
|
|
2612
3918
|
process.exitCode = await run();
|
|
2613
3919
|
} catch (err) {
|
|
2614
3920
|
if (isExpected(err)) {
|
|
2615
|
-
process.stderr.write(`${
|
|
3921
|
+
process.stderr.write(`${pc14.red("error")} ${err.message}
|
|
2616
3922
|
`);
|
|
2617
3923
|
process.exitCode = 1;
|
|
2618
3924
|
return;
|
|
@@ -2628,7 +3934,11 @@ program.command("init").description("Create the .nexusmem workspace and database
|
|
|
2628
3934
|
() => runInit({ cwd: options.cwd, force: options.force, hook: options.hook, enableConversation: options.enableConversation })
|
|
2629
3935
|
)()
|
|
2630
3936
|
);
|
|
2631
|
-
program.command("sync").description("Ingest new history into the local database").option("-C, --cwd <path>", "repository path", process.cwd()).option("--full", "ignore the stored cursor and re-walk all history", false).option("--rebuild", "drop this project's nodes and re-ingest from scratch", false).option("--since <date>", "override the configured git cutoff, e.g. 1.year.ago").option("--shell-lines <count>", "override the configured shell tail-window size", (v) => Number.parseInt(v, 10)).option("--conversation", "force the conversation source on for this run, without persisting it to config", false).option("--no-embed", "skip the vector-embedding pass for this run").option(
|
|
3937
|
+
program.command("sync").description("Ingest new history into the local database").option("-C, --cwd <path>", "repository path", process.cwd()).option("--full", "ignore the stored cursor and re-walk all history", false).option("--rebuild", "drop this project's nodes and re-ingest from scratch", false).option("--since <date>", "override the configured git cutoff, e.g. 1.year.ago").option("--shell-lines <count>", "override the configured shell tail-window size", (v) => Number.parseInt(v, 10)).option("--conversation", "force the conversation source on for this run, without persisting it to config", false).option("--no-embed", "skip the vector-embedding pass for this run").option(
|
|
3938
|
+
"--embed-limit <count>",
|
|
3939
|
+
"stop embedding after this many nodes (default: embed everything pending)",
|
|
3940
|
+
(v) => Number.parseInt(v, 10)
|
|
3941
|
+
).option("-q, --quiet", "only print the final summary", false).action(
|
|
2632
3942
|
(options) => guard(
|
|
2633
3943
|
() => runSync({
|
|
2634
3944
|
cwd: options.cwd,
|
|
@@ -2638,6 +3948,7 @@ program.command("sync").description("Ingest new history into the local database"
|
|
|
2638
3948
|
shellTailLines: options.shellLines,
|
|
2639
3949
|
conversationOverride: options.conversation ? true : void 0,
|
|
2640
3950
|
noEmbed: !options.embed,
|
|
3951
|
+
embedLimit: options.embedLimit,
|
|
2641
3952
|
quiet: options.quiet
|
|
2642
3953
|
})
|
|
2643
3954
|
)()
|
|
@@ -2650,7 +3961,7 @@ program.command("hook").description("Manage the opt-in PowerShell hook that logs
|
|
|
2650
3961
|
new Command("status").description("Show whether the hook is installed").option("--profile <path>", "override the auto-detected $PROFILE path").action((options) => guard(() => runHookStatus({ profile: options.profile }))())
|
|
2651
3962
|
);
|
|
2652
3963
|
program.command("status").description("Show what is currently remembered for this repository").option("-C, --cwd <path>", "repository path", process.cwd()).action((options) => guard(() => runStatus({ cwd: options.cwd }))());
|
|
2653
|
-
program.command("query").description("Search remembered history and print a token-budgeted context block").argument("<text>", "free-text query").option("-C, --cwd <path>", "repository path", process.cwd()).option("-b, --budget <tokens>", "max tokens in the packed context", (v) => Number.parseInt(v, 10), 2e3).option("-n, --candidates <count>", "how many search hits to rank before packing", (v) => Number.parseInt(v, 10), 30).option("--half-life <days>", "days for a node's recency weight to halve", (v) => Number.parseFloat(v)).option("--no-vector", "BM25 only -- skip embedding the query and vector search").option("--json", "emit the packed result as JSON on stdout", false).action(
|
|
3964
|
+
program.command("query").description("Search remembered history and print a token-budgeted context block").argument("<text>", "free-text query").option("-C, --cwd <path>", "repository path", process.cwd()).option("-b, --budget <tokens>", "max tokens in the packed context", (v) => Number.parseInt(v, 10), 2e3).option("-n, --candidates <count>", "how many search hits to rank before packing", (v) => Number.parseInt(v, 10), 30).option("--half-life <days>", "days for a node's recency weight to halve", (v) => Number.parseFloat(v)).option("--no-vector", "BM25 only -- skip embedding the query and vector search").option("-a, --all-projects", "search every registered repository, not just this one", false).option("--json", "emit the packed result as JSON on stdout", false).action(
|
|
2654
3965
|
(text, options) => guard(
|
|
2655
3966
|
() => runQuery({
|
|
2656
3967
|
cwd: options.cwd,
|
|
@@ -2659,10 +3970,12 @@ program.command("query").description("Search remembered history and print a toke
|
|
|
2659
3970
|
candidates: options.candidates,
|
|
2660
3971
|
halfLifeDays: options.halfLife,
|
|
2661
3972
|
noVector: !options.vector,
|
|
3973
|
+
allProjects: options.allProjects,
|
|
2662
3974
|
json: options.json
|
|
2663
3975
|
})
|
|
2664
3976
|
)()
|
|
2665
3977
|
);
|
|
3978
|
+
program.command("projects").description("List the repositories `query --all-projects` would search").option("--prune", "forget registered projects whose database is no longer on disk", false).option("--json", "emit the registry as JSON on stdout", false).action((options) => guard(() => runProjects({ prune: options.prune, json: options.json }))());
|
|
2666
3979
|
program.command("scan-git").description("Preview the MemoryNodes git history would produce (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("--since <date>", "only commits newer than this git date expression, e.g. 90.days.ago").option("-n, --limit <count>", "stop after N commits", (v) => Number.parseInt(v, 10)).option("--no-merges", "skip merge commits").option("--min-signal <score>", "drop nodes below this signal", (v) => Number.parseFloat(v), 0).option("--json", "emit MemoryNodes as JSON on stdout", false).action(
|
|
2667
3980
|
(options) => guard(
|
|
2668
3981
|
() => runScanGit({
|
|
@@ -2675,6 +3988,17 @@ program.command("scan-git").description("Preview the MemoryNodes git history wou
|
|
|
2675
3988
|
})
|
|
2676
3989
|
)()
|
|
2677
3990
|
);
|
|
3991
|
+
program.command("scan-diff").description("Preview the MemoryNodes commit patches would produce, one per changed file (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("--since <date>", "only commits newer than this git date expression, e.g. 90.days.ago").option("-n, --limit <count>", "stop after N commits (not N nodes)", (v) => Number.parseInt(v, 10)).option("--min-signal <score>", "drop nodes below this signal", (v) => Number.parseFloat(v), 0).option("--json", "emit MemoryNodes as JSON on stdout", false).action(
|
|
3992
|
+
(options) => guard(
|
|
3993
|
+
() => runScanDiff({
|
|
3994
|
+
cwd: options.cwd,
|
|
3995
|
+
since: options.since,
|
|
3996
|
+
limit: options.limit,
|
|
3997
|
+
json: options.json,
|
|
3998
|
+
minSignal: options.minSignal
|
|
3999
|
+
})
|
|
4000
|
+
)()
|
|
4001
|
+
);
|
|
2678
4002
|
program.command("scan-shell").description("Preview the MemoryNodes shell history would produce (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("-n, --tail-lines <count>", "lines kept from each scrape-based source", (v) => Number.parseInt(v, 10), 300).option("--min-signal <score>", "drop nodes below this signal", (v) => Number.parseFloat(v), 0).option("--json", "emit MemoryNodes as JSON on stdout", false).action(
|
|
2679
4003
|
(options) => guard(
|
|
2680
4004
|
() => runScanShell({ cwd: options.cwd, tailLines: options.tailLines, minSignal: options.minSignal, json: options.json })
|
|
@@ -2683,11 +4007,23 @@ program.command("scan-shell").description("Preview the MemoryNodes shell history
|
|
|
2683
4007
|
program.command("scan-conversation").description("Preview the MemoryNodes the conversation transcript would produce (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("--min-signal <score>", "drop nodes below this signal", (v) => Number.parseFloat(v), 0).option("--json", "emit MemoryNodes as JSON on stdout", false).action(
|
|
2684
4008
|
(options) => guard(() => runScanConversation({ cwd: options.cwd, minSignal: options.minSignal, json: options.json }))()
|
|
2685
4009
|
);
|
|
4010
|
+
program.command("scan-session").description("Preview the session summaries a local model would produce (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("--model <name>", "Ollama model to summarize with", SCAN_SESSION_DEFAULT_MODEL).option("--settle-minutes <count>", "minutes of quiet before a session counts as finished", (v) => Number.parseInt(v, 10), 30).option("-n, --max-sessions <count>", "how many sessions to summarize", (v) => Number.parseInt(v, 10), 3).option("--dry-run", "print the prompts instead of calling the model", false).option("--json", "emit as JSON on stdout", false).action(
|
|
4011
|
+
(options) => guard(
|
|
4012
|
+
() => runScanSession({
|
|
4013
|
+
cwd: options.cwd,
|
|
4014
|
+
model: options.model,
|
|
4015
|
+
settleMinutes: options.settleMinutes,
|
|
4016
|
+
maxSessions: options.maxSessions,
|
|
4017
|
+
dryRun: options.dryRun,
|
|
4018
|
+
json: options.json
|
|
4019
|
+
})
|
|
4020
|
+
)()
|
|
4021
|
+
);
|
|
2686
4022
|
program.command("scan-docs").description("Preview the MemoryNodes tracked .md files would produce (writes nothing)").option("-C, --cwd <path>", "repository path", process.cwd()).option("--min-signal <score>", "drop nodes below this signal", (v) => Number.parseFloat(v), 0).option("--json", "emit MemoryNodes as JSON on stdout", false).action((options) => guard(() => runScanDocs({ cwd: options.cwd, minSignal: options.minSignal, json: options.json }))());
|
|
2687
4023
|
program.command("mcp").description("Start the MCP server (stdio transport) for Claude Desktop, Cursor, Windsurf, etc.").action(() => guard(() => runMcpServer().then(() => 0))());
|
|
2688
4024
|
program.parseAsync(process.argv).catch((err) => {
|
|
2689
4025
|
const message = err instanceof Error ? err.message : String(err);
|
|
2690
|
-
process.stderr.write(`${
|
|
4026
|
+
process.stderr.write(`${pc14.red("error")} ${message}
|
|
2691
4027
|
`);
|
|
2692
4028
|
process.exitCode = 1;
|
|
2693
4029
|
});
|