@threadbase-sh/scanner 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -10
- package/dist/cli.js +2059 -338
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +2114 -376
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +154 -6
- package/dist/index.d.ts +154 -6
- package/dist/index.js +2111 -377
- package/dist/index.js.map +1 -1
- package/package.json +4 -1
package/dist/index.cjs
CHANGED
|
@@ -30,9 +30,11 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/index.ts
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
|
+
CodexCliProvider: () => CodexCliProvider,
|
|
33
34
|
ConversationScanner: () => ConversationScanner,
|
|
34
35
|
DEFAULT_TIERS: () => DEFAULT_TIERS,
|
|
35
36
|
SearchIndexer: () => SearchIndexer,
|
|
37
|
+
ThreadbaseProvider: () => ThreadbaseProvider,
|
|
36
38
|
VALID_SORT_ORDERS: () => VALID_SORT_ORDERS,
|
|
37
39
|
applyAccountFilter: () => applyAccountFilter,
|
|
38
40
|
applyIncludeFilter: () => applyIncludeFilter,
|
|
@@ -48,36 +50,40 @@ __export(index_exports, {
|
|
|
48
50
|
getProjectsDir: () => getProjectsDir,
|
|
49
51
|
loadProfiles: () => loadProfiles,
|
|
50
52
|
readGitBranch: () => readGitBranch,
|
|
53
|
+
readSidecar: () => readSidecar,
|
|
51
54
|
resetDefaultScanner: () => resetDefaultScanner,
|
|
52
55
|
resolveConfigDir: () => resolveConfigDir,
|
|
53
56
|
resolveTier: () => resolveTier,
|
|
54
57
|
saveProfiles: () => saveProfiles,
|
|
55
58
|
scan: () => scan,
|
|
56
59
|
search: () => search,
|
|
57
|
-
setLogger: () => setLogger
|
|
60
|
+
setLogger: () => setLogger,
|
|
61
|
+
sidecarPath: () => sidecarPath
|
|
58
62
|
});
|
|
59
63
|
module.exports = __toCommonJS(index_exports);
|
|
60
64
|
|
|
61
65
|
// src/filters.ts
|
|
62
66
|
function applySort(metas, order) {
|
|
63
67
|
const out = [...metas];
|
|
68
|
+
const tie = (a, b) => a.id.localeCompare(b.id);
|
|
64
69
|
switch (order) {
|
|
65
70
|
case "recent":
|
|
66
|
-
out.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
71
|
+
out.sort((a, b) => b.timestamp.localeCompare(a.timestamp) || tie(a, b));
|
|
67
72
|
break;
|
|
68
73
|
case "oldest":
|
|
69
|
-
out.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
74
|
+
out.sort((a, b) => a.timestamp.localeCompare(b.timestamp) || tie(a, b));
|
|
70
75
|
break;
|
|
71
76
|
case "messages-desc":
|
|
72
|
-
out.sort((a, b) => b.messageCount - a.messageCount);
|
|
77
|
+
out.sort((a, b) => b.messageCount - a.messageCount || tie(a, b));
|
|
73
78
|
break;
|
|
74
79
|
case "messages-asc":
|
|
75
|
-
out.sort((a, b) => a.messageCount - b.messageCount);
|
|
80
|
+
out.sort((a, b) => a.messageCount - b.messageCount || tie(a, b));
|
|
76
81
|
break;
|
|
77
82
|
case "alpha":
|
|
78
83
|
out.sort((a, b) => {
|
|
79
84
|
const cmp = a.projectName.localeCompare(b.projectName);
|
|
80
|
-
|
|
85
|
+
if (cmp !== 0) return cmp;
|
|
86
|
+
return a.preview.localeCompare(b.preview) || tie(a, b);
|
|
81
87
|
});
|
|
82
88
|
break;
|
|
83
89
|
}
|
|
@@ -202,6 +208,36 @@ function readGitBranch(projectPath) {
|
|
|
202
208
|
|
|
203
209
|
// src/indexer.ts
|
|
204
210
|
var import_flexsearch = __toESM(require("flexsearch"), 1);
|
|
211
|
+
|
|
212
|
+
// src/search-matches.ts
|
|
213
|
+
function generateMatches(meta, query) {
|
|
214
|
+
const matches = [];
|
|
215
|
+
const lowerQuery = query.toLowerCase();
|
|
216
|
+
const fields = [
|
|
217
|
+
["contentSnippet", meta.contentSnippet],
|
|
218
|
+
["projectName", meta.projectName],
|
|
219
|
+
["sessionId", meta.sessionId],
|
|
220
|
+
["sessionName", meta.sessionName],
|
|
221
|
+
["account", meta.account],
|
|
222
|
+
["model", meta.model || ""],
|
|
223
|
+
["gitBranch", meta.gitBranch || ""],
|
|
224
|
+
["toolNames", meta.toolNames.join(" ")]
|
|
225
|
+
];
|
|
226
|
+
for (const [field, value] of fields) {
|
|
227
|
+
const idx = value.toLowerCase().indexOf(lowerQuery);
|
|
228
|
+
if (idx !== -1) {
|
|
229
|
+
const start = Math.max(0, idx - 80);
|
|
230
|
+
const end = Math.min(value.length, idx + query.length + 120);
|
|
231
|
+
let snippet = value.slice(start, end);
|
|
232
|
+
if (start > 0) snippet = `...${snippet}`;
|
|
233
|
+
if (end < value.length) snippet = `${snippet}...`;
|
|
234
|
+
matches.push({ field, snippet });
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return matches.length > 0 ? matches : [{ field: "preview", snippet: meta.preview }];
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// src/indexer.ts
|
|
205
241
|
var FlexSearch = import_flexsearch.default.default ?? import_flexsearch.default;
|
|
206
242
|
var SearchIndexer = class {
|
|
207
243
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -270,7 +306,7 @@ var SearchIndexer = class {
|
|
|
270
306
|
seen.add(id);
|
|
271
307
|
const meta = this.documents.get(id);
|
|
272
308
|
if (!meta) continue;
|
|
273
|
-
const matches =
|
|
309
|
+
const matches = generateMatches(meta, query);
|
|
274
310
|
searchResults.push({ meta, score: 1, matches });
|
|
275
311
|
if (searchResults.length >= limit) break;
|
|
276
312
|
}
|
|
@@ -285,32 +321,6 @@ var SearchIndexer = class {
|
|
|
285
321
|
matches: [{ field: "timestamp", snippet: meta.preview }]
|
|
286
322
|
}));
|
|
287
323
|
}
|
|
288
|
-
generateMatches(meta, query) {
|
|
289
|
-
const matches = [];
|
|
290
|
-
const lowerQuery = query.toLowerCase();
|
|
291
|
-
const fields = [
|
|
292
|
-
["contentSnippet", meta.contentSnippet],
|
|
293
|
-
["projectName", meta.projectName],
|
|
294
|
-
["sessionId", meta.sessionId],
|
|
295
|
-
["sessionName", meta.sessionName],
|
|
296
|
-
["account", meta.account],
|
|
297
|
-
["model", meta.model || ""],
|
|
298
|
-
["gitBranch", meta.gitBranch || ""],
|
|
299
|
-
["toolNames", meta.toolNames.join(" ")]
|
|
300
|
-
];
|
|
301
|
-
for (const [field, value] of fields) {
|
|
302
|
-
const idx = value.toLowerCase().indexOf(lowerQuery);
|
|
303
|
-
if (idx !== -1) {
|
|
304
|
-
const start = Math.max(0, idx - 80);
|
|
305
|
-
const end = Math.min(value.length, idx + query.length + 120);
|
|
306
|
-
let snippet = value.slice(start, end);
|
|
307
|
-
if (start > 0) snippet = `...${snippet}`;
|
|
308
|
-
if (end < value.length) snippet = `${snippet}...`;
|
|
309
|
-
matches.push({ field, snippet });
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
return matches.length > 0 ? matches : [{ field: "preview", snippet: meta.preview }];
|
|
313
|
-
}
|
|
314
324
|
getDocumentCount() {
|
|
315
325
|
return this.documents.size;
|
|
316
326
|
}
|
|
@@ -343,6 +353,46 @@ var SearchIndexer = class {
|
|
|
343
353
|
}
|
|
344
354
|
};
|
|
345
355
|
|
|
356
|
+
// src/persistent/sidecar.ts
|
|
357
|
+
var import_fs2 = require("fs");
|
|
358
|
+
var SIDECAR_VERSION = 1;
|
|
359
|
+
function sidecarPath(jsonlPath) {
|
|
360
|
+
return `${jsonlPath}.idx.json`;
|
|
361
|
+
}
|
|
362
|
+
function buildSidecar(meta, cursor, updatedAt) {
|
|
363
|
+
return {
|
|
364
|
+
version: SIDECAR_VERSION,
|
|
365
|
+
sourcePath: meta.filePath,
|
|
366
|
+
sizeBytes: cursor.sizeBytes,
|
|
367
|
+
mtimeMs: cursor.mtimeMs,
|
|
368
|
+
lastIndexedOffset: cursor.offset,
|
|
369
|
+
lastIndexedLine: cursor.line,
|
|
370
|
+
messageCount: meta.messageCount,
|
|
371
|
+
projectPath: meta.projectPath,
|
|
372
|
+
projectName: meta.projectName,
|
|
373
|
+
branch: meta.gitBranch,
|
|
374
|
+
firstSentAt: meta.firstMessage?.timestamp ?? null,
|
|
375
|
+
firstSentText: meta.firstMessage?.text ?? null,
|
|
376
|
+
lastSentAt: meta.lastMessage?.timestamp ?? null,
|
|
377
|
+
lastSentText: meta.lastMessage?.text ?? null,
|
|
378
|
+
updatedAt
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
function writeSidecar(jsonlPath, sidecar) {
|
|
382
|
+
try {
|
|
383
|
+
(0, import_fs2.writeFileSync)(sidecarPath(jsonlPath), JSON.stringify(sidecar, null, 2));
|
|
384
|
+
} catch (err) {
|
|
385
|
+
getLogger().warn({ jsonlPath, err }, "sidecar: write failed");
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
function readSidecar(jsonlPath) {
|
|
389
|
+
try {
|
|
390
|
+
return JSON.parse((0, import_fs2.readFileSync)(sidecarPath(jsonlPath), "utf-8"));
|
|
391
|
+
} catch {
|
|
392
|
+
return null;
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
346
396
|
// src/profiles.ts
|
|
347
397
|
var import_promises = require("fs/promises");
|
|
348
398
|
var import_os = require("os");
|
|
@@ -384,48 +434,272 @@ async function saveProfiles(profiles, configPath) {
|
|
|
384
434
|
getLogger().debug({ configPath, count: profiles.length }, "profiles: saved");
|
|
385
435
|
}
|
|
386
436
|
|
|
387
|
-
// src/
|
|
437
|
+
// src/providers/codex-cli.ts
|
|
438
|
+
var import_fast_glob = __toESM(require("fast-glob"), 1);
|
|
388
439
|
var import_fs3 = require("fs");
|
|
440
|
+
var import_promises2 = require("fs/promises");
|
|
441
|
+
var import_path3 = require("path");
|
|
442
|
+
var import_readline = require("readline");
|
|
389
443
|
|
|
390
|
-
// src/
|
|
391
|
-
var
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
444
|
+
// src/tags.ts
|
|
445
|
+
var SYSTEM_TAGS = [
|
|
446
|
+
"system-reminder",
|
|
447
|
+
"command-name",
|
|
448
|
+
"command-message",
|
|
449
|
+
"command-args",
|
|
450
|
+
"ide_selection",
|
|
451
|
+
"ide_opened_file",
|
|
452
|
+
"local-command-stdout",
|
|
453
|
+
"local-command-caveat",
|
|
454
|
+
"retrieval_status",
|
|
455
|
+
"task_id",
|
|
456
|
+
"task_type",
|
|
457
|
+
"task-id",
|
|
458
|
+
"task-notification",
|
|
459
|
+
"fast_mode_info",
|
|
460
|
+
"persisted-output",
|
|
461
|
+
"tool_use_error",
|
|
462
|
+
"user-prompt-submit-hook",
|
|
463
|
+
"thinking",
|
|
464
|
+
"ask_user",
|
|
465
|
+
"teammate-message"
|
|
466
|
+
];
|
|
467
|
+
var SYSTEM_TAG_RE = new RegExp(`<(${SYSTEM_TAGS.join("|")})[^>]*>[\\s\\S]*?<\\/\\1>`, "g");
|
|
468
|
+
function cleanSystemTags(text) {
|
|
469
|
+
return text.replace(SYSTEM_TAG_RE, "").replace(/[^\S\n]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
// src/providers/provider.ts
|
|
473
|
+
var CLAUDE_CODE_PROVIDER = "claude-code";
|
|
474
|
+
var CODEX_CLI_PROVIDER = "codex-cli";
|
|
475
|
+
|
|
476
|
+
// src/providers/codex-cli.ts
|
|
477
|
+
var CodexCliProvider = class {
|
|
478
|
+
name = CODEX_CLI_PROVIDER;
|
|
479
|
+
async discover(roots) {
|
|
480
|
+
const log = getLogger();
|
|
481
|
+
const results = [];
|
|
482
|
+
for (const root of roots) {
|
|
483
|
+
let paths;
|
|
484
|
+
try {
|
|
485
|
+
paths = await (0, import_fast_glob.default)(["**/rollout-*.jsonl", "**/*.jsonl"], {
|
|
486
|
+
cwd: root,
|
|
487
|
+
absolute: true,
|
|
488
|
+
dot: false,
|
|
489
|
+
unique: true
|
|
490
|
+
});
|
|
491
|
+
} catch (err) {
|
|
492
|
+
log.warn({ root, err }, "codex discovery: glob failed");
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
495
|
+
for (const filePath of paths) {
|
|
496
|
+
try {
|
|
497
|
+
const s = await (0, import_promises2.stat)(filePath);
|
|
498
|
+
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
499
|
+
} catch (err) {
|
|
500
|
+
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
return results;
|
|
396
505
|
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
506
|
+
// Codex rollout lines carry distinctive top-level types.
|
|
507
|
+
canParse(_filePath, sample) {
|
|
508
|
+
for (const line of sample.split("\n")) {
|
|
509
|
+
if (!line.trim()) continue;
|
|
510
|
+
try {
|
|
511
|
+
const e = JSON.parse(line);
|
|
512
|
+
if (e.type === "session_meta" || e.type === "response_item" || e.type === "event_msg") {
|
|
513
|
+
return true;
|
|
514
|
+
}
|
|
515
|
+
if (e.type === "user" || e.type === "assistant") return false;
|
|
516
|
+
} catch {
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
return false;
|
|
403
520
|
}
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
521
|
+
createEmptyAccumulator() {
|
|
522
|
+
return {
|
|
523
|
+
sessionId: "",
|
|
524
|
+
cwd: "",
|
|
525
|
+
gitBranch: null,
|
|
526
|
+
model: null,
|
|
527
|
+
latestTimestamp: "",
|
|
528
|
+
messageCount: 0,
|
|
529
|
+
lastMessageSender: "user",
|
|
530
|
+
firstUser: null,
|
|
531
|
+
lastUser: null,
|
|
532
|
+
lastAssistant: null,
|
|
533
|
+
toolNames: [],
|
|
534
|
+
previewParts: [],
|
|
535
|
+
previewLength: 0,
|
|
536
|
+
snippetParts: [],
|
|
537
|
+
snippetLength: 0
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
reduceEntry(acc, entry, tier) {
|
|
541
|
+
reduceCodexEntry(acc, entry, tier);
|
|
542
|
+
}
|
|
543
|
+
finalize(acc, filePath, account, tier) {
|
|
544
|
+
return finalizeCodexMeta(acc, filePath, account, tier);
|
|
545
|
+
}
|
|
546
|
+
};
|
|
547
|
+
var asString = (v) => typeof v === "string" ? v : "";
|
|
548
|
+
function extractCodexText(content) {
|
|
549
|
+
if (typeof content === "string") return cleanSystemTags(content);
|
|
550
|
+
if (!Array.isArray(content)) return "";
|
|
551
|
+
return content.map((item) => {
|
|
552
|
+
if (typeof item === "string") return item;
|
|
553
|
+
const t = item?.type;
|
|
554
|
+
if ((t === "input_text" || t === "output_text" || t === "text") && item?.text) {
|
|
555
|
+
return item.text;
|
|
410
556
|
}
|
|
557
|
+
return "";
|
|
558
|
+
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
559
|
+
}
|
|
560
|
+
function reduceCodexEntry(acc, entry, tier) {
|
|
561
|
+
const ts = asString(entry.timestamp);
|
|
562
|
+
if (ts && (!acc.latestTimestamp || ts > acc.latestTimestamp)) acc.latestTimestamp = ts;
|
|
563
|
+
const payload = entry.payload;
|
|
564
|
+
if (!payload || typeof payload !== "object") return;
|
|
565
|
+
const type = entry.type;
|
|
566
|
+
if (type === "session_meta") {
|
|
567
|
+
if (!acc.sessionId) acc.sessionId = asString(payload.id);
|
|
568
|
+
if (!acc.cwd) acc.cwd = asString(payload.cwd);
|
|
569
|
+
const git = payload.git;
|
|
570
|
+
if (acc.gitBranch === null && git?.branch) acc.gitBranch = asString(git.branch) || null;
|
|
571
|
+
return;
|
|
411
572
|
}
|
|
412
|
-
|
|
413
|
-
|
|
573
|
+
if (acc.model === null && payload.model) acc.model = asString(payload.model) || null;
|
|
574
|
+
if (type !== "response_item") return;
|
|
575
|
+
const ptype = payload.type;
|
|
576
|
+
if (ptype === "function_call" || ptype === "custom_tool_call") {
|
|
577
|
+
const name = asString(payload.name);
|
|
578
|
+
if (name && !acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
579
|
+
return;
|
|
414
580
|
}
|
|
415
|
-
|
|
416
|
-
|
|
581
|
+
if (ptype !== "message") return;
|
|
582
|
+
const role = payload.role;
|
|
583
|
+
if (role !== "user" && role !== "assistant") return;
|
|
584
|
+
const text = extractCodexText(payload.content);
|
|
585
|
+
if (!text) return;
|
|
586
|
+
const sender = role;
|
|
587
|
+
acc.messageCount++;
|
|
588
|
+
acc.lastMessageSender = sender;
|
|
589
|
+
const snapshot = { text: text.slice(0, 200), timestamp: ts };
|
|
590
|
+
if (sender === "user") {
|
|
591
|
+
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
592
|
+
acc.lastUser = snapshot;
|
|
593
|
+
} else {
|
|
594
|
+
acc.lastAssistant = snapshot;
|
|
417
595
|
}
|
|
418
|
-
|
|
419
|
-
|
|
596
|
+
if (acc.previewLength < tier.previewMax) {
|
|
597
|
+
acc.previewParts.push(text);
|
|
598
|
+
acc.previewLength += text.length;
|
|
420
599
|
}
|
|
421
|
-
|
|
422
|
-
|
|
600
|
+
if (acc.snippetLength < tier.snippetMax) {
|
|
601
|
+
const remaining = tier.snippetMax - acc.snippetLength;
|
|
602
|
+
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
603
|
+
acc.snippetParts.push(chunk);
|
|
604
|
+
acc.snippetLength += chunk.length;
|
|
423
605
|
}
|
|
424
|
-
}
|
|
606
|
+
}
|
|
607
|
+
function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
608
|
+
if (acc.messageCount === 0) return null;
|
|
609
|
+
const sessionId = acc.sessionId || (0, import_path3.basename)(filePath, ".jsonl");
|
|
610
|
+
const projectPath = acc.cwd;
|
|
611
|
+
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
612
|
+
return {
|
|
613
|
+
id: filePath,
|
|
614
|
+
filePath,
|
|
615
|
+
provider: CODEX_CLI_PROVIDER,
|
|
616
|
+
kind,
|
|
617
|
+
externalSessionId: acc.sessionId || void 0,
|
|
618
|
+
sessionId,
|
|
619
|
+
sessionName: "",
|
|
620
|
+
projectPath,
|
|
621
|
+
projectName: getShortProjectName(projectPath),
|
|
622
|
+
account,
|
|
623
|
+
timestamp: acc.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
624
|
+
messageCount: acc.messageCount,
|
|
625
|
+
lastMessageSender: acc.lastMessageSender,
|
|
626
|
+
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
627
|
+
contentSnippet: acc.snippetParts.join(" "),
|
|
628
|
+
gitBranch: acc.gitBranch,
|
|
629
|
+
model: acc.model,
|
|
630
|
+
isSubagent: false,
|
|
631
|
+
parentSessionId: null,
|
|
632
|
+
isTeammate: false,
|
|
633
|
+
teamName: null,
|
|
634
|
+
toolNames: acc.toolNames,
|
|
635
|
+
firstMessage: acc.firstUser,
|
|
636
|
+
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
637
|
+
lastPrompt: acc.lastUser?.text || void 0
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
function getShortProjectName(fullPath) {
|
|
641
|
+
return fullPath.split("/").filter(Boolean).slice(-3).join("/");
|
|
642
|
+
}
|
|
643
|
+
async function parseCodexConversation(filePath, account) {
|
|
644
|
+
const log = getLogger();
|
|
645
|
+
const messages = [];
|
|
646
|
+
const textParts = [];
|
|
647
|
+
let sessionId = "";
|
|
648
|
+
let cwd = "";
|
|
649
|
+
let latestTimestamp = "";
|
|
650
|
+
let lastUserText = "";
|
|
651
|
+
const rl = (0, import_readline.createInterface)({ input: (0, import_fs3.createReadStream)(filePath), crlfDelay: Infinity });
|
|
652
|
+
try {
|
|
653
|
+
for await (const line of rl) {
|
|
654
|
+
if (!line.trim()) continue;
|
|
655
|
+
let entry;
|
|
656
|
+
try {
|
|
657
|
+
entry = JSON.parse(line);
|
|
658
|
+
} catch {
|
|
659
|
+
continue;
|
|
660
|
+
}
|
|
661
|
+
const ts = asString(entry.timestamp);
|
|
662
|
+
if (ts && (!latestTimestamp || ts > latestTimestamp)) latestTimestamp = ts;
|
|
663
|
+
const payload = entry.payload;
|
|
664
|
+
if (!payload || typeof payload !== "object") continue;
|
|
665
|
+
if (entry.type === "session_meta") {
|
|
666
|
+
if (!sessionId) sessionId = asString(payload.id);
|
|
667
|
+
if (!cwd) cwd = asString(payload.cwd);
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
if (entry.type !== "response_item" || payload.type !== "message") continue;
|
|
671
|
+
const role = payload.role;
|
|
672
|
+
if (role !== "user" && role !== "assistant") continue;
|
|
673
|
+
const text = extractCodexText(payload.content);
|
|
674
|
+
if (!text) continue;
|
|
675
|
+
messages.push({ role, text, timestamp: ts });
|
|
676
|
+
textParts.push(text);
|
|
677
|
+
if (role === "user") lastUserText = text;
|
|
678
|
+
}
|
|
679
|
+
} catch (err) {
|
|
680
|
+
log.warn({ filePath, err }, "parseCodexConversation: read failed");
|
|
681
|
+
return null;
|
|
682
|
+
}
|
|
683
|
+
if (messages.length === 0) return null;
|
|
684
|
+
return {
|
|
685
|
+
id: filePath,
|
|
686
|
+
filePath,
|
|
687
|
+
projectPath: cwd,
|
|
688
|
+
projectName: getShortProjectName(cwd),
|
|
689
|
+
sessionId: sessionId || (0, import_path3.basename)(filePath, ".jsonl"),
|
|
690
|
+
sessionName: "",
|
|
691
|
+
messages,
|
|
692
|
+
fullText: textParts.join(" "),
|
|
693
|
+
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
694
|
+
messageCount: messages.length,
|
|
695
|
+
account,
|
|
696
|
+
lastPrompt: lastUserText || void 0
|
|
697
|
+
};
|
|
698
|
+
}
|
|
425
699
|
|
|
426
700
|
// src/discovery.ts
|
|
427
|
-
var
|
|
428
|
-
var
|
|
701
|
+
var import_fast_glob2 = __toESM(require("fast-glob"), 1);
|
|
702
|
+
var import_promises3 = require("fs/promises");
|
|
429
703
|
var EXCLUDED_SEGMENTS = ["/memory/", "/tool-results/"];
|
|
430
704
|
var STAT_CONCURRENCY = 32;
|
|
431
705
|
async function discoverJsonlFiles(dirs, onProgress) {
|
|
@@ -434,7 +708,7 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
434
708
|
for (const { projectsDir, account } of dirs) {
|
|
435
709
|
let filePaths;
|
|
436
710
|
try {
|
|
437
|
-
filePaths = await (0,
|
|
711
|
+
filePaths = await (0, import_fast_glob2.default)("**/*.jsonl", {
|
|
438
712
|
cwd: projectsDir,
|
|
439
713
|
absolute: true,
|
|
440
714
|
dot: false
|
|
@@ -452,7 +726,7 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
452
726
|
const statted = await Promise.all(
|
|
453
727
|
chunk.map(async (filePath) => {
|
|
454
728
|
try {
|
|
455
|
-
const s = await (0,
|
|
729
|
+
const s = await (0, import_promises3.stat)(filePath);
|
|
456
730
|
return { filePath, size: s.size };
|
|
457
731
|
} catch (err) {
|
|
458
732
|
log.warn({ filePath, err }, "discovery: stat failed");
|
|
@@ -489,64 +763,115 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
489
763
|
return results;
|
|
490
764
|
}
|
|
491
765
|
|
|
766
|
+
// src/persistent/metadata-reducer.ts
|
|
767
|
+
var import_path5 = require("path");
|
|
768
|
+
|
|
492
769
|
// src/parser.ts
|
|
493
|
-
var
|
|
494
|
-
var
|
|
495
|
-
var
|
|
770
|
+
var import_fs4 = require("fs");
|
|
771
|
+
var import_path4 = require("path");
|
|
772
|
+
var import_readline2 = require("readline");
|
|
496
773
|
|
|
497
|
-
// src/
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
"
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
774
|
+
// src/persistent/conversation-reducer.ts
|
|
775
|
+
function initialConvState() {
|
|
776
|
+
return {
|
|
777
|
+
cwd: "",
|
|
778
|
+
sessionId: "",
|
|
779
|
+
sessionName: "",
|
|
780
|
+
latestTimestamp: "",
|
|
781
|
+
lastPrompt: "",
|
|
782
|
+
pendingToolUses: {},
|
|
783
|
+
teamInfo: {}
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
function reduceConvLine(state, entry) {
|
|
787
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
788
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
789
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
790
|
+
if (entry.timestamp) {
|
|
791
|
+
const ts = entry.timestamp;
|
|
792
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
793
|
+
}
|
|
794
|
+
const type = entry.type;
|
|
795
|
+
if (type === "last-prompt") {
|
|
796
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
797
|
+
return null;
|
|
798
|
+
}
|
|
799
|
+
if (type !== "user" && type !== "assistant") return null;
|
|
800
|
+
if (entry.isMeta) return null;
|
|
801
|
+
const msg = entry.message;
|
|
802
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
803
|
+
for (const block of toolUseBlocks) state.pendingToolUses[block.id] = block;
|
|
804
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
805
|
+
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
806
|
+
const content = extractTextContent(msg?.content);
|
|
807
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
808
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
809
|
+
if (!(content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking)) return null;
|
|
810
|
+
const metadata = {};
|
|
811
|
+
if (msg?.model) metadata.model = msg.model;
|
|
812
|
+
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
813
|
+
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
814
|
+
if (entry.version) metadata.version = entry.version;
|
|
815
|
+
const usage = msg?.usage;
|
|
816
|
+
if (usage) {
|
|
817
|
+
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
818
|
+
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
819
|
+
if (usage.cache_read_input_tokens) metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
820
|
+
if (usage.cache_creation_input_tokens)
|
|
821
|
+
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
822
|
+
}
|
|
823
|
+
const toolUseNames = extractToolUseNames(msg?.content);
|
|
824
|
+
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
825
|
+
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
826
|
+
if (isToolResultOnly) {
|
|
827
|
+
const pending = new Map(Object.entries(state.pendingToolUses));
|
|
828
|
+
const toolResultBlocks = extractToolResultBlocks(msg?.content, pending);
|
|
829
|
+
if (toolResultBlocks.length > 0) metadata.toolResults = toolResultBlocks;
|
|
830
|
+
}
|
|
831
|
+
if (entry.teamName) {
|
|
832
|
+
metadata.teamName = entry.teamName;
|
|
833
|
+
if (!state.teamInfo[metadata.teamName] && content) {
|
|
834
|
+
const info = parseTeammateMessageTag(content);
|
|
835
|
+
if (info) state.teamInfo[metadata.teamName] = info;
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
const thinkingContent = thinking?.content || void 0;
|
|
839
|
+
const thinkingSignature = thinking?.signature || void 0;
|
|
840
|
+
const hasMetadata = Object.keys(metadata).length > 0;
|
|
841
|
+
return {
|
|
842
|
+
role: type,
|
|
843
|
+
text: content || "",
|
|
844
|
+
timestamp: entry.timestamp || "",
|
|
845
|
+
uuid: entry.uuid || void 0,
|
|
846
|
+
metadata: hasMetadata ? metadata : void 0,
|
|
847
|
+
isToolResult: isToolResultOnly || void 0,
|
|
848
|
+
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
849
|
+
thinkingContent,
|
|
850
|
+
thinkingSignature,
|
|
851
|
+
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
852
|
+
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
853
|
+
promptId: type === "user" ? entry.promptId : void 0,
|
|
854
|
+
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
855
|
+
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
856
|
+
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
857
|
+
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
function applyTeamInfo(messages, state) {
|
|
861
|
+
if (Object.keys(state.teamInfo).length === 0) return;
|
|
862
|
+
for (const m of messages) {
|
|
863
|
+
const name = m.metadata?.teamName;
|
|
864
|
+
if (name && state.teamInfo[name] && m.metadata) m.metadata.teamInfo = state.teamInfo[name];
|
|
865
|
+
}
|
|
523
866
|
}
|
|
524
867
|
|
|
525
868
|
// src/parser.ts
|
|
526
869
|
async function parseMeta(filePath, account, tier) {
|
|
527
870
|
const log = getLogger();
|
|
528
871
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
let latestTimestamp = "";
|
|
533
|
-
let cwd = "";
|
|
534
|
-
let teamName = "";
|
|
535
|
-
let model = null;
|
|
536
|
-
let messageCount = 0;
|
|
537
|
-
let lastMessageSender = "user";
|
|
538
|
-
let isTeammate = false;
|
|
539
|
-
let firstUserSeen = false;
|
|
540
|
-
let firstMessage = null;
|
|
541
|
-
let lastMessage = null;
|
|
542
|
-
let lastPrompt = "";
|
|
543
|
-
const toolNameSet = /* @__PURE__ */ new Set();
|
|
544
|
-
const previewParts = [];
|
|
545
|
-
const snippetParts = [];
|
|
546
|
-
let snippetLength = 0;
|
|
547
|
-
let previewLength = 0;
|
|
548
|
-
const fileStream = (0, import_fs2.createReadStream)(filePath);
|
|
549
|
-
const rl = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
872
|
+
const state = initialReducerState();
|
|
873
|
+
const fileStream = (0, import_fs4.createReadStream)(filePath);
|
|
874
|
+
const rl = (0, import_readline2.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
550
875
|
try {
|
|
551
876
|
for await (const line of rl) {
|
|
552
877
|
if (!line.trim()) continue;
|
|
@@ -554,121 +879,35 @@ async function parseMeta(filePath, account, tier) {
|
|
|
554
879
|
try {
|
|
555
880
|
entry = JSON.parse(line);
|
|
556
881
|
} catch {
|
|
557
|
-
badJsonLines++;
|
|
558
|
-
continue;
|
|
559
|
-
}
|
|
560
|
-
if (entry.cwd && !cwd) cwd = entry.cwd;
|
|
561
|
-
if (entry.sessionId && !sessionId) sessionId = entry.sessionId;
|
|
562
|
-
if (entry.slug && !sessionName) sessionName = entry.slug;
|
|
563
|
-
if (entry.teamName && !teamName) teamName = entry.teamName;
|
|
564
|
-
if (entry.timestamp) {
|
|
565
|
-
const ts = entry.timestamp;
|
|
566
|
-
if (!latestTimestamp || ts > latestTimestamp) latestTimestamp = ts;
|
|
567
|
-
}
|
|
568
|
-
const type = entry.type;
|
|
569
|
-
if (type === "last-prompt") {
|
|
570
|
-
if (entry.lastPrompt && !lastPrompt) lastPrompt = entry.lastPrompt;
|
|
882
|
+
state.badJsonLines++;
|
|
571
883
|
continue;
|
|
572
884
|
}
|
|
573
|
-
|
|
574
|
-
if (entry.isMeta) continue;
|
|
575
|
-
if (model === null) {
|
|
576
|
-
const msg2 = entry.message;
|
|
577
|
-
if (msg2?.model) model = msg2.model;
|
|
578
|
-
}
|
|
579
|
-
if (type === "user" && !firstUserSeen) {
|
|
580
|
-
firstUserSeen = true;
|
|
581
|
-
if (isTeammateContent(entry.message?.content)) {
|
|
582
|
-
isTeammate = true;
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
const msg = entry.message;
|
|
586
|
-
const content = extractTextContent(msg?.content);
|
|
587
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
588
|
-
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
589
|
-
collectToolNames(msg?.content, toolNameSet);
|
|
590
|
-
if (content || isOnlyToolResult) {
|
|
591
|
-
messageCount++;
|
|
592
|
-
lastMessageSender = type;
|
|
593
|
-
if (content) {
|
|
594
|
-
const ts = entry.timestamp || "";
|
|
595
|
-
if (!firstMessage) {
|
|
596
|
-
firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
597
|
-
}
|
|
598
|
-
lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
599
|
-
if (previewLength < tier.previewMax) {
|
|
600
|
-
previewParts.push(content);
|
|
601
|
-
previewLength += content.length;
|
|
602
|
-
}
|
|
603
|
-
if (snippetLength < tier.snippetMax) {
|
|
604
|
-
const remaining = tier.snippetMax - snippetLength;
|
|
605
|
-
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
606
|
-
snippetParts.push(chunk);
|
|
607
|
-
snippetLength += chunk.length;
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
}
|
|
885
|
+
reduceLine(state, entry, tier);
|
|
611
886
|
}
|
|
612
887
|
} catch (err) {
|
|
613
888
|
log.warn({ filePath, err }, "parseMeta: read failed");
|
|
614
889
|
return null;
|
|
615
890
|
}
|
|
616
|
-
if (badJsonLines > 0) {
|
|
617
|
-
log.warn(
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
return null;
|
|
622
|
-
}
|
|
623
|
-
const isSubagent = filePath.includes("/subagents/");
|
|
624
|
-
let parentSessionId = null;
|
|
625
|
-
if (isSubagent) {
|
|
626
|
-
const uuidDir = (0, import_path3.dirname)((0, import_path3.dirname)(filePath));
|
|
627
|
-
parentSessionId = (0, import_path3.join)((0, import_path3.dirname)(uuidDir), `${(0, import_path3.basename)(uuidDir)}.jsonl`);
|
|
891
|
+
if (state.badJsonLines > 0) {
|
|
892
|
+
log.warn(
|
|
893
|
+
{ filePath, badJsonLines: state.badJsonLines },
|
|
894
|
+
"parseMeta: skipped malformed JSON lines"
|
|
895
|
+
);
|
|
628
896
|
}
|
|
629
|
-
const
|
|
630
|
-
|
|
631
|
-
return
|
|
632
|
-
id: filePath,
|
|
633
|
-
filePath,
|
|
634
|
-
sessionId: sessionId || (0, import_path3.basename)(filePath, ".jsonl"),
|
|
635
|
-
sessionName,
|
|
636
|
-
projectPath,
|
|
637
|
-
projectName: getShortProjectName(projectPath),
|
|
638
|
-
account,
|
|
639
|
-
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
640
|
-
messageCount,
|
|
641
|
-
lastMessageSender,
|
|
642
|
-
preview,
|
|
643
|
-
contentSnippet: snippetParts.join(" "),
|
|
644
|
-
gitBranch: null,
|
|
645
|
-
model,
|
|
646
|
-
isSubagent,
|
|
647
|
-
parentSessionId,
|
|
648
|
-
isTeammate,
|
|
649
|
-
teamName: teamName || null,
|
|
650
|
-
toolNames: Array.from(toolNameSet),
|
|
651
|
-
firstMessage,
|
|
652
|
-
lastMessage,
|
|
653
|
-
lastPrompt: lastPrompt || void 0
|
|
654
|
-
};
|
|
897
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
898
|
+
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
899
|
+
return meta;
|
|
655
900
|
}
|
|
656
901
|
async function parseConversation(filePath, account) {
|
|
657
902
|
const log = getLogger();
|
|
658
903
|
log.trace({ filePath, account }, "parseConversation: start");
|
|
659
904
|
const messages = [];
|
|
660
905
|
let badJsonLines = 0;
|
|
661
|
-
let sessionId = "";
|
|
662
|
-
let sessionName = "";
|
|
663
|
-
let latestTimestamp = "";
|
|
664
|
-
let cwd = "";
|
|
665
906
|
const textParts = [];
|
|
666
|
-
const pendingToolUses = /* @__PURE__ */ new Map();
|
|
667
|
-
const teamInfoMap = /* @__PURE__ */ new Map();
|
|
668
907
|
const turnDurations = [];
|
|
669
|
-
|
|
670
|
-
const fileStream = (0,
|
|
671
|
-
const rl = (0,
|
|
908
|
+
const state = initialConvState();
|
|
909
|
+
const fileStream = (0, import_fs4.createReadStream)(filePath);
|
|
910
|
+
const rl = (0, import_readline2.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
672
911
|
try {
|
|
673
912
|
for await (const line of rl) {
|
|
674
913
|
if (!line.trim()) continue;
|
|
@@ -679,91 +918,18 @@ async function parseConversation(filePath, account) {
|
|
|
679
918
|
badJsonLines++;
|
|
680
919
|
continue;
|
|
681
920
|
}
|
|
682
|
-
if (entry.
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
}
|
|
689
|
-
const type = entry.type;
|
|
690
|
-
if (type === "last-prompt") {
|
|
691
|
-
if (entry.lastPrompt && !lastPrompt) lastPrompt = entry.lastPrompt;
|
|
692
|
-
continue;
|
|
693
|
-
}
|
|
694
|
-
if (type === "system") {
|
|
695
|
-
if (entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
696
|
-
turnDurations.push({
|
|
697
|
-
durationMs: entry.durationMs,
|
|
698
|
-
messageCount: entry.messageCount || 0,
|
|
699
|
-
uuid: entry.uuid
|
|
700
|
-
});
|
|
701
|
-
}
|
|
921
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
922
|
+
turnDurations.push({
|
|
923
|
+
durationMs: entry.durationMs,
|
|
924
|
+
messageCount: entry.messageCount || 0,
|
|
925
|
+
uuid: entry.uuid
|
|
926
|
+
});
|
|
702
927
|
continue;
|
|
703
928
|
}
|
|
704
|
-
|
|
705
|
-
if (
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
for (const block of toolUseBlocks) {
|
|
709
|
-
pendingToolUses.set(block.id, block);
|
|
710
|
-
}
|
|
711
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
712
|
-
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
713
|
-
const content = extractTextContent(msg?.content);
|
|
714
|
-
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
715
|
-
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
716
|
-
if (content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking) {
|
|
717
|
-
const metadata = {};
|
|
718
|
-
if (msg?.model) metadata.model = msg.model;
|
|
719
|
-
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
720
|
-
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
721
|
-
if (entry.version) metadata.version = entry.version;
|
|
722
|
-
const usage = msg?.usage;
|
|
723
|
-
if (usage) {
|
|
724
|
-
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
725
|
-
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
726
|
-
if (usage.cache_read_input_tokens)
|
|
727
|
-
metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
728
|
-
if (usage.cache_creation_input_tokens)
|
|
729
|
-
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
730
|
-
}
|
|
731
|
-
const toolUseNames = extractToolUseNames(msg?.content);
|
|
732
|
-
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
733
|
-
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
734
|
-
if (isToolResultOnly) {
|
|
735
|
-
const toolResultBlocks = extractToolResultBlocks(msg?.content, pendingToolUses);
|
|
736
|
-
if (toolResultBlocks.length > 0) metadata.toolResults = toolResultBlocks;
|
|
737
|
-
}
|
|
738
|
-
if (entry.teamName) {
|
|
739
|
-
metadata.teamName = entry.teamName;
|
|
740
|
-
if (!teamInfoMap.has(metadata.teamName) && content) {
|
|
741
|
-
const info = parseTeammateMessageTag(content);
|
|
742
|
-
if (info) teamInfoMap.set(metadata.teamName, info);
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
const thinkingContent = thinking?.content || void 0;
|
|
746
|
-
const thinkingSignature = thinking?.signature || void 0;
|
|
747
|
-
const hasMetadata = Object.keys(metadata).length > 0;
|
|
748
|
-
messages.push({
|
|
749
|
-
role: type,
|
|
750
|
-
text: content || "",
|
|
751
|
-
timestamp: entry.timestamp || "",
|
|
752
|
-
uuid: entry.uuid || void 0,
|
|
753
|
-
metadata: hasMetadata ? metadata : void 0,
|
|
754
|
-
isToolResult: isToolResultOnly || void 0,
|
|
755
|
-
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
756
|
-
thinkingContent,
|
|
757
|
-
thinkingSignature,
|
|
758
|
-
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
759
|
-
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
760
|
-
promptId: type === "user" ? entry.promptId : void 0,
|
|
761
|
-
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
762
|
-
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
763
|
-
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
764
|
-
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
765
|
-
});
|
|
766
|
-
if (content) textParts.push(content);
|
|
929
|
+
const message = reduceConvLine(state, entry);
|
|
930
|
+
if (message) {
|
|
931
|
+
messages.push(message);
|
|
932
|
+
if (message.text) textParts.push(message.text);
|
|
767
933
|
}
|
|
768
934
|
}
|
|
769
935
|
} catch (err) {
|
|
@@ -778,28 +944,21 @@ async function parseConversation(filePath, account) {
|
|
|
778
944
|
return null;
|
|
779
945
|
}
|
|
780
946
|
log.debug({ filePath, messageCount: messages.length }, "parseConversation: complete");
|
|
781
|
-
|
|
782
|
-
for (const msg of messages) {
|
|
783
|
-
if (msg.metadata?.teamName) {
|
|
784
|
-
const info = teamInfoMap.get(msg.metadata.teamName);
|
|
785
|
-
if (info) msg.metadata.teamInfo = info;
|
|
786
|
-
}
|
|
787
|
-
}
|
|
788
|
-
}
|
|
947
|
+
applyTeamInfo(messages, state);
|
|
789
948
|
return {
|
|
790
949
|
id: filePath,
|
|
791
950
|
filePath,
|
|
792
|
-
projectPath: cwd,
|
|
793
|
-
projectName:
|
|
794
|
-
sessionId: sessionId || (0,
|
|
795
|
-
sessionName,
|
|
951
|
+
projectPath: state.cwd,
|
|
952
|
+
projectName: getShortProjectName2(state.cwd),
|
|
953
|
+
sessionId: state.sessionId || (0, import_path4.basename)(filePath, ".jsonl"),
|
|
954
|
+
sessionName: state.sessionName,
|
|
796
955
|
messages,
|
|
797
956
|
fullText: textParts.join(" "),
|
|
798
|
-
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
957
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
799
958
|
messageCount: messages.length,
|
|
800
959
|
account,
|
|
801
960
|
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
802
|
-
lastPrompt: lastPrompt || void 0
|
|
961
|
+
lastPrompt: state.lastPrompt || void 0
|
|
803
962
|
};
|
|
804
963
|
}
|
|
805
964
|
function extractTextContent(content) {
|
|
@@ -892,12 +1051,293 @@ function parseTeammateMessageTag(content) {
|
|
|
892
1051
|
const color = attrs.match(/color="([^"]*)"/)?.[1];
|
|
893
1052
|
return { teammateId: id, summary, color };
|
|
894
1053
|
}
|
|
895
|
-
function
|
|
1054
|
+
function getShortProjectName2(fullPath) {
|
|
896
1055
|
const parts = fullPath.split("/").filter(Boolean);
|
|
897
1056
|
return parts.slice(-3).join("/");
|
|
898
1057
|
}
|
|
899
1058
|
|
|
900
|
-
// src/
|
|
1059
|
+
// src/persistent/metadata-reducer.ts
|
|
1060
|
+
function initialReducerState() {
|
|
1061
|
+
return {
|
|
1062
|
+
sessionId: "",
|
|
1063
|
+
sessionName: "",
|
|
1064
|
+
latestTimestamp: "",
|
|
1065
|
+
cwd: "",
|
|
1066
|
+
teamName: "",
|
|
1067
|
+
model: null,
|
|
1068
|
+
messageCount: 0,
|
|
1069
|
+
lastMessageSender: "user",
|
|
1070
|
+
isTeammate: false,
|
|
1071
|
+
firstUserSeen: false,
|
|
1072
|
+
firstMessage: null,
|
|
1073
|
+
lastMessage: null,
|
|
1074
|
+
lastPrompt: "",
|
|
1075
|
+
pageMessageCount: 0,
|
|
1076
|
+
toolNames: [],
|
|
1077
|
+
previewParts: [],
|
|
1078
|
+
snippetParts: [],
|
|
1079
|
+
previewLength: 0,
|
|
1080
|
+
snippetLength: 0,
|
|
1081
|
+
badJsonLines: 0
|
|
1082
|
+
};
|
|
1083
|
+
}
|
|
1084
|
+
function reduceLine(state, entry, tier) {
|
|
1085
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
1086
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
1087
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
1088
|
+
if (entry.teamName && !state.teamName) state.teamName = entry.teamName;
|
|
1089
|
+
if (entry.timestamp) {
|
|
1090
|
+
const ts = entry.timestamp;
|
|
1091
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
1092
|
+
}
|
|
1093
|
+
const type = entry.type;
|
|
1094
|
+
if (type === "last-prompt") {
|
|
1095
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
1096
|
+
return;
|
|
1097
|
+
}
|
|
1098
|
+
if (type !== "user" && type !== "assistant") return;
|
|
1099
|
+
if (entry.isMeta) return;
|
|
1100
|
+
const msg = entry.message;
|
|
1101
|
+
if (state.model === null && msg?.model) state.model = msg.model;
|
|
1102
|
+
if (type === "user" && !state.firstUserSeen) {
|
|
1103
|
+
state.firstUserSeen = true;
|
|
1104
|
+
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
1105
|
+
}
|
|
1106
|
+
const content = extractTextContent(msg?.content);
|
|
1107
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
1108
|
+
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
1109
|
+
const toolSet = new Set(state.toolNames);
|
|
1110
|
+
collectToolNames(msg?.content, toolSet);
|
|
1111
|
+
state.toolNames = Array.from(toolSet);
|
|
1112
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
1113
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
1114
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
1115
|
+
if (content || isOnlyToolResult || toolUseBlocks.length > 0 || hasThinking) {
|
|
1116
|
+
state.pageMessageCount++;
|
|
1117
|
+
}
|
|
1118
|
+
if (content || isOnlyToolResult) {
|
|
1119
|
+
state.messageCount++;
|
|
1120
|
+
state.lastMessageSender = type;
|
|
1121
|
+
if (content) {
|
|
1122
|
+
const ts = entry.timestamp || "";
|
|
1123
|
+
if (!state.firstMessage) state.firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
1124
|
+
state.lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
1125
|
+
if (state.previewLength < tier.previewMax) {
|
|
1126
|
+
state.previewParts.push(content);
|
|
1127
|
+
state.previewLength += content.length;
|
|
1128
|
+
}
|
|
1129
|
+
if (state.snippetLength < tier.snippetMax) {
|
|
1130
|
+
const remaining = tier.snippetMax - state.snippetLength;
|
|
1131
|
+
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
1132
|
+
state.snippetParts.push(chunk);
|
|
1133
|
+
state.snippetLength += chunk.length;
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
function finalizeMeta(state, filePath, account, tier) {
|
|
1139
|
+
if (state.messageCount === 0) return null;
|
|
1140
|
+
const isSubagent = filePath.includes("/subagents/");
|
|
1141
|
+
let parentSessionId = null;
|
|
1142
|
+
if (isSubagent) {
|
|
1143
|
+
const uuidDir = (0, import_path5.dirname)((0, import_path5.dirname)(filePath));
|
|
1144
|
+
parentSessionId = (0, import_path5.join)((0, import_path5.dirname)(uuidDir), `${(0, import_path5.basename)(uuidDir)}.jsonl`);
|
|
1145
|
+
}
|
|
1146
|
+
const projectPath = state.cwd;
|
|
1147
|
+
return {
|
|
1148
|
+
id: filePath,
|
|
1149
|
+
filePath,
|
|
1150
|
+
provider: CLAUDE_CODE_PROVIDER,
|
|
1151
|
+
sessionId: state.sessionId || (0, import_path5.basename)(filePath, ".jsonl"),
|
|
1152
|
+
sessionName: state.sessionName,
|
|
1153
|
+
projectPath,
|
|
1154
|
+
projectName: getShortProjectName3(projectPath),
|
|
1155
|
+
account,
|
|
1156
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1157
|
+
messageCount: state.messageCount,
|
|
1158
|
+
lastMessageSender: state.lastMessageSender,
|
|
1159
|
+
preview: state.previewParts.join(" ").slice(0, tier.previewMax),
|
|
1160
|
+
contentSnippet: state.snippetParts.join(" "),
|
|
1161
|
+
gitBranch: null,
|
|
1162
|
+
model: state.model,
|
|
1163
|
+
isSubagent,
|
|
1164
|
+
parentSessionId,
|
|
1165
|
+
isTeammate: state.isTeammate,
|
|
1166
|
+
teamName: state.teamName || null,
|
|
1167
|
+
toolNames: state.toolNames,
|
|
1168
|
+
firstMessage: state.firstMessage,
|
|
1169
|
+
lastMessage: state.lastMessage,
|
|
1170
|
+
lastPrompt: state.lastPrompt || void 0
|
|
1171
|
+
};
|
|
1172
|
+
}
|
|
1173
|
+
function getShortProjectName3(fullPath) {
|
|
1174
|
+
const parts = fullPath.split("/").filter(Boolean);
|
|
1175
|
+
return parts.slice(-3).join("/");
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
// src/providers/threadbase.ts
|
|
1179
|
+
var ThreadbaseProvider = class {
|
|
1180
|
+
name = CLAUDE_CODE_PROVIDER;
|
|
1181
|
+
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
1182
|
+
// per-root account through the shared interface. The scanner builds these.
|
|
1183
|
+
async discover(roots) {
|
|
1184
|
+
const dirs = roots.map((r) => {
|
|
1185
|
+
const [projectsDir, account = "default"] = r.split("\0");
|
|
1186
|
+
return { projectsDir, account };
|
|
1187
|
+
});
|
|
1188
|
+
return discoverJsonlFiles(dirs);
|
|
1189
|
+
}
|
|
1190
|
+
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
1191
|
+
canParse(_filePath, sample) {
|
|
1192
|
+
for (const line of sample.split("\n")) {
|
|
1193
|
+
if (!line.trim()) continue;
|
|
1194
|
+
try {
|
|
1195
|
+
const e = JSON.parse(line);
|
|
1196
|
+
if (e.type === "user" || e.type === "assistant") return true;
|
|
1197
|
+
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
1198
|
+
} catch {
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
return false;
|
|
1202
|
+
}
|
|
1203
|
+
createEmptyAccumulator() {
|
|
1204
|
+
return initialReducerState();
|
|
1205
|
+
}
|
|
1206
|
+
reduceEntry(acc, entry, tier) {
|
|
1207
|
+
reduceLine(acc, entry, tier);
|
|
1208
|
+
}
|
|
1209
|
+
finalize(acc, filePath, account, tier) {
|
|
1210
|
+
return finalizeMeta(acc, filePath, account, tier);
|
|
1211
|
+
}
|
|
1212
|
+
};
|
|
1213
|
+
|
|
1214
|
+
// src/scanner.ts
|
|
1215
|
+
var import_events = require("events");
|
|
1216
|
+
var import_fs10 = require("fs");
|
|
1217
|
+
var import_os2 = require("os");
|
|
1218
|
+
var import_path8 = require("path");
|
|
1219
|
+
|
|
1220
|
+
// src/cache.ts
|
|
1221
|
+
var LRUCache = class {
|
|
1222
|
+
map = /* @__PURE__ */ new Map();
|
|
1223
|
+
capacity;
|
|
1224
|
+
constructor(capacity) {
|
|
1225
|
+
this.capacity = capacity;
|
|
1226
|
+
}
|
|
1227
|
+
get(key) {
|
|
1228
|
+
const value = this.map.get(key);
|
|
1229
|
+
if (value === void 0) return void 0;
|
|
1230
|
+
this.map.delete(key);
|
|
1231
|
+
this.map.set(key, value);
|
|
1232
|
+
return value;
|
|
1233
|
+
}
|
|
1234
|
+
set(key, value) {
|
|
1235
|
+
this.map.delete(key);
|
|
1236
|
+
this.map.set(key, value);
|
|
1237
|
+
if (this.map.size > this.capacity) {
|
|
1238
|
+
const oldest = this.map.keys().next();
|
|
1239
|
+
if (!oldest.done) this.map.delete(oldest.value);
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
has(key) {
|
|
1243
|
+
return this.map.has(key);
|
|
1244
|
+
}
|
|
1245
|
+
delete(key) {
|
|
1246
|
+
return this.map.delete(key);
|
|
1247
|
+
}
|
|
1248
|
+
clear() {
|
|
1249
|
+
this.map.clear();
|
|
1250
|
+
}
|
|
1251
|
+
get size() {
|
|
1252
|
+
return this.map.size;
|
|
1253
|
+
}
|
|
1254
|
+
};
|
|
1255
|
+
|
|
1256
|
+
// src/persistent/cursor.ts
|
|
1257
|
+
var import_crypto = require("crypto");
|
|
1258
|
+
var import_fs5 = require("fs");
|
|
1259
|
+
var FP_BYTES = 4096;
|
|
1260
|
+
function fingerprint(filePath, size) {
|
|
1261
|
+
const hash = (0, import_crypto.createHash)("sha1");
|
|
1262
|
+
hash.update(String(size));
|
|
1263
|
+
const fd = (0, import_fs5.openSync)(filePath, "r");
|
|
1264
|
+
try {
|
|
1265
|
+
const head = Buffer.alloc(Math.min(FP_BYTES, size));
|
|
1266
|
+
if (head.length > 0) {
|
|
1267
|
+
(0, import_fs5.readSync)(fd, head, 0, head.length, 0);
|
|
1268
|
+
hash.update(head);
|
|
1269
|
+
}
|
|
1270
|
+
if (size > FP_BYTES) {
|
|
1271
|
+
const tailLen = Math.min(FP_BYTES, size);
|
|
1272
|
+
const tail = Buffer.alloc(tailLen);
|
|
1273
|
+
(0, import_fs5.readSync)(fd, tail, 0, tailLen, size - tailLen);
|
|
1274
|
+
hash.update(tail);
|
|
1275
|
+
}
|
|
1276
|
+
} finally {
|
|
1277
|
+
(0, import_fs5.closeSync)(fd);
|
|
1278
|
+
}
|
|
1279
|
+
return hash.digest("hex");
|
|
1280
|
+
}
|
|
1281
|
+
function classify(filePath, existing) {
|
|
1282
|
+
let stat3;
|
|
1283
|
+
try {
|
|
1284
|
+
const s = (0, import_fs5.statSync)(filePath);
|
|
1285
|
+
stat3 = { size: s.size, mtimeMs: s.mtimeMs };
|
|
1286
|
+
} catch {
|
|
1287
|
+
return { change: "vanished" };
|
|
1288
|
+
}
|
|
1289
|
+
if (!existing || existing.status !== "active" || existing.last_indexed_offset === 0) {
|
|
1290
|
+
return { change: "reindex", stat: stat3 };
|
|
1291
|
+
}
|
|
1292
|
+
if (stat3.size < existing.last_indexed_offset) {
|
|
1293
|
+
return { change: "reindex", stat: stat3 };
|
|
1294
|
+
}
|
|
1295
|
+
if (stat3.size === existing.size_bytes && stat3.mtimeMs === existing.mtime_ms) {
|
|
1296
|
+
return { change: "unchanged", stat: stat3 };
|
|
1297
|
+
}
|
|
1298
|
+
if (stat3.size === existing.last_indexed_offset) {
|
|
1299
|
+
const fp = fingerprint(filePath, stat3.size);
|
|
1300
|
+
if (existing.content_fingerprint && fp !== existing.content_fingerprint) {
|
|
1301
|
+
return { change: "reindex", stat: stat3 };
|
|
1302
|
+
}
|
|
1303
|
+
return { change: "unchanged", stat: stat3 };
|
|
1304
|
+
}
|
|
1305
|
+
return { change: "appended", stat: stat3 };
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
// src/providers/parse.ts
|
|
1309
|
+
var import_fs6 = require("fs");
|
|
1310
|
+
var import_readline3 = require("readline");
|
|
1311
|
+
async function parseMetaWithProvider(provider, filePath, account, tier) {
|
|
1312
|
+
const log = getLogger();
|
|
1313
|
+
const acc = provider.createEmptyAccumulator();
|
|
1314
|
+
const rl = (0, import_readline3.createInterface)({
|
|
1315
|
+
input: (0, import_fs6.createReadStream)(filePath),
|
|
1316
|
+
crlfDelay: Infinity
|
|
1317
|
+
});
|
|
1318
|
+
try {
|
|
1319
|
+
for await (const line of rl) {
|
|
1320
|
+
if (!line.trim()) continue;
|
|
1321
|
+
let entry;
|
|
1322
|
+
try {
|
|
1323
|
+
entry = JSON.parse(line);
|
|
1324
|
+
} catch {
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
try {
|
|
1328
|
+
provider.reduceEntry(acc, entry, tier);
|
|
1329
|
+
} catch (err) {
|
|
1330
|
+
log.warn({ filePath, provider: provider.name, err }, "provider reduce threw; line skipped");
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
} catch (err) {
|
|
1334
|
+
log.warn({ filePath, provider: provider.name, err }, "provider parse: read failed");
|
|
1335
|
+
return null;
|
|
1336
|
+
}
|
|
1337
|
+
return provider.finalize(acc, filePath, account, tier);
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
// src/tiers.ts
|
|
901
1341
|
var DEFAULT_TIERS = {
|
|
902
1342
|
standard: { name: "standard", previewMax: 200, snippetMax: 5e3 },
|
|
903
1343
|
full: { name: "full", previewMax: 1200, snippetMax: 5e4 }
|
|
@@ -912,28 +1352,1108 @@ function resolveTier(tierName, customTiers) {
|
|
|
912
1352
|
return tier;
|
|
913
1353
|
}
|
|
914
1354
|
|
|
915
|
-
// src/
|
|
1355
|
+
// src/persistent/db.ts
|
|
1356
|
+
var import_better_sqlite3 = __toESM(require("better-sqlite3"), 1);
|
|
1357
|
+
var import_fs7 = require("fs");
|
|
1358
|
+
var import_path6 = require("path");
|
|
1359
|
+
|
|
1360
|
+
// src/persistent/schema.ts
|
|
1361
|
+
var SCHEMA_VERSION = 3;
|
|
1362
|
+
var SCHEMA_SQL = `
|
|
1363
|
+
CREATE TABLE IF NOT EXISTS conversation_files (
|
|
1364
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1365
|
+
|
|
1366
|
+
absolute_path TEXT NOT NULL UNIQUE,
|
|
1367
|
+
parent_dir TEXT NOT NULL,
|
|
1368
|
+
file_name TEXT NOT NULL,
|
|
1369
|
+
|
|
1370
|
+
account TEXT NOT NULL DEFAULT 'default',
|
|
1371
|
+
|
|
1372
|
+
size_bytes INTEGER NOT NULL DEFAULT 0,
|
|
1373
|
+
mtime_ms INTEGER NOT NULL DEFAULT 0,
|
|
1374
|
+
|
|
1375
|
+
last_indexed_offset INTEGER NOT NULL DEFAULT 0,
|
|
1376
|
+
last_indexed_line INTEGER NOT NULL DEFAULT 0,
|
|
1377
|
+
|
|
1378
|
+
reducer_state TEXT,
|
|
1379
|
+
|
|
1380
|
+
content_fingerprint TEXT,
|
|
1381
|
+
|
|
1382
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
1383
|
+
|
|
1384
|
+
last_indexed_at TEXT,
|
|
1385
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1386
|
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1387
|
+
deleted_at TEXT
|
|
1388
|
+
);
|
|
1389
|
+
|
|
1390
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_status ON conversation_files(status);
|
|
1391
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_parent_dir ON conversation_files(parent_dir);
|
|
1392
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_account ON conversation_files(account);
|
|
1393
|
+
|
|
1394
|
+
CREATE TABLE IF NOT EXISTS conversations (
|
|
1395
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1396
|
+
|
|
1397
|
+
file_id INTEGER NOT NULL UNIQUE,
|
|
1398
|
+
|
|
1399
|
+
source_path TEXT NOT NULL UNIQUE,
|
|
1400
|
+
-- Which provider produced this row. Canonical identity is (provider,
|
|
1401
|
+
-- source_path); session_id stays non-unique across providers too.
|
|
1402
|
+
provider TEXT NOT NULL DEFAULT 'claude-code',
|
|
1403
|
+
kind TEXT,
|
|
1404
|
+
external_session_id TEXT,
|
|
1405
|
+
session_id TEXT NOT NULL,
|
|
1406
|
+
session_name TEXT,
|
|
1407
|
+
|
|
1408
|
+
project_path TEXT,
|
|
1409
|
+
project_name TEXT,
|
|
1410
|
+
account TEXT NOT NULL DEFAULT 'default',
|
|
1411
|
+
branch TEXT,
|
|
1412
|
+
|
|
1413
|
+
preview TEXT,
|
|
1414
|
+
content_snippet TEXT,
|
|
1415
|
+
|
|
1416
|
+
message_count INTEGER NOT NULL DEFAULT 0,
|
|
1417
|
+
-- Count of messages parseConversation() produces (broader than message_count;
|
|
1418
|
+
-- includes tool_use-only and thinking-only lines). The total for bounded paging.
|
|
1419
|
+
page_message_count INTEGER NOT NULL DEFAULT 0,
|
|
1420
|
+
last_message_sender TEXT NOT NULL DEFAULT 'user',
|
|
1421
|
+
timestamp TEXT,
|
|
1422
|
+
|
|
1423
|
+
-- Monotonic write counter; the highest value is the most recently indexed
|
|
1424
|
+
-- row. Drives last-writer-wins resolution for shared session_ids (matching
|
|
1425
|
+
-- the in-memory sessionId map), at the sub-second precision updated_at lacks.
|
|
1426
|
+
index_seq INTEGER NOT NULL DEFAULT 0,
|
|
1427
|
+
|
|
1428
|
+
first_sent_at TEXT,
|
|
1429
|
+
first_sent_text TEXT,
|
|
1430
|
+
|
|
1431
|
+
last_sent_at TEXT,
|
|
1432
|
+
last_sent_text TEXT,
|
|
1433
|
+
|
|
1434
|
+
model TEXT,
|
|
1435
|
+
is_subagent INTEGER NOT NULL DEFAULT 0,
|
|
1436
|
+
parent_session_id TEXT,
|
|
1437
|
+
is_teammate INTEGER NOT NULL DEFAULT 0,
|
|
1438
|
+
team_name TEXT,
|
|
1439
|
+
tool_names_json TEXT,
|
|
1440
|
+
last_prompt TEXT,
|
|
1441
|
+
|
|
1442
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
1443
|
+
|
|
1444
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1445
|
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1446
|
+
|
|
1447
|
+
FOREIGN KEY (file_id) REFERENCES conversation_files(id)
|
|
1448
|
+
);
|
|
1449
|
+
|
|
1450
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_session_id ON conversations(session_id);
|
|
1451
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_session ON conversations(provider, session_id);
|
|
1452
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_recent ON conversations(provider, timestamp DESC);
|
|
1453
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_project_branch ON conversations(provider, project_path, branch);
|
|
1454
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_recent ON conversations(timestamp DESC);
|
|
1455
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_project_recent ON conversations(project_path, timestamp DESC);
|
|
1456
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_project_branch_recent ON conversations(project_path, branch, timestamp DESC);
|
|
1457
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_account_recent ON conversations(account, timestamp DESC);
|
|
1458
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_subagent_recent ON conversations(is_subagent, timestamp DESC);
|
|
1459
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_team_recent ON conversations(team_name, timestamp DESC);
|
|
1460
|
+
|
|
1461
|
+
-- Full-text search index over conversation content + metadata. Kept separate
|
|
1462
|
+
-- from the metadata tables so list-screen queries stay small and fast.
|
|
1463
|
+
-- source_path is UNINDEXED (stored, not tokenized) and links back to a
|
|
1464
|
+
-- conversations row. One FTS row per conversation, replaced on each upsert.
|
|
1465
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS conversation_messages_fts USING fts5(
|
|
1466
|
+
source_path UNINDEXED,
|
|
1467
|
+
content,
|
|
1468
|
+
project_name,
|
|
1469
|
+
session_id,
|
|
1470
|
+
session_name,
|
|
1471
|
+
account,
|
|
1472
|
+
model,
|
|
1473
|
+
branch,
|
|
1474
|
+
tool_names,
|
|
1475
|
+
tokenize = 'unicode61'
|
|
1476
|
+
);
|
|
1477
|
+
|
|
1478
|
+
-- Seek index for large conversations: every N messages, record the byte offset
|
|
1479
|
+
-- where the next message's line begins plus the parser's cross-line state
|
|
1480
|
+
-- (pending tool_use blocks, team info) needed to resume an equivalent parse
|
|
1481
|
+
-- from that point. Lets getConversationPage() read a window near the end of a
|
|
1482
|
+
-- huge file without parsing from byte 0.
|
|
1483
|
+
CREATE TABLE IF NOT EXISTS message_checkpoints (
|
|
1484
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1485
|
+
source_path TEXT NOT NULL,
|
|
1486
|
+
message_index INTEGER NOT NULL,
|
|
1487
|
+
byte_offset INTEGER NOT NULL,
|
|
1488
|
+
line_number INTEGER NOT NULL,
|
|
1489
|
+
parser_state TEXT NOT NULL,
|
|
1490
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
1491
|
+
);
|
|
1492
|
+
|
|
1493
|
+
CREATE INDEX IF NOT EXISTS idx_message_checkpoints_lookup
|
|
1494
|
+
ON message_checkpoints(source_path, message_index);
|
|
1495
|
+
`;
|
|
1496
|
+
|
|
1497
|
+
// src/persistent/migrations.ts
|
|
1498
|
+
function runMigrations(db) {
|
|
1499
|
+
const current = db.pragma("user_version", { simple: true });
|
|
1500
|
+
if (current >= SCHEMA_VERSION) return;
|
|
1501
|
+
const log = getLogger();
|
|
1502
|
+
log.info({ from: current, to: SCHEMA_VERSION }, "migrations: applying");
|
|
1503
|
+
if (current >= 1 && current < 2 && tableExists(db, "conversations")) {
|
|
1504
|
+
for (const [col, ddl] of [
|
|
1505
|
+
[
|
|
1506
|
+
"provider",
|
|
1507
|
+
// Keep the historical DEFAULT 'threadbase' — v2→v3 below updates these rows.
|
|
1508
|
+
"ALTER TABLE conversations ADD COLUMN provider TEXT NOT NULL DEFAULT 'threadbase'"
|
|
1509
|
+
],
|
|
1510
|
+
["kind", "ALTER TABLE conversations ADD COLUMN kind TEXT"],
|
|
1511
|
+
["external_session_id", "ALTER TABLE conversations ADD COLUMN external_session_id TEXT"]
|
|
1512
|
+
]) {
|
|
1513
|
+
if (!hasColumn(db, "conversations", col)) db.exec(ddl);
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
if (current >= 1 && current < 3 && tableExists(db, "conversations")) {
|
|
1517
|
+
db.exec("UPDATE conversations SET provider = 'claude-code' WHERE provider = 'threadbase'");
|
|
1518
|
+
}
|
|
1519
|
+
db.exec(SCHEMA_SQL);
|
|
1520
|
+
db.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
1521
|
+
}
|
|
1522
|
+
function tableExists(db, table) {
|
|
1523
|
+
return db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== void 0;
|
|
1524
|
+
}
|
|
1525
|
+
function hasColumn(db, table, column) {
|
|
1526
|
+
const cols = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
1527
|
+
return cols.some((c) => c.name === column);
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
// src/persistent/db.ts
|
|
1531
|
+
function openDatabase(dbPath) {
|
|
1532
|
+
if (dbPath !== ":memory:") {
|
|
1533
|
+
(0, import_fs7.mkdirSync)((0, import_path6.dirname)(dbPath), { recursive: true });
|
|
1534
|
+
}
|
|
1535
|
+
const db = new import_better_sqlite3.default(dbPath);
|
|
1536
|
+
db.pragma("journal_mode = WAL");
|
|
1537
|
+
db.pragma("synchronous = NORMAL");
|
|
1538
|
+
db.pragma("temp_store = MEMORY");
|
|
1539
|
+
db.pragma("foreign_keys = ON");
|
|
1540
|
+
runMigrations(db);
|
|
1541
|
+
getLogger().debug({ dbPath }, "db: opened");
|
|
1542
|
+
return db;
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
// src/persistent/jsonl-tail-reader.ts
|
|
1546
|
+
var import_fs8 = require("fs");
|
|
1547
|
+
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1548
|
+
const stream = (0, import_fs8.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1549
|
+
let buffer = "";
|
|
1550
|
+
let offset = startOffset;
|
|
1551
|
+
let line = startLine;
|
|
1552
|
+
let parsedLines = 0;
|
|
1553
|
+
for await (const chunk of stream) {
|
|
1554
|
+
buffer += chunk;
|
|
1555
|
+
let nl;
|
|
1556
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1557
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1558
|
+
const text = lineWithNewline.trimEnd();
|
|
1559
|
+
buffer = buffer.slice(nl + 1);
|
|
1560
|
+
if (text.length > 0) {
|
|
1561
|
+
try {
|
|
1562
|
+
reduceLine(state, JSON.parse(text), tier);
|
|
1563
|
+
} catch {
|
|
1564
|
+
state.badJsonLines++;
|
|
1565
|
+
}
|
|
1566
|
+
parsedLines++;
|
|
1567
|
+
}
|
|
1568
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1569
|
+
line++;
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
// src/persistent/paged-reader.ts
|
|
1576
|
+
var import_fs9 = require("fs");
|
|
1577
|
+
var CHECKPOINT_INTERVAL = 500;
|
|
1578
|
+
async function streamMessages(filePath, startOffset, startLine, state, onMessage) {
|
|
1579
|
+
const stream = (0, import_fs9.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1580
|
+
let buffer = "";
|
|
1581
|
+
let offset = startOffset;
|
|
1582
|
+
let line = startLine;
|
|
1583
|
+
for await (const chunk of stream) {
|
|
1584
|
+
buffer += chunk;
|
|
1585
|
+
let nl;
|
|
1586
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1587
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1588
|
+
const text = lineWithNewline.trimEnd();
|
|
1589
|
+
buffer = buffer.slice(nl + 1);
|
|
1590
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1591
|
+
line += 1;
|
|
1592
|
+
if (text.length === 0) continue;
|
|
1593
|
+
let entry;
|
|
1594
|
+
try {
|
|
1595
|
+
entry = JSON.parse(text);
|
|
1596
|
+
} catch {
|
|
1597
|
+
continue;
|
|
1598
|
+
}
|
|
1599
|
+
const message = reduceConvLine(state, entry);
|
|
1600
|
+
if (message && onMessage(message, offset, line)) {
|
|
1601
|
+
stream.destroy();
|
|
1602
|
+
return;
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL) {
|
|
1608
|
+
const checkpoints = [];
|
|
1609
|
+
const state = initialConvState();
|
|
1610
|
+
let index = 0;
|
|
1611
|
+
await streamMessages(filePath, 0, 0, state, (_msg, nextOffset, nextLine) => {
|
|
1612
|
+
index += 1;
|
|
1613
|
+
if (index % interval === 0) {
|
|
1614
|
+
checkpoints.push({
|
|
1615
|
+
messageIndex: index,
|
|
1616
|
+
byteOffset: nextOffset,
|
|
1617
|
+
lineNumber: nextLine,
|
|
1618
|
+
state: structuredClone(state)
|
|
1619
|
+
});
|
|
1620
|
+
}
|
|
1621
|
+
return false;
|
|
1622
|
+
});
|
|
1623
|
+
return checkpoints;
|
|
1624
|
+
}
|
|
1625
|
+
async function readPage(filePath, total, options, floor) {
|
|
1626
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
1627
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1628
|
+
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1629
|
+
const startOffset = floor ? floor.byteOffset : 0;
|
|
1630
|
+
const startLine = floor ? floor.lineNumber : 0;
|
|
1631
|
+
let index = floor ? floor.messageIndex : 0;
|
|
1632
|
+
const window = [];
|
|
1633
|
+
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1634
|
+
const current = index;
|
|
1635
|
+
index += 1;
|
|
1636
|
+
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1637
|
+
return index >= beforeIndex;
|
|
1638
|
+
});
|
|
1639
|
+
applyTeamInfo(window, state);
|
|
1640
|
+
return { messages: window, total, fromIndex };
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
// src/persistent/repositories/checkpoints.repo.ts
|
|
1644
|
+
var CheckpointsRepo = class {
|
|
1645
|
+
constructor(db) {
|
|
1646
|
+
this.db = db;
|
|
1647
|
+
}
|
|
1648
|
+
db;
|
|
1649
|
+
replaceAll(sourcePath, checkpoints) {
|
|
1650
|
+
const tx = this.db.transaction(() => {
|
|
1651
|
+
this.db.prepare("DELETE FROM message_checkpoints WHERE source_path = ?").run(sourcePath);
|
|
1652
|
+
const insert = this.db.prepare(
|
|
1653
|
+
`INSERT INTO message_checkpoints
|
|
1654
|
+
(source_path, message_index, byte_offset, line_number, parser_state)
|
|
1655
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
1656
|
+
);
|
|
1657
|
+
for (const c of checkpoints) {
|
|
1658
|
+
insert.run(sourcePath, c.messageIndex, c.byteOffset, c.lineNumber, JSON.stringify(c.state));
|
|
1659
|
+
}
|
|
1660
|
+
});
|
|
1661
|
+
tx();
|
|
1662
|
+
}
|
|
1663
|
+
// The latest checkpoint at or before `messageIndex`, or null if none (read
|
|
1664
|
+
// from the file start). Lets a page seek to the nearest prior anchor.
|
|
1665
|
+
floor(sourcePath, messageIndex) {
|
|
1666
|
+
const row = this.db.prepare(
|
|
1667
|
+
`SELECT message_index, byte_offset, line_number, parser_state
|
|
1668
|
+
FROM message_checkpoints
|
|
1669
|
+
WHERE source_path = ? AND message_index <= ?
|
|
1670
|
+
ORDER BY message_index DESC LIMIT 1`
|
|
1671
|
+
).get(sourcePath, messageIndex);
|
|
1672
|
+
return row ? toCheckpoint(row) : null;
|
|
1673
|
+
}
|
|
1674
|
+
count(sourcePath) {
|
|
1675
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM message_checkpoints WHERE source_path = ?").get(sourcePath).n;
|
|
1676
|
+
}
|
|
1677
|
+
remove(sourcePath) {
|
|
1678
|
+
this.db.prepare("DELETE FROM message_checkpoints WHERE source_path = ?").run(sourcePath);
|
|
1679
|
+
}
|
|
1680
|
+
};
|
|
1681
|
+
function toCheckpoint(row) {
|
|
1682
|
+
return {
|
|
1683
|
+
messageIndex: row.message_index,
|
|
1684
|
+
byteOffset: row.byte_offset,
|
|
1685
|
+
lineNumber: row.line_number,
|
|
1686
|
+
state: JSON.parse(row.parser_state)
|
|
1687
|
+
};
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
// src/persistent/repositories/conversation-files.repo.ts
|
|
1691
|
+
var import_path7 = require("path");
|
|
1692
|
+
var ConversationFilesRepo = class {
|
|
1693
|
+
constructor(db) {
|
|
1694
|
+
this.db = db;
|
|
1695
|
+
}
|
|
1696
|
+
db;
|
|
1697
|
+
getByPath(absolutePath) {
|
|
1698
|
+
return this.db.prepare("SELECT * FROM conversation_files WHERE absolute_path = ?").get(absolutePath);
|
|
1699
|
+
}
|
|
1700
|
+
// Insert a freshly-discovered file at offset 0; returns its row id. Existing
|
|
1701
|
+
// path is left untouched (returns the existing id) so a re-discovery is safe.
|
|
1702
|
+
ensure(absolutePath, account) {
|
|
1703
|
+
const existing = this.getByPath(absolutePath);
|
|
1704
|
+
if (existing) return existing.id;
|
|
1705
|
+
const info = this.db.prepare(
|
|
1706
|
+
`INSERT INTO conversation_files (absolute_path, parent_dir, file_name, account)
|
|
1707
|
+
VALUES (?, ?, ?, ?)`
|
|
1708
|
+
).run(absolutePath, (0, import_path7.dirname)(absolutePath), (0, import_path7.basename)(absolutePath), account);
|
|
1709
|
+
return Number(info.lastInsertRowid);
|
|
1710
|
+
}
|
|
1711
|
+
// Advance the cursor + persisted reducer state after a successful index pass.
|
|
1712
|
+
updateCursor(id, fields) {
|
|
1713
|
+
this.db.prepare(
|
|
1714
|
+
`UPDATE conversation_files
|
|
1715
|
+
SET size_bytes = ?, mtime_ms = ?, last_indexed_offset = ?, last_indexed_line = ?,
|
|
1716
|
+
reducer_state = ?, content_fingerprint = ?, status = ?,
|
|
1717
|
+
last_indexed_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
|
|
1718
|
+
WHERE id = ?`
|
|
1719
|
+
).run(
|
|
1720
|
+
fields.sizeBytes,
|
|
1721
|
+
fields.mtimeMs,
|
|
1722
|
+
fields.offset,
|
|
1723
|
+
fields.line,
|
|
1724
|
+
fields.reducerState,
|
|
1725
|
+
fields.fingerprint,
|
|
1726
|
+
fields.status ?? "active",
|
|
1727
|
+
id
|
|
1728
|
+
);
|
|
1729
|
+
}
|
|
1730
|
+
// Reset the cursor to 0 for a truncated/replaced file before a full reindex.
|
|
1731
|
+
resetCursor(id) {
|
|
1732
|
+
this.db.prepare(
|
|
1733
|
+
`UPDATE conversation_files
|
|
1734
|
+
SET last_indexed_offset = 0, last_indexed_line = 0, reducer_state = NULL,
|
|
1735
|
+
status = 'needs_reindex', updated_at = CURRENT_TIMESTAMP
|
|
1736
|
+
WHERE id = ?`
|
|
1737
|
+
).run(id);
|
|
1738
|
+
}
|
|
1739
|
+
setStatus(id, status) {
|
|
1740
|
+
const deletedAt = status === "deleted" ? "CURRENT_TIMESTAMP" : "deleted_at";
|
|
1741
|
+
this.db.prepare(
|
|
1742
|
+
`UPDATE conversation_files
|
|
1743
|
+
SET status = ?, deleted_at = ${deletedAt}, updated_at = CURRENT_TIMESTAMP
|
|
1744
|
+
WHERE id = ?`
|
|
1745
|
+
).run(status, id);
|
|
1746
|
+
}
|
|
1747
|
+
allActivePaths() {
|
|
1748
|
+
const rows = this.db.prepare("SELECT absolute_path FROM conversation_files WHERE status != 'deleted'").all();
|
|
1749
|
+
return rows.map((r) => r.absolute_path);
|
|
1750
|
+
}
|
|
1751
|
+
};
|
|
1752
|
+
|
|
1753
|
+
// src/persistent/repositories/conversations.repo.ts
|
|
1754
|
+
function rowToMeta(row) {
|
|
1755
|
+
return {
|
|
1756
|
+
id: row.source_path,
|
|
1757
|
+
filePath: row.source_path,
|
|
1758
|
+
provider: row.provider ?? CLAUDE_CODE_PROVIDER,
|
|
1759
|
+
kind: row.kind ?? void 0,
|
|
1760
|
+
externalSessionId: row.external_session_id ?? void 0,
|
|
1761
|
+
sessionId: row.session_id,
|
|
1762
|
+
sessionName: row.session_name ?? "",
|
|
1763
|
+
projectPath: row.project_path ?? "",
|
|
1764
|
+
projectName: row.project_name ?? "",
|
|
1765
|
+
account: row.account,
|
|
1766
|
+
timestamp: row.timestamp ?? "",
|
|
1767
|
+
messageCount: row.message_count,
|
|
1768
|
+
lastMessageSender: row.last_message_sender,
|
|
1769
|
+
preview: row.preview ?? "",
|
|
1770
|
+
contentSnippet: row.content_snippet ?? "",
|
|
1771
|
+
gitBranch: row.branch,
|
|
1772
|
+
model: row.model,
|
|
1773
|
+
isSubagent: row.is_subagent === 1,
|
|
1774
|
+
parentSessionId: row.parent_session_id,
|
|
1775
|
+
isTeammate: row.is_teammate === 1,
|
|
1776
|
+
teamName: row.team_name,
|
|
1777
|
+
toolNames: row.tool_names_json ? JSON.parse(row.tool_names_json) : [],
|
|
1778
|
+
firstMessage: row.first_sent_text ? { text: row.first_sent_text, timestamp: row.first_sent_at ?? "" } : null,
|
|
1779
|
+
lastMessage: row.last_sent_text ? { text: row.last_sent_text, timestamp: row.last_sent_at ?? "" } : null,
|
|
1780
|
+
lastPrompt: row.last_prompt ?? void 0
|
|
1781
|
+
};
|
|
1782
|
+
}
|
|
1783
|
+
var ConversationsRepo = class {
|
|
1784
|
+
constructor(db) {
|
|
1785
|
+
this.db = db;
|
|
1786
|
+
}
|
|
1787
|
+
db;
|
|
1788
|
+
// Upsert by file_id (1 file = 1 conversation). Keyed on the unique file_id so
|
|
1789
|
+
// a re-index overwrites the prior summary in place.
|
|
1790
|
+
upsert(fileId, meta, pageMessageCount = meta.messageCount) {
|
|
1791
|
+
this.db.prepare(
|
|
1792
|
+
`INSERT INTO conversations (
|
|
1793
|
+
file_id, source_path, provider, kind, external_session_id,
|
|
1794
|
+
session_id, session_name, project_path, project_name,
|
|
1795
|
+
account, branch, preview, content_snippet, message_count, page_message_count,
|
|
1796
|
+
last_message_sender,
|
|
1797
|
+
timestamp, index_seq, first_sent_at, first_sent_text, last_sent_at, last_sent_text,
|
|
1798
|
+
model, is_subagent, parent_session_id, is_teammate, team_name, tool_names_json,
|
|
1799
|
+
last_prompt, status, updated_at
|
|
1800
|
+
) VALUES (
|
|
1801
|
+
@file_id, @source_path, @provider, @kind, @external_session_id,
|
|
1802
|
+
@session_id, @session_name, @project_path, @project_name,
|
|
1803
|
+
@account, @branch, @preview, @content_snippet, @message_count, @page_message_count,
|
|
1804
|
+
@last_message_sender,
|
|
1805
|
+
@timestamp,
|
|
1806
|
+
(SELECT COALESCE(MAX(index_seq), 0) + 1 FROM conversations),
|
|
1807
|
+
@first_sent_at, @first_sent_text, @last_sent_at, @last_sent_text,
|
|
1808
|
+
@model, @is_subagent, @parent_session_id, @is_teammate, @team_name, @tool_names_json,
|
|
1809
|
+
@last_prompt, 'active', CURRENT_TIMESTAMP
|
|
1810
|
+
)
|
|
1811
|
+
ON CONFLICT(file_id) DO UPDATE SET
|
|
1812
|
+
source_path = excluded.source_path,
|
|
1813
|
+
provider = excluded.provider,
|
|
1814
|
+
kind = excluded.kind,
|
|
1815
|
+
external_session_id = excluded.external_session_id,
|
|
1816
|
+
session_id = excluded.session_id,
|
|
1817
|
+
session_name = excluded.session_name,
|
|
1818
|
+
project_path = excluded.project_path,
|
|
1819
|
+
project_name = excluded.project_name,
|
|
1820
|
+
account = excluded.account,
|
|
1821
|
+
branch = excluded.branch,
|
|
1822
|
+
preview = excluded.preview,
|
|
1823
|
+
content_snippet = excluded.content_snippet,
|
|
1824
|
+
message_count = excluded.message_count,
|
|
1825
|
+
page_message_count = excluded.page_message_count,
|
|
1826
|
+
last_message_sender = excluded.last_message_sender,
|
|
1827
|
+
timestamp = excluded.timestamp,
|
|
1828
|
+
first_sent_at = excluded.first_sent_at,
|
|
1829
|
+
first_sent_text = excluded.first_sent_text,
|
|
1830
|
+
last_sent_at = excluded.last_sent_at,
|
|
1831
|
+
last_sent_text = excluded.last_sent_text,
|
|
1832
|
+
model = excluded.model,
|
|
1833
|
+
is_subagent = excluded.is_subagent,
|
|
1834
|
+
parent_session_id = excluded.parent_session_id,
|
|
1835
|
+
is_teammate = excluded.is_teammate,
|
|
1836
|
+
team_name = excluded.team_name,
|
|
1837
|
+
tool_names_json = excluded.tool_names_json,
|
|
1838
|
+
last_prompt = excluded.last_prompt,
|
|
1839
|
+
status = 'active',
|
|
1840
|
+
index_seq = (SELECT COALESCE(MAX(index_seq), 0) + 1 FROM conversations),
|
|
1841
|
+
updated_at = CURRENT_TIMESTAMP`
|
|
1842
|
+
).run({
|
|
1843
|
+
file_id: fileId,
|
|
1844
|
+
source_path: meta.id,
|
|
1845
|
+
provider: meta.provider ?? CLAUDE_CODE_PROVIDER,
|
|
1846
|
+
kind: meta.kind ?? null,
|
|
1847
|
+
external_session_id: meta.externalSessionId ?? null,
|
|
1848
|
+
session_id: meta.sessionId,
|
|
1849
|
+
session_name: meta.sessionName || null,
|
|
1850
|
+
project_path: meta.projectPath || null,
|
|
1851
|
+
project_name: meta.projectName || null,
|
|
1852
|
+
account: meta.account,
|
|
1853
|
+
branch: meta.gitBranch,
|
|
1854
|
+
preview: meta.preview || null,
|
|
1855
|
+
content_snippet: meta.contentSnippet || null,
|
|
1856
|
+
message_count: meta.messageCount,
|
|
1857
|
+
page_message_count: pageMessageCount,
|
|
1858
|
+
last_message_sender: meta.lastMessageSender,
|
|
1859
|
+
timestamp: meta.timestamp || null,
|
|
1860
|
+
first_sent_at: meta.firstMessage?.timestamp ?? null,
|
|
1861
|
+
first_sent_text: meta.firstMessage?.text ?? null,
|
|
1862
|
+
last_sent_at: meta.lastMessage?.timestamp ?? null,
|
|
1863
|
+
last_sent_text: meta.lastMessage?.text ?? null,
|
|
1864
|
+
model: meta.model,
|
|
1865
|
+
is_subagent: meta.isSubagent ? 1 : 0,
|
|
1866
|
+
parent_session_id: meta.parentSessionId,
|
|
1867
|
+
is_teammate: meta.isTeammate ? 1 : 0,
|
|
1868
|
+
team_name: meta.teamName,
|
|
1869
|
+
tool_names_json: JSON.stringify(meta.toolNames),
|
|
1870
|
+
last_prompt: meta.lastPrompt ?? null
|
|
1871
|
+
});
|
|
1872
|
+
}
|
|
1873
|
+
getBySourcePath(sourcePath) {
|
|
1874
|
+
const row = this.db.prepare("SELECT * FROM conversations WHERE source_path = ? AND status = 'active'").get(sourcePath);
|
|
1875
|
+
return row ? rowToMeta(row) : null;
|
|
1876
|
+
}
|
|
1877
|
+
// Dual lookup matching scanner.getConversation: resolve by source_path (the
|
|
1878
|
+
// canonical id) first, then by session_id.
|
|
1879
|
+
//
|
|
1880
|
+
// session_id is NOT unique (see schema header) — the sessionId form is a
|
|
1881
|
+
// compatibility convenience. Resolution is deterministic and matches the
|
|
1882
|
+
// in-memory scanner: among active rows sharing the session_id, newest
|
|
1883
|
+
// timestamp wins, index_seq breaks sub-second ties (precision updated_at
|
|
1884
|
+
// lacks), then source_path ascending. Collision-safe callers should use
|
|
1885
|
+
// getAllBySessionId() instead.
|
|
1886
|
+
getByIdOrSession(id) {
|
|
1887
|
+
const direct = this.getBySourcePath(id);
|
|
1888
|
+
if (direct) return direct;
|
|
1889
|
+
const row = this.db.prepare(
|
|
1890
|
+
`SELECT * FROM conversations WHERE session_id = ? AND status = 'active'
|
|
1891
|
+
ORDER BY COALESCE(timestamp, '') DESC, index_seq DESC, source_path ASC LIMIT 1`
|
|
1892
|
+
).get(id);
|
|
1893
|
+
return row ? rowToMeta(row) : null;
|
|
1894
|
+
}
|
|
1895
|
+
// All active conversations sharing a session_id, newest first (same ordering
|
|
1896
|
+
// as getByIdOrSession). Collision-safe counterpart to the convenience
|
|
1897
|
+
// getByIdOrSession() sessionId lookup.
|
|
1898
|
+
getAllBySessionId(sessionId) {
|
|
1899
|
+
const rows = this.db.prepare(
|
|
1900
|
+
`SELECT * FROM conversations WHERE session_id = ? AND status = 'active'
|
|
1901
|
+
ORDER BY COALESCE(timestamp, '') DESC, index_seq DESC, source_path ASC`
|
|
1902
|
+
).all(sessionId);
|
|
1903
|
+
return rows.map(rowToMeta);
|
|
1904
|
+
}
|
|
1905
|
+
// All active metas (unsorted/unfiltered) — callers apply the existing
|
|
1906
|
+
// filters/view transforms. Used by scan() before SQL filtering is wired in.
|
|
1907
|
+
allActive() {
|
|
1908
|
+
const rows = this.db.prepare("SELECT * FROM conversations WHERE status = 'active'").all();
|
|
1909
|
+
return rows.map(rowToMeta);
|
|
1910
|
+
}
|
|
1911
|
+
// Most recent active conversations, newest first. Backs the empty-query
|
|
1912
|
+
// search path (mirrors the in-memory indexer's getRecent).
|
|
1913
|
+
recent(limit) {
|
|
1914
|
+
const rows = this.db.prepare(
|
|
1915
|
+
`SELECT * FROM conversations WHERE status = 'active'
|
|
1916
|
+
ORDER BY COALESCE(timestamp, '') DESC, source_path ASC LIMIT ?`
|
|
1917
|
+
).all(limit);
|
|
1918
|
+
return rows.map(rowToMeta);
|
|
1919
|
+
}
|
|
1920
|
+
distinctProjects() {
|
|
1921
|
+
const rows = this.db.prepare(
|
|
1922
|
+
`SELECT DISTINCT project_path FROM conversations
|
|
1923
|
+
WHERE status = 'active' AND project_path IS NOT NULL AND project_path != ''
|
|
1924
|
+
ORDER BY project_path ASC`
|
|
1925
|
+
).all();
|
|
1926
|
+
return rows.map((r) => r.project_path);
|
|
1927
|
+
}
|
|
1928
|
+
deleteByFileId(fileId) {
|
|
1929
|
+
this.db.prepare(
|
|
1930
|
+
"UPDATE conversations SET status = 'deleted', updated_at = CURRENT_TIMESTAMP WHERE file_id = ?"
|
|
1931
|
+
).run(fileId);
|
|
1932
|
+
}
|
|
1933
|
+
// The parseConversation message total for a file (for bounded paging), or 0
|
|
1934
|
+
// if not indexed.
|
|
1935
|
+
pageMessageCount(sourcePath) {
|
|
1936
|
+
const row = this.db.prepare(
|
|
1937
|
+
"SELECT page_message_count AS n FROM conversations WHERE source_path = ? AND status = 'active'"
|
|
1938
|
+
).get(sourcePath);
|
|
1939
|
+
return row?.n ?? 0;
|
|
1940
|
+
}
|
|
1941
|
+
count() {
|
|
1942
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM conversations WHERE status = 'active'").get().n;
|
|
1943
|
+
}
|
|
1944
|
+
};
|
|
1945
|
+
|
|
1946
|
+
// src/persistent/repositories/fts.repo.ts
|
|
1947
|
+
var FtsRepo = class {
|
|
1948
|
+
constructor(db) {
|
|
1949
|
+
this.db = db;
|
|
1950
|
+
}
|
|
1951
|
+
db;
|
|
1952
|
+
upsert(meta) {
|
|
1953
|
+
const tx = this.db.transaction(() => {
|
|
1954
|
+
this.db.prepare("DELETE FROM conversation_messages_fts WHERE source_path = ?").run(meta.id);
|
|
1955
|
+
this.db.prepare(
|
|
1956
|
+
`INSERT INTO conversation_messages_fts
|
|
1957
|
+
(source_path, content, project_name, session_id, session_name, account, model, branch, tool_names)
|
|
1958
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1959
|
+
).run(
|
|
1960
|
+
meta.id,
|
|
1961
|
+
meta.contentSnippet ?? "",
|
|
1962
|
+
meta.projectName ?? "",
|
|
1963
|
+
meta.sessionId ?? "",
|
|
1964
|
+
meta.sessionName ?? "",
|
|
1965
|
+
meta.account ?? "",
|
|
1966
|
+
meta.model ?? "",
|
|
1967
|
+
meta.gitBranch ?? "",
|
|
1968
|
+
meta.toolNames.join(" ")
|
|
1969
|
+
);
|
|
1970
|
+
});
|
|
1971
|
+
tx();
|
|
1972
|
+
}
|
|
1973
|
+
remove(sourcePath) {
|
|
1974
|
+
this.db.prepare("DELETE FROM conversation_messages_fts WHERE source_path = ?").run(sourcePath);
|
|
1975
|
+
}
|
|
1976
|
+
// Ranked source_paths matching the query, best first. Returns [] on an empty
|
|
1977
|
+
// query (callers fall back to a recency listing).
|
|
1978
|
+
search(query, limit) {
|
|
1979
|
+
const match = toMatchQuery(query);
|
|
1980
|
+
if (!match) return [];
|
|
1981
|
+
const rows = this.db.prepare(
|
|
1982
|
+
`SELECT source_path FROM conversation_messages_fts
|
|
1983
|
+
WHERE conversation_messages_fts MATCH ?
|
|
1984
|
+
ORDER BY rank
|
|
1985
|
+
LIMIT ?`
|
|
1986
|
+
).all(match, limit);
|
|
1987
|
+
return rows.map((r) => r.source_path);
|
|
1988
|
+
}
|
|
1989
|
+
count() {
|
|
1990
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM conversation_messages_fts").get().n;
|
|
1991
|
+
}
|
|
1992
|
+
};
|
|
1993
|
+
function toMatchQuery(query) {
|
|
1994
|
+
const terms = query.trim().split(/\s+/).map((t) => t.replace(/"/g, "").trim()).filter(Boolean);
|
|
1995
|
+
if (terms.length === 0) return "";
|
|
1996
|
+
return terms.map((t) => `"${t}"*`).join(" AND ");
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1999
|
+
// src/persistent/index-engine.ts
|
|
916
2000
|
var BATCH_SIZE = 12;
|
|
2001
|
+
var PersistentEngine = class {
|
|
2002
|
+
db;
|
|
2003
|
+
files;
|
|
2004
|
+
conversations;
|
|
2005
|
+
fts;
|
|
2006
|
+
checkpoints;
|
|
2007
|
+
// When true, write a portable <file>.idx.json sidecar next to each indexed
|
|
2008
|
+
// JSONL. Off by default.
|
|
2009
|
+
sidecar;
|
|
2010
|
+
constructor(dbPath, options = {}) {
|
|
2011
|
+
this.db = openDatabase(dbPath);
|
|
2012
|
+
this.files = new ConversationFilesRepo(this.db);
|
|
2013
|
+
this.conversations = new ConversationsRepo(this.db);
|
|
2014
|
+
this.fts = new FtsRepo(this.db);
|
|
2015
|
+
this.checkpoints = new CheckpointsRepo(this.db);
|
|
2016
|
+
this.sidecar = options.sidecar ?? false;
|
|
2017
|
+
}
|
|
2018
|
+
close() {
|
|
2019
|
+
this.db.close();
|
|
2020
|
+
}
|
|
2021
|
+
// Discover all JSONL files under the active profiles, (re)parse files that
|
|
2022
|
+
// changed since the last index, and upsert their metadata. Returns the number
|
|
2023
|
+
// of files seen on disk this pass (the scan "scanned" count).
|
|
2024
|
+
async indexAll(activeProfiles, options) {
|
|
2025
|
+
const log = getLogger();
|
|
2026
|
+
const tier = resolveTier(options.tier ?? "standard", options.tiers);
|
|
2027
|
+
const enabled = options.providers ?? [CLAUDE_CODE_PROVIDER];
|
|
2028
|
+
const discovered = [];
|
|
2029
|
+
if (enabled.includes(CLAUDE_CODE_PROVIDER)) {
|
|
2030
|
+
const configDirs = activeProfiles.map((p) => ({
|
|
2031
|
+
projectsDir: getProjectsDir(p),
|
|
2032
|
+
account: p.id
|
|
2033
|
+
}));
|
|
2034
|
+
for (const f of await discoverJsonlFiles(configDirs)) discovered.push(f);
|
|
2035
|
+
}
|
|
2036
|
+
const codex = new CodexCliProvider();
|
|
2037
|
+
if (enabled.includes(CODEX_CLI_PROVIDER) && (options.codexRoots?.length ?? 0) > 0) {
|
|
2038
|
+
for (const f of await codex.discover(options.codexRoots)) {
|
|
2039
|
+
discovered.push({ ...f, provider: codex });
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
let scanned = 0;
|
|
2043
|
+
const gitBranchMemo = /* @__PURE__ */ new Map();
|
|
2044
|
+
const resolveGitBranch = (projectPath) => {
|
|
2045
|
+
if (gitBranchMemo.has(projectPath)) {
|
|
2046
|
+
return gitBranchMemo.get(projectPath) ?? null;
|
|
2047
|
+
}
|
|
2048
|
+
const branch = readGitBranch(projectPath);
|
|
2049
|
+
gitBranchMemo.set(projectPath, branch);
|
|
2050
|
+
return branch;
|
|
2051
|
+
};
|
|
2052
|
+
for (let i = 0; i < discovered.length; i += BATCH_SIZE) {
|
|
2053
|
+
const batch = discovered.slice(i, i + BATCH_SIZE);
|
|
2054
|
+
const results = await Promise.all(
|
|
2055
|
+
batch.map(async ({ filePath, account, provider }) => {
|
|
2056
|
+
const meta = await this.indexFile(
|
|
2057
|
+
filePath,
|
|
2058
|
+
account,
|
|
2059
|
+
tier.name,
|
|
2060
|
+
options.tiers,
|
|
2061
|
+
resolveGitBranch,
|
|
2062
|
+
false,
|
|
2063
|
+
provider
|
|
2064
|
+
);
|
|
2065
|
+
return meta;
|
|
2066
|
+
})
|
|
2067
|
+
);
|
|
2068
|
+
const kept = results.filter((m) => m != null);
|
|
2069
|
+
if (kept.length > 0) options.onBatch?.(kept);
|
|
2070
|
+
scanned += batch.length;
|
|
2071
|
+
options.onProgress?.(scanned, discovered.length);
|
|
2072
|
+
}
|
|
2073
|
+
const seen = new Set(discovered.map((d) => d.filePath));
|
|
2074
|
+
for (const path of this.files.allActivePaths()) {
|
|
2075
|
+
if (!seen.has(path)) this.markDeleted(path);
|
|
2076
|
+
}
|
|
2077
|
+
log.info({ scanned, indexed: this.conversations.count() }, "persistent: indexAll complete");
|
|
2078
|
+
return { scanned };
|
|
2079
|
+
}
|
|
2080
|
+
// (Re)index a single file. Classifies the change vs. the persisted cursor:
|
|
2081
|
+
// unchanged → return the stored summary; appended → resume the fold and read
|
|
2082
|
+
// only new bytes; reindex/force → fold from offset 0. Writes the summary +
|
|
2083
|
+
// cursor + reducer state in one transaction so a crash never leaves a
|
|
2084
|
+
// half-written row or an over-advanced cursor.
|
|
2085
|
+
async indexFile(filePath, account, tierName, customTiers, resolveGitBranch, force = false, provider) {
|
|
2086
|
+
const log = getLogger();
|
|
2087
|
+
const tier = resolveTier(tierName, customTiers);
|
|
2088
|
+
const existing = this.files.getByPath(filePath);
|
|
2089
|
+
const { change, stat: stat3 } = classify(filePath, existing);
|
|
2090
|
+
if (change === "vanished" || !stat3) {
|
|
2091
|
+
this.markDeleted(filePath);
|
|
2092
|
+
return null;
|
|
2093
|
+
}
|
|
2094
|
+
if (change === "unchanged" && !force) {
|
|
2095
|
+
return this.conversations.getBySourcePath(filePath);
|
|
2096
|
+
}
|
|
2097
|
+
if (provider && provider.name !== CLAUDE_CODE_PROVIDER) {
|
|
2098
|
+
return this.indexFileWithProvider(provider, filePath, account, tier, stat3, resolveGitBranch);
|
|
2099
|
+
}
|
|
2100
|
+
const resume = change === "appended" && !force && existing?.reducer_state;
|
|
2101
|
+
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
2102
|
+
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
2103
|
+
const startLine = resume ? existing.last_indexed_line : 0;
|
|
2104
|
+
let result;
|
|
2105
|
+
try {
|
|
2106
|
+
result = await tailReduce(filePath, startOffset, startLine, state, tier);
|
|
2107
|
+
} catch (err) {
|
|
2108
|
+
log.warn({ filePath, err }, "persistent: tail read failed");
|
|
2109
|
+
return null;
|
|
2110
|
+
}
|
|
2111
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
2112
|
+
if (!meta) {
|
|
2113
|
+
this.markDeleted(filePath);
|
|
2114
|
+
return null;
|
|
2115
|
+
}
|
|
2116
|
+
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2117
|
+
const fp = stat3.size > 0 ? fingerprint(filePath, stat3.size) : null;
|
|
2118
|
+
const fileId = this.files.ensure(filePath, account);
|
|
2119
|
+
const upsert = this.db.transaction(() => {
|
|
2120
|
+
this.conversations.upsert(fileId, meta, state.pageMessageCount);
|
|
2121
|
+
this.fts.upsert(meta);
|
|
2122
|
+
this.checkpoints.remove(filePath);
|
|
2123
|
+
this.files.updateCursor(fileId, {
|
|
2124
|
+
sizeBytes: stat3.size,
|
|
2125
|
+
mtimeMs: stat3.mtimeMs,
|
|
2126
|
+
// Advance only to the last fully-parsed line; a trailing partial line
|
|
2127
|
+
// is left for the next pass.
|
|
2128
|
+
offset: result.newOffset,
|
|
2129
|
+
line: result.newLine,
|
|
2130
|
+
reducerState: JSON.stringify(state),
|
|
2131
|
+
fingerprint: fp,
|
|
2132
|
+
status: "active"
|
|
2133
|
+
});
|
|
2134
|
+
});
|
|
2135
|
+
upsert();
|
|
2136
|
+
if (this.sidecar) {
|
|
2137
|
+
writeSidecar(
|
|
2138
|
+
filePath,
|
|
2139
|
+
buildSidecar(
|
|
2140
|
+
meta,
|
|
2141
|
+
{
|
|
2142
|
+
sizeBytes: stat3.size,
|
|
2143
|
+
mtimeMs: stat3.mtimeMs,
|
|
2144
|
+
offset: result.newOffset,
|
|
2145
|
+
line: result.newLine
|
|
2146
|
+
},
|
|
2147
|
+
(/* @__PURE__ */ new Date()).toISOString()
|
|
2148
|
+
)
|
|
2149
|
+
);
|
|
2150
|
+
}
|
|
2151
|
+
log.debug(
|
|
2152
|
+
{ filePath, change, bytesRead: result.newOffset - startOffset, msgs: meta.messageCount },
|
|
2153
|
+
"persistent: indexed file"
|
|
2154
|
+
);
|
|
2155
|
+
return meta;
|
|
2156
|
+
}
|
|
2157
|
+
// Index a non-Threadbase provider file: full reparse from offset 0 through the
|
|
2158
|
+
// provider's reducer/finalize, then the same upsert + FTS write + cursor bump
|
|
2159
|
+
// the Threadbase path uses. The cursor records size/mtime (and offset = size)
|
|
2160
|
+
// so the next pass classifies an unchanged file as "unchanged" and skips it;
|
|
2161
|
+
// any change reparses from 0 again. No reducer_state is persisted.
|
|
2162
|
+
async indexFileWithProvider(provider, filePath, account, tier, stat3, resolveGitBranch) {
|
|
2163
|
+
const log = getLogger();
|
|
2164
|
+
const meta = await parseMetaWithProvider(provider, filePath, account, tier);
|
|
2165
|
+
if (!meta) {
|
|
2166
|
+
this.markDeleted(filePath);
|
|
2167
|
+
return null;
|
|
2168
|
+
}
|
|
2169
|
+
if (meta.gitBranch === null && meta.projectPath) {
|
|
2170
|
+
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2171
|
+
}
|
|
2172
|
+
const fp = stat3.size > 0 ? fingerprint(filePath, stat3.size) : null;
|
|
2173
|
+
const fileId = this.files.ensure(filePath, account);
|
|
2174
|
+
const upsert = this.db.transaction(() => {
|
|
2175
|
+
this.conversations.upsert(fileId, meta, meta.messageCount);
|
|
2176
|
+
this.fts.upsert(meta);
|
|
2177
|
+
this.checkpoints.remove(filePath);
|
|
2178
|
+
this.files.updateCursor(fileId, {
|
|
2179
|
+
sizeBytes: stat3.size,
|
|
2180
|
+
mtimeMs: stat3.mtimeMs,
|
|
2181
|
+
// offset = size marks the file fully consumed (non-zero so the next pass
|
|
2182
|
+
// can classify it "unchanged"); no resumable reducer state is kept.
|
|
2183
|
+
offset: stat3.size,
|
|
2184
|
+
line: 0,
|
|
2185
|
+
reducerState: null,
|
|
2186
|
+
fingerprint: fp,
|
|
2187
|
+
status: "active"
|
|
2188
|
+
});
|
|
2189
|
+
});
|
|
2190
|
+
upsert();
|
|
2191
|
+
log.debug(
|
|
2192
|
+
{ filePath, provider: provider.name, msgs: meta.messageCount },
|
|
2193
|
+
"persistent: indexed provider file"
|
|
2194
|
+
);
|
|
2195
|
+
return meta;
|
|
2196
|
+
}
|
|
2197
|
+
markDeleted(filePath) {
|
|
2198
|
+
const existing = this.files.getByPath(filePath);
|
|
2199
|
+
if (!existing) return;
|
|
2200
|
+
const tx = this.db.transaction(() => {
|
|
2201
|
+
this.conversations.deleteByFileId(existing.id);
|
|
2202
|
+
this.fts.remove(filePath);
|
|
2203
|
+
this.checkpoints.remove(filePath);
|
|
2204
|
+
this.files.setStatus(existing.id, "deleted");
|
|
2205
|
+
});
|
|
2206
|
+
tx();
|
|
2207
|
+
}
|
|
2208
|
+
// ── Query helpers (read straight from SQLite) ───────────────────────────
|
|
2209
|
+
allActive() {
|
|
2210
|
+
return this.conversations.allActive();
|
|
2211
|
+
}
|
|
2212
|
+
getByIdOrSession(id) {
|
|
2213
|
+
return this.conversations.getByIdOrSession(id);
|
|
2214
|
+
}
|
|
2215
|
+
getAllBySessionId(sessionId) {
|
|
2216
|
+
return this.conversations.getAllBySessionId(sessionId);
|
|
2217
|
+
}
|
|
2218
|
+
// Ranked metas matching the FTS query, best first. Empty query returns the
|
|
2219
|
+
// most recent conversations (mirroring the in-memory indexer's empty-query
|
|
2220
|
+
// behavior). Resolves each FTS hit to its active conversation row.
|
|
2221
|
+
searchMetas(query, limit) {
|
|
2222
|
+
if (!query.trim()) {
|
|
2223
|
+
return this.conversations.recent(limit);
|
|
2224
|
+
}
|
|
2225
|
+
const paths = this.fts.search(query, limit);
|
|
2226
|
+
const metas = [];
|
|
2227
|
+
for (const path of paths) {
|
|
2228
|
+
const meta = this.conversations.getBySourcePath(path);
|
|
2229
|
+
if (meta) metas.push(meta);
|
|
2230
|
+
}
|
|
2231
|
+
return metas;
|
|
2232
|
+
}
|
|
2233
|
+
getProjects() {
|
|
2234
|
+
return this.conversations.distinctProjects();
|
|
2235
|
+
}
|
|
2236
|
+
// Bounded conversation page: read only the requested window from the file,
|
|
2237
|
+
// seeking from the nearest checkpoint. Returns null if the id can't be
|
|
2238
|
+
// resolved to an indexed conversation. For conversations large enough to span
|
|
2239
|
+
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
2240
|
+
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
2241
|
+
async getPage(id, options) {
|
|
2242
|
+
const meta = this.conversations.getByIdOrSession(id);
|
|
2243
|
+
if (!meta) return null;
|
|
2244
|
+
const filePath = meta.filePath;
|
|
2245
|
+
const total = this.conversations.pageMessageCount(filePath);
|
|
2246
|
+
if (total > CHECKPOINT_INTERVAL && this.checkpoints.count(filePath) === 0) {
|
|
2247
|
+
const built = await buildCheckpoints(filePath);
|
|
2248
|
+
if (built.length > 0) this.checkpoints.replaceAll(filePath, built);
|
|
2249
|
+
}
|
|
2250
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
2251
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2252
|
+
const floor = this.checkpoints.floor(filePath, fromIndex);
|
|
2253
|
+
return readPage(filePath, total, options, floor);
|
|
2254
|
+
}
|
|
2255
|
+
};
|
|
2256
|
+
|
|
2257
|
+
// src/watcher/file-watcher.ts
|
|
2258
|
+
var import_chokidar = __toESM(require("chokidar"), 1);
|
|
2259
|
+
var EXCLUDED_SEGMENTS2 = ["/memory/", "/tool-results/"];
|
|
2260
|
+
var FileWatcher = class {
|
|
2261
|
+
constructor(profiles, onEvent, options = {}) {
|
|
2262
|
+
this.profiles = profiles;
|
|
2263
|
+
this.onEvent = onEvent;
|
|
2264
|
+
this.debounceMs = options.debounceMs ?? 400;
|
|
2265
|
+
}
|
|
2266
|
+
profiles;
|
|
2267
|
+
onEvent;
|
|
2268
|
+
watchers = [];
|
|
2269
|
+
timers = /* @__PURE__ */ new Map();
|
|
2270
|
+
debounceMs;
|
|
2271
|
+
// Resolves once every underlying chokidar watcher has finished its initial
|
|
2272
|
+
// scan, so the caller knows subsequent FS changes will be observed.
|
|
2273
|
+
async start() {
|
|
2274
|
+
const log = getLogger();
|
|
2275
|
+
const ready = [];
|
|
2276
|
+
for (const profile of this.profiles) {
|
|
2277
|
+
const dir = getProjectsDir(profile);
|
|
2278
|
+
const watcher = import_chokidar.default.watch(dir, {
|
|
2279
|
+
ignoreInitial: true,
|
|
2280
|
+
ignored: (p) => EXCLUDED_SEGMENTS2.some((seg) => p.includes(seg)),
|
|
2281
|
+
awaitWriteFinish: { stabilityThreshold: this.debounceMs, pollInterval: 100 }
|
|
2282
|
+
});
|
|
2283
|
+
watcher.on("add", (p) => this.dispatch(p, profile.id, "add")).on("change", (p) => this.dispatch(p, profile.id, "change")).on("unlink", (p) => this.dispatch(p, profile.id, "unlink"));
|
|
2284
|
+
ready.push(new Promise((resolve) => watcher.once("ready", () => resolve())));
|
|
2285
|
+
this.watchers.push(watcher);
|
|
2286
|
+
log.debug({ dir, account: profile.id }, "watcher: watching");
|
|
2287
|
+
}
|
|
2288
|
+
await Promise.all(ready);
|
|
2289
|
+
}
|
|
2290
|
+
async stop() {
|
|
2291
|
+
for (const t of this.timers.values()) clearTimeout(t);
|
|
2292
|
+
this.timers.clear();
|
|
2293
|
+
await Promise.all(this.watchers.map((w) => w.close()));
|
|
2294
|
+
this.watchers = [];
|
|
2295
|
+
}
|
|
2296
|
+
dispatch(filePath, account, type) {
|
|
2297
|
+
if (!filePath.endsWith(".jsonl")) return;
|
|
2298
|
+
const existing = this.timers.get(filePath);
|
|
2299
|
+
if (existing) clearTimeout(existing);
|
|
2300
|
+
this.timers.set(
|
|
2301
|
+
filePath,
|
|
2302
|
+
setTimeout(() => {
|
|
2303
|
+
this.timers.delete(filePath);
|
|
2304
|
+
this.onEvent({ filePath, account, type });
|
|
2305
|
+
}, this.debounceMs)
|
|
2306
|
+
);
|
|
2307
|
+
}
|
|
2308
|
+
};
|
|
2309
|
+
|
|
2310
|
+
// src/watcher/index-queue.ts
|
|
2311
|
+
var IndexQueue = class {
|
|
2312
|
+
constructor(process2) {
|
|
2313
|
+
this.process = process2;
|
|
2314
|
+
}
|
|
2315
|
+
process;
|
|
2316
|
+
pending = /* @__PURE__ */ new Map();
|
|
2317
|
+
running = false;
|
|
2318
|
+
idleResolvers = [];
|
|
2319
|
+
enqueue(job) {
|
|
2320
|
+
this.pending.set(job.filePath, job);
|
|
2321
|
+
void this.drain();
|
|
2322
|
+
}
|
|
2323
|
+
// Resolves when the queue has fully drained — useful for tests and for a
|
|
2324
|
+
// clean shutdown.
|
|
2325
|
+
onIdle() {
|
|
2326
|
+
if (!this.running && this.pending.size === 0) return Promise.resolve();
|
|
2327
|
+
return new Promise((resolve) => this.idleResolvers.push(resolve));
|
|
2328
|
+
}
|
|
2329
|
+
get size() {
|
|
2330
|
+
return this.pending.size;
|
|
2331
|
+
}
|
|
2332
|
+
async drain() {
|
|
2333
|
+
if (this.running) return;
|
|
2334
|
+
this.running = true;
|
|
2335
|
+
const log = getLogger();
|
|
2336
|
+
while (this.pending.size > 0) {
|
|
2337
|
+
const [path, job] = this.pending.entries().next().value;
|
|
2338
|
+
this.pending.delete(path);
|
|
2339
|
+
try {
|
|
2340
|
+
await this.process(job);
|
|
2341
|
+
} catch (err) {
|
|
2342
|
+
log.warn({ filePath: job.filePath, err }, "index-queue: job failed");
|
|
2343
|
+
}
|
|
2344
|
+
}
|
|
2345
|
+
this.running = false;
|
|
2346
|
+
const resolvers = this.idleResolvers;
|
|
2347
|
+
this.idleResolvers = [];
|
|
2348
|
+
for (const r of resolvers) r();
|
|
2349
|
+
}
|
|
2350
|
+
};
|
|
2351
|
+
|
|
2352
|
+
// src/scanner.ts
|
|
2353
|
+
var BATCH_SIZE2 = 12;
|
|
917
2354
|
var DEFAULT_CONFIG_PATH = "~/.config/threadbase-scanner";
|
|
2355
|
+
function defaultDbPath() {
|
|
2356
|
+
return process.env.TB_SCANNER_DB ?? (0, import_path8.join)((0, import_os2.homedir)(), ".config", "threadbase-scanner", "index.db");
|
|
2357
|
+
}
|
|
918
2358
|
var ConversationScanner = class {
|
|
919
2359
|
metadataCache = /* @__PURE__ */ new Map();
|
|
920
2360
|
conversationLRU;
|
|
2361
|
+
// session_id is NOT unique, so this maps a sessionId to every active meta that
|
|
2362
|
+
// carries it. Resolution picks deterministically (newest timestamp, then path
|
|
2363
|
+
// ascending) so dropping one file never hides another with the same id.
|
|
921
2364
|
sessionIdIndex = /* @__PURE__ */ new Map();
|
|
922
2365
|
projects = /* @__PURE__ */ new Set();
|
|
923
2366
|
indexer = new SearchIndexer();
|
|
924
2367
|
// Tier the most recent scan() ran with, so refreshFile() re-parses a single
|
|
925
2368
|
// file at the same content depth. Defaults to the standard tier.
|
|
926
2369
|
lastTier = resolveTier("standard");
|
|
2370
|
+
// null when persistent mode is disabled (legacy in-memory path). Lazily
|
|
2371
|
+
// opened on first use so merely constructing a scanner never touches disk.
|
|
2372
|
+
dbPath;
|
|
2373
|
+
sidecarEnabled;
|
|
2374
|
+
engineInstance = null;
|
|
2375
|
+
emitter = new import_events.EventEmitter();
|
|
2376
|
+
watcher = null;
|
|
2377
|
+
queue = null;
|
|
2378
|
+
periodicTimer = null;
|
|
927
2379
|
constructor(options) {
|
|
928
2380
|
this.conversationLRU = new LRUCache(options?.conversationCacheSize ?? 5);
|
|
2381
|
+
if (options?.persistent === false) {
|
|
2382
|
+
this.dbPath = null;
|
|
2383
|
+
this.sidecarEnabled = false;
|
|
2384
|
+
} else {
|
|
2385
|
+
this.dbPath = options?.persistent?.dbPath ?? defaultDbPath();
|
|
2386
|
+
this.sidecarEnabled = options?.persistent?.sidecar ?? false;
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
get persistent() {
|
|
2390
|
+
return this.dbPath !== null;
|
|
2391
|
+
}
|
|
2392
|
+
engine() {
|
|
2393
|
+
if (!this.engineInstance) {
|
|
2394
|
+
this.engineInstance = new PersistentEngine(this.dbPath, {
|
|
2395
|
+
sidecar: this.sidecarEnabled
|
|
2396
|
+
});
|
|
2397
|
+
}
|
|
2398
|
+
return this.engineInstance;
|
|
2399
|
+
}
|
|
2400
|
+
// Release the SQLite connection. No-op in legacy mode. Safe to call
|
|
2401
|
+
// repeatedly. Stops the watcher first if one is running; call unwatch()
|
|
2402
|
+
// explicitly beforehand if you need to await watcher teardown.
|
|
2403
|
+
close() {
|
|
2404
|
+
if (this.watcher || this.queue || this.periodicTimer) void this.unwatch();
|
|
2405
|
+
this.engineInstance?.close();
|
|
2406
|
+
this.engineInstance = null;
|
|
929
2407
|
}
|
|
930
2408
|
async scan(options = {}) {
|
|
931
|
-
const log = getLogger();
|
|
932
|
-
const startedAt = Date.now();
|
|
933
2409
|
const profiles = await this.resolveProfiles(options.profiles);
|
|
934
2410
|
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
935
|
-
|
|
936
|
-
this.
|
|
2411
|
+
this.lastTier = resolveTier(options.tier ?? "standard", options.tiers);
|
|
2412
|
+
if (this.persistent) {
|
|
2413
|
+
return this.scanPersistent(activeProfiles, options);
|
|
2414
|
+
}
|
|
2415
|
+
return this.scanInMemory(activeProfiles, options);
|
|
2416
|
+
}
|
|
2417
|
+
// SQLite-backed scan: (re)index changed files into the DB, then query all
|
|
2418
|
+
// active metas and run the identical filter/sort/view/paginate pipeline as
|
|
2419
|
+
// the in-memory path — guaranteeing an identical ScanResult shape.
|
|
2420
|
+
async scanPersistent(activeProfiles, options) {
|
|
2421
|
+
const log = getLogger();
|
|
2422
|
+
const startedAt = Date.now();
|
|
2423
|
+
const engine = this.engine();
|
|
2424
|
+
const { scanned } = await engine.indexAll(activeProfiles, options);
|
|
2425
|
+
const allMetas = engine.allActive();
|
|
2426
|
+
const { conversations, total } = this.finalize(allMetas, options);
|
|
2427
|
+
log.info(
|
|
2428
|
+
{ scanned, kept: allMetas.length, filteredTotal: total, elapsedMs: Date.now() - startedAt },
|
|
2429
|
+
"scan: complete (persistent)"
|
|
2430
|
+
);
|
|
2431
|
+
return { conversations, total, scanned };
|
|
2432
|
+
}
|
|
2433
|
+
// Apply include/project/account/since filters, sort, view transform, and
|
|
2434
|
+
// pagination. Shared by both backends so results never diverge.
|
|
2435
|
+
finalize(allMetas, options) {
|
|
2436
|
+
let filtered = allMetas;
|
|
2437
|
+
if (options.include && options.include !== "all") {
|
|
2438
|
+
filtered = applyIncludeFilter(filtered, options.include);
|
|
2439
|
+
}
|
|
2440
|
+
if (options.project) filtered = applyProjectFilter(filtered, options.project);
|
|
2441
|
+
if (options.account) filtered = applyAccountFilter(filtered, options.account);
|
|
2442
|
+
if (options.since) filtered = applySinceFilter(filtered, options.since);
|
|
2443
|
+
filtered = applySort(filtered, options.sort ?? "recent");
|
|
2444
|
+
const total = filtered.length;
|
|
2445
|
+
const conversations = this.transformView(filtered, options);
|
|
2446
|
+
if (Array.isArray(conversations)) {
|
|
2447
|
+
const limit = options.limit ?? 50;
|
|
2448
|
+
const offset = options.offset ?? 0;
|
|
2449
|
+
return { conversations: applyPagination(conversations, limit, offset).items, total };
|
|
2450
|
+
}
|
|
2451
|
+
return { conversations, total };
|
|
2452
|
+
}
|
|
2453
|
+
async scanInMemory(activeProfiles, options) {
|
|
2454
|
+
const log = getLogger();
|
|
2455
|
+
const startedAt = Date.now();
|
|
2456
|
+
const tier = this.lastTier;
|
|
937
2457
|
log.info(
|
|
938
2458
|
{
|
|
939
2459
|
activeProfiles: activeProfiles.length,
|
|
@@ -949,11 +2469,7 @@ var ConversationScanner = class {
|
|
|
949
2469
|
this.sessionIdIndex.clear();
|
|
950
2470
|
this.projects.clear();
|
|
951
2471
|
this.indexer.clear();
|
|
952
|
-
const
|
|
953
|
-
projectsDir: getProjectsDir(p),
|
|
954
|
-
account: p.id
|
|
955
|
-
}));
|
|
956
|
-
const files = await discoverJsonlFiles(configDirs);
|
|
2472
|
+
const files = await this.discoverWithProviders(activeProfiles, options);
|
|
957
2473
|
const totalFiles = files.length;
|
|
958
2474
|
let scanned = 0;
|
|
959
2475
|
let parseFailures = 0;
|
|
@@ -968,15 +2484,15 @@ var ConversationScanner = class {
|
|
|
968
2484
|
return branch;
|
|
969
2485
|
};
|
|
970
2486
|
const { statCache } = options;
|
|
971
|
-
for (let i = 0; i < files.length; i +=
|
|
972
|
-
const batch = files.slice(i, i +
|
|
2487
|
+
for (let i = 0; i < files.length; i += BATCH_SIZE2) {
|
|
2488
|
+
const batch = files.slice(i, i + BATCH_SIZE2);
|
|
973
2489
|
const results = await Promise.all(
|
|
974
|
-
batch.map(async ({ filePath, account }) => {
|
|
2490
|
+
batch.map(async ({ filePath, account, provider }) => {
|
|
975
2491
|
if (statCache) {
|
|
976
2492
|
const cached = statCache.get(filePath);
|
|
977
2493
|
if (cached) {
|
|
978
2494
|
try {
|
|
979
|
-
const s = (0,
|
|
2495
|
+
const s = (0, import_fs10.statSync)(filePath);
|
|
980
2496
|
if (s.mtimeMs === cached.stat.mtimeMs && s.size === cached.stat.size) {
|
|
981
2497
|
return cached.meta;
|
|
982
2498
|
}
|
|
@@ -985,14 +2501,14 @@ var ConversationScanner = class {
|
|
|
985
2501
|
}
|
|
986
2502
|
}
|
|
987
2503
|
try {
|
|
988
|
-
const meta = await
|
|
989
|
-
if (meta) {
|
|
2504
|
+
const meta = await parseMetaWithProvider(provider, filePath, account, tier);
|
|
2505
|
+
if (meta && meta.gitBranch === null && meta.projectPath) {
|
|
990
2506
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
991
2507
|
}
|
|
992
2508
|
return meta;
|
|
993
2509
|
} catch (err) {
|
|
994
2510
|
parseFailures++;
|
|
995
|
-
log.warn({ filePath, account, err }, "scan:
|
|
2511
|
+
log.warn({ filePath, account, provider: provider.name, err }, "scan: parse threw");
|
|
996
2512
|
return null;
|
|
997
2513
|
}
|
|
998
2514
|
})
|
|
@@ -1001,7 +2517,7 @@ var ConversationScanner = class {
|
|
|
1001
2517
|
for (const meta of results) {
|
|
1002
2518
|
if (meta && meta.messageCount > 0) {
|
|
1003
2519
|
this.metadataCache.set(meta.id, meta);
|
|
1004
|
-
this.
|
|
2520
|
+
this.addToSessionIndex(meta);
|
|
1005
2521
|
this.projects.add(meta.projectPath);
|
|
1006
2522
|
allMetas.push(meta);
|
|
1007
2523
|
batchMetas.push(meta);
|
|
@@ -1015,22 +2531,7 @@ var ConversationScanner = class {
|
|
|
1015
2531
|
log.debug({ scanned, totalFiles, batchKept: batchMetas.length }, "scan: batch complete");
|
|
1016
2532
|
options.onProgress?.(scanned, totalFiles);
|
|
1017
2533
|
}
|
|
1018
|
-
|
|
1019
|
-
if (options.include && options.include !== "all") {
|
|
1020
|
-
filtered = applyIncludeFilter(filtered, options.include);
|
|
1021
|
-
}
|
|
1022
|
-
if (options.project) {
|
|
1023
|
-
filtered = applyProjectFilter(filtered, options.project);
|
|
1024
|
-
}
|
|
1025
|
-
if (options.account) {
|
|
1026
|
-
filtered = applyAccountFilter(filtered, options.account);
|
|
1027
|
-
}
|
|
1028
|
-
if (options.since) {
|
|
1029
|
-
filtered = applySinceFilter(filtered, options.since);
|
|
1030
|
-
}
|
|
1031
|
-
filtered = applySort(filtered, options.sort ?? "recent");
|
|
1032
|
-
const total = filtered.length;
|
|
1033
|
-
const conversations = this.transformView(filtered, options);
|
|
2534
|
+
const { conversations, total } = this.finalize(allMetas, options);
|
|
1034
2535
|
const elapsedMs = Date.now() - startedAt;
|
|
1035
2536
|
log.info(
|
|
1036
2537
|
{
|
|
@@ -1043,25 +2544,36 @@ var ConversationScanner = class {
|
|
|
1043
2544
|
},
|
|
1044
2545
|
"scan: complete"
|
|
1045
2546
|
);
|
|
1046
|
-
if (Array.isArray(conversations)) {
|
|
1047
|
-
const limit = options.limit ?? 50;
|
|
1048
|
-
const offset = options.offset ?? 0;
|
|
1049
|
-
const paginated = applyPagination(conversations, limit, offset);
|
|
1050
|
-
return { conversations: paginated.items, total, scanned };
|
|
1051
|
-
}
|
|
1052
2547
|
return { conversations, total, scanned };
|
|
1053
2548
|
}
|
|
1054
2549
|
async search(query, options = {}) {
|
|
1055
2550
|
const log = getLogger();
|
|
1056
2551
|
log.debug({ query, indexSize: this.indexer.getDocumentCount() }, "search: start");
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
2552
|
+
let results;
|
|
2553
|
+
if (this.persistent) {
|
|
2554
|
+
const engine = this.engine();
|
|
2555
|
+
if (engine.conversations.count() === 0) {
|
|
2556
|
+
log.debug("search: persistent index empty, triggering scan");
|
|
2557
|
+
const profiles = await this.resolveProfiles(options.profiles);
|
|
2558
|
+
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
2559
|
+
await engine.indexAll(activeProfiles, { ...options, limit: void 0, offset: void 0 });
|
|
2560
|
+
}
|
|
2561
|
+
const metas = engine.searchMetas(query, (options.limit ?? 50) * 2);
|
|
2562
|
+
results = query.trim() ? metas.map((meta) => ({ meta, score: 1, matches: generateMatches(meta, query) })) : metas.map((meta) => ({
|
|
2563
|
+
meta,
|
|
2564
|
+
score: 1,
|
|
2565
|
+
matches: [{ field: "timestamp", snippet: meta.preview }]
|
|
2566
|
+
}));
|
|
2567
|
+
} else {
|
|
2568
|
+
if (this.indexer.getDocumentCount() === 0) {
|
|
2569
|
+
log.debug("search: index empty, triggering scan");
|
|
2570
|
+
await this.scan({ ...options, limit: void 0, offset: void 0 });
|
|
2571
|
+
}
|
|
2572
|
+
results = this.indexer.search(query, {
|
|
2573
|
+
fields: options.fields,
|
|
2574
|
+
limit: (options.limit ?? 50) * 2
|
|
2575
|
+
});
|
|
1060
2576
|
}
|
|
1061
|
-
let results = this.indexer.search(query, {
|
|
1062
|
-
fields: options.fields,
|
|
1063
|
-
limit: (options.limit ?? 50) * 2
|
|
1064
|
-
});
|
|
1065
2577
|
if (options.include && options.include !== "all") {
|
|
1066
2578
|
results = results.filter((r) => {
|
|
1067
2579
|
switch (options.include) {
|
|
@@ -1085,6 +2597,11 @@ var ConversationScanner = class {
|
|
|
1085
2597
|
if (options.account) {
|
|
1086
2598
|
results = results.filter((r) => r.meta.account === options.account);
|
|
1087
2599
|
}
|
|
2600
|
+
if (options.provider) {
|
|
2601
|
+
results = results.filter(
|
|
2602
|
+
(r) => (r.meta.provider ?? CLAUDE_CODE_PROVIDER) === options.provider
|
|
2603
|
+
);
|
|
2604
|
+
}
|
|
1088
2605
|
if (options.since) {
|
|
1089
2606
|
const cutoff = parseSinceCutoff(options.since);
|
|
1090
2607
|
results = results.filter((r) => new Date(r.meta.timestamp).getTime() >= cutoff.getTime());
|
|
@@ -1102,14 +2619,14 @@ var ConversationScanner = class {
|
|
|
1102
2619
|
log.debug({ id }, "getConversation: cache hit");
|
|
1103
2620
|
return cached;
|
|
1104
2621
|
}
|
|
1105
|
-
const meta = this.metadataCache.get(id) ?? this.
|
|
2622
|
+
const meta = this.persistent ? this.engine().getByIdOrSession(id) : this.metadataCache.get(id) ?? this.resolveSessionId(id);
|
|
1106
2623
|
if (!meta) {
|
|
1107
2624
|
log.debug({ id }, "getConversation: not found in metadata");
|
|
1108
2625
|
return null;
|
|
1109
2626
|
}
|
|
1110
2627
|
log.debug({ id, filePath: meta.filePath }, "getConversation: cache miss, parsing");
|
|
1111
2628
|
try {
|
|
1112
|
-
const conversation = await parseConversation(meta.filePath, meta.account);
|
|
2629
|
+
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
1113
2630
|
if (conversation) {
|
|
1114
2631
|
this.conversationLRU.set(id, conversation);
|
|
1115
2632
|
}
|
|
@@ -1129,14 +2646,18 @@ var ConversationScanner = class {
|
|
|
1129
2646
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
1130
2647
|
// same contract as getConversation.
|
|
1131
2648
|
//
|
|
1132
|
-
//
|
|
1133
|
-
//
|
|
1134
|
-
//
|
|
1135
|
-
// parseConversation() by
|
|
1136
|
-
//
|
|
1137
|
-
//
|
|
1138
|
-
//
|
|
2649
|
+
// Persistent mode: a true bounded read — the engine seeks from the nearest
|
|
2650
|
+
// checkpoint and parses only the requested window (checkpoints are built
|
|
2651
|
+
// lazily for large conversations). Windowed message indices are proven
|
|
2652
|
+
// identical to parseConversation().messages.slice(...) by the paged-reader
|
|
2653
|
+
// equivalence test.
|
|
2654
|
+
//
|
|
2655
|
+
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
2656
|
+
// which is identical by construction.
|
|
1139
2657
|
async getConversationPage(id, options) {
|
|
2658
|
+
if (this.persistent) {
|
|
2659
|
+
return this.engine().getPage(id, options);
|
|
2660
|
+
}
|
|
1140
2661
|
const conversation = await this.getConversation(id);
|
|
1141
2662
|
if (!conversation) return null;
|
|
1142
2663
|
const { messages } = conversation;
|
|
@@ -1181,6 +2702,30 @@ var ConversationScanner = class {
|
|
|
1181
2702
|
// dropped from all indexes.
|
|
1182
2703
|
async refreshFile(filePath, account) {
|
|
1183
2704
|
const log = getLogger();
|
|
2705
|
+
if (this.persistent) {
|
|
2706
|
+
const engine = this.engine();
|
|
2707
|
+
const previous2 = engine.getByIdOrSession(filePath);
|
|
2708
|
+
const resolvedAccount2 = account ?? previous2?.account ?? "default";
|
|
2709
|
+
const evict2 = (m) => {
|
|
2710
|
+
if (!m) return;
|
|
2711
|
+
this.conversationLRU.delete(m.id);
|
|
2712
|
+
this.conversationLRU.delete(m.sessionId);
|
|
2713
|
+
};
|
|
2714
|
+
evict2(previous2);
|
|
2715
|
+
const provider = await this.resolveProviderForFile(filePath, previous2);
|
|
2716
|
+
const meta2 = await engine.indexFile(
|
|
2717
|
+
filePath,
|
|
2718
|
+
resolvedAccount2,
|
|
2719
|
+
this.lastTier.name,
|
|
2720
|
+
void 0,
|
|
2721
|
+
readGitBranch,
|
|
2722
|
+
true,
|
|
2723
|
+
provider
|
|
2724
|
+
);
|
|
2725
|
+
evict2(meta2);
|
|
2726
|
+
log.debug({ filePath, kept: !!meta2 }, "refreshFile: updated persistent index");
|
|
2727
|
+
return meta2;
|
|
2728
|
+
}
|
|
1184
2729
|
const previous = this.metadataCache.get(filePath);
|
|
1185
2730
|
const resolvedAccount = account ?? previous?.account ?? "default";
|
|
1186
2731
|
let meta = null;
|
|
@@ -1200,18 +2745,16 @@ var ConversationScanner = class {
|
|
|
1200
2745
|
if (!meta || meta.messageCount === 0) {
|
|
1201
2746
|
if (previous) {
|
|
1202
2747
|
this.metadataCache.delete(previous.id);
|
|
1203
|
-
this.
|
|
2748
|
+
this.removeFromSessionIndex(previous);
|
|
1204
2749
|
this.indexer.removeDocument(previous.id);
|
|
1205
2750
|
}
|
|
1206
2751
|
log.debug({ filePath }, "refreshFile: dropped (no parseable messages)");
|
|
1207
2752
|
return null;
|
|
1208
2753
|
}
|
|
1209
2754
|
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
1210
|
-
if (previous
|
|
1211
|
-
this.sessionIdIndex.delete(previous.sessionId);
|
|
1212
|
-
}
|
|
2755
|
+
if (previous) this.removeFromSessionIndex(previous);
|
|
1213
2756
|
this.metadataCache.set(meta.id, meta);
|
|
1214
|
-
this.
|
|
2757
|
+
this.addToSessionIndex(meta);
|
|
1215
2758
|
this.projects.add(meta.projectPath);
|
|
1216
2759
|
if (previous) {
|
|
1217
2760
|
this.indexer.updateDocument(meta);
|
|
@@ -1225,15 +2768,206 @@ var ConversationScanner = class {
|
|
|
1225
2768
|
return meta;
|
|
1226
2769
|
}
|
|
1227
2770
|
getMetadataCache() {
|
|
2771
|
+
if (this.persistent) {
|
|
2772
|
+
const map = /* @__PURE__ */ new Map();
|
|
2773
|
+
for (const meta of this.engine().allActive()) map.set(meta.id, meta);
|
|
2774
|
+
return map;
|
|
2775
|
+
}
|
|
1228
2776
|
return this.metadataCache;
|
|
1229
2777
|
}
|
|
2778
|
+
// Collision-safe sessionId lookup. session_id is NOT unique (the parser falls
|
|
2779
|
+
// back to the file basename, and resumed/subagent sessions repeat ids), so
|
|
2780
|
+
// getConversation(sessionId) is a convenience that resolves to one match;
|
|
2781
|
+
// this returns every active conversation sharing the id, newest first.
|
|
2782
|
+
getConversationsBySessionId(sessionId) {
|
|
2783
|
+
if (this.persistent) {
|
|
2784
|
+
return this.engine().getAllBySessionId(sessionId);
|
|
2785
|
+
}
|
|
2786
|
+
return this.sortBySessionPriority(this.sessionIdIndex.get(sessionId) ?? []);
|
|
2787
|
+
}
|
|
1230
2788
|
getProjects() {
|
|
2789
|
+
const source = this.persistent ? this.engine().getProjects() : this.projects;
|
|
1231
2790
|
const normalized = /* @__PURE__ */ new Set();
|
|
1232
|
-
for (const p of
|
|
2791
|
+
for (const p of source) {
|
|
1233
2792
|
normalized.add(p.replace(/\/+$/, ""));
|
|
1234
2793
|
}
|
|
1235
2794
|
return Array.from(normalized).sort();
|
|
1236
2795
|
}
|
|
2796
|
+
on(event, listener) {
|
|
2797
|
+
this.emitter.on(event, listener);
|
|
2798
|
+
return this;
|
|
2799
|
+
}
|
|
2800
|
+
off(event, listener) {
|
|
2801
|
+
this.emitter.off(event, listener);
|
|
2802
|
+
return this;
|
|
2803
|
+
}
|
|
2804
|
+
// Start watching the active profiles' project dirs. A filesystem watcher
|
|
2805
|
+
// feeds debounced, path-deduplicated index jobs through a single-writer
|
|
2806
|
+
// queue; a periodic full rescan backstops any events the watcher misses
|
|
2807
|
+
// (sleep/wake, network FS, restarts). Emits "change" per indexed file.
|
|
2808
|
+
// Persistent mode only — throws in legacy mode (no durable index to update).
|
|
2809
|
+
async watch(options = {}) {
|
|
2810
|
+
if (!this.persistent) {
|
|
2811
|
+
throw new Error("watch() requires persistent mode; construct with persistent enabled");
|
|
2812
|
+
}
|
|
2813
|
+
if (this.watcher) return;
|
|
2814
|
+
const profiles = await this.resolveProfiles(options.profiles);
|
|
2815
|
+
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
2816
|
+
this.queue = new IndexQueue(async (job) => {
|
|
2817
|
+
try {
|
|
2818
|
+
const meta = await this.refreshFile(job.filePath, job.account);
|
|
2819
|
+
this.emitter.emit("change", {
|
|
2820
|
+
filePath: job.filePath,
|
|
2821
|
+
account: job.account,
|
|
2822
|
+
meta,
|
|
2823
|
+
reason: job.reason
|
|
2824
|
+
});
|
|
2825
|
+
} catch (err) {
|
|
2826
|
+
this.emitter.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
2827
|
+
}
|
|
2828
|
+
});
|
|
2829
|
+
this.watcher = new FileWatcher(
|
|
2830
|
+
activeProfiles,
|
|
2831
|
+
(e) => {
|
|
2832
|
+
this.queue?.enqueue({
|
|
2833
|
+
filePath: e.filePath,
|
|
2834
|
+
account: e.account,
|
|
2835
|
+
reason: "watcher"
|
|
2836
|
+
});
|
|
2837
|
+
},
|
|
2838
|
+
{ debounceMs: options.debounceMs }
|
|
2839
|
+
);
|
|
2840
|
+
await this.watcher.start();
|
|
2841
|
+
const periodicMs = options.periodicMs ?? 6e4;
|
|
2842
|
+
if (periodicMs > 0) {
|
|
2843
|
+
this.periodicTimer = setInterval(() => {
|
|
2844
|
+
void this.periodicReconcile(activeProfiles);
|
|
2845
|
+
}, periodicMs);
|
|
2846
|
+
this.periodicTimer.unref?.();
|
|
2847
|
+
}
|
|
2848
|
+
getLogger().info({ profiles: activeProfiles.length, periodicMs }, "watch: started");
|
|
2849
|
+
}
|
|
2850
|
+
// Periodic correctness backstop: re-discover all files and enqueue them
|
|
2851
|
+
// through the same queue the watcher uses, so any add/change the watcher
|
|
2852
|
+
// missed still gets indexed and emits a "change" event. Vanished files
|
|
2853
|
+
// (active in the DB but no longer on disk) are enqueued too — refreshFile
|
|
2854
|
+
// drops them. Routing through the queue (not a direct scan) keeps event
|
|
2855
|
+
// emission and single-writer serialization unified.
|
|
2856
|
+
async periodicReconcile(activeProfiles) {
|
|
2857
|
+
if (!this.queue) return;
|
|
2858
|
+
try {
|
|
2859
|
+
const engine = this.engine();
|
|
2860
|
+
const configDirs = activeProfiles.map((p) => ({
|
|
2861
|
+
projectsDir: getProjectsDir(p),
|
|
2862
|
+
account: p.id
|
|
2863
|
+
}));
|
|
2864
|
+
const discovered = await discoverJsonlFiles(configDirs);
|
|
2865
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2866
|
+
for (const { filePath, account } of discovered) {
|
|
2867
|
+
seen.add(filePath);
|
|
2868
|
+
const { change } = classify(filePath, engine.files.getByPath(filePath));
|
|
2869
|
+
if (change !== "unchanged") {
|
|
2870
|
+
this.queue.enqueue({ filePath, account, reason: "periodic" });
|
|
2871
|
+
}
|
|
2872
|
+
}
|
|
2873
|
+
for (const path of engine.files.allActivePaths()) {
|
|
2874
|
+
if (!seen.has(path)) {
|
|
2875
|
+
this.queue.enqueue({ filePath: path, account: "default", reason: "periodic" });
|
|
2876
|
+
}
|
|
2877
|
+
}
|
|
2878
|
+
} catch (err) {
|
|
2879
|
+
this.emitter.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
2880
|
+
}
|
|
2881
|
+
}
|
|
2882
|
+
// Stop watching and drain any in-flight index jobs.
|
|
2883
|
+
async unwatch() {
|
|
2884
|
+
if (this.periodicTimer) {
|
|
2885
|
+
clearInterval(this.periodicTimer);
|
|
2886
|
+
this.periodicTimer = null;
|
|
2887
|
+
}
|
|
2888
|
+
if (this.watcher) {
|
|
2889
|
+
await this.watcher.stop();
|
|
2890
|
+
this.watcher = null;
|
|
2891
|
+
}
|
|
2892
|
+
if (this.queue) {
|
|
2893
|
+
await this.queue.onIdle();
|
|
2894
|
+
this.queue = null;
|
|
2895
|
+
}
|
|
2896
|
+
}
|
|
2897
|
+
// ── Provider plumbing (in-memory path) ──────────────────────────────────
|
|
2898
|
+
// Build the flat parse worklist for the enabled providers. Threadbase
|
|
2899
|
+
// discovers under the active profiles' project dirs; Codex discovers under
|
|
2900
|
+
// the explicit codexRoots (opt-in — no default home scan).
|
|
2901
|
+
async discoverWithProviders(activeProfiles, options) {
|
|
2902
|
+
const enabled = options.providers ?? [CLAUDE_CODE_PROVIDER];
|
|
2903
|
+
const work = [];
|
|
2904
|
+
if (enabled.includes(CLAUDE_CODE_PROVIDER)) {
|
|
2905
|
+
const provider = new ThreadbaseProvider();
|
|
2906
|
+
const roots = activeProfiles.map((p) => `${getProjectsDir(p)}\0${p.id}`);
|
|
2907
|
+
for (const f of await provider.discover(roots)) {
|
|
2908
|
+
work.push({ ...f, provider });
|
|
2909
|
+
}
|
|
2910
|
+
}
|
|
2911
|
+
if (enabled.includes(CODEX_CLI_PROVIDER) && (options.codexRoots?.length ?? 0) > 0) {
|
|
2912
|
+
const provider = new CodexCliProvider();
|
|
2913
|
+
for (const f of await provider.discover(options.codexRoots)) {
|
|
2914
|
+
work.push({ ...f, provider });
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2917
|
+
return work;
|
|
2918
|
+
}
|
|
2919
|
+
// Resolve which provider should (re)parse a file. Stored metadata wins; for a
|
|
2920
|
+
// file the index has not seen, sniff the first lines with each non-Threadbase
|
|
2921
|
+
// provider's canParse. Returns undefined for Threadbase (the engine's default
|
|
2922
|
+
// tail-read path).
|
|
2923
|
+
async resolveProviderForFile(filePath, previous) {
|
|
2924
|
+
if (previous?.provider === CODEX_CLI_PROVIDER) return new CodexCliProvider();
|
|
2925
|
+
if (previous?.provider) return void 0;
|
|
2926
|
+
let sample = "";
|
|
2927
|
+
try {
|
|
2928
|
+
const fd = (0, import_fs10.openSync)(filePath, "r");
|
|
2929
|
+
try {
|
|
2930
|
+
const buf = Buffer.alloc(8192);
|
|
2931
|
+
const n = (0, import_fs10.readSync)(fd, buf, 0, buf.length, 0);
|
|
2932
|
+
sample = buf.subarray(0, n).toString("utf8");
|
|
2933
|
+
} finally {
|
|
2934
|
+
(0, import_fs10.closeSync)(fd);
|
|
2935
|
+
}
|
|
2936
|
+
} catch {
|
|
2937
|
+
return void 0;
|
|
2938
|
+
}
|
|
2939
|
+
const codex = new CodexCliProvider();
|
|
2940
|
+
if (codex.canParse(filePath, sample)) return codex;
|
|
2941
|
+
return void 0;
|
|
2942
|
+
}
|
|
2943
|
+
addToSessionIndex(meta) {
|
|
2944
|
+
const list = this.sessionIdIndex.get(meta.sessionId);
|
|
2945
|
+
if (list) {
|
|
2946
|
+
const i = list.findIndex((m) => m.id === meta.id);
|
|
2947
|
+
if (i >= 0) list[i] = meta;
|
|
2948
|
+
else list.push(meta);
|
|
2949
|
+
} else {
|
|
2950
|
+
this.sessionIdIndex.set(meta.sessionId, [meta]);
|
|
2951
|
+
}
|
|
2952
|
+
}
|
|
2953
|
+
removeFromSessionIndex(meta) {
|
|
2954
|
+
const list = this.sessionIdIndex.get(meta.sessionId);
|
|
2955
|
+
if (!list) return;
|
|
2956
|
+
const next = list.filter((m) => m.id !== meta.id);
|
|
2957
|
+
if (next.length > 0) this.sessionIdIndex.set(meta.sessionId, next);
|
|
2958
|
+
else this.sessionIdIndex.delete(meta.sessionId);
|
|
2959
|
+
}
|
|
2960
|
+
// Deterministic single-result sessionId resolution: newest timestamp first,
|
|
2961
|
+
// tie-broken by absolute path ascending.
|
|
2962
|
+
resolveSessionId(sessionId) {
|
|
2963
|
+
return this.sortBySessionPriority(this.sessionIdIndex.get(sessionId) ?? [])[0];
|
|
2964
|
+
}
|
|
2965
|
+
sortBySessionPriority(metas) {
|
|
2966
|
+
return [...metas].sort((a, b) => {
|
|
2967
|
+
if (a.timestamp !== b.timestamp) return a.timestamp < b.timestamp ? 1 : -1;
|
|
2968
|
+
return a.filePath < b.filePath ? -1 : a.filePath > b.filePath ? 1 : 0;
|
|
2969
|
+
});
|
|
2970
|
+
}
|
|
1237
2971
|
async resolveProfiles(profiles) {
|
|
1238
2972
|
if (profiles && profiles.length > 0) return profiles;
|
|
1239
2973
|
return loadProfiles(DEFAULT_CONFIG_PATH);
|
|
@@ -1311,9 +3045,11 @@ async function getConversation(id, options, scanner) {
|
|
|
1311
3045
|
}
|
|
1312
3046
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1313
3047
|
0 && (module.exports = {
|
|
3048
|
+
CodexCliProvider,
|
|
1314
3049
|
ConversationScanner,
|
|
1315
3050
|
DEFAULT_TIERS,
|
|
1316
3051
|
SearchIndexer,
|
|
3052
|
+
ThreadbaseProvider,
|
|
1317
3053
|
VALID_SORT_ORDERS,
|
|
1318
3054
|
applyAccountFilter,
|
|
1319
3055
|
applyIncludeFilter,
|
|
@@ -1329,12 +3065,14 @@ async function getConversation(id, options, scanner) {
|
|
|
1329
3065
|
getProjectsDir,
|
|
1330
3066
|
loadProfiles,
|
|
1331
3067
|
readGitBranch,
|
|
3068
|
+
readSidecar,
|
|
1332
3069
|
resetDefaultScanner,
|
|
1333
3070
|
resolveConfigDir,
|
|
1334
3071
|
resolveTier,
|
|
1335
3072
|
saveProfiles,
|
|
1336
3073
|
scan,
|
|
1337
3074
|
search,
|
|
1338
|
-
setLogger
|
|
3075
|
+
setLogger,
|
|
3076
|
+
sidecarPath
|
|
1339
3077
|
});
|
|
1340
3078
|
//# sourceMappingURL=index.cjs.map
|