@threadbase-sh/scanner 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -10
- package/dist/cli.js +2059 -338
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +2114 -376
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +154 -6
- package/dist/index.d.ts +154 -6
- package/dist/index.js +2111 -377
- package/dist/index.js.map +1 -1
- package/package.json +4 -1
package/dist/cli.js
CHANGED
|
@@ -5,7 +5,7 @@ import { Command } from "commander";
|
|
|
5
5
|
import pino2 from "pino";
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.8.0";
|
|
9
9
|
|
|
10
10
|
// src/logger.ts
|
|
11
11
|
import pino from "pino";
|
|
@@ -59,7 +59,10 @@ async function saveProfiles(profiles, configPath) {
|
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
// src/scanner.ts
|
|
62
|
-
import {
|
|
62
|
+
import { EventEmitter } from "events";
|
|
63
|
+
import { closeSync as closeSync2, openSync as openSync2, readSync as readSync2, statSync as statSync2 } from "fs";
|
|
64
|
+
import { homedir as homedir2 } from "os";
|
|
65
|
+
import { join as join4 } from "path";
|
|
63
66
|
|
|
64
67
|
// src/cache.ts
|
|
65
68
|
var LRUCache = class {
|
|
@@ -166,23 +169,25 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
166
169
|
// src/filters.ts
|
|
167
170
|
function applySort(metas, order) {
|
|
168
171
|
const out = [...metas];
|
|
172
|
+
const tie = (a, b) => a.id.localeCompare(b.id);
|
|
169
173
|
switch (order) {
|
|
170
174
|
case "recent":
|
|
171
|
-
out.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
175
|
+
out.sort((a, b) => b.timestamp.localeCompare(a.timestamp) || tie(a, b));
|
|
172
176
|
break;
|
|
173
177
|
case "oldest":
|
|
174
|
-
out.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
178
|
+
out.sort((a, b) => a.timestamp.localeCompare(b.timestamp) || tie(a, b));
|
|
175
179
|
break;
|
|
176
180
|
case "messages-desc":
|
|
177
|
-
out.sort((a, b) => b.messageCount - a.messageCount);
|
|
181
|
+
out.sort((a, b) => b.messageCount - a.messageCount || tie(a, b));
|
|
178
182
|
break;
|
|
179
183
|
case "messages-asc":
|
|
180
|
-
out.sort((a, b) => a.messageCount - b.messageCount);
|
|
184
|
+
out.sort((a, b) => a.messageCount - b.messageCount || tie(a, b));
|
|
181
185
|
break;
|
|
182
186
|
case "alpha":
|
|
183
187
|
out.sort((a, b) => {
|
|
184
188
|
const cmp = a.projectName.localeCompare(b.projectName);
|
|
185
|
-
|
|
189
|
+
if (cmp !== 0) return cmp;
|
|
190
|
+
return a.preview.localeCompare(b.preview) || tie(a, b);
|
|
186
191
|
});
|
|
187
192
|
break;
|
|
188
193
|
}
|
|
@@ -289,6 +294,36 @@ function readGitBranch(projectPath) {
|
|
|
289
294
|
|
|
290
295
|
// src/indexer.ts
|
|
291
296
|
import FlexSearchModule from "flexsearch";
|
|
297
|
+
|
|
298
|
+
// src/search-matches.ts
|
|
299
|
+
function generateMatches(meta, query) {
|
|
300
|
+
const matches = [];
|
|
301
|
+
const lowerQuery = query.toLowerCase();
|
|
302
|
+
const fields = [
|
|
303
|
+
["contentSnippet", meta.contentSnippet],
|
|
304
|
+
["projectName", meta.projectName],
|
|
305
|
+
["sessionId", meta.sessionId],
|
|
306
|
+
["sessionName", meta.sessionName],
|
|
307
|
+
["account", meta.account],
|
|
308
|
+
["model", meta.model || ""],
|
|
309
|
+
["gitBranch", meta.gitBranch || ""],
|
|
310
|
+
["toolNames", meta.toolNames.join(" ")]
|
|
311
|
+
];
|
|
312
|
+
for (const [field, value] of fields) {
|
|
313
|
+
const idx = value.toLowerCase().indexOf(lowerQuery);
|
|
314
|
+
if (idx !== -1) {
|
|
315
|
+
const start = Math.max(0, idx - 80);
|
|
316
|
+
const end = Math.min(value.length, idx + query.length + 120);
|
|
317
|
+
let snippet = value.slice(start, end);
|
|
318
|
+
if (start > 0) snippet = `...${snippet}`;
|
|
319
|
+
if (end < value.length) snippet = `${snippet}...`;
|
|
320
|
+
matches.push({ field, snippet });
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
return matches.length > 0 ? matches : [{ field: "preview", snippet: meta.preview }];
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// src/indexer.ts
|
|
292
327
|
var FlexSearch = FlexSearchModule.default ?? FlexSearchModule;
|
|
293
328
|
var SearchIndexer = class {
|
|
294
329
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -357,7 +392,7 @@ var SearchIndexer = class {
|
|
|
357
392
|
seen.add(id);
|
|
358
393
|
const meta = this.documents.get(id);
|
|
359
394
|
if (!meta) continue;
|
|
360
|
-
const matches =
|
|
395
|
+
const matches = generateMatches(meta, query);
|
|
361
396
|
searchResults.push({ meta, score: 1, matches });
|
|
362
397
|
if (searchResults.length >= limit) break;
|
|
363
398
|
}
|
|
@@ -372,32 +407,6 @@ var SearchIndexer = class {
|
|
|
372
407
|
matches: [{ field: "timestamp", snippet: meta.preview }]
|
|
373
408
|
}));
|
|
374
409
|
}
|
|
375
|
-
generateMatches(meta, query) {
|
|
376
|
-
const matches = [];
|
|
377
|
-
const lowerQuery = query.toLowerCase();
|
|
378
|
-
const fields = [
|
|
379
|
-
["contentSnippet", meta.contentSnippet],
|
|
380
|
-
["projectName", meta.projectName],
|
|
381
|
-
["sessionId", meta.sessionId],
|
|
382
|
-
["sessionName", meta.sessionName],
|
|
383
|
-
["account", meta.account],
|
|
384
|
-
["model", meta.model || ""],
|
|
385
|
-
["gitBranch", meta.gitBranch || ""],
|
|
386
|
-
["toolNames", meta.toolNames.join(" ")]
|
|
387
|
-
];
|
|
388
|
-
for (const [field, value] of fields) {
|
|
389
|
-
const idx = value.toLowerCase().indexOf(lowerQuery);
|
|
390
|
-
if (idx !== -1) {
|
|
391
|
-
const start = Math.max(0, idx - 80);
|
|
392
|
-
const end = Math.min(value.length, idx + query.length + 120);
|
|
393
|
-
let snippet = value.slice(start, end);
|
|
394
|
-
if (start > 0) snippet = `...${snippet}`;
|
|
395
|
-
if (end < value.length) snippet = `${snippet}...`;
|
|
396
|
-
matches.push({ field, snippet });
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
return matches.length > 0 ? matches : [{ field: "preview", snippet: meta.preview }];
|
|
400
|
-
}
|
|
401
410
|
getDocumentCount() {
|
|
402
411
|
return this.documents.size;
|
|
403
412
|
}
|
|
@@ -432,9 +441,229 @@ var SearchIndexer = class {
|
|
|
432
441
|
|
|
433
442
|
// src/parser.ts
|
|
434
443
|
import { createReadStream } from "fs";
|
|
435
|
-
import { basename
|
|
444
|
+
import { basename as basename2 } from "path";
|
|
436
445
|
import { createInterface } from "readline";
|
|
437
446
|
|
|
447
|
+
// src/persistent/conversation-reducer.ts
|
|
448
|
+
function initialConvState() {
|
|
449
|
+
return {
|
|
450
|
+
cwd: "",
|
|
451
|
+
sessionId: "",
|
|
452
|
+
sessionName: "",
|
|
453
|
+
latestTimestamp: "",
|
|
454
|
+
lastPrompt: "",
|
|
455
|
+
pendingToolUses: {},
|
|
456
|
+
teamInfo: {}
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
function reduceConvLine(state, entry) {
|
|
460
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
461
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
462
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
463
|
+
if (entry.timestamp) {
|
|
464
|
+
const ts = entry.timestamp;
|
|
465
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
466
|
+
}
|
|
467
|
+
const type = entry.type;
|
|
468
|
+
if (type === "last-prompt") {
|
|
469
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
470
|
+
return null;
|
|
471
|
+
}
|
|
472
|
+
if (type !== "user" && type !== "assistant") return null;
|
|
473
|
+
if (entry.isMeta) return null;
|
|
474
|
+
const msg = entry.message;
|
|
475
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
476
|
+
for (const block of toolUseBlocks) state.pendingToolUses[block.id] = block;
|
|
477
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
478
|
+
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
479
|
+
const content = extractTextContent(msg?.content);
|
|
480
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
481
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
482
|
+
if (!(content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking)) return null;
|
|
483
|
+
const metadata = {};
|
|
484
|
+
if (msg?.model) metadata.model = msg.model;
|
|
485
|
+
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
486
|
+
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
487
|
+
if (entry.version) metadata.version = entry.version;
|
|
488
|
+
const usage = msg?.usage;
|
|
489
|
+
if (usage) {
|
|
490
|
+
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
491
|
+
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
492
|
+
if (usage.cache_read_input_tokens) metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
493
|
+
if (usage.cache_creation_input_tokens)
|
|
494
|
+
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
495
|
+
}
|
|
496
|
+
const toolUseNames = extractToolUseNames(msg?.content);
|
|
497
|
+
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
498
|
+
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
499
|
+
if (isToolResultOnly) {
|
|
500
|
+
const pending = new Map(Object.entries(state.pendingToolUses));
|
|
501
|
+
const toolResultBlocks = extractToolResultBlocks(msg?.content, pending);
|
|
502
|
+
if (toolResultBlocks.length > 0) metadata.toolResults = toolResultBlocks;
|
|
503
|
+
}
|
|
504
|
+
if (entry.teamName) {
|
|
505
|
+
metadata.teamName = entry.teamName;
|
|
506
|
+
if (!state.teamInfo[metadata.teamName] && content) {
|
|
507
|
+
const info = parseTeammateMessageTag(content);
|
|
508
|
+
if (info) state.teamInfo[metadata.teamName] = info;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
const thinkingContent = thinking?.content || void 0;
|
|
512
|
+
const thinkingSignature = thinking?.signature || void 0;
|
|
513
|
+
const hasMetadata = Object.keys(metadata).length > 0;
|
|
514
|
+
return {
|
|
515
|
+
role: type,
|
|
516
|
+
text: content || "",
|
|
517
|
+
timestamp: entry.timestamp || "",
|
|
518
|
+
uuid: entry.uuid || void 0,
|
|
519
|
+
metadata: hasMetadata ? metadata : void 0,
|
|
520
|
+
isToolResult: isToolResultOnly || void 0,
|
|
521
|
+
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
522
|
+
thinkingContent,
|
|
523
|
+
thinkingSignature,
|
|
524
|
+
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
525
|
+
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
526
|
+
promptId: type === "user" ? entry.promptId : void 0,
|
|
527
|
+
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
528
|
+
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
529
|
+
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
530
|
+
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
function applyTeamInfo(messages, state) {
|
|
534
|
+
if (Object.keys(state.teamInfo).length === 0) return;
|
|
535
|
+
for (const m of messages) {
|
|
536
|
+
const name = m.metadata?.teamName;
|
|
537
|
+
if (name && state.teamInfo[name] && m.metadata) m.metadata.teamInfo = state.teamInfo[name];
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// src/persistent/metadata-reducer.ts
|
|
542
|
+
import { basename, dirname as dirname2, join as join3 } from "path";
|
|
543
|
+
|
|
544
|
+
// src/providers/provider.ts
|
|
545
|
+
var CLAUDE_CODE_PROVIDER = "claude-code";
|
|
546
|
+
var CODEX_CLI_PROVIDER = "codex-cli";
|
|
547
|
+
|
|
548
|
+
// src/persistent/metadata-reducer.ts
|
|
549
|
+
function initialReducerState() {
|
|
550
|
+
return {
|
|
551
|
+
sessionId: "",
|
|
552
|
+
sessionName: "",
|
|
553
|
+
latestTimestamp: "",
|
|
554
|
+
cwd: "",
|
|
555
|
+
teamName: "",
|
|
556
|
+
model: null,
|
|
557
|
+
messageCount: 0,
|
|
558
|
+
lastMessageSender: "user",
|
|
559
|
+
isTeammate: false,
|
|
560
|
+
firstUserSeen: false,
|
|
561
|
+
firstMessage: null,
|
|
562
|
+
lastMessage: null,
|
|
563
|
+
lastPrompt: "",
|
|
564
|
+
pageMessageCount: 0,
|
|
565
|
+
toolNames: [],
|
|
566
|
+
previewParts: [],
|
|
567
|
+
snippetParts: [],
|
|
568
|
+
previewLength: 0,
|
|
569
|
+
snippetLength: 0,
|
|
570
|
+
badJsonLines: 0
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
function reduceLine(state, entry, tier) {
|
|
574
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
575
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
576
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
577
|
+
if (entry.teamName && !state.teamName) state.teamName = entry.teamName;
|
|
578
|
+
if (entry.timestamp) {
|
|
579
|
+
const ts = entry.timestamp;
|
|
580
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
581
|
+
}
|
|
582
|
+
const type = entry.type;
|
|
583
|
+
if (type === "last-prompt") {
|
|
584
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
if (type !== "user" && type !== "assistant") return;
|
|
588
|
+
if (entry.isMeta) return;
|
|
589
|
+
const msg = entry.message;
|
|
590
|
+
if (state.model === null && msg?.model) state.model = msg.model;
|
|
591
|
+
if (type === "user" && !state.firstUserSeen) {
|
|
592
|
+
state.firstUserSeen = true;
|
|
593
|
+
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
594
|
+
}
|
|
595
|
+
const content = extractTextContent(msg?.content);
|
|
596
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
597
|
+
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
598
|
+
const toolSet = new Set(state.toolNames);
|
|
599
|
+
collectToolNames(msg?.content, toolSet);
|
|
600
|
+
state.toolNames = Array.from(toolSet);
|
|
601
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
602
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
603
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
604
|
+
if (content || isOnlyToolResult || toolUseBlocks.length > 0 || hasThinking) {
|
|
605
|
+
state.pageMessageCount++;
|
|
606
|
+
}
|
|
607
|
+
if (content || isOnlyToolResult) {
|
|
608
|
+
state.messageCount++;
|
|
609
|
+
state.lastMessageSender = type;
|
|
610
|
+
if (content) {
|
|
611
|
+
const ts = entry.timestamp || "";
|
|
612
|
+
if (!state.firstMessage) state.firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
613
|
+
state.lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
614
|
+
if (state.previewLength < tier.previewMax) {
|
|
615
|
+
state.previewParts.push(content);
|
|
616
|
+
state.previewLength += content.length;
|
|
617
|
+
}
|
|
618
|
+
if (state.snippetLength < tier.snippetMax) {
|
|
619
|
+
const remaining = tier.snippetMax - state.snippetLength;
|
|
620
|
+
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
621
|
+
state.snippetParts.push(chunk);
|
|
622
|
+
state.snippetLength += chunk.length;
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
function finalizeMeta(state, filePath, account, tier) {
|
|
628
|
+
if (state.messageCount === 0) return null;
|
|
629
|
+
const isSubagent = filePath.includes("/subagents/");
|
|
630
|
+
let parentSessionId = null;
|
|
631
|
+
if (isSubagent) {
|
|
632
|
+
const uuidDir = dirname2(dirname2(filePath));
|
|
633
|
+
parentSessionId = join3(dirname2(uuidDir), `${basename(uuidDir)}.jsonl`);
|
|
634
|
+
}
|
|
635
|
+
const projectPath = state.cwd;
|
|
636
|
+
return {
|
|
637
|
+
id: filePath,
|
|
638
|
+
filePath,
|
|
639
|
+
provider: CLAUDE_CODE_PROVIDER,
|
|
640
|
+
sessionId: state.sessionId || basename(filePath, ".jsonl"),
|
|
641
|
+
sessionName: state.sessionName,
|
|
642
|
+
projectPath,
|
|
643
|
+
projectName: getShortProjectName(projectPath),
|
|
644
|
+
account,
|
|
645
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
646
|
+
messageCount: state.messageCount,
|
|
647
|
+
lastMessageSender: state.lastMessageSender,
|
|
648
|
+
preview: state.previewParts.join(" ").slice(0, tier.previewMax),
|
|
649
|
+
contentSnippet: state.snippetParts.join(" "),
|
|
650
|
+
gitBranch: null,
|
|
651
|
+
model: state.model,
|
|
652
|
+
isSubagent,
|
|
653
|
+
parentSessionId,
|
|
654
|
+
isTeammate: state.isTeammate,
|
|
655
|
+
teamName: state.teamName || null,
|
|
656
|
+
toolNames: state.toolNames,
|
|
657
|
+
firstMessage: state.firstMessage,
|
|
658
|
+
lastMessage: state.lastMessage,
|
|
659
|
+
lastPrompt: state.lastPrompt || void 0
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
function getShortProjectName(fullPath) {
|
|
663
|
+
const parts = fullPath.split("/").filter(Boolean);
|
|
664
|
+
return parts.slice(-3).join("/");
|
|
665
|
+
}
|
|
666
|
+
|
|
438
667
|
// src/tags.ts
|
|
439
668
|
var SYSTEM_TAGS = [
|
|
440
669
|
"system-reminder",
|
|
@@ -467,25 +696,7 @@ function cleanSystemTags(text) {
|
|
|
467
696
|
async function parseMeta(filePath, account, tier) {
|
|
468
697
|
const log = getLogger();
|
|
469
698
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
470
|
-
|
|
471
|
-
let badJsonLines = 0;
|
|
472
|
-
let sessionName = "";
|
|
473
|
-
let latestTimestamp = "";
|
|
474
|
-
let cwd = "";
|
|
475
|
-
let teamName = "";
|
|
476
|
-
let model = null;
|
|
477
|
-
let messageCount = 0;
|
|
478
|
-
let lastMessageSender = "user";
|
|
479
|
-
let isTeammate = false;
|
|
480
|
-
let firstUserSeen = false;
|
|
481
|
-
let firstMessage = null;
|
|
482
|
-
let lastMessage = null;
|
|
483
|
-
let lastPrompt = "";
|
|
484
|
-
const toolNameSet = /* @__PURE__ */ new Set();
|
|
485
|
-
const previewParts = [];
|
|
486
|
-
const snippetParts = [];
|
|
487
|
-
let snippetLength = 0;
|
|
488
|
-
let previewLength = 0;
|
|
699
|
+
const state = initialReducerState();
|
|
489
700
|
const fileStream = createReadStream(filePath);
|
|
490
701
|
const rl = createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
491
702
|
try {
|
|
@@ -495,119 +706,33 @@ async function parseMeta(filePath, account, tier) {
|
|
|
495
706
|
try {
|
|
496
707
|
entry = JSON.parse(line);
|
|
497
708
|
} catch {
|
|
498
|
-
badJsonLines++;
|
|
709
|
+
state.badJsonLines++;
|
|
499
710
|
continue;
|
|
500
711
|
}
|
|
501
|
-
|
|
502
|
-
if (entry.sessionId && !sessionId) sessionId = entry.sessionId;
|
|
503
|
-
if (entry.slug && !sessionName) sessionName = entry.slug;
|
|
504
|
-
if (entry.teamName && !teamName) teamName = entry.teamName;
|
|
505
|
-
if (entry.timestamp) {
|
|
506
|
-
const ts = entry.timestamp;
|
|
507
|
-
if (!latestTimestamp || ts > latestTimestamp) latestTimestamp = ts;
|
|
508
|
-
}
|
|
509
|
-
const type = entry.type;
|
|
510
|
-
if (type === "last-prompt") {
|
|
511
|
-
if (entry.lastPrompt && !lastPrompt) lastPrompt = entry.lastPrompt;
|
|
512
|
-
continue;
|
|
513
|
-
}
|
|
514
|
-
if (type !== "user" && type !== "assistant") continue;
|
|
515
|
-
if (entry.isMeta) continue;
|
|
516
|
-
if (model === null) {
|
|
517
|
-
const msg2 = entry.message;
|
|
518
|
-
if (msg2?.model) model = msg2.model;
|
|
519
|
-
}
|
|
520
|
-
if (type === "user" && !firstUserSeen) {
|
|
521
|
-
firstUserSeen = true;
|
|
522
|
-
if (isTeammateContent(entry.message?.content)) {
|
|
523
|
-
isTeammate = true;
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
const msg = entry.message;
|
|
527
|
-
const content = extractTextContent(msg?.content);
|
|
528
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
529
|
-
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
530
|
-
collectToolNames(msg?.content, toolNameSet);
|
|
531
|
-
if (content || isOnlyToolResult) {
|
|
532
|
-
messageCount++;
|
|
533
|
-
lastMessageSender = type;
|
|
534
|
-
if (content) {
|
|
535
|
-
const ts = entry.timestamp || "";
|
|
536
|
-
if (!firstMessage) {
|
|
537
|
-
firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
538
|
-
}
|
|
539
|
-
lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
540
|
-
if (previewLength < tier.previewMax) {
|
|
541
|
-
previewParts.push(content);
|
|
542
|
-
previewLength += content.length;
|
|
543
|
-
}
|
|
544
|
-
if (snippetLength < tier.snippetMax) {
|
|
545
|
-
const remaining = tier.snippetMax - snippetLength;
|
|
546
|
-
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
547
|
-
snippetParts.push(chunk);
|
|
548
|
-
snippetLength += chunk.length;
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
}
|
|
712
|
+
reduceLine(state, entry, tier);
|
|
552
713
|
}
|
|
553
714
|
} catch (err) {
|
|
554
715
|
log.warn({ filePath, err }, "parseMeta: read failed");
|
|
555
716
|
return null;
|
|
556
717
|
}
|
|
557
|
-
if (badJsonLines > 0) {
|
|
558
|
-
log.warn(
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
return null;
|
|
563
|
-
}
|
|
564
|
-
const isSubagent = filePath.includes("/subagents/");
|
|
565
|
-
let parentSessionId = null;
|
|
566
|
-
if (isSubagent) {
|
|
567
|
-
const uuidDir = dirname2(dirname2(filePath));
|
|
568
|
-
parentSessionId = join3(dirname2(uuidDir), `${basename(uuidDir)}.jsonl`);
|
|
718
|
+
if (state.badJsonLines > 0) {
|
|
719
|
+
log.warn(
|
|
720
|
+
{ filePath, badJsonLines: state.badJsonLines },
|
|
721
|
+
"parseMeta: skipped malformed JSON lines"
|
|
722
|
+
);
|
|
569
723
|
}
|
|
570
|
-
const
|
|
571
|
-
|
|
572
|
-
return
|
|
573
|
-
id: filePath,
|
|
574
|
-
filePath,
|
|
575
|
-
sessionId: sessionId || basename(filePath, ".jsonl"),
|
|
576
|
-
sessionName,
|
|
577
|
-
projectPath,
|
|
578
|
-
projectName: getShortProjectName(projectPath),
|
|
579
|
-
account,
|
|
580
|
-
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
581
|
-
messageCount,
|
|
582
|
-
lastMessageSender,
|
|
583
|
-
preview,
|
|
584
|
-
contentSnippet: snippetParts.join(" "),
|
|
585
|
-
gitBranch: null,
|
|
586
|
-
model,
|
|
587
|
-
isSubagent,
|
|
588
|
-
parentSessionId,
|
|
589
|
-
isTeammate,
|
|
590
|
-
teamName: teamName || null,
|
|
591
|
-
toolNames: Array.from(toolNameSet),
|
|
592
|
-
firstMessage,
|
|
593
|
-
lastMessage,
|
|
594
|
-
lastPrompt: lastPrompt || void 0
|
|
595
|
-
};
|
|
724
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
725
|
+
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
726
|
+
return meta;
|
|
596
727
|
}
|
|
597
728
|
async function parseConversation(filePath, account) {
|
|
598
729
|
const log = getLogger();
|
|
599
730
|
log.trace({ filePath, account }, "parseConversation: start");
|
|
600
731
|
const messages = [];
|
|
601
732
|
let badJsonLines = 0;
|
|
602
|
-
let sessionId = "";
|
|
603
|
-
let sessionName = "";
|
|
604
|
-
let latestTimestamp = "";
|
|
605
|
-
let cwd = "";
|
|
606
733
|
const textParts = [];
|
|
607
|
-
const pendingToolUses = /* @__PURE__ */ new Map();
|
|
608
|
-
const teamInfoMap = /* @__PURE__ */ new Map();
|
|
609
734
|
const turnDurations = [];
|
|
610
|
-
|
|
735
|
+
const state = initialConvState();
|
|
611
736
|
const fileStream = createReadStream(filePath);
|
|
612
737
|
const rl = createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
613
738
|
try {
|
|
@@ -620,91 +745,18 @@ async function parseConversation(filePath, account) {
|
|
|
620
745
|
badJsonLines++;
|
|
621
746
|
continue;
|
|
622
747
|
}
|
|
623
|
-
if (entry.
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
}
|
|
630
|
-
const type = entry.type;
|
|
631
|
-
if (type === "last-prompt") {
|
|
632
|
-
if (entry.lastPrompt && !lastPrompt) lastPrompt = entry.lastPrompt;
|
|
633
|
-
continue;
|
|
634
|
-
}
|
|
635
|
-
if (type === "system") {
|
|
636
|
-
if (entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
637
|
-
turnDurations.push({
|
|
638
|
-
durationMs: entry.durationMs,
|
|
639
|
-
messageCount: entry.messageCount || 0,
|
|
640
|
-
uuid: entry.uuid
|
|
641
|
-
});
|
|
642
|
-
}
|
|
748
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
749
|
+
turnDurations.push({
|
|
750
|
+
durationMs: entry.durationMs,
|
|
751
|
+
messageCount: entry.messageCount || 0,
|
|
752
|
+
uuid: entry.uuid
|
|
753
|
+
});
|
|
643
754
|
continue;
|
|
644
755
|
}
|
|
645
|
-
|
|
646
|
-
if (
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
for (const block of toolUseBlocks) {
|
|
650
|
-
pendingToolUses.set(block.id, block);
|
|
651
|
-
}
|
|
652
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
653
|
-
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
654
|
-
const content = extractTextContent(msg?.content);
|
|
655
|
-
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
656
|
-
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
657
|
-
if (content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking) {
|
|
658
|
-
const metadata = {};
|
|
659
|
-
if (msg?.model) metadata.model = msg.model;
|
|
660
|
-
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
661
|
-
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
662
|
-
if (entry.version) metadata.version = entry.version;
|
|
663
|
-
const usage = msg?.usage;
|
|
664
|
-
if (usage) {
|
|
665
|
-
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
666
|
-
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
667
|
-
if (usage.cache_read_input_tokens)
|
|
668
|
-
metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
669
|
-
if (usage.cache_creation_input_tokens)
|
|
670
|
-
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
671
|
-
}
|
|
672
|
-
const toolUseNames = extractToolUseNames(msg?.content);
|
|
673
|
-
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
674
|
-
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
675
|
-
if (isToolResultOnly) {
|
|
676
|
-
const toolResultBlocks = extractToolResultBlocks(msg?.content, pendingToolUses);
|
|
677
|
-
if (toolResultBlocks.length > 0) metadata.toolResults = toolResultBlocks;
|
|
678
|
-
}
|
|
679
|
-
if (entry.teamName) {
|
|
680
|
-
metadata.teamName = entry.teamName;
|
|
681
|
-
if (!teamInfoMap.has(metadata.teamName) && content) {
|
|
682
|
-
const info = parseTeammateMessageTag(content);
|
|
683
|
-
if (info) teamInfoMap.set(metadata.teamName, info);
|
|
684
|
-
}
|
|
685
|
-
}
|
|
686
|
-
const thinkingContent = thinking?.content || void 0;
|
|
687
|
-
const thinkingSignature = thinking?.signature || void 0;
|
|
688
|
-
const hasMetadata = Object.keys(metadata).length > 0;
|
|
689
|
-
messages.push({
|
|
690
|
-
role: type,
|
|
691
|
-
text: content || "",
|
|
692
|
-
timestamp: entry.timestamp || "",
|
|
693
|
-
uuid: entry.uuid || void 0,
|
|
694
|
-
metadata: hasMetadata ? metadata : void 0,
|
|
695
|
-
isToolResult: isToolResultOnly || void 0,
|
|
696
|
-
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
697
|
-
thinkingContent,
|
|
698
|
-
thinkingSignature,
|
|
699
|
-
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
700
|
-
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
701
|
-
promptId: type === "user" ? entry.promptId : void 0,
|
|
702
|
-
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
703
|
-
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
704
|
-
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
705
|
-
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
706
|
-
});
|
|
707
|
-
if (content) textParts.push(content);
|
|
756
|
+
const message = reduceConvLine(state, entry);
|
|
757
|
+
if (message) {
|
|
758
|
+
messages.push(message);
|
|
759
|
+
if (message.text) textParts.push(message.text);
|
|
708
760
|
}
|
|
709
761
|
}
|
|
710
762
|
} catch (err) {
|
|
@@ -719,28 +771,21 @@ async function parseConversation(filePath, account) {
|
|
|
719
771
|
return null;
|
|
720
772
|
}
|
|
721
773
|
log.debug({ filePath, messageCount: messages.length }, "parseConversation: complete");
|
|
722
|
-
|
|
723
|
-
for (const msg of messages) {
|
|
724
|
-
if (msg.metadata?.teamName) {
|
|
725
|
-
const info = teamInfoMap.get(msg.metadata.teamName);
|
|
726
|
-
if (info) msg.metadata.teamInfo = info;
|
|
727
|
-
}
|
|
728
|
-
}
|
|
729
|
-
}
|
|
774
|
+
applyTeamInfo(messages, state);
|
|
730
775
|
return {
|
|
731
776
|
id: filePath,
|
|
732
777
|
filePath,
|
|
733
|
-
projectPath: cwd,
|
|
734
|
-
projectName:
|
|
735
|
-
sessionId: sessionId ||
|
|
736
|
-
sessionName,
|
|
778
|
+
projectPath: state.cwd,
|
|
779
|
+
projectName: getShortProjectName2(state.cwd),
|
|
780
|
+
sessionId: state.sessionId || basename2(filePath, ".jsonl"),
|
|
781
|
+
sessionName: state.sessionName,
|
|
737
782
|
messages,
|
|
738
783
|
fullText: textParts.join(" "),
|
|
739
|
-
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
784
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
740
785
|
messageCount: messages.length,
|
|
741
786
|
account,
|
|
742
787
|
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
743
|
-
lastPrompt: lastPrompt || void 0
|
|
788
|
+
lastPrompt: state.lastPrompt || void 0
|
|
744
789
|
};
|
|
745
790
|
}
|
|
746
791
|
function extractTextContent(content) {
|
|
@@ -833,11 +878,324 @@ function parseTeammateMessageTag(content) {
|
|
|
833
878
|
const color = attrs.match(/color="([^"]*)"/)?.[1];
|
|
834
879
|
return { teammateId: id, summary, color };
|
|
835
880
|
}
|
|
836
|
-
function
|
|
881
|
+
function getShortProjectName2(fullPath) {
|
|
837
882
|
const parts = fullPath.split("/").filter(Boolean);
|
|
838
883
|
return parts.slice(-3).join("/");
|
|
839
884
|
}
|
|
840
885
|
|
|
886
|
+
// src/persistent/cursor.ts
|
|
887
|
+
import { createHash } from "crypto";
|
|
888
|
+
import { closeSync, openSync, readSync, statSync } from "fs";
|
|
889
|
+
var FP_BYTES = 4096;
|
|
890
|
+
function fingerprint(filePath, size) {
|
|
891
|
+
const hash = createHash("sha1");
|
|
892
|
+
hash.update(String(size));
|
|
893
|
+
const fd = openSync(filePath, "r");
|
|
894
|
+
try {
|
|
895
|
+
const head = Buffer.alloc(Math.min(FP_BYTES, size));
|
|
896
|
+
if (head.length > 0) {
|
|
897
|
+
readSync(fd, head, 0, head.length, 0);
|
|
898
|
+
hash.update(head);
|
|
899
|
+
}
|
|
900
|
+
if (size > FP_BYTES) {
|
|
901
|
+
const tailLen = Math.min(FP_BYTES, size);
|
|
902
|
+
const tail = Buffer.alloc(tailLen);
|
|
903
|
+
readSync(fd, tail, 0, tailLen, size - tailLen);
|
|
904
|
+
hash.update(tail);
|
|
905
|
+
}
|
|
906
|
+
} finally {
|
|
907
|
+
closeSync(fd);
|
|
908
|
+
}
|
|
909
|
+
return hash.digest("hex");
|
|
910
|
+
}
|
|
911
|
+
function classify(filePath, existing) {
|
|
912
|
+
let stat3;
|
|
913
|
+
try {
|
|
914
|
+
const s = statSync(filePath);
|
|
915
|
+
stat3 = { size: s.size, mtimeMs: s.mtimeMs };
|
|
916
|
+
} catch {
|
|
917
|
+
return { change: "vanished" };
|
|
918
|
+
}
|
|
919
|
+
if (!existing || existing.status !== "active" || existing.last_indexed_offset === 0) {
|
|
920
|
+
return { change: "reindex", stat: stat3 };
|
|
921
|
+
}
|
|
922
|
+
if (stat3.size < existing.last_indexed_offset) {
|
|
923
|
+
return { change: "reindex", stat: stat3 };
|
|
924
|
+
}
|
|
925
|
+
if (stat3.size === existing.size_bytes && stat3.mtimeMs === existing.mtime_ms) {
|
|
926
|
+
return { change: "unchanged", stat: stat3 };
|
|
927
|
+
}
|
|
928
|
+
if (stat3.size === existing.last_indexed_offset) {
|
|
929
|
+
const fp = fingerprint(filePath, stat3.size);
|
|
930
|
+
if (existing.content_fingerprint && fp !== existing.content_fingerprint) {
|
|
931
|
+
return { change: "reindex", stat: stat3 };
|
|
932
|
+
}
|
|
933
|
+
return { change: "unchanged", stat: stat3 };
|
|
934
|
+
}
|
|
935
|
+
return { change: "appended", stat: stat3 };
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
// src/providers/codex-cli.ts
|
|
939
|
+
import fg2 from "fast-glob";
|
|
940
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
941
|
+
import { stat as stat2 } from "fs/promises";
|
|
942
|
+
import { basename as basename3 } from "path";
|
|
943
|
+
import { createInterface as createInterface2 } from "readline";
|
|
944
|
+
var CodexCliProvider = class {
|
|
945
|
+
name = CODEX_CLI_PROVIDER;
|
|
946
|
+
async discover(roots) {
|
|
947
|
+
const log = getLogger();
|
|
948
|
+
const results = [];
|
|
949
|
+
for (const root of roots) {
|
|
950
|
+
let paths;
|
|
951
|
+
try {
|
|
952
|
+
paths = await fg2(["**/rollout-*.jsonl", "**/*.jsonl"], {
|
|
953
|
+
cwd: root,
|
|
954
|
+
absolute: true,
|
|
955
|
+
dot: false,
|
|
956
|
+
unique: true
|
|
957
|
+
});
|
|
958
|
+
} catch (err) {
|
|
959
|
+
log.warn({ root, err }, "codex discovery: glob failed");
|
|
960
|
+
continue;
|
|
961
|
+
}
|
|
962
|
+
for (const filePath of paths) {
|
|
963
|
+
try {
|
|
964
|
+
const s = await stat2(filePath);
|
|
965
|
+
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
966
|
+
} catch (err) {
|
|
967
|
+
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
return results;
|
|
972
|
+
}
|
|
973
|
+
// Codex rollout lines carry distinctive top-level types.
|
|
974
|
+
canParse(_filePath, sample) {
|
|
975
|
+
for (const line of sample.split("\n")) {
|
|
976
|
+
if (!line.trim()) continue;
|
|
977
|
+
try {
|
|
978
|
+
const e = JSON.parse(line);
|
|
979
|
+
if (e.type === "session_meta" || e.type === "response_item" || e.type === "event_msg") {
|
|
980
|
+
return true;
|
|
981
|
+
}
|
|
982
|
+
if (e.type === "user" || e.type === "assistant") return false;
|
|
983
|
+
} catch {
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
return false;
|
|
987
|
+
}
|
|
988
|
+
createEmptyAccumulator() {
|
|
989
|
+
return {
|
|
990
|
+
sessionId: "",
|
|
991
|
+
cwd: "",
|
|
992
|
+
gitBranch: null,
|
|
993
|
+
model: null,
|
|
994
|
+
latestTimestamp: "",
|
|
995
|
+
messageCount: 0,
|
|
996
|
+
lastMessageSender: "user",
|
|
997
|
+
firstUser: null,
|
|
998
|
+
lastUser: null,
|
|
999
|
+
lastAssistant: null,
|
|
1000
|
+
toolNames: [],
|
|
1001
|
+
previewParts: [],
|
|
1002
|
+
previewLength: 0,
|
|
1003
|
+
snippetParts: [],
|
|
1004
|
+
snippetLength: 0
|
|
1005
|
+
};
|
|
1006
|
+
}
|
|
1007
|
+
reduceEntry(acc, entry, tier) {
|
|
1008
|
+
reduceCodexEntry(acc, entry, tier);
|
|
1009
|
+
}
|
|
1010
|
+
finalize(acc, filePath, account, tier) {
|
|
1011
|
+
return finalizeCodexMeta(acc, filePath, account, tier);
|
|
1012
|
+
}
|
|
1013
|
+
};
|
|
1014
|
+
var asString = (v) => typeof v === "string" ? v : "";
|
|
1015
|
+
function extractCodexText(content) {
|
|
1016
|
+
if (typeof content === "string") return cleanSystemTags(content);
|
|
1017
|
+
if (!Array.isArray(content)) return "";
|
|
1018
|
+
return content.map((item) => {
|
|
1019
|
+
if (typeof item === "string") return item;
|
|
1020
|
+
const t = item?.type;
|
|
1021
|
+
if ((t === "input_text" || t === "output_text" || t === "text") && item?.text) {
|
|
1022
|
+
return item.text;
|
|
1023
|
+
}
|
|
1024
|
+
return "";
|
|
1025
|
+
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
1026
|
+
}
|
|
1027
|
+
function reduceCodexEntry(acc, entry, tier) {
|
|
1028
|
+
const ts = asString(entry.timestamp);
|
|
1029
|
+
if (ts && (!acc.latestTimestamp || ts > acc.latestTimestamp)) acc.latestTimestamp = ts;
|
|
1030
|
+
const payload = entry.payload;
|
|
1031
|
+
if (!payload || typeof payload !== "object") return;
|
|
1032
|
+
const type = entry.type;
|
|
1033
|
+
if (type === "session_meta") {
|
|
1034
|
+
if (!acc.sessionId) acc.sessionId = asString(payload.id);
|
|
1035
|
+
if (!acc.cwd) acc.cwd = asString(payload.cwd);
|
|
1036
|
+
const git = payload.git;
|
|
1037
|
+
if (acc.gitBranch === null && git?.branch) acc.gitBranch = asString(git.branch) || null;
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
if (acc.model === null && payload.model) acc.model = asString(payload.model) || null;
|
|
1041
|
+
if (type !== "response_item") return;
|
|
1042
|
+
const ptype = payload.type;
|
|
1043
|
+
if (ptype === "function_call" || ptype === "custom_tool_call") {
|
|
1044
|
+
const name = asString(payload.name);
|
|
1045
|
+
if (name && !acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
1046
|
+
return;
|
|
1047
|
+
}
|
|
1048
|
+
if (ptype !== "message") return;
|
|
1049
|
+
const role = payload.role;
|
|
1050
|
+
if (role !== "user" && role !== "assistant") return;
|
|
1051
|
+
const text = extractCodexText(payload.content);
|
|
1052
|
+
if (!text) return;
|
|
1053
|
+
const sender = role;
|
|
1054
|
+
acc.messageCount++;
|
|
1055
|
+
acc.lastMessageSender = sender;
|
|
1056
|
+
const snapshot = { text: text.slice(0, 200), timestamp: ts };
|
|
1057
|
+
if (sender === "user") {
|
|
1058
|
+
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
1059
|
+
acc.lastUser = snapshot;
|
|
1060
|
+
} else {
|
|
1061
|
+
acc.lastAssistant = snapshot;
|
|
1062
|
+
}
|
|
1063
|
+
if (acc.previewLength < tier.previewMax) {
|
|
1064
|
+
acc.previewParts.push(text);
|
|
1065
|
+
acc.previewLength += text.length;
|
|
1066
|
+
}
|
|
1067
|
+
if (acc.snippetLength < tier.snippetMax) {
|
|
1068
|
+
const remaining = tier.snippetMax - acc.snippetLength;
|
|
1069
|
+
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
1070
|
+
acc.snippetParts.push(chunk);
|
|
1071
|
+
acc.snippetLength += chunk.length;
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
1075
|
+
if (acc.messageCount === 0) return null;
|
|
1076
|
+
const sessionId = acc.sessionId || basename3(filePath, ".jsonl");
|
|
1077
|
+
const projectPath = acc.cwd;
|
|
1078
|
+
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
1079
|
+
return {
|
|
1080
|
+
id: filePath,
|
|
1081
|
+
filePath,
|
|
1082
|
+
provider: CODEX_CLI_PROVIDER,
|
|
1083
|
+
kind,
|
|
1084
|
+
externalSessionId: acc.sessionId || void 0,
|
|
1085
|
+
sessionId,
|
|
1086
|
+
sessionName: "",
|
|
1087
|
+
projectPath,
|
|
1088
|
+
projectName: getShortProjectName3(projectPath),
|
|
1089
|
+
account,
|
|
1090
|
+
timestamp: acc.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1091
|
+
messageCount: acc.messageCount,
|
|
1092
|
+
lastMessageSender: acc.lastMessageSender,
|
|
1093
|
+
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
1094
|
+
contentSnippet: acc.snippetParts.join(" "),
|
|
1095
|
+
gitBranch: acc.gitBranch,
|
|
1096
|
+
model: acc.model,
|
|
1097
|
+
isSubagent: false,
|
|
1098
|
+
parentSessionId: null,
|
|
1099
|
+
isTeammate: false,
|
|
1100
|
+
teamName: null,
|
|
1101
|
+
toolNames: acc.toolNames,
|
|
1102
|
+
firstMessage: acc.firstUser,
|
|
1103
|
+
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
1104
|
+
lastPrompt: acc.lastUser?.text || void 0
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
function getShortProjectName3(fullPath) {
|
|
1108
|
+
return fullPath.split("/").filter(Boolean).slice(-3).join("/");
|
|
1109
|
+
}
|
|
1110
|
+
async function parseCodexConversation(filePath, account) {
|
|
1111
|
+
const log = getLogger();
|
|
1112
|
+
const messages = [];
|
|
1113
|
+
const textParts = [];
|
|
1114
|
+
let sessionId = "";
|
|
1115
|
+
let cwd = "";
|
|
1116
|
+
let latestTimestamp = "";
|
|
1117
|
+
let lastUserText = "";
|
|
1118
|
+
const rl = createInterface2({ input: createReadStream2(filePath), crlfDelay: Infinity });
|
|
1119
|
+
try {
|
|
1120
|
+
for await (const line of rl) {
|
|
1121
|
+
if (!line.trim()) continue;
|
|
1122
|
+
let entry;
|
|
1123
|
+
try {
|
|
1124
|
+
entry = JSON.parse(line);
|
|
1125
|
+
} catch {
|
|
1126
|
+
continue;
|
|
1127
|
+
}
|
|
1128
|
+
const ts = asString(entry.timestamp);
|
|
1129
|
+
if (ts && (!latestTimestamp || ts > latestTimestamp)) latestTimestamp = ts;
|
|
1130
|
+
const payload = entry.payload;
|
|
1131
|
+
if (!payload || typeof payload !== "object") continue;
|
|
1132
|
+
if (entry.type === "session_meta") {
|
|
1133
|
+
if (!sessionId) sessionId = asString(payload.id);
|
|
1134
|
+
if (!cwd) cwd = asString(payload.cwd);
|
|
1135
|
+
continue;
|
|
1136
|
+
}
|
|
1137
|
+
if (entry.type !== "response_item" || payload.type !== "message") continue;
|
|
1138
|
+
const role = payload.role;
|
|
1139
|
+
if (role !== "user" && role !== "assistant") continue;
|
|
1140
|
+
const text = extractCodexText(payload.content);
|
|
1141
|
+
if (!text) continue;
|
|
1142
|
+
messages.push({ role, text, timestamp: ts });
|
|
1143
|
+
textParts.push(text);
|
|
1144
|
+
if (role === "user") lastUserText = text;
|
|
1145
|
+
}
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
log.warn({ filePath, err }, "parseCodexConversation: read failed");
|
|
1148
|
+
return null;
|
|
1149
|
+
}
|
|
1150
|
+
if (messages.length === 0) return null;
|
|
1151
|
+
return {
|
|
1152
|
+
id: filePath,
|
|
1153
|
+
filePath,
|
|
1154
|
+
projectPath: cwd,
|
|
1155
|
+
projectName: getShortProjectName3(cwd),
|
|
1156
|
+
sessionId: sessionId || basename3(filePath, ".jsonl"),
|
|
1157
|
+
sessionName: "",
|
|
1158
|
+
messages,
|
|
1159
|
+
fullText: textParts.join(" "),
|
|
1160
|
+
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1161
|
+
messageCount: messages.length,
|
|
1162
|
+
account,
|
|
1163
|
+
lastPrompt: lastUserText || void 0
|
|
1164
|
+
};
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
// src/providers/parse.ts
|
|
1168
|
+
import { createReadStream as createReadStream3 } from "fs";
|
|
1169
|
+
import { createInterface as createInterface3 } from "readline";
|
|
1170
|
+
async function parseMetaWithProvider(provider, filePath, account, tier) {
|
|
1171
|
+
const log = getLogger();
|
|
1172
|
+
const acc = provider.createEmptyAccumulator();
|
|
1173
|
+
const rl = createInterface3({
|
|
1174
|
+
input: createReadStream3(filePath),
|
|
1175
|
+
crlfDelay: Infinity
|
|
1176
|
+
});
|
|
1177
|
+
try {
|
|
1178
|
+
for await (const line of rl) {
|
|
1179
|
+
if (!line.trim()) continue;
|
|
1180
|
+
let entry;
|
|
1181
|
+
try {
|
|
1182
|
+
entry = JSON.parse(line);
|
|
1183
|
+
} catch {
|
|
1184
|
+
continue;
|
|
1185
|
+
}
|
|
1186
|
+
try {
|
|
1187
|
+
provider.reduceEntry(acc, entry, tier);
|
|
1188
|
+
} catch (err) {
|
|
1189
|
+
log.warn({ filePath, provider: provider.name, err }, "provider reduce threw; line skipped");
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
} catch (err) {
|
|
1193
|
+
log.warn({ filePath, provider: provider.name, err }, "provider parse: read failed");
|
|
1194
|
+
return null;
|
|
1195
|
+
}
|
|
1196
|
+
return provider.finalize(acc, filePath, account, tier);
|
|
1197
|
+
}
|
|
1198
|
+
|
|
841
1199
|
// src/tiers.ts
|
|
842
1200
|
var DEFAULT_TIERS = {
|
|
843
1201
|
standard: { name: "standard", previewMax: 200, snippetMax: 5e3 },
|
|
@@ -853,48 +1211,1193 @@ function resolveTier(tierName, customTiers) {
|
|
|
853
1211
|
return tier;
|
|
854
1212
|
}
|
|
855
1213
|
|
|
856
|
-
// src/
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
1214
|
+
// src/persistent/db.ts
|
|
1215
|
+
import Database from "better-sqlite3";
|
|
1216
|
+
import { mkdirSync } from "fs";
|
|
1217
|
+
import { dirname as dirname3 } from "path";
|
|
1218
|
+
|
|
1219
|
+
// src/persistent/schema.ts
|
|
1220
|
+
var SCHEMA_VERSION = 3;
|
|
1221
|
+
var SCHEMA_SQL = `
|
|
1222
|
+
CREATE TABLE IF NOT EXISTS conversation_files (
|
|
1223
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1224
|
+
|
|
1225
|
+
absolute_path TEXT NOT NULL UNIQUE,
|
|
1226
|
+
parent_dir TEXT NOT NULL,
|
|
1227
|
+
file_name TEXT NOT NULL,
|
|
1228
|
+
|
|
1229
|
+
account TEXT NOT NULL DEFAULT 'default',
|
|
1230
|
+
|
|
1231
|
+
size_bytes INTEGER NOT NULL DEFAULT 0,
|
|
1232
|
+
mtime_ms INTEGER NOT NULL DEFAULT 0,
|
|
1233
|
+
|
|
1234
|
+
last_indexed_offset INTEGER NOT NULL DEFAULT 0,
|
|
1235
|
+
last_indexed_line INTEGER NOT NULL DEFAULT 0,
|
|
1236
|
+
|
|
1237
|
+
reducer_state TEXT,
|
|
1238
|
+
|
|
1239
|
+
content_fingerprint TEXT,
|
|
1240
|
+
|
|
1241
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
1242
|
+
|
|
1243
|
+
last_indexed_at TEXT,
|
|
1244
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1245
|
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1246
|
+
deleted_at TEXT
|
|
1247
|
+
);
|
|
1248
|
+
|
|
1249
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_status ON conversation_files(status);
|
|
1250
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_parent_dir ON conversation_files(parent_dir);
|
|
1251
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_account ON conversation_files(account);
|
|
1252
|
+
|
|
1253
|
+
CREATE TABLE IF NOT EXISTS conversations (
|
|
1254
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1255
|
+
|
|
1256
|
+
file_id INTEGER NOT NULL UNIQUE,
|
|
1257
|
+
|
|
1258
|
+
source_path TEXT NOT NULL UNIQUE,
|
|
1259
|
+
-- Which provider produced this row. Canonical identity is (provider,
|
|
1260
|
+
-- source_path); session_id stays non-unique across providers too.
|
|
1261
|
+
provider TEXT NOT NULL DEFAULT 'claude-code',
|
|
1262
|
+
kind TEXT,
|
|
1263
|
+
external_session_id TEXT,
|
|
1264
|
+
session_id TEXT NOT NULL,
|
|
1265
|
+
session_name TEXT,
|
|
1266
|
+
|
|
1267
|
+
project_path TEXT,
|
|
1268
|
+
project_name TEXT,
|
|
1269
|
+
account TEXT NOT NULL DEFAULT 'default',
|
|
1270
|
+
branch TEXT,
|
|
1271
|
+
|
|
1272
|
+
preview TEXT,
|
|
1273
|
+
content_snippet TEXT,
|
|
1274
|
+
|
|
1275
|
+
message_count INTEGER NOT NULL DEFAULT 0,
|
|
1276
|
+
-- Count of messages parseConversation() produces (broader than message_count;
|
|
1277
|
+
-- includes tool_use-only and thinking-only lines). The total for bounded paging.
|
|
1278
|
+
page_message_count INTEGER NOT NULL DEFAULT 0,
|
|
1279
|
+
last_message_sender TEXT NOT NULL DEFAULT 'user',
|
|
1280
|
+
timestamp TEXT,
|
|
1281
|
+
|
|
1282
|
+
-- Monotonic write counter; the highest value is the most recently indexed
|
|
1283
|
+
-- row. Drives last-writer-wins resolution for shared session_ids (matching
|
|
1284
|
+
-- the in-memory sessionId map), at the sub-second precision updated_at lacks.
|
|
1285
|
+
index_seq INTEGER NOT NULL DEFAULT 0,
|
|
1286
|
+
|
|
1287
|
+
first_sent_at TEXT,
|
|
1288
|
+
first_sent_text TEXT,
|
|
1289
|
+
|
|
1290
|
+
last_sent_at TEXT,
|
|
1291
|
+
last_sent_text TEXT,
|
|
1292
|
+
|
|
1293
|
+
model TEXT,
|
|
1294
|
+
is_subagent INTEGER NOT NULL DEFAULT 0,
|
|
1295
|
+
parent_session_id TEXT,
|
|
1296
|
+
is_teammate INTEGER NOT NULL DEFAULT 0,
|
|
1297
|
+
team_name TEXT,
|
|
1298
|
+
tool_names_json TEXT,
|
|
1299
|
+
last_prompt TEXT,
|
|
1300
|
+
|
|
1301
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
1302
|
+
|
|
1303
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1304
|
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1305
|
+
|
|
1306
|
+
FOREIGN KEY (file_id) REFERENCES conversation_files(id)
|
|
1307
|
+
);
|
|
1308
|
+
|
|
1309
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_session_id ON conversations(session_id);
|
|
1310
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_session ON conversations(provider, session_id);
|
|
1311
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_recent ON conversations(provider, timestamp DESC);
|
|
1312
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_project_branch ON conversations(provider, project_path, branch);
|
|
1313
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_recent ON conversations(timestamp DESC);
|
|
1314
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_project_recent ON conversations(project_path, timestamp DESC);
|
|
1315
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_project_branch_recent ON conversations(project_path, branch, timestamp DESC);
|
|
1316
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_account_recent ON conversations(account, timestamp DESC);
|
|
1317
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_subagent_recent ON conversations(is_subagent, timestamp DESC);
|
|
1318
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_team_recent ON conversations(team_name, timestamp DESC);
|
|
1319
|
+
|
|
1320
|
+
-- Full-text search index over conversation content + metadata. Kept separate
|
|
1321
|
+
-- from the metadata tables so list-screen queries stay small and fast.
|
|
1322
|
+
-- source_path is UNINDEXED (stored, not tokenized) and links back to a
|
|
1323
|
+
-- conversations row. One FTS row per conversation, replaced on each upsert.
|
|
1324
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS conversation_messages_fts USING fts5(
|
|
1325
|
+
source_path UNINDEXED,
|
|
1326
|
+
content,
|
|
1327
|
+
project_name,
|
|
1328
|
+
session_id,
|
|
1329
|
+
session_name,
|
|
1330
|
+
account,
|
|
1331
|
+
model,
|
|
1332
|
+
branch,
|
|
1333
|
+
tool_names,
|
|
1334
|
+
tokenize = 'unicode61'
|
|
1335
|
+
);
|
|
1336
|
+
|
|
1337
|
+
-- Seek index for large conversations: every N messages, record the byte offset
|
|
1338
|
+
-- where the next message's line begins plus the parser's cross-line state
|
|
1339
|
+
-- (pending tool_use blocks, team info) needed to resume an equivalent parse
|
|
1340
|
+
-- from that point. Lets getConversationPage() read a window near the end of a
|
|
1341
|
+
-- huge file without parsing from byte 0.
|
|
1342
|
+
CREATE TABLE IF NOT EXISTS message_checkpoints (
|
|
1343
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1344
|
+
source_path TEXT NOT NULL,
|
|
1345
|
+
message_index INTEGER NOT NULL,
|
|
1346
|
+
byte_offset INTEGER NOT NULL,
|
|
1347
|
+
line_number INTEGER NOT NULL,
|
|
1348
|
+
parser_state TEXT NOT NULL,
|
|
1349
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
1350
|
+
);
|
|
1351
|
+
|
|
1352
|
+
CREATE INDEX IF NOT EXISTS idx_message_checkpoints_lookup
|
|
1353
|
+
ON message_checkpoints(source_path, message_index);
|
|
1354
|
+
`;
|
|
1355
|
+
|
|
1356
|
+
// src/persistent/migrations.ts
|
|
1357
|
+
function runMigrations(db) {
|
|
1358
|
+
const current = db.pragma("user_version", { simple: true });
|
|
1359
|
+
if (current >= SCHEMA_VERSION) return;
|
|
1360
|
+
const log = getLogger();
|
|
1361
|
+
log.info({ from: current, to: SCHEMA_VERSION }, "migrations: applying");
|
|
1362
|
+
if (current >= 1 && current < 2 && tableExists(db, "conversations")) {
|
|
1363
|
+
for (const [col, ddl] of [
|
|
1364
|
+
[
|
|
1365
|
+
"provider",
|
|
1366
|
+
// Keep the historical DEFAULT 'threadbase' — v2→v3 below updates these rows.
|
|
1367
|
+
"ALTER TABLE conversations ADD COLUMN provider TEXT NOT NULL DEFAULT 'threadbase'"
|
|
1368
|
+
],
|
|
1369
|
+
["kind", "ALTER TABLE conversations ADD COLUMN kind TEXT"],
|
|
1370
|
+
["external_session_id", "ALTER TABLE conversations ADD COLUMN external_session_id TEXT"]
|
|
1371
|
+
]) {
|
|
1372
|
+
if (!hasColumn(db, "conversations", col)) db.exec(ddl);
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
if (current >= 1 && current < 3 && tableExists(db, "conversations")) {
|
|
1376
|
+
db.exec("UPDATE conversations SET provider = 'claude-code' WHERE provider = 'threadbase'");
|
|
1377
|
+
}
|
|
1378
|
+
db.exec(SCHEMA_SQL);
|
|
1379
|
+
db.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
1380
|
+
}
|
|
1381
|
+
function tableExists(db, table) {
|
|
1382
|
+
return db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== void 0;
|
|
1383
|
+
}
|
|
1384
|
+
function hasColumn(db, table, column) {
|
|
1385
|
+
const cols = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
1386
|
+
return cols.some((c) => c.name === column);
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
// src/persistent/db.ts
|
|
1390
|
+
function openDatabase(dbPath) {
|
|
1391
|
+
if (dbPath !== ":memory:") {
|
|
1392
|
+
mkdirSync(dirname3(dbPath), { recursive: true });
|
|
1393
|
+
}
|
|
1394
|
+
const db = new Database(dbPath);
|
|
1395
|
+
db.pragma("journal_mode = WAL");
|
|
1396
|
+
db.pragma("synchronous = NORMAL");
|
|
1397
|
+
db.pragma("temp_store = MEMORY");
|
|
1398
|
+
db.pragma("foreign_keys = ON");
|
|
1399
|
+
runMigrations(db);
|
|
1400
|
+
getLogger().debug({ dbPath }, "db: opened");
|
|
1401
|
+
return db;
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
// src/persistent/jsonl-tail-reader.ts
|
|
1405
|
+
import { createReadStream as createReadStream4 } from "fs";
|
|
1406
|
+
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1407
|
+
const stream = createReadStream4(filePath, { start: startOffset, encoding: "utf8" });
|
|
1408
|
+
let buffer = "";
|
|
1409
|
+
let offset = startOffset;
|
|
1410
|
+
let line = startLine;
|
|
1411
|
+
let parsedLines = 0;
|
|
1412
|
+
for await (const chunk of stream) {
|
|
1413
|
+
buffer += chunk;
|
|
1414
|
+
let nl;
|
|
1415
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1416
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1417
|
+
const text = lineWithNewline.trimEnd();
|
|
1418
|
+
buffer = buffer.slice(nl + 1);
|
|
1419
|
+
if (text.length > 0) {
|
|
1420
|
+
try {
|
|
1421
|
+
reduceLine(state, JSON.parse(text), tier);
|
|
1422
|
+
} catch {
|
|
1423
|
+
state.badJsonLines++;
|
|
1424
|
+
}
|
|
1425
|
+
parsedLines++;
|
|
1426
|
+
}
|
|
1427
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1428
|
+
line++;
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
// src/persistent/paged-reader.ts
|
|
1435
|
+
import { createReadStream as createReadStream5 } from "fs";
|
|
1436
|
+
var CHECKPOINT_INTERVAL = 500;
|
|
1437
|
+
async function streamMessages(filePath, startOffset, startLine, state, onMessage) {
|
|
1438
|
+
const stream = createReadStream5(filePath, { start: startOffset, encoding: "utf8" });
|
|
1439
|
+
let buffer = "";
|
|
1440
|
+
let offset = startOffset;
|
|
1441
|
+
let line = startLine;
|
|
1442
|
+
for await (const chunk of stream) {
|
|
1443
|
+
buffer += chunk;
|
|
1444
|
+
let nl;
|
|
1445
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1446
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1447
|
+
const text = lineWithNewline.trimEnd();
|
|
1448
|
+
buffer = buffer.slice(nl + 1);
|
|
1449
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1450
|
+
line += 1;
|
|
1451
|
+
if (text.length === 0) continue;
|
|
1452
|
+
let entry;
|
|
1453
|
+
try {
|
|
1454
|
+
entry = JSON.parse(text);
|
|
1455
|
+
} catch {
|
|
1456
|
+
continue;
|
|
1457
|
+
}
|
|
1458
|
+
const message = reduceConvLine(state, entry);
|
|
1459
|
+
if (message && onMessage(message, offset, line)) {
|
|
1460
|
+
stream.destroy();
|
|
1461
|
+
return;
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL) {
|
|
1467
|
+
const checkpoints = [];
|
|
1468
|
+
const state = initialConvState();
|
|
1469
|
+
let index = 0;
|
|
1470
|
+
await streamMessages(filePath, 0, 0, state, (_msg, nextOffset, nextLine) => {
|
|
1471
|
+
index += 1;
|
|
1472
|
+
if (index % interval === 0) {
|
|
1473
|
+
checkpoints.push({
|
|
1474
|
+
messageIndex: index,
|
|
1475
|
+
byteOffset: nextOffset,
|
|
1476
|
+
lineNumber: nextLine,
|
|
1477
|
+
state: structuredClone(state)
|
|
1478
|
+
});
|
|
1479
|
+
}
|
|
1480
|
+
return false;
|
|
1481
|
+
});
|
|
1482
|
+
return checkpoints;
|
|
1483
|
+
}
|
|
1484
|
+
async function readPage(filePath, total, options, floor) {
|
|
1485
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
1486
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1487
|
+
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1488
|
+
const startOffset = floor ? floor.byteOffset : 0;
|
|
1489
|
+
const startLine = floor ? floor.lineNumber : 0;
|
|
1490
|
+
let index = floor ? floor.messageIndex : 0;
|
|
1491
|
+
const window = [];
|
|
1492
|
+
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1493
|
+
const current = index;
|
|
1494
|
+
index += 1;
|
|
1495
|
+
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1496
|
+
return index >= beforeIndex;
|
|
1497
|
+
});
|
|
1498
|
+
applyTeamInfo(window, state);
|
|
1499
|
+
return { messages: window, total, fromIndex };
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
// src/persistent/repositories/checkpoints.repo.ts
|
|
1503
|
+
var CheckpointsRepo = class {
|
|
1504
|
+
constructor(db) {
|
|
1505
|
+
this.db = db;
|
|
1506
|
+
}
|
|
1507
|
+
db;
|
|
1508
|
+
replaceAll(sourcePath, checkpoints) {
|
|
1509
|
+
const tx = this.db.transaction(() => {
|
|
1510
|
+
this.db.prepare("DELETE FROM message_checkpoints WHERE source_path = ?").run(sourcePath);
|
|
1511
|
+
const insert = this.db.prepare(
|
|
1512
|
+
`INSERT INTO message_checkpoints
|
|
1513
|
+
(source_path, message_index, byte_offset, line_number, parser_state)
|
|
1514
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
1515
|
+
);
|
|
1516
|
+
for (const c of checkpoints) {
|
|
1517
|
+
insert.run(sourcePath, c.messageIndex, c.byteOffset, c.lineNumber, JSON.stringify(c.state));
|
|
1518
|
+
}
|
|
1519
|
+
});
|
|
1520
|
+
tx();
|
|
1521
|
+
}
|
|
1522
|
+
// The latest checkpoint at or before `messageIndex`, or null if none (read
|
|
1523
|
+
// from the file start). Lets a page seek to the nearest prior anchor.
|
|
1524
|
+
floor(sourcePath, messageIndex) {
|
|
1525
|
+
const row = this.db.prepare(
|
|
1526
|
+
`SELECT message_index, byte_offset, line_number, parser_state
|
|
1527
|
+
FROM message_checkpoints
|
|
1528
|
+
WHERE source_path = ? AND message_index <= ?
|
|
1529
|
+
ORDER BY message_index DESC LIMIT 1`
|
|
1530
|
+
).get(sourcePath, messageIndex);
|
|
1531
|
+
return row ? toCheckpoint(row) : null;
|
|
1532
|
+
}
|
|
1533
|
+
count(sourcePath) {
|
|
1534
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM message_checkpoints WHERE source_path = ?").get(sourcePath).n;
|
|
1535
|
+
}
|
|
1536
|
+
remove(sourcePath) {
|
|
1537
|
+
this.db.prepare("DELETE FROM message_checkpoints WHERE source_path = ?").run(sourcePath);
|
|
1538
|
+
}
|
|
1539
|
+
};
|
|
1540
|
+
function toCheckpoint(row) {
|
|
1541
|
+
return {
|
|
1542
|
+
messageIndex: row.message_index,
|
|
1543
|
+
byteOffset: row.byte_offset,
|
|
1544
|
+
lineNumber: row.line_number,
|
|
1545
|
+
state: JSON.parse(row.parser_state)
|
|
1546
|
+
};
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
// src/persistent/repositories/conversation-files.repo.ts
|
|
1550
|
+
import { basename as basename4, dirname as dirname4 } from "path";
|
|
1551
|
+
var ConversationFilesRepo = class {
|
|
1552
|
+
constructor(db) {
|
|
1553
|
+
this.db = db;
|
|
1554
|
+
}
|
|
1555
|
+
db;
|
|
1556
|
+
getByPath(absolutePath) {
|
|
1557
|
+
return this.db.prepare("SELECT * FROM conversation_files WHERE absolute_path = ?").get(absolutePath);
|
|
1558
|
+
}
|
|
1559
|
+
// Insert a freshly-discovered file at offset 0; returns its row id. Existing
|
|
1560
|
+
// path is left untouched (returns the existing id) so a re-discovery is safe.
|
|
1561
|
+
ensure(absolutePath, account) {
|
|
1562
|
+
const existing = this.getByPath(absolutePath);
|
|
1563
|
+
if (existing) return existing.id;
|
|
1564
|
+
const info = this.db.prepare(
|
|
1565
|
+
`INSERT INTO conversation_files (absolute_path, parent_dir, file_name, account)
|
|
1566
|
+
VALUES (?, ?, ?, ?)`
|
|
1567
|
+
).run(absolutePath, dirname4(absolutePath), basename4(absolutePath), account);
|
|
1568
|
+
return Number(info.lastInsertRowid);
|
|
1569
|
+
}
|
|
1570
|
+
// Advance the cursor + persisted reducer state after a successful index pass.
|
|
1571
|
+
updateCursor(id, fields) {
|
|
1572
|
+
this.db.prepare(
|
|
1573
|
+
`UPDATE conversation_files
|
|
1574
|
+
SET size_bytes = ?, mtime_ms = ?, last_indexed_offset = ?, last_indexed_line = ?,
|
|
1575
|
+
reducer_state = ?, content_fingerprint = ?, status = ?,
|
|
1576
|
+
last_indexed_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
|
|
1577
|
+
WHERE id = ?`
|
|
1578
|
+
).run(
|
|
1579
|
+
fields.sizeBytes,
|
|
1580
|
+
fields.mtimeMs,
|
|
1581
|
+
fields.offset,
|
|
1582
|
+
fields.line,
|
|
1583
|
+
fields.reducerState,
|
|
1584
|
+
fields.fingerprint,
|
|
1585
|
+
fields.status ?? "active",
|
|
1586
|
+
id
|
|
1587
|
+
);
|
|
1588
|
+
}
|
|
1589
|
+
// Reset the cursor to 0 for a truncated/replaced file before a full reindex.
|
|
1590
|
+
resetCursor(id) {
|
|
1591
|
+
this.db.prepare(
|
|
1592
|
+
`UPDATE conversation_files
|
|
1593
|
+
SET last_indexed_offset = 0, last_indexed_line = 0, reducer_state = NULL,
|
|
1594
|
+
status = 'needs_reindex', updated_at = CURRENT_TIMESTAMP
|
|
1595
|
+
WHERE id = ?`
|
|
1596
|
+
).run(id);
|
|
1597
|
+
}
|
|
1598
|
+
setStatus(id, status) {
|
|
1599
|
+
const deletedAt = status === "deleted" ? "CURRENT_TIMESTAMP" : "deleted_at";
|
|
1600
|
+
this.db.prepare(
|
|
1601
|
+
`UPDATE conversation_files
|
|
1602
|
+
SET status = ?, deleted_at = ${deletedAt}, updated_at = CURRENT_TIMESTAMP
|
|
1603
|
+
WHERE id = ?`
|
|
1604
|
+
).run(status, id);
|
|
1605
|
+
}
|
|
1606
|
+
allActivePaths() {
|
|
1607
|
+
const rows = this.db.prepare("SELECT absolute_path FROM conversation_files WHERE status != 'deleted'").all();
|
|
1608
|
+
return rows.map((r) => r.absolute_path);
|
|
1609
|
+
}
|
|
1610
|
+
};
|
|
1611
|
+
|
|
1612
|
+
// src/persistent/repositories/conversations.repo.ts
|
|
1613
|
+
function rowToMeta(row) {
|
|
1614
|
+
return {
|
|
1615
|
+
id: row.source_path,
|
|
1616
|
+
filePath: row.source_path,
|
|
1617
|
+
provider: row.provider ?? CLAUDE_CODE_PROVIDER,
|
|
1618
|
+
kind: row.kind ?? void 0,
|
|
1619
|
+
externalSessionId: row.external_session_id ?? void 0,
|
|
1620
|
+
sessionId: row.session_id,
|
|
1621
|
+
sessionName: row.session_name ?? "",
|
|
1622
|
+
projectPath: row.project_path ?? "",
|
|
1623
|
+
projectName: row.project_name ?? "",
|
|
1624
|
+
account: row.account,
|
|
1625
|
+
timestamp: row.timestamp ?? "",
|
|
1626
|
+
messageCount: row.message_count,
|
|
1627
|
+
lastMessageSender: row.last_message_sender,
|
|
1628
|
+
preview: row.preview ?? "",
|
|
1629
|
+
contentSnippet: row.content_snippet ?? "",
|
|
1630
|
+
gitBranch: row.branch,
|
|
1631
|
+
model: row.model,
|
|
1632
|
+
isSubagent: row.is_subagent === 1,
|
|
1633
|
+
parentSessionId: row.parent_session_id,
|
|
1634
|
+
isTeammate: row.is_teammate === 1,
|
|
1635
|
+
teamName: row.team_name,
|
|
1636
|
+
toolNames: row.tool_names_json ? JSON.parse(row.tool_names_json) : [],
|
|
1637
|
+
firstMessage: row.first_sent_text ? { text: row.first_sent_text, timestamp: row.first_sent_at ?? "" } : null,
|
|
1638
|
+
lastMessage: row.last_sent_text ? { text: row.last_sent_text, timestamp: row.last_sent_at ?? "" } : null,
|
|
1639
|
+
lastPrompt: row.last_prompt ?? void 0
|
|
1640
|
+
};
|
|
1641
|
+
}
|
|
1642
|
+
var ConversationsRepo = class {
|
|
1643
|
+
constructor(db) {
|
|
1644
|
+
this.db = db;
|
|
1645
|
+
}
|
|
1646
|
+
db;
|
|
1647
|
+
// Upsert by file_id (1 file = 1 conversation). Keyed on the unique file_id so
|
|
1648
|
+
// a re-index overwrites the prior summary in place.
|
|
1649
|
+
upsert(fileId, meta, pageMessageCount = meta.messageCount) {
|
|
1650
|
+
this.db.prepare(
|
|
1651
|
+
`INSERT INTO conversations (
|
|
1652
|
+
file_id, source_path, provider, kind, external_session_id,
|
|
1653
|
+
session_id, session_name, project_path, project_name,
|
|
1654
|
+
account, branch, preview, content_snippet, message_count, page_message_count,
|
|
1655
|
+
last_message_sender,
|
|
1656
|
+
timestamp, index_seq, first_sent_at, first_sent_text, last_sent_at, last_sent_text,
|
|
1657
|
+
model, is_subagent, parent_session_id, is_teammate, team_name, tool_names_json,
|
|
1658
|
+
last_prompt, status, updated_at
|
|
1659
|
+
) VALUES (
|
|
1660
|
+
@file_id, @source_path, @provider, @kind, @external_session_id,
|
|
1661
|
+
@session_id, @session_name, @project_path, @project_name,
|
|
1662
|
+
@account, @branch, @preview, @content_snippet, @message_count, @page_message_count,
|
|
1663
|
+
@last_message_sender,
|
|
1664
|
+
@timestamp,
|
|
1665
|
+
(SELECT COALESCE(MAX(index_seq), 0) + 1 FROM conversations),
|
|
1666
|
+
@first_sent_at, @first_sent_text, @last_sent_at, @last_sent_text,
|
|
1667
|
+
@model, @is_subagent, @parent_session_id, @is_teammate, @team_name, @tool_names_json,
|
|
1668
|
+
@last_prompt, 'active', CURRENT_TIMESTAMP
|
|
1669
|
+
)
|
|
1670
|
+
ON CONFLICT(file_id) DO UPDATE SET
|
|
1671
|
+
source_path = excluded.source_path,
|
|
1672
|
+
provider = excluded.provider,
|
|
1673
|
+
kind = excluded.kind,
|
|
1674
|
+
external_session_id = excluded.external_session_id,
|
|
1675
|
+
session_id = excluded.session_id,
|
|
1676
|
+
session_name = excluded.session_name,
|
|
1677
|
+
project_path = excluded.project_path,
|
|
1678
|
+
project_name = excluded.project_name,
|
|
1679
|
+
account = excluded.account,
|
|
1680
|
+
branch = excluded.branch,
|
|
1681
|
+
preview = excluded.preview,
|
|
1682
|
+
content_snippet = excluded.content_snippet,
|
|
1683
|
+
message_count = excluded.message_count,
|
|
1684
|
+
page_message_count = excluded.page_message_count,
|
|
1685
|
+
last_message_sender = excluded.last_message_sender,
|
|
1686
|
+
timestamp = excluded.timestamp,
|
|
1687
|
+
first_sent_at = excluded.first_sent_at,
|
|
1688
|
+
first_sent_text = excluded.first_sent_text,
|
|
1689
|
+
last_sent_at = excluded.last_sent_at,
|
|
1690
|
+
last_sent_text = excluded.last_sent_text,
|
|
1691
|
+
model = excluded.model,
|
|
1692
|
+
is_subagent = excluded.is_subagent,
|
|
1693
|
+
parent_session_id = excluded.parent_session_id,
|
|
1694
|
+
is_teammate = excluded.is_teammate,
|
|
1695
|
+
team_name = excluded.team_name,
|
|
1696
|
+
tool_names_json = excluded.tool_names_json,
|
|
1697
|
+
last_prompt = excluded.last_prompt,
|
|
1698
|
+
status = 'active',
|
|
1699
|
+
index_seq = (SELECT COALESCE(MAX(index_seq), 0) + 1 FROM conversations),
|
|
1700
|
+
updated_at = CURRENT_TIMESTAMP`
|
|
1701
|
+
).run({
|
|
1702
|
+
file_id: fileId,
|
|
1703
|
+
source_path: meta.id,
|
|
1704
|
+
provider: meta.provider ?? CLAUDE_CODE_PROVIDER,
|
|
1705
|
+
kind: meta.kind ?? null,
|
|
1706
|
+
external_session_id: meta.externalSessionId ?? null,
|
|
1707
|
+
session_id: meta.sessionId,
|
|
1708
|
+
session_name: meta.sessionName || null,
|
|
1709
|
+
project_path: meta.projectPath || null,
|
|
1710
|
+
project_name: meta.projectName || null,
|
|
1711
|
+
account: meta.account,
|
|
1712
|
+
branch: meta.gitBranch,
|
|
1713
|
+
preview: meta.preview || null,
|
|
1714
|
+
content_snippet: meta.contentSnippet || null,
|
|
1715
|
+
message_count: meta.messageCount,
|
|
1716
|
+
page_message_count: pageMessageCount,
|
|
1717
|
+
last_message_sender: meta.lastMessageSender,
|
|
1718
|
+
timestamp: meta.timestamp || null,
|
|
1719
|
+
first_sent_at: meta.firstMessage?.timestamp ?? null,
|
|
1720
|
+
first_sent_text: meta.firstMessage?.text ?? null,
|
|
1721
|
+
last_sent_at: meta.lastMessage?.timestamp ?? null,
|
|
1722
|
+
last_sent_text: meta.lastMessage?.text ?? null,
|
|
1723
|
+
model: meta.model,
|
|
1724
|
+
is_subagent: meta.isSubagent ? 1 : 0,
|
|
1725
|
+
parent_session_id: meta.parentSessionId,
|
|
1726
|
+
is_teammate: meta.isTeammate ? 1 : 0,
|
|
1727
|
+
team_name: meta.teamName,
|
|
1728
|
+
tool_names_json: JSON.stringify(meta.toolNames),
|
|
1729
|
+
last_prompt: meta.lastPrompt ?? null
|
|
1730
|
+
});
|
|
1731
|
+
}
|
|
1732
|
+
getBySourcePath(sourcePath) {
|
|
1733
|
+
const row = this.db.prepare("SELECT * FROM conversations WHERE source_path = ? AND status = 'active'").get(sourcePath);
|
|
1734
|
+
return row ? rowToMeta(row) : null;
|
|
1735
|
+
}
|
|
1736
|
+
// Dual lookup matching scanner.getConversation: resolve by source_path (the
|
|
1737
|
+
// canonical id) first, then by session_id.
|
|
1738
|
+
//
|
|
1739
|
+
// session_id is NOT unique (see schema header) — the sessionId form is a
|
|
1740
|
+
// compatibility convenience. Resolution is deterministic and matches the
|
|
1741
|
+
// in-memory scanner: among active rows sharing the session_id, newest
|
|
1742
|
+
// timestamp wins, index_seq breaks sub-second ties (precision updated_at
|
|
1743
|
+
// lacks), then source_path ascending. Collision-safe callers should use
|
|
1744
|
+
// getAllBySessionId() instead.
|
|
1745
|
+
getByIdOrSession(id) {
|
|
1746
|
+
const direct = this.getBySourcePath(id);
|
|
1747
|
+
if (direct) return direct;
|
|
1748
|
+
const row = this.db.prepare(
|
|
1749
|
+
`SELECT * FROM conversations WHERE session_id = ? AND status = 'active'
|
|
1750
|
+
ORDER BY COALESCE(timestamp, '') DESC, index_seq DESC, source_path ASC LIMIT 1`
|
|
1751
|
+
).get(id);
|
|
1752
|
+
return row ? rowToMeta(row) : null;
|
|
1753
|
+
}
|
|
1754
|
+
// All active conversations sharing a session_id, newest first (same ordering
|
|
1755
|
+
// as getByIdOrSession). Collision-safe counterpart to the convenience
|
|
1756
|
+
// getByIdOrSession() sessionId lookup.
|
|
1757
|
+
getAllBySessionId(sessionId) {
|
|
1758
|
+
const rows = this.db.prepare(
|
|
1759
|
+
`SELECT * FROM conversations WHERE session_id = ? AND status = 'active'
|
|
1760
|
+
ORDER BY COALESCE(timestamp, '') DESC, index_seq DESC, source_path ASC`
|
|
1761
|
+
).all(sessionId);
|
|
1762
|
+
return rows.map(rowToMeta);
|
|
1763
|
+
}
|
|
1764
|
+
// All active metas (unsorted/unfiltered) — callers apply the existing
|
|
1765
|
+
// filters/view transforms. Used by scan() before SQL filtering is wired in.
|
|
1766
|
+
allActive() {
|
|
1767
|
+
const rows = this.db.prepare("SELECT * FROM conversations WHERE status = 'active'").all();
|
|
1768
|
+
return rows.map(rowToMeta);
|
|
1769
|
+
}
|
|
1770
|
+
// Most recent active conversations, newest first. Backs the empty-query
|
|
1771
|
+
// search path (mirrors the in-memory indexer's getRecent).
|
|
1772
|
+
recent(limit) {
|
|
1773
|
+
const rows = this.db.prepare(
|
|
1774
|
+
`SELECT * FROM conversations WHERE status = 'active'
|
|
1775
|
+
ORDER BY COALESCE(timestamp, '') DESC, source_path ASC LIMIT ?`
|
|
1776
|
+
).all(limit);
|
|
1777
|
+
return rows.map(rowToMeta);
|
|
1778
|
+
}
|
|
1779
|
+
distinctProjects() {
|
|
1780
|
+
const rows = this.db.prepare(
|
|
1781
|
+
`SELECT DISTINCT project_path FROM conversations
|
|
1782
|
+
WHERE status = 'active' AND project_path IS NOT NULL AND project_path != ''
|
|
1783
|
+
ORDER BY project_path ASC`
|
|
1784
|
+
).all();
|
|
1785
|
+
return rows.map((r) => r.project_path);
|
|
1786
|
+
}
|
|
1787
|
+
deleteByFileId(fileId) {
|
|
1788
|
+
this.db.prepare(
|
|
1789
|
+
"UPDATE conversations SET status = 'deleted', updated_at = CURRENT_TIMESTAMP WHERE file_id = ?"
|
|
1790
|
+
).run(fileId);
|
|
1791
|
+
}
|
|
1792
|
+
// The parseConversation message total for a file (for bounded paging), or 0
|
|
1793
|
+
// if not indexed.
|
|
1794
|
+
pageMessageCount(sourcePath) {
|
|
1795
|
+
const row = this.db.prepare(
|
|
1796
|
+
"SELECT page_message_count AS n FROM conversations WHERE source_path = ? AND status = 'active'"
|
|
1797
|
+
).get(sourcePath);
|
|
1798
|
+
return row?.n ?? 0;
|
|
1799
|
+
}
|
|
1800
|
+
count() {
|
|
1801
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM conversations WHERE status = 'active'").get().n;
|
|
1802
|
+
}
|
|
1803
|
+
};
|
|
1804
|
+
|
|
1805
|
+
// src/persistent/repositories/fts.repo.ts
|
|
1806
|
+
var FtsRepo = class {
|
|
1807
|
+
constructor(db) {
|
|
1808
|
+
this.db = db;
|
|
1809
|
+
}
|
|
1810
|
+
db;
|
|
1811
|
+
upsert(meta) {
|
|
1812
|
+
const tx = this.db.transaction(() => {
|
|
1813
|
+
this.db.prepare("DELETE FROM conversation_messages_fts WHERE source_path = ?").run(meta.id);
|
|
1814
|
+
this.db.prepare(
|
|
1815
|
+
`INSERT INTO conversation_messages_fts
|
|
1816
|
+
(source_path, content, project_name, session_id, session_name, account, model, branch, tool_names)
|
|
1817
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1818
|
+
).run(
|
|
1819
|
+
meta.id,
|
|
1820
|
+
meta.contentSnippet ?? "",
|
|
1821
|
+
meta.projectName ?? "",
|
|
1822
|
+
meta.sessionId ?? "",
|
|
1823
|
+
meta.sessionName ?? "",
|
|
1824
|
+
meta.account ?? "",
|
|
1825
|
+
meta.model ?? "",
|
|
1826
|
+
meta.gitBranch ?? "",
|
|
1827
|
+
meta.toolNames.join(" ")
|
|
1828
|
+
);
|
|
1829
|
+
});
|
|
1830
|
+
tx();
|
|
1831
|
+
}
|
|
1832
|
+
remove(sourcePath) {
|
|
1833
|
+
this.db.prepare("DELETE FROM conversation_messages_fts WHERE source_path = ?").run(sourcePath);
|
|
1834
|
+
}
|
|
1835
|
+
// Ranked source_paths matching the query, best first. Returns [] on an empty
|
|
1836
|
+
// query (callers fall back to a recency listing).
|
|
1837
|
+
search(query, limit) {
|
|
1838
|
+
const match = toMatchQuery(query);
|
|
1839
|
+
if (!match) return [];
|
|
1840
|
+
const rows = this.db.prepare(
|
|
1841
|
+
`SELECT source_path FROM conversation_messages_fts
|
|
1842
|
+
WHERE conversation_messages_fts MATCH ?
|
|
1843
|
+
ORDER BY rank
|
|
1844
|
+
LIMIT ?`
|
|
1845
|
+
).all(match, limit);
|
|
1846
|
+
return rows.map((r) => r.source_path);
|
|
1847
|
+
}
|
|
1848
|
+
count() {
|
|
1849
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM conversation_messages_fts").get().n;
|
|
1850
|
+
}
|
|
1851
|
+
};
|
|
1852
|
+
function toMatchQuery(query) {
|
|
1853
|
+
const terms = query.trim().split(/\s+/).map((t) => t.replace(/"/g, "").trim()).filter(Boolean);
|
|
1854
|
+
if (terms.length === 0) return "";
|
|
1855
|
+
return terms.map((t) => `"${t}"*`).join(" AND ");
|
|
1856
|
+
}
|
|
1857
|
+
|
|
1858
|
+
// src/persistent/sidecar.ts
|
|
1859
|
+
import { readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
1860
|
+
var SIDECAR_VERSION = 1;
|
|
1861
|
+
function sidecarPath(jsonlPath) {
|
|
1862
|
+
return `${jsonlPath}.idx.json`;
|
|
1863
|
+
}
|
|
1864
|
+
function buildSidecar(meta, cursor, updatedAt) {
|
|
1865
|
+
return {
|
|
1866
|
+
version: SIDECAR_VERSION,
|
|
1867
|
+
sourcePath: meta.filePath,
|
|
1868
|
+
sizeBytes: cursor.sizeBytes,
|
|
1869
|
+
mtimeMs: cursor.mtimeMs,
|
|
1870
|
+
lastIndexedOffset: cursor.offset,
|
|
1871
|
+
lastIndexedLine: cursor.line,
|
|
1872
|
+
messageCount: meta.messageCount,
|
|
1873
|
+
projectPath: meta.projectPath,
|
|
1874
|
+
projectName: meta.projectName,
|
|
1875
|
+
branch: meta.gitBranch,
|
|
1876
|
+
firstSentAt: meta.firstMessage?.timestamp ?? null,
|
|
1877
|
+
firstSentText: meta.firstMessage?.text ?? null,
|
|
1878
|
+
lastSentAt: meta.lastMessage?.timestamp ?? null,
|
|
1879
|
+
lastSentText: meta.lastMessage?.text ?? null,
|
|
1880
|
+
updatedAt
|
|
1881
|
+
};
|
|
1882
|
+
}
|
|
1883
|
+
function writeSidecar(jsonlPath, sidecar) {
|
|
1884
|
+
try {
|
|
1885
|
+
writeFileSync(sidecarPath(jsonlPath), JSON.stringify(sidecar, null, 2));
|
|
1886
|
+
} catch (err) {
|
|
1887
|
+
getLogger().warn({ jsonlPath, err }, "sidecar: write failed");
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
// src/persistent/index-engine.ts
|
|
1892
|
+
var BATCH_SIZE = 12;
|
|
1893
|
+
var PersistentEngine = class {
|
|
1894
|
+
db;
|
|
1895
|
+
files;
|
|
1896
|
+
conversations;
|
|
1897
|
+
fts;
|
|
1898
|
+
checkpoints;
|
|
1899
|
+
// When true, write a portable <file>.idx.json sidecar next to each indexed
|
|
1900
|
+
// JSONL. Off by default.
|
|
1901
|
+
sidecar;
|
|
1902
|
+
constructor(dbPath, options = {}) {
|
|
1903
|
+
this.db = openDatabase(dbPath);
|
|
1904
|
+
this.files = new ConversationFilesRepo(this.db);
|
|
1905
|
+
this.conversations = new ConversationsRepo(this.db);
|
|
1906
|
+
this.fts = new FtsRepo(this.db);
|
|
1907
|
+
this.checkpoints = new CheckpointsRepo(this.db);
|
|
1908
|
+
this.sidecar = options.sidecar ?? false;
|
|
1909
|
+
}
|
|
1910
|
+
close() {
|
|
1911
|
+
this.db.close();
|
|
1912
|
+
}
|
|
1913
|
+
// Discover all JSONL files under the active profiles, (re)parse files that
|
|
1914
|
+
// changed since the last index, and upsert their metadata. Returns the number
|
|
1915
|
+
// of files seen on disk this pass (the scan "scanned" count).
|
|
1916
|
+
async indexAll(activeProfiles, options) {
|
|
1917
|
+
const log = getLogger();
|
|
1918
|
+
const tier = resolveTier(options.tier ?? "standard", options.tiers);
|
|
1919
|
+
const enabled = options.providers ?? [CLAUDE_CODE_PROVIDER];
|
|
1920
|
+
const discovered = [];
|
|
1921
|
+
if (enabled.includes(CLAUDE_CODE_PROVIDER)) {
|
|
1922
|
+
const configDirs = activeProfiles.map((p) => ({
|
|
1923
|
+
projectsDir: getProjectsDir(p),
|
|
1924
|
+
account: p.id
|
|
1925
|
+
}));
|
|
1926
|
+
for (const f of await discoverJsonlFiles(configDirs)) discovered.push(f);
|
|
1927
|
+
}
|
|
1928
|
+
const codex = new CodexCliProvider();
|
|
1929
|
+
if (enabled.includes(CODEX_CLI_PROVIDER) && (options.codexRoots?.length ?? 0) > 0) {
|
|
1930
|
+
for (const f of await codex.discover(options.codexRoots)) {
|
|
1931
|
+
discovered.push({ ...f, provider: codex });
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
let scanned = 0;
|
|
1935
|
+
const gitBranchMemo = /* @__PURE__ */ new Map();
|
|
1936
|
+
const resolveGitBranch = (projectPath) => {
|
|
1937
|
+
if (gitBranchMemo.has(projectPath)) {
|
|
1938
|
+
return gitBranchMemo.get(projectPath) ?? null;
|
|
1939
|
+
}
|
|
1940
|
+
const branch = readGitBranch(projectPath);
|
|
1941
|
+
gitBranchMemo.set(projectPath, branch);
|
|
1942
|
+
return branch;
|
|
1943
|
+
};
|
|
1944
|
+
for (let i = 0; i < discovered.length; i += BATCH_SIZE) {
|
|
1945
|
+
const batch = discovered.slice(i, i + BATCH_SIZE);
|
|
1946
|
+
const results = await Promise.all(
|
|
1947
|
+
batch.map(async ({ filePath, account, provider }) => {
|
|
1948
|
+
const meta = await this.indexFile(
|
|
1949
|
+
filePath,
|
|
1950
|
+
account,
|
|
1951
|
+
tier.name,
|
|
1952
|
+
options.tiers,
|
|
1953
|
+
resolveGitBranch,
|
|
1954
|
+
false,
|
|
1955
|
+
provider
|
|
1956
|
+
);
|
|
1957
|
+
return meta;
|
|
1958
|
+
})
|
|
1959
|
+
);
|
|
1960
|
+
const kept = results.filter((m) => m != null);
|
|
1961
|
+
if (kept.length > 0) options.onBatch?.(kept);
|
|
1962
|
+
scanned += batch.length;
|
|
1963
|
+
options.onProgress?.(scanned, discovered.length);
|
|
1964
|
+
}
|
|
1965
|
+
const seen = new Set(discovered.map((d) => d.filePath));
|
|
1966
|
+
for (const path of this.files.allActivePaths()) {
|
|
1967
|
+
if (!seen.has(path)) this.markDeleted(path);
|
|
1968
|
+
}
|
|
1969
|
+
log.info({ scanned, indexed: this.conversations.count() }, "persistent: indexAll complete");
|
|
1970
|
+
return { scanned };
|
|
1971
|
+
}
|
|
1972
|
+
// (Re)index a single file. Classifies the change vs. the persisted cursor:
|
|
1973
|
+
// unchanged → return the stored summary; appended → resume the fold and read
|
|
1974
|
+
// only new bytes; reindex/force → fold from offset 0. Writes the summary +
|
|
1975
|
+
// cursor + reducer state in one transaction so a crash never leaves a
|
|
1976
|
+
// half-written row or an over-advanced cursor.
|
|
1977
|
+
async indexFile(filePath, account, tierName, customTiers, resolveGitBranch, force = false, provider) {
|
|
1978
|
+
const log = getLogger();
|
|
1979
|
+
const tier = resolveTier(tierName, customTiers);
|
|
1980
|
+
const existing = this.files.getByPath(filePath);
|
|
1981
|
+
const { change, stat: stat3 } = classify(filePath, existing);
|
|
1982
|
+
if (change === "vanished" || !stat3) {
|
|
1983
|
+
this.markDeleted(filePath);
|
|
1984
|
+
return null;
|
|
1985
|
+
}
|
|
1986
|
+
if (change === "unchanged" && !force) {
|
|
1987
|
+
return this.conversations.getBySourcePath(filePath);
|
|
1988
|
+
}
|
|
1989
|
+
if (provider && provider.name !== CLAUDE_CODE_PROVIDER) {
|
|
1990
|
+
return this.indexFileWithProvider(provider, filePath, account, tier, stat3, resolveGitBranch);
|
|
1991
|
+
}
|
|
1992
|
+
const resume = change === "appended" && !force && existing?.reducer_state;
|
|
1993
|
+
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
1994
|
+
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
1995
|
+
const startLine = resume ? existing.last_indexed_line : 0;
|
|
1996
|
+
let result;
|
|
1997
|
+
try {
|
|
1998
|
+
result = await tailReduce(filePath, startOffset, startLine, state, tier);
|
|
1999
|
+
} catch (err) {
|
|
2000
|
+
log.warn({ filePath, err }, "persistent: tail read failed");
|
|
2001
|
+
return null;
|
|
2002
|
+
}
|
|
2003
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
2004
|
+
if (!meta) {
|
|
2005
|
+
this.markDeleted(filePath);
|
|
2006
|
+
return null;
|
|
2007
|
+
}
|
|
2008
|
+
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2009
|
+
const fp = stat3.size > 0 ? fingerprint(filePath, stat3.size) : null;
|
|
2010
|
+
const fileId = this.files.ensure(filePath, account);
|
|
2011
|
+
const upsert = this.db.transaction(() => {
|
|
2012
|
+
this.conversations.upsert(fileId, meta, state.pageMessageCount);
|
|
2013
|
+
this.fts.upsert(meta);
|
|
2014
|
+
this.checkpoints.remove(filePath);
|
|
2015
|
+
this.files.updateCursor(fileId, {
|
|
2016
|
+
sizeBytes: stat3.size,
|
|
2017
|
+
mtimeMs: stat3.mtimeMs,
|
|
2018
|
+
// Advance only to the last fully-parsed line; a trailing partial line
|
|
2019
|
+
// is left for the next pass.
|
|
2020
|
+
offset: result.newOffset,
|
|
2021
|
+
line: result.newLine,
|
|
2022
|
+
reducerState: JSON.stringify(state),
|
|
2023
|
+
fingerprint: fp,
|
|
2024
|
+
status: "active"
|
|
2025
|
+
});
|
|
2026
|
+
});
|
|
2027
|
+
upsert();
|
|
2028
|
+
if (this.sidecar) {
|
|
2029
|
+
writeSidecar(
|
|
2030
|
+
filePath,
|
|
2031
|
+
buildSidecar(
|
|
2032
|
+
meta,
|
|
2033
|
+
{
|
|
2034
|
+
sizeBytes: stat3.size,
|
|
2035
|
+
mtimeMs: stat3.mtimeMs,
|
|
2036
|
+
offset: result.newOffset,
|
|
2037
|
+
line: result.newLine
|
|
2038
|
+
},
|
|
2039
|
+
(/* @__PURE__ */ new Date()).toISOString()
|
|
2040
|
+
)
|
|
2041
|
+
);
|
|
2042
|
+
}
|
|
2043
|
+
log.debug(
|
|
2044
|
+
{ filePath, change, bytesRead: result.newOffset - startOffset, msgs: meta.messageCount },
|
|
2045
|
+
"persistent: indexed file"
|
|
2046
|
+
);
|
|
2047
|
+
return meta;
|
|
2048
|
+
}
|
|
2049
|
+
// Index a non-Threadbase provider file: full reparse from offset 0 through the
|
|
2050
|
+
// provider's reducer/finalize, then the same upsert + FTS write + cursor bump
|
|
2051
|
+
// the Threadbase path uses. The cursor records size/mtime (and offset = size)
|
|
2052
|
+
// so the next pass classifies an unchanged file as "unchanged" and skips it;
|
|
2053
|
+
// any change reparses from 0 again. No reducer_state is persisted.
|
|
2054
|
+
async indexFileWithProvider(provider, filePath, account, tier, stat3, resolveGitBranch) {
|
|
2055
|
+
const log = getLogger();
|
|
2056
|
+
const meta = await parseMetaWithProvider(provider, filePath, account, tier);
|
|
2057
|
+
if (!meta) {
|
|
2058
|
+
this.markDeleted(filePath);
|
|
2059
|
+
return null;
|
|
2060
|
+
}
|
|
2061
|
+
if (meta.gitBranch === null && meta.projectPath) {
|
|
2062
|
+
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2063
|
+
}
|
|
2064
|
+
const fp = stat3.size > 0 ? fingerprint(filePath, stat3.size) : null;
|
|
2065
|
+
const fileId = this.files.ensure(filePath, account);
|
|
2066
|
+
const upsert = this.db.transaction(() => {
|
|
2067
|
+
this.conversations.upsert(fileId, meta, meta.messageCount);
|
|
2068
|
+
this.fts.upsert(meta);
|
|
2069
|
+
this.checkpoints.remove(filePath);
|
|
2070
|
+
this.files.updateCursor(fileId, {
|
|
2071
|
+
sizeBytes: stat3.size,
|
|
2072
|
+
mtimeMs: stat3.mtimeMs,
|
|
2073
|
+
// offset = size marks the file fully consumed (non-zero so the next pass
|
|
2074
|
+
// can classify it "unchanged"); no resumable reducer state is kept.
|
|
2075
|
+
offset: stat3.size,
|
|
2076
|
+
line: 0,
|
|
2077
|
+
reducerState: null,
|
|
2078
|
+
fingerprint: fp,
|
|
2079
|
+
status: "active"
|
|
2080
|
+
});
|
|
2081
|
+
});
|
|
2082
|
+
upsert();
|
|
2083
|
+
log.debug(
|
|
2084
|
+
{ filePath, provider: provider.name, msgs: meta.messageCount },
|
|
2085
|
+
"persistent: indexed provider file"
|
|
2086
|
+
);
|
|
2087
|
+
return meta;
|
|
2088
|
+
}
|
|
2089
|
+
markDeleted(filePath) {
|
|
2090
|
+
const existing = this.files.getByPath(filePath);
|
|
2091
|
+
if (!existing) return;
|
|
2092
|
+
const tx = this.db.transaction(() => {
|
|
2093
|
+
this.conversations.deleteByFileId(existing.id);
|
|
2094
|
+
this.fts.remove(filePath);
|
|
2095
|
+
this.checkpoints.remove(filePath);
|
|
2096
|
+
this.files.setStatus(existing.id, "deleted");
|
|
2097
|
+
});
|
|
2098
|
+
tx();
|
|
2099
|
+
}
|
|
2100
|
+
// ── Query helpers (read straight from SQLite) ───────────────────────────
|
|
2101
|
+
allActive() {
|
|
2102
|
+
return this.conversations.allActive();
|
|
2103
|
+
}
|
|
2104
|
+
getByIdOrSession(id) {
|
|
2105
|
+
return this.conversations.getByIdOrSession(id);
|
|
2106
|
+
}
|
|
2107
|
+
getAllBySessionId(sessionId) {
|
|
2108
|
+
return this.conversations.getAllBySessionId(sessionId);
|
|
2109
|
+
}
|
|
2110
|
+
// Ranked metas matching the FTS query, best first. Empty query returns the
|
|
2111
|
+
// most recent conversations (mirroring the in-memory indexer's empty-query
|
|
2112
|
+
// behavior). Resolves each FTS hit to its active conversation row.
|
|
2113
|
+
searchMetas(query, limit) {
|
|
2114
|
+
if (!query.trim()) {
|
|
2115
|
+
return this.conversations.recent(limit);
|
|
2116
|
+
}
|
|
2117
|
+
const paths = this.fts.search(query, limit);
|
|
2118
|
+
const metas = [];
|
|
2119
|
+
for (const path of paths) {
|
|
2120
|
+
const meta = this.conversations.getBySourcePath(path);
|
|
2121
|
+
if (meta) metas.push(meta);
|
|
2122
|
+
}
|
|
2123
|
+
return metas;
|
|
2124
|
+
}
|
|
2125
|
+
getProjects() {
|
|
2126
|
+
return this.conversations.distinctProjects();
|
|
2127
|
+
}
|
|
2128
|
+
// Bounded conversation page: read only the requested window from the file,
|
|
2129
|
+
// seeking from the nearest checkpoint. Returns null if the id can't be
|
|
2130
|
+
// resolved to an indexed conversation. For conversations large enough to span
|
|
2131
|
+
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
2132
|
+
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
2133
|
+
async getPage(id, options) {
|
|
2134
|
+
const meta = this.conversations.getByIdOrSession(id);
|
|
2135
|
+
if (!meta) return null;
|
|
2136
|
+
const filePath = meta.filePath;
|
|
2137
|
+
const total = this.conversations.pageMessageCount(filePath);
|
|
2138
|
+
if (total > CHECKPOINT_INTERVAL && this.checkpoints.count(filePath) === 0) {
|
|
2139
|
+
const built = await buildCheckpoints(filePath);
|
|
2140
|
+
if (built.length > 0) this.checkpoints.replaceAll(filePath, built);
|
|
2141
|
+
}
|
|
2142
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
2143
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2144
|
+
const floor = this.checkpoints.floor(filePath, fromIndex);
|
|
2145
|
+
return readPage(filePath, total, options, floor);
|
|
2146
|
+
}
|
|
2147
|
+
};
|
|
2148
|
+
|
|
2149
|
+
// src/providers/threadbase.ts
|
|
2150
|
+
var ThreadbaseProvider = class {
|
|
2151
|
+
name = CLAUDE_CODE_PROVIDER;
|
|
2152
|
+
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
2153
|
+
// per-root account through the shared interface. The scanner builds these.
|
|
2154
|
+
async discover(roots) {
|
|
2155
|
+
const dirs = roots.map((r) => {
|
|
2156
|
+
const [projectsDir, account = "default"] = r.split("\0");
|
|
2157
|
+
return { projectsDir, account };
|
|
2158
|
+
});
|
|
2159
|
+
return discoverJsonlFiles(dirs);
|
|
2160
|
+
}
|
|
2161
|
+
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
2162
|
+
canParse(_filePath, sample) {
|
|
2163
|
+
for (const line of sample.split("\n")) {
|
|
2164
|
+
if (!line.trim()) continue;
|
|
2165
|
+
try {
|
|
2166
|
+
const e = JSON.parse(line);
|
|
2167
|
+
if (e.type === "user" || e.type === "assistant") return true;
|
|
2168
|
+
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
2169
|
+
} catch {
|
|
2170
|
+
}
|
|
2171
|
+
}
|
|
2172
|
+
return false;
|
|
2173
|
+
}
|
|
2174
|
+
createEmptyAccumulator() {
|
|
2175
|
+
return initialReducerState();
|
|
2176
|
+
}
|
|
2177
|
+
reduceEntry(acc, entry, tier) {
|
|
2178
|
+
reduceLine(acc, entry, tier);
|
|
2179
|
+
}
|
|
2180
|
+
finalize(acc, filePath, account, tier) {
|
|
2181
|
+
return finalizeMeta(acc, filePath, account, tier);
|
|
2182
|
+
}
|
|
2183
|
+
};
|
|
2184
|
+
|
|
2185
|
+
// src/watcher/file-watcher.ts
|
|
2186
|
+
import chokidar from "chokidar";
|
|
2187
|
+
var EXCLUDED_SEGMENTS2 = ["/memory/", "/tool-results/"];
|
|
2188
|
+
var FileWatcher = class {
|
|
2189
|
+
constructor(profiles, onEvent, options = {}) {
|
|
2190
|
+
this.profiles = profiles;
|
|
2191
|
+
this.onEvent = onEvent;
|
|
2192
|
+
this.debounceMs = options.debounceMs ?? 400;
|
|
2193
|
+
}
|
|
2194
|
+
profiles;
|
|
2195
|
+
onEvent;
|
|
2196
|
+
watchers = [];
|
|
2197
|
+
timers = /* @__PURE__ */ new Map();
|
|
2198
|
+
debounceMs;
|
|
2199
|
+
// Resolves once every underlying chokidar watcher has finished its initial
|
|
2200
|
+
// scan, so the caller knows subsequent FS changes will be observed.
|
|
2201
|
+
async start() {
|
|
2202
|
+
const log = getLogger();
|
|
2203
|
+
const ready = [];
|
|
2204
|
+
for (const profile of this.profiles) {
|
|
2205
|
+
const dir = getProjectsDir(profile);
|
|
2206
|
+
const watcher = chokidar.watch(dir, {
|
|
2207
|
+
ignoreInitial: true,
|
|
2208
|
+
ignored: (p) => EXCLUDED_SEGMENTS2.some((seg) => p.includes(seg)),
|
|
2209
|
+
awaitWriteFinish: { stabilityThreshold: this.debounceMs, pollInterval: 100 }
|
|
2210
|
+
});
|
|
2211
|
+
watcher.on("add", (p) => this.dispatch(p, profile.id, "add")).on("change", (p) => this.dispatch(p, profile.id, "change")).on("unlink", (p) => this.dispatch(p, profile.id, "unlink"));
|
|
2212
|
+
ready.push(new Promise((resolve) => watcher.once("ready", () => resolve())));
|
|
2213
|
+
this.watchers.push(watcher);
|
|
2214
|
+
log.debug({ dir, account: profile.id }, "watcher: watching");
|
|
2215
|
+
}
|
|
2216
|
+
await Promise.all(ready);
|
|
2217
|
+
}
|
|
2218
|
+
async stop() {
|
|
2219
|
+
for (const t of this.timers.values()) clearTimeout(t);
|
|
2220
|
+
this.timers.clear();
|
|
2221
|
+
await Promise.all(this.watchers.map((w) => w.close()));
|
|
2222
|
+
this.watchers = [];
|
|
2223
|
+
}
|
|
2224
|
+
dispatch(filePath, account, type) {
|
|
2225
|
+
if (!filePath.endsWith(".jsonl")) return;
|
|
2226
|
+
const existing = this.timers.get(filePath);
|
|
2227
|
+
if (existing) clearTimeout(existing);
|
|
2228
|
+
this.timers.set(
|
|
2229
|
+
filePath,
|
|
2230
|
+
setTimeout(() => {
|
|
2231
|
+
this.timers.delete(filePath);
|
|
2232
|
+
this.onEvent({ filePath, account, type });
|
|
2233
|
+
}, this.debounceMs)
|
|
2234
|
+
);
|
|
2235
|
+
}
|
|
2236
|
+
};
|
|
2237
|
+
|
|
2238
|
+
// src/watcher/index-queue.ts
|
|
2239
|
+
var IndexQueue = class {
|
|
2240
|
+
constructor(process2) {
|
|
2241
|
+
this.process = process2;
|
|
2242
|
+
}
|
|
2243
|
+
process;
|
|
2244
|
+
pending = /* @__PURE__ */ new Map();
|
|
2245
|
+
running = false;
|
|
2246
|
+
idleResolvers = [];
|
|
2247
|
+
enqueue(job) {
|
|
2248
|
+
this.pending.set(job.filePath, job);
|
|
2249
|
+
void this.drain();
|
|
2250
|
+
}
|
|
2251
|
+
// Resolves when the queue has fully drained — useful for tests and for a
|
|
2252
|
+
// clean shutdown.
|
|
2253
|
+
onIdle() {
|
|
2254
|
+
if (!this.running && this.pending.size === 0) return Promise.resolve();
|
|
2255
|
+
return new Promise((resolve) => this.idleResolvers.push(resolve));
|
|
2256
|
+
}
|
|
2257
|
+
get size() {
|
|
2258
|
+
return this.pending.size;
|
|
2259
|
+
}
|
|
2260
|
+
async drain() {
|
|
2261
|
+
if (this.running) return;
|
|
2262
|
+
this.running = true;
|
|
2263
|
+
const log = getLogger();
|
|
2264
|
+
while (this.pending.size > 0) {
|
|
2265
|
+
const [path, job] = this.pending.entries().next().value;
|
|
2266
|
+
this.pending.delete(path);
|
|
2267
|
+
try {
|
|
2268
|
+
await this.process(job);
|
|
2269
|
+
} catch (err) {
|
|
2270
|
+
log.warn({ filePath: job.filePath, err }, "index-queue: job failed");
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2273
|
+
this.running = false;
|
|
2274
|
+
const resolvers = this.idleResolvers;
|
|
2275
|
+
this.idleResolvers = [];
|
|
2276
|
+
for (const r of resolvers) r();
|
|
2277
|
+
}
|
|
2278
|
+
};
|
|
2279
|
+
|
|
2280
|
+
// src/scanner.ts
|
|
2281
|
+
var BATCH_SIZE2 = 12;
|
|
2282
|
+
var DEFAULT_CONFIG_PATH = "~/.config/threadbase-scanner";
|
|
2283
|
+
function defaultDbPath() {
|
|
2284
|
+
return process.env.TB_SCANNER_DB ?? join4(homedir2(), ".config", "threadbase-scanner", "index.db");
|
|
2285
|
+
}
|
|
2286
|
+
var ConversationScanner = class {
|
|
2287
|
+
metadataCache = /* @__PURE__ */ new Map();
|
|
2288
|
+
conversationLRU;
|
|
2289
|
+
// session_id is NOT unique, so this maps a sessionId to every active meta that
|
|
2290
|
+
// carries it. Resolution picks deterministically (newest timestamp, then path
|
|
2291
|
+
// ascending) so dropping one file never hides another with the same id.
|
|
2292
|
+
sessionIdIndex = /* @__PURE__ */ new Map();
|
|
2293
|
+
projects = /* @__PURE__ */ new Set();
|
|
2294
|
+
indexer = new SearchIndexer();
|
|
2295
|
+
// Tier the most recent scan() ran with, so refreshFile() re-parses a single
|
|
2296
|
+
// file at the same content depth. Defaults to the standard tier.
|
|
2297
|
+
lastTier = resolveTier("standard");
|
|
2298
|
+
// null when persistent mode is disabled (legacy in-memory path). Lazily
|
|
2299
|
+
// opened on first use so merely constructing a scanner never touches disk.
|
|
2300
|
+
dbPath;
|
|
2301
|
+
sidecarEnabled;
|
|
2302
|
+
engineInstance = null;
|
|
2303
|
+
emitter = new EventEmitter();
|
|
2304
|
+
watcher = null;
|
|
2305
|
+
queue = null;
|
|
2306
|
+
periodicTimer = null;
|
|
2307
|
+
constructor(options) {
|
|
2308
|
+
this.conversationLRU = new LRUCache(options?.conversationCacheSize ?? 5);
|
|
2309
|
+
if (options?.persistent === false) {
|
|
2310
|
+
this.dbPath = null;
|
|
2311
|
+
this.sidecarEnabled = false;
|
|
2312
|
+
} else {
|
|
2313
|
+
this.dbPath = options?.persistent?.dbPath ?? defaultDbPath();
|
|
2314
|
+
this.sidecarEnabled = options?.persistent?.sidecar ?? false;
|
|
2315
|
+
}
|
|
2316
|
+
}
|
|
2317
|
+
get persistent() {
|
|
2318
|
+
return this.dbPath !== null;
|
|
2319
|
+
}
|
|
2320
|
+
engine() {
|
|
2321
|
+
if (!this.engineInstance) {
|
|
2322
|
+
this.engineInstance = new PersistentEngine(this.dbPath, {
|
|
2323
|
+
sidecar: this.sidecarEnabled
|
|
2324
|
+
});
|
|
2325
|
+
}
|
|
2326
|
+
return this.engineInstance;
|
|
2327
|
+
}
|
|
2328
|
+
// Release the SQLite connection. No-op in legacy mode. Safe to call
|
|
2329
|
+
// repeatedly. Stops the watcher first if one is running; call unwatch()
|
|
2330
|
+
// explicitly beforehand if you need to await watcher teardown.
|
|
2331
|
+
close() {
|
|
2332
|
+
if (this.watcher || this.queue || this.periodicTimer) void this.unwatch();
|
|
2333
|
+
this.engineInstance?.close();
|
|
2334
|
+
this.engineInstance = null;
|
|
2335
|
+
}
|
|
2336
|
+
async scan(options = {}) {
|
|
2337
|
+
const profiles = await this.resolveProfiles(options.profiles);
|
|
2338
|
+
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
2339
|
+
this.lastTier = resolveTier(options.tier ?? "standard", options.tiers);
|
|
2340
|
+
if (this.persistent) {
|
|
2341
|
+
return this.scanPersistent(activeProfiles, options);
|
|
2342
|
+
}
|
|
2343
|
+
return this.scanInMemory(activeProfiles, options);
|
|
2344
|
+
}
|
|
2345
|
+
// SQLite-backed scan: (re)index changed files into the DB, then query all
|
|
2346
|
+
// active metas and run the identical filter/sort/view/paginate pipeline as
|
|
2347
|
+
// the in-memory path — guaranteeing an identical ScanResult shape.
|
|
2348
|
+
async scanPersistent(activeProfiles, options) {
|
|
2349
|
+
const log = getLogger();
|
|
2350
|
+
const startedAt = Date.now();
|
|
2351
|
+
const engine = this.engine();
|
|
2352
|
+
const { scanned } = await engine.indexAll(activeProfiles, options);
|
|
2353
|
+
const allMetas = engine.allActive();
|
|
2354
|
+
const { conversations, total } = this.finalize(allMetas, options);
|
|
2355
|
+
log.info(
|
|
2356
|
+
{ scanned, kept: allMetas.length, filteredTotal: total, elapsedMs: Date.now() - startedAt },
|
|
2357
|
+
"scan: complete (persistent)"
|
|
2358
|
+
);
|
|
2359
|
+
return { conversations, total, scanned };
|
|
2360
|
+
}
|
|
2361
|
+
// Apply include/project/account/since filters, sort, view transform, and
|
|
2362
|
+
// pagination. Shared by both backends so results never diverge.
|
|
2363
|
+
finalize(allMetas, options) {
|
|
2364
|
+
let filtered = allMetas;
|
|
2365
|
+
if (options.include && options.include !== "all") {
|
|
2366
|
+
filtered = applyIncludeFilter(filtered, options.include);
|
|
2367
|
+
}
|
|
2368
|
+
if (options.project) filtered = applyProjectFilter(filtered, options.project);
|
|
2369
|
+
if (options.account) filtered = applyAccountFilter(filtered, options.account);
|
|
2370
|
+
if (options.since) filtered = applySinceFilter(filtered, options.since);
|
|
2371
|
+
filtered = applySort(filtered, options.sort ?? "recent");
|
|
2372
|
+
const total = filtered.length;
|
|
2373
|
+
const conversations = this.transformView(filtered, options);
|
|
2374
|
+
if (Array.isArray(conversations)) {
|
|
2375
|
+
const limit = options.limit ?? 50;
|
|
2376
|
+
const offset = options.offset ?? 0;
|
|
2377
|
+
return { conversations: applyPagination(conversations, limit, offset).items, total };
|
|
2378
|
+
}
|
|
2379
|
+
return { conversations, total };
|
|
2380
|
+
}
|
|
2381
|
+
async scanInMemory(activeProfiles, options) {
|
|
2382
|
+
const log = getLogger();
|
|
2383
|
+
const startedAt = Date.now();
|
|
2384
|
+
const tier = this.lastTier;
|
|
2385
|
+
log.info(
|
|
2386
|
+
{
|
|
2387
|
+
activeProfiles: activeProfiles.length,
|
|
2388
|
+
tier: tier.name,
|
|
2389
|
+
sort: options.sort ?? "recent",
|
|
2390
|
+
include: options.include ?? "all",
|
|
2391
|
+
view: options.view ?? "flat"
|
|
2392
|
+
},
|
|
2393
|
+
"scan: start"
|
|
2394
|
+
);
|
|
888
2395
|
this.metadataCache.clear();
|
|
889
2396
|
this.conversationLRU.clear();
|
|
890
2397
|
this.sessionIdIndex.clear();
|
|
891
2398
|
this.projects.clear();
|
|
892
2399
|
this.indexer.clear();
|
|
893
|
-
const
|
|
894
|
-
projectsDir: getProjectsDir(p),
|
|
895
|
-
account: p.id
|
|
896
|
-
}));
|
|
897
|
-
const files = await discoverJsonlFiles(configDirs);
|
|
2400
|
+
const files = await this.discoverWithProviders(activeProfiles, options);
|
|
898
2401
|
const totalFiles = files.length;
|
|
899
2402
|
let scanned = 0;
|
|
900
2403
|
let parseFailures = 0;
|
|
@@ -909,15 +2412,15 @@ var ConversationScanner = class {
|
|
|
909
2412
|
return branch;
|
|
910
2413
|
};
|
|
911
2414
|
const { statCache } = options;
|
|
912
|
-
for (let i = 0; i < files.length; i +=
|
|
913
|
-
const batch = files.slice(i, i +
|
|
2415
|
+
for (let i = 0; i < files.length; i += BATCH_SIZE2) {
|
|
2416
|
+
const batch = files.slice(i, i + BATCH_SIZE2);
|
|
914
2417
|
const results = await Promise.all(
|
|
915
|
-
batch.map(async ({ filePath, account }) => {
|
|
2418
|
+
batch.map(async ({ filePath, account, provider }) => {
|
|
916
2419
|
if (statCache) {
|
|
917
2420
|
const cached = statCache.get(filePath);
|
|
918
2421
|
if (cached) {
|
|
919
2422
|
try {
|
|
920
|
-
const s =
|
|
2423
|
+
const s = statSync2(filePath);
|
|
921
2424
|
if (s.mtimeMs === cached.stat.mtimeMs && s.size === cached.stat.size) {
|
|
922
2425
|
return cached.meta;
|
|
923
2426
|
}
|
|
@@ -926,14 +2429,14 @@ var ConversationScanner = class {
|
|
|
926
2429
|
}
|
|
927
2430
|
}
|
|
928
2431
|
try {
|
|
929
|
-
const meta = await
|
|
930
|
-
if (meta) {
|
|
2432
|
+
const meta = await parseMetaWithProvider(provider, filePath, account, tier);
|
|
2433
|
+
if (meta && meta.gitBranch === null && meta.projectPath) {
|
|
931
2434
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
932
2435
|
}
|
|
933
2436
|
return meta;
|
|
934
2437
|
} catch (err) {
|
|
935
2438
|
parseFailures++;
|
|
936
|
-
log.warn({ filePath, account, err }, "scan:
|
|
2439
|
+
log.warn({ filePath, account, provider: provider.name, err }, "scan: parse threw");
|
|
937
2440
|
return null;
|
|
938
2441
|
}
|
|
939
2442
|
})
|
|
@@ -942,7 +2445,7 @@ var ConversationScanner = class {
|
|
|
942
2445
|
for (const meta of results) {
|
|
943
2446
|
if (meta && meta.messageCount > 0) {
|
|
944
2447
|
this.metadataCache.set(meta.id, meta);
|
|
945
|
-
this.
|
|
2448
|
+
this.addToSessionIndex(meta);
|
|
946
2449
|
this.projects.add(meta.projectPath);
|
|
947
2450
|
allMetas.push(meta);
|
|
948
2451
|
batchMetas.push(meta);
|
|
@@ -956,22 +2459,7 @@ var ConversationScanner = class {
|
|
|
956
2459
|
log.debug({ scanned, totalFiles, batchKept: batchMetas.length }, "scan: batch complete");
|
|
957
2460
|
options.onProgress?.(scanned, totalFiles);
|
|
958
2461
|
}
|
|
959
|
-
|
|
960
|
-
if (options.include && options.include !== "all") {
|
|
961
|
-
filtered = applyIncludeFilter(filtered, options.include);
|
|
962
|
-
}
|
|
963
|
-
if (options.project) {
|
|
964
|
-
filtered = applyProjectFilter(filtered, options.project);
|
|
965
|
-
}
|
|
966
|
-
if (options.account) {
|
|
967
|
-
filtered = applyAccountFilter(filtered, options.account);
|
|
968
|
-
}
|
|
969
|
-
if (options.since) {
|
|
970
|
-
filtered = applySinceFilter(filtered, options.since);
|
|
971
|
-
}
|
|
972
|
-
filtered = applySort(filtered, options.sort ?? "recent");
|
|
973
|
-
const total = filtered.length;
|
|
974
|
-
const conversations = this.transformView(filtered, options);
|
|
2462
|
+
const { conversations, total } = this.finalize(allMetas, options);
|
|
975
2463
|
const elapsedMs = Date.now() - startedAt;
|
|
976
2464
|
log.info(
|
|
977
2465
|
{
|
|
@@ -984,25 +2472,36 @@ var ConversationScanner = class {
|
|
|
984
2472
|
},
|
|
985
2473
|
"scan: complete"
|
|
986
2474
|
);
|
|
987
|
-
if (Array.isArray(conversations)) {
|
|
988
|
-
const limit = options.limit ?? 50;
|
|
989
|
-
const offset = options.offset ?? 0;
|
|
990
|
-
const paginated = applyPagination(conversations, limit, offset);
|
|
991
|
-
return { conversations: paginated.items, total, scanned };
|
|
992
|
-
}
|
|
993
2475
|
return { conversations, total, scanned };
|
|
994
2476
|
}
|
|
995
2477
|
async search(query, options = {}) {
|
|
996
2478
|
const log = getLogger();
|
|
997
2479
|
log.debug({ query, indexSize: this.indexer.getDocumentCount() }, "search: start");
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
2480
|
+
let results;
|
|
2481
|
+
if (this.persistent) {
|
|
2482
|
+
const engine = this.engine();
|
|
2483
|
+
if (engine.conversations.count() === 0) {
|
|
2484
|
+
log.debug("search: persistent index empty, triggering scan");
|
|
2485
|
+
const profiles = await this.resolveProfiles(options.profiles);
|
|
2486
|
+
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
2487
|
+
await engine.indexAll(activeProfiles, { ...options, limit: void 0, offset: void 0 });
|
|
2488
|
+
}
|
|
2489
|
+
const metas = engine.searchMetas(query, (options.limit ?? 50) * 2);
|
|
2490
|
+
results = query.trim() ? metas.map((meta) => ({ meta, score: 1, matches: generateMatches(meta, query) })) : metas.map((meta) => ({
|
|
2491
|
+
meta,
|
|
2492
|
+
score: 1,
|
|
2493
|
+
matches: [{ field: "timestamp", snippet: meta.preview }]
|
|
2494
|
+
}));
|
|
2495
|
+
} else {
|
|
2496
|
+
if (this.indexer.getDocumentCount() === 0) {
|
|
2497
|
+
log.debug("search: index empty, triggering scan");
|
|
2498
|
+
await this.scan({ ...options, limit: void 0, offset: void 0 });
|
|
2499
|
+
}
|
|
2500
|
+
results = this.indexer.search(query, {
|
|
2501
|
+
fields: options.fields,
|
|
2502
|
+
limit: (options.limit ?? 50) * 2
|
|
2503
|
+
});
|
|
1001
2504
|
}
|
|
1002
|
-
let results = this.indexer.search(query, {
|
|
1003
|
-
fields: options.fields,
|
|
1004
|
-
limit: (options.limit ?? 50) * 2
|
|
1005
|
-
});
|
|
1006
2505
|
if (options.include && options.include !== "all") {
|
|
1007
2506
|
results = results.filter((r) => {
|
|
1008
2507
|
switch (options.include) {
|
|
@@ -1026,6 +2525,11 @@ var ConversationScanner = class {
|
|
|
1026
2525
|
if (options.account) {
|
|
1027
2526
|
results = results.filter((r) => r.meta.account === options.account);
|
|
1028
2527
|
}
|
|
2528
|
+
if (options.provider) {
|
|
2529
|
+
results = results.filter(
|
|
2530
|
+
(r) => (r.meta.provider ?? CLAUDE_CODE_PROVIDER) === options.provider
|
|
2531
|
+
);
|
|
2532
|
+
}
|
|
1029
2533
|
if (options.since) {
|
|
1030
2534
|
const cutoff = parseSinceCutoff(options.since);
|
|
1031
2535
|
results = results.filter((r) => new Date(r.meta.timestamp).getTime() >= cutoff.getTime());
|
|
@@ -1043,14 +2547,14 @@ var ConversationScanner = class {
|
|
|
1043
2547
|
log.debug({ id }, "getConversation: cache hit");
|
|
1044
2548
|
return cached;
|
|
1045
2549
|
}
|
|
1046
|
-
const meta = this.metadataCache.get(id) ?? this.
|
|
2550
|
+
const meta = this.persistent ? this.engine().getByIdOrSession(id) : this.metadataCache.get(id) ?? this.resolveSessionId(id);
|
|
1047
2551
|
if (!meta) {
|
|
1048
2552
|
log.debug({ id }, "getConversation: not found in metadata");
|
|
1049
2553
|
return null;
|
|
1050
2554
|
}
|
|
1051
2555
|
log.debug({ id, filePath: meta.filePath }, "getConversation: cache miss, parsing");
|
|
1052
2556
|
try {
|
|
1053
|
-
const conversation = await parseConversation(meta.filePath, meta.account);
|
|
2557
|
+
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
1054
2558
|
if (conversation) {
|
|
1055
2559
|
this.conversationLRU.set(id, conversation);
|
|
1056
2560
|
}
|
|
@@ -1070,14 +2574,18 @@ var ConversationScanner = class {
|
|
|
1070
2574
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
1071
2575
|
// same contract as getConversation.
|
|
1072
2576
|
//
|
|
1073
|
-
//
|
|
1074
|
-
//
|
|
1075
|
-
//
|
|
1076
|
-
// parseConversation() by
|
|
1077
|
-
//
|
|
1078
|
-
//
|
|
1079
|
-
//
|
|
2577
|
+
// Persistent mode: a true bounded read — the engine seeks from the nearest
|
|
2578
|
+
// checkpoint and parses only the requested window (checkpoints are built
|
|
2579
|
+
// lazily for large conversations). Windowed message indices are proven
|
|
2580
|
+
// identical to parseConversation().messages.slice(...) by the paged-reader
|
|
2581
|
+
// equivalence test.
|
|
2582
|
+
//
|
|
2583
|
+
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
2584
|
+
// which is identical by construction.
|
|
1080
2585
|
async getConversationPage(id, options) {
|
|
2586
|
+
if (this.persistent) {
|
|
2587
|
+
return this.engine().getPage(id, options);
|
|
2588
|
+
}
|
|
1081
2589
|
const conversation = await this.getConversation(id);
|
|
1082
2590
|
if (!conversation) return null;
|
|
1083
2591
|
const { messages } = conversation;
|
|
@@ -1122,6 +2630,30 @@ var ConversationScanner = class {
|
|
|
1122
2630
|
// dropped from all indexes.
|
|
1123
2631
|
async refreshFile(filePath, account) {
|
|
1124
2632
|
const log = getLogger();
|
|
2633
|
+
if (this.persistent) {
|
|
2634
|
+
const engine = this.engine();
|
|
2635
|
+
const previous2 = engine.getByIdOrSession(filePath);
|
|
2636
|
+
const resolvedAccount2 = account ?? previous2?.account ?? "default";
|
|
2637
|
+
const evict2 = (m) => {
|
|
2638
|
+
if (!m) return;
|
|
2639
|
+
this.conversationLRU.delete(m.id);
|
|
2640
|
+
this.conversationLRU.delete(m.sessionId);
|
|
2641
|
+
};
|
|
2642
|
+
evict2(previous2);
|
|
2643
|
+
const provider = await this.resolveProviderForFile(filePath, previous2);
|
|
2644
|
+
const meta2 = await engine.indexFile(
|
|
2645
|
+
filePath,
|
|
2646
|
+
resolvedAccount2,
|
|
2647
|
+
this.lastTier.name,
|
|
2648
|
+
void 0,
|
|
2649
|
+
readGitBranch,
|
|
2650
|
+
true,
|
|
2651
|
+
provider
|
|
2652
|
+
);
|
|
2653
|
+
evict2(meta2);
|
|
2654
|
+
log.debug({ filePath, kept: !!meta2 }, "refreshFile: updated persistent index");
|
|
2655
|
+
return meta2;
|
|
2656
|
+
}
|
|
1125
2657
|
const previous = this.metadataCache.get(filePath);
|
|
1126
2658
|
const resolvedAccount = account ?? previous?.account ?? "default";
|
|
1127
2659
|
let meta = null;
|
|
@@ -1141,18 +2673,16 @@ var ConversationScanner = class {
|
|
|
1141
2673
|
if (!meta || meta.messageCount === 0) {
|
|
1142
2674
|
if (previous) {
|
|
1143
2675
|
this.metadataCache.delete(previous.id);
|
|
1144
|
-
this.
|
|
2676
|
+
this.removeFromSessionIndex(previous);
|
|
1145
2677
|
this.indexer.removeDocument(previous.id);
|
|
1146
2678
|
}
|
|
1147
2679
|
log.debug({ filePath }, "refreshFile: dropped (no parseable messages)");
|
|
1148
2680
|
return null;
|
|
1149
2681
|
}
|
|
1150
2682
|
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
1151
|
-
if (previous
|
|
1152
|
-
this.sessionIdIndex.delete(previous.sessionId);
|
|
1153
|
-
}
|
|
2683
|
+
if (previous) this.removeFromSessionIndex(previous);
|
|
1154
2684
|
this.metadataCache.set(meta.id, meta);
|
|
1155
|
-
this.
|
|
2685
|
+
this.addToSessionIndex(meta);
|
|
1156
2686
|
this.projects.add(meta.projectPath);
|
|
1157
2687
|
if (previous) {
|
|
1158
2688
|
this.indexer.updateDocument(meta);
|
|
@@ -1166,15 +2696,206 @@ var ConversationScanner = class {
|
|
|
1166
2696
|
return meta;
|
|
1167
2697
|
}
|
|
1168
2698
|
getMetadataCache() {
|
|
2699
|
+
if (this.persistent) {
|
|
2700
|
+
const map = /* @__PURE__ */ new Map();
|
|
2701
|
+
for (const meta of this.engine().allActive()) map.set(meta.id, meta);
|
|
2702
|
+
return map;
|
|
2703
|
+
}
|
|
1169
2704
|
return this.metadataCache;
|
|
1170
2705
|
}
|
|
2706
|
+
// Collision-safe sessionId lookup. session_id is NOT unique (the parser falls
|
|
2707
|
+
// back to the file basename, and resumed/subagent sessions repeat ids), so
|
|
2708
|
+
// getConversation(sessionId) is a convenience that resolves to one match;
|
|
2709
|
+
// this returns every active conversation sharing the id, newest first.
|
|
2710
|
+
getConversationsBySessionId(sessionId) {
|
|
2711
|
+
if (this.persistent) {
|
|
2712
|
+
return this.engine().getAllBySessionId(sessionId);
|
|
2713
|
+
}
|
|
2714
|
+
return this.sortBySessionPriority(this.sessionIdIndex.get(sessionId) ?? []);
|
|
2715
|
+
}
|
|
1171
2716
|
getProjects() {
|
|
2717
|
+
const source = this.persistent ? this.engine().getProjects() : this.projects;
|
|
1172
2718
|
const normalized = /* @__PURE__ */ new Set();
|
|
1173
|
-
for (const p of
|
|
2719
|
+
for (const p of source) {
|
|
1174
2720
|
normalized.add(p.replace(/\/+$/, ""));
|
|
1175
2721
|
}
|
|
1176
2722
|
return Array.from(normalized).sort();
|
|
1177
2723
|
}
|
|
2724
|
+
on(event, listener) {
|
|
2725
|
+
this.emitter.on(event, listener);
|
|
2726
|
+
return this;
|
|
2727
|
+
}
|
|
2728
|
+
off(event, listener) {
|
|
2729
|
+
this.emitter.off(event, listener);
|
|
2730
|
+
return this;
|
|
2731
|
+
}
|
|
2732
|
+
// Start watching the active profiles' project dirs. A filesystem watcher
|
|
2733
|
+
// feeds debounced, path-deduplicated index jobs through a single-writer
|
|
2734
|
+
// queue; a periodic full rescan backstops any events the watcher misses
|
|
2735
|
+
// (sleep/wake, network FS, restarts). Emits "change" per indexed file.
|
|
2736
|
+
// Persistent mode only — throws in legacy mode (no durable index to update).
|
|
2737
|
+
async watch(options = {}) {
|
|
2738
|
+
if (!this.persistent) {
|
|
2739
|
+
throw new Error("watch() requires persistent mode; construct with persistent enabled");
|
|
2740
|
+
}
|
|
2741
|
+
if (this.watcher) return;
|
|
2742
|
+
const profiles = await this.resolveProfiles(options.profiles);
|
|
2743
|
+
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
2744
|
+
this.queue = new IndexQueue(async (job) => {
|
|
2745
|
+
try {
|
|
2746
|
+
const meta = await this.refreshFile(job.filePath, job.account);
|
|
2747
|
+
this.emitter.emit("change", {
|
|
2748
|
+
filePath: job.filePath,
|
|
2749
|
+
account: job.account,
|
|
2750
|
+
meta,
|
|
2751
|
+
reason: job.reason
|
|
2752
|
+
});
|
|
2753
|
+
} catch (err) {
|
|
2754
|
+
this.emitter.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
2755
|
+
}
|
|
2756
|
+
});
|
|
2757
|
+
this.watcher = new FileWatcher(
|
|
2758
|
+
activeProfiles,
|
|
2759
|
+
(e) => {
|
|
2760
|
+
this.queue?.enqueue({
|
|
2761
|
+
filePath: e.filePath,
|
|
2762
|
+
account: e.account,
|
|
2763
|
+
reason: "watcher"
|
|
2764
|
+
});
|
|
2765
|
+
},
|
|
2766
|
+
{ debounceMs: options.debounceMs }
|
|
2767
|
+
);
|
|
2768
|
+
await this.watcher.start();
|
|
2769
|
+
const periodicMs = options.periodicMs ?? 6e4;
|
|
2770
|
+
if (periodicMs > 0) {
|
|
2771
|
+
this.periodicTimer = setInterval(() => {
|
|
2772
|
+
void this.periodicReconcile(activeProfiles);
|
|
2773
|
+
}, periodicMs);
|
|
2774
|
+
this.periodicTimer.unref?.();
|
|
2775
|
+
}
|
|
2776
|
+
getLogger().info({ profiles: activeProfiles.length, periodicMs }, "watch: started");
|
|
2777
|
+
}
|
|
2778
|
+
// Periodic correctness backstop: re-discover all files and enqueue them
|
|
2779
|
+
// through the same queue the watcher uses, so any add/change the watcher
|
|
2780
|
+
// missed still gets indexed and emits a "change" event. Vanished files
|
|
2781
|
+
// (active in the DB but no longer on disk) are enqueued too — refreshFile
|
|
2782
|
+
// drops them. Routing through the queue (not a direct scan) keeps event
|
|
2783
|
+
// emission and single-writer serialization unified.
|
|
2784
|
+
async periodicReconcile(activeProfiles) {
|
|
2785
|
+
if (!this.queue) return;
|
|
2786
|
+
try {
|
|
2787
|
+
const engine = this.engine();
|
|
2788
|
+
const configDirs = activeProfiles.map((p) => ({
|
|
2789
|
+
projectsDir: getProjectsDir(p),
|
|
2790
|
+
account: p.id
|
|
2791
|
+
}));
|
|
2792
|
+
const discovered = await discoverJsonlFiles(configDirs);
|
|
2793
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2794
|
+
for (const { filePath, account } of discovered) {
|
|
2795
|
+
seen.add(filePath);
|
|
2796
|
+
const { change } = classify(filePath, engine.files.getByPath(filePath));
|
|
2797
|
+
if (change !== "unchanged") {
|
|
2798
|
+
this.queue.enqueue({ filePath, account, reason: "periodic" });
|
|
2799
|
+
}
|
|
2800
|
+
}
|
|
2801
|
+
for (const path of engine.files.allActivePaths()) {
|
|
2802
|
+
if (!seen.has(path)) {
|
|
2803
|
+
this.queue.enqueue({ filePath: path, account: "default", reason: "periodic" });
|
|
2804
|
+
}
|
|
2805
|
+
}
|
|
2806
|
+
} catch (err) {
|
|
2807
|
+
this.emitter.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2810
|
+
// Stop watching and drain any in-flight index jobs.
|
|
2811
|
+
async unwatch() {
|
|
2812
|
+
if (this.periodicTimer) {
|
|
2813
|
+
clearInterval(this.periodicTimer);
|
|
2814
|
+
this.periodicTimer = null;
|
|
2815
|
+
}
|
|
2816
|
+
if (this.watcher) {
|
|
2817
|
+
await this.watcher.stop();
|
|
2818
|
+
this.watcher = null;
|
|
2819
|
+
}
|
|
2820
|
+
if (this.queue) {
|
|
2821
|
+
await this.queue.onIdle();
|
|
2822
|
+
this.queue = null;
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
// ── Provider plumbing (in-memory path) ──────────────────────────────────
|
|
2826
|
+
// Build the flat parse worklist for the enabled providers. Threadbase
|
|
2827
|
+
// discovers under the active profiles' project dirs; Codex discovers under
|
|
2828
|
+
// the explicit codexRoots (opt-in — no default home scan).
|
|
2829
|
+
async discoverWithProviders(activeProfiles, options) {
|
|
2830
|
+
const enabled = options.providers ?? [CLAUDE_CODE_PROVIDER];
|
|
2831
|
+
const work = [];
|
|
2832
|
+
if (enabled.includes(CLAUDE_CODE_PROVIDER)) {
|
|
2833
|
+
const provider = new ThreadbaseProvider();
|
|
2834
|
+
const roots = activeProfiles.map((p) => `${getProjectsDir(p)}\0${p.id}`);
|
|
2835
|
+
for (const f of await provider.discover(roots)) {
|
|
2836
|
+
work.push({ ...f, provider });
|
|
2837
|
+
}
|
|
2838
|
+
}
|
|
2839
|
+
if (enabled.includes(CODEX_CLI_PROVIDER) && (options.codexRoots?.length ?? 0) > 0) {
|
|
2840
|
+
const provider = new CodexCliProvider();
|
|
2841
|
+
for (const f of await provider.discover(options.codexRoots)) {
|
|
2842
|
+
work.push({ ...f, provider });
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2845
|
+
return work;
|
|
2846
|
+
}
|
|
2847
|
+
// Resolve which provider should (re)parse a file. Stored metadata wins; for a
|
|
2848
|
+
// file the index has not seen, sniff the first lines with each non-Threadbase
|
|
2849
|
+
// provider's canParse. Returns undefined for Threadbase (the engine's default
|
|
2850
|
+
// tail-read path).
|
|
2851
|
+
async resolveProviderForFile(filePath, previous) {
|
|
2852
|
+
if (previous?.provider === CODEX_CLI_PROVIDER) return new CodexCliProvider();
|
|
2853
|
+
if (previous?.provider) return void 0;
|
|
2854
|
+
let sample = "";
|
|
2855
|
+
try {
|
|
2856
|
+
const fd = openSync2(filePath, "r");
|
|
2857
|
+
try {
|
|
2858
|
+
const buf = Buffer.alloc(8192);
|
|
2859
|
+
const n = readSync2(fd, buf, 0, buf.length, 0);
|
|
2860
|
+
sample = buf.subarray(0, n).toString("utf8");
|
|
2861
|
+
} finally {
|
|
2862
|
+
closeSync2(fd);
|
|
2863
|
+
}
|
|
2864
|
+
} catch {
|
|
2865
|
+
return void 0;
|
|
2866
|
+
}
|
|
2867
|
+
const codex = new CodexCliProvider();
|
|
2868
|
+
if (codex.canParse(filePath, sample)) return codex;
|
|
2869
|
+
return void 0;
|
|
2870
|
+
}
|
|
2871
|
+
addToSessionIndex(meta) {
|
|
2872
|
+
const list = this.sessionIdIndex.get(meta.sessionId);
|
|
2873
|
+
if (list) {
|
|
2874
|
+
const i = list.findIndex((m) => m.id === meta.id);
|
|
2875
|
+
if (i >= 0) list[i] = meta;
|
|
2876
|
+
else list.push(meta);
|
|
2877
|
+
} else {
|
|
2878
|
+
this.sessionIdIndex.set(meta.sessionId, [meta]);
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
removeFromSessionIndex(meta) {
|
|
2882
|
+
const list = this.sessionIdIndex.get(meta.sessionId);
|
|
2883
|
+
if (!list) return;
|
|
2884
|
+
const next = list.filter((m) => m.id !== meta.id);
|
|
2885
|
+
if (next.length > 0) this.sessionIdIndex.set(meta.sessionId, next);
|
|
2886
|
+
else this.sessionIdIndex.delete(meta.sessionId);
|
|
2887
|
+
}
|
|
2888
|
+
// Deterministic single-result sessionId resolution: newest timestamp first,
|
|
2889
|
+
// tie-broken by absolute path ascending.
|
|
2890
|
+
resolveSessionId(sessionId) {
|
|
2891
|
+
return this.sortBySessionPriority(this.sessionIdIndex.get(sessionId) ?? [])[0];
|
|
2892
|
+
}
|
|
2893
|
+
sortBySessionPriority(metas) {
|
|
2894
|
+
return [...metas].sort((a, b) => {
|
|
2895
|
+
if (a.timestamp !== b.timestamp) return a.timestamp < b.timestamp ? 1 : -1;
|
|
2896
|
+
return a.filePath < b.filePath ? -1 : a.filePath > b.filePath ? 1 : 0;
|
|
2897
|
+
});
|
|
2898
|
+
}
|
|
1178
2899
|
async resolveProfiles(profiles) {
|
|
1179
2900
|
if (profiles && profiles.length > 0) return profiles;
|
|
1180
2901
|
return loadProfiles(DEFAULT_CONFIG_PATH);
|