@threadbase-sh/scanner 0.9.4 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +376 -138
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +963 -712
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +16 -1
- package/dist/index.d.ts +16 -1
- package/dist/index.js +1059 -810
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -43,12 +43,14 @@ __export(index_exports, {
|
|
|
43
43
|
applySinceFilter: () => applySinceFilter,
|
|
44
44
|
applySort: () => applySort,
|
|
45
45
|
cleanSystemTags: () => cleanSystemTags,
|
|
46
|
+
createJsonlParseState: () => initialConvState,
|
|
46
47
|
createLogger: () => createLogger,
|
|
47
48
|
detectDefaultProfile: () => detectDefaultProfile,
|
|
48
49
|
getConversation: () => getConversation,
|
|
49
50
|
getLogger: () => getLogger,
|
|
50
51
|
getProjectsDir: () => getProjectsDir,
|
|
51
52
|
loadProfiles: () => loadProfiles,
|
|
53
|
+
parseJsonlLine: () => parseJsonlLine,
|
|
52
54
|
readGitBranch: () => readGitBranch,
|
|
53
55
|
readSidecar: () => readSidecar,
|
|
54
56
|
resetDefaultScanner: () => resetDefaultScanner,
|
|
@@ -353,94 +355,137 @@ var SearchIndexer = class {
|
|
|
353
355
|
}
|
|
354
356
|
};
|
|
355
357
|
|
|
356
|
-
// src/
|
|
358
|
+
// src/parser.ts
|
|
357
359
|
var import_fs2 = require("fs");
|
|
358
|
-
var
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
360
|
+
var import_path3 = require("path");
|
|
361
|
+
var import_readline = require("readline");
|
|
362
|
+
|
|
363
|
+
// src/persistent/metadata-reducer.ts
|
|
364
|
+
var import_path2 = require("path");
|
|
365
|
+
|
|
366
|
+
// src/providers/provider.ts
|
|
367
|
+
var CLAUDE_CODE_PROVIDER = "claude-code";
|
|
368
|
+
var CODEX_CLI_PROVIDER = "codex-cli";
|
|
369
|
+
|
|
370
|
+
// src/persistent/metadata-reducer.ts
|
|
371
|
+
function initialReducerState() {
|
|
363
372
|
return {
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
messageCount:
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
373
|
+
sessionId: "",
|
|
374
|
+
sessionName: "",
|
|
375
|
+
latestTimestamp: "",
|
|
376
|
+
cwd: "",
|
|
377
|
+
teamName: "",
|
|
378
|
+
model: null,
|
|
379
|
+
messageCount: 0,
|
|
380
|
+
lastMessageSender: "user",
|
|
381
|
+
isTeammate: false,
|
|
382
|
+
firstUserSeen: false,
|
|
383
|
+
firstMessage: null,
|
|
384
|
+
lastMessage: null,
|
|
385
|
+
lastPrompt: "",
|
|
386
|
+
pageMessageCount: 0,
|
|
387
|
+
toolNames: [],
|
|
388
|
+
previewParts: [],
|
|
389
|
+
snippetParts: [],
|
|
390
|
+
previewLength: 0,
|
|
391
|
+
snippetLength: 0,
|
|
392
|
+
badJsonLines: 0
|
|
379
393
|
};
|
|
380
394
|
}
|
|
381
|
-
function
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
395
|
+
function reduceLine(state, entry, tier) {
|
|
396
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
397
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
398
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
399
|
+
if (entry.teamName && !state.teamName) state.teamName = entry.teamName;
|
|
400
|
+
if (entry.timestamp) {
|
|
401
|
+
const ts = entry.timestamp;
|
|
402
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
386
403
|
}
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
return
|
|
391
|
-
}
|
|
392
|
-
|
|
404
|
+
const type = entry.type;
|
|
405
|
+
if (type === "last-prompt") {
|
|
406
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
if (type !== "user" && type !== "assistant") return;
|
|
410
|
+
if (entry.isMeta) return;
|
|
411
|
+
const msg = entry.message;
|
|
412
|
+
if (state.model === null && msg?.model) state.model = msg.model;
|
|
413
|
+
if (type === "user" && !state.firstUserSeen) {
|
|
414
|
+
state.firstUserSeen = true;
|
|
415
|
+
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
416
|
+
}
|
|
417
|
+
const content = extractTextContent(msg?.content);
|
|
418
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
419
|
+
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
420
|
+
const toolSet = new Set(state.toolNames);
|
|
421
|
+
collectToolNames(msg?.content, toolSet);
|
|
422
|
+
state.toolNames = Array.from(toolSet);
|
|
423
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
424
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
425
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
426
|
+
if (content || isOnlyToolResult || toolUseBlocks.length > 0 || hasThinking) {
|
|
427
|
+
state.pageMessageCount++;
|
|
428
|
+
}
|
|
429
|
+
if (content || isOnlyToolResult) {
|
|
430
|
+
state.messageCount++;
|
|
431
|
+
state.lastMessageSender = type;
|
|
432
|
+
if (content) {
|
|
433
|
+
const ts = entry.timestamp || "";
|
|
434
|
+
if (!state.firstMessage) state.firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
435
|
+
state.lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
436
|
+
if (state.previewLength < tier.previewMax) {
|
|
437
|
+
state.previewParts.push(content);
|
|
438
|
+
state.previewLength += content.length;
|
|
439
|
+
}
|
|
440
|
+
if (state.snippetLength < tier.snippetMax) {
|
|
441
|
+
const remaining = tier.snippetMax - state.snippetLength;
|
|
442
|
+
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
443
|
+
state.snippetParts.push(chunk);
|
|
444
|
+
state.snippetLength += chunk.length;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
393
447
|
}
|
|
394
448
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
function getProjectsDir(profile) {
|
|
405
|
-
return (0, import_path2.join)(resolveConfigDir(profile.configDir), "projects");
|
|
406
|
-
}
|
|
407
|
-
async function detectDefaultProfile() {
|
|
449
|
+
function finalizeMeta(state, filePath, account, tier) {
|
|
450
|
+
if (state.messageCount === 0) return null;
|
|
451
|
+
const isSubagent = filePath.includes("/subagents/");
|
|
452
|
+
let parentSessionId = null;
|
|
453
|
+
if (isSubagent) {
|
|
454
|
+
const uuidDir = (0, import_path2.dirname)((0, import_path2.dirname)(filePath));
|
|
455
|
+
parentSessionId = (0, import_path2.join)((0, import_path2.dirname)(uuidDir), `${(0, import_path2.basename)(uuidDir)}.jsonl`);
|
|
456
|
+
}
|
|
457
|
+
const projectPath = state.cwd;
|
|
408
458
|
return {
|
|
409
|
-
id:
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
459
|
+
id: filePath,
|
|
460
|
+
filePath,
|
|
461
|
+
provider: CLAUDE_CODE_PROVIDER,
|
|
462
|
+
sessionId: state.sessionId || (0, import_path2.basename)(filePath, ".jsonl"),
|
|
463
|
+
sessionName: state.sessionName,
|
|
464
|
+
projectPath,
|
|
465
|
+
projectName: getShortProjectName(projectPath),
|
|
466
|
+
account,
|
|
467
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
468
|
+
messageCount: state.messageCount,
|
|
469
|
+
lastMessageSender: state.lastMessageSender,
|
|
470
|
+
preview: state.previewParts.join(" ").slice(0, tier.previewMax),
|
|
471
|
+
contentSnippet: state.snippetParts.join(" "),
|
|
472
|
+
gitBranch: null,
|
|
473
|
+
model: state.model,
|
|
474
|
+
isSubagent,
|
|
475
|
+
parentSessionId,
|
|
476
|
+
isTeammate: state.isTeammate,
|
|
477
|
+
teamName: state.teamName || null,
|
|
478
|
+
toolNames: state.toolNames,
|
|
479
|
+
firstMessage: state.firstMessage,
|
|
480
|
+
lastMessage: state.lastMessage,
|
|
481
|
+
lastPrompt: state.lastPrompt || void 0
|
|
414
482
|
};
|
|
415
483
|
}
|
|
416
|
-
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
const resolved = resolveConfigDir(configPath);
|
|
420
|
-
const data = await (0, import_promises.readFile)((0, import_path2.join)(resolved, PROFILES_FILE), "utf-8");
|
|
421
|
-
const profiles = JSON.parse(data);
|
|
422
|
-
log.debug({ configPath, count: profiles.length }, "profiles: loaded");
|
|
423
|
-
return profiles;
|
|
424
|
-
} catch (err) {
|
|
425
|
-
log.debug({ configPath, err }, "profiles: load failed, using default");
|
|
426
|
-
const defaultProfile = await detectDefaultProfile();
|
|
427
|
-
return [defaultProfile];
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
async function saveProfiles(profiles, configPath) {
|
|
431
|
-
const resolved = resolveConfigDir(configPath);
|
|
432
|
-
await (0, import_promises.mkdir)(resolved, { recursive: true });
|
|
433
|
-
await (0, import_promises.writeFile)((0, import_path2.join)(resolved, PROFILES_FILE), JSON.stringify(profiles, null, 2));
|
|
434
|
-
getLogger().debug({ configPath, count: profiles.length }, "profiles: saved");
|
|
484
|
+
function getShortProjectName(fullPath) {
|
|
485
|
+
const parts = fullPath.split("/").filter(Boolean);
|
|
486
|
+
return parts.slice(-3).join("/");
|
|
435
487
|
}
|
|
436
488
|
|
|
437
|
-
// src/providers/codex-cli.ts
|
|
438
|
-
var import_fast_glob = __toESM(require("fast-glob"), 1);
|
|
439
|
-
var import_fs3 = require("fs");
|
|
440
|
-
var import_promises2 = require("fs/promises");
|
|
441
|
-
var import_path3 = require("path");
|
|
442
|
-
var import_readline = require("readline");
|
|
443
|
-
|
|
444
489
|
// src/tags.ts
|
|
445
490
|
var SYSTEM_TAGS = [
|
|
446
491
|
"system-reminder",
|
|
@@ -469,46 +514,427 @@ function cleanSystemTags(text) {
|
|
|
469
514
|
return text.replace(SYSTEM_TAG_RE, "").replace(/[^\S\n]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
|
|
470
515
|
}
|
|
471
516
|
|
|
472
|
-
// src/
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
const
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
let paths;
|
|
517
|
+
// src/parser.ts
|
|
518
|
+
async function parseMeta(filePath, account, tier) {
|
|
519
|
+
const log = getLogger();
|
|
520
|
+
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
521
|
+
const state = initialReducerState();
|
|
522
|
+
const fileStream = (0, import_fs2.createReadStream)(filePath);
|
|
523
|
+
const rl = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
524
|
+
try {
|
|
525
|
+
for await (const line of rl) {
|
|
526
|
+
if (!line.trim()) continue;
|
|
527
|
+
let entry;
|
|
484
528
|
try {
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
dot: false,
|
|
489
|
-
unique: true
|
|
490
|
-
});
|
|
491
|
-
} catch (err) {
|
|
492
|
-
log.warn({ root, err }, "codex discovery: glob failed");
|
|
529
|
+
entry = JSON.parse(line);
|
|
530
|
+
} catch {
|
|
531
|
+
state.badJsonLines++;
|
|
493
532
|
continue;
|
|
494
533
|
}
|
|
495
|
-
|
|
496
|
-
try {
|
|
497
|
-
const s = await (0, import_promises2.stat)(filePath);
|
|
498
|
-
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
499
|
-
} catch (err) {
|
|
500
|
-
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
501
|
-
}
|
|
502
|
-
}
|
|
534
|
+
reduceLine(state, entry, tier);
|
|
503
535
|
}
|
|
504
|
-
|
|
536
|
+
} catch (err) {
|
|
537
|
+
log.warn({ filePath, err }, "parseMeta: read failed");
|
|
538
|
+
return null;
|
|
505
539
|
}
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
540
|
+
if (state.badJsonLines > 0) {
|
|
541
|
+
log.warn(
|
|
542
|
+
{ filePath, badJsonLines: state.badJsonLines },
|
|
543
|
+
"parseMeta: skipped malformed JSON lines"
|
|
544
|
+
);
|
|
545
|
+
}
|
|
546
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
547
|
+
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
548
|
+
return meta;
|
|
549
|
+
}
|
|
550
|
+
async function parseConversation(filePath, account) {
|
|
551
|
+
const log = getLogger();
|
|
552
|
+
log.trace({ filePath, account }, "parseConversation: start");
|
|
553
|
+
const messages = [];
|
|
554
|
+
let badJsonLines = 0;
|
|
555
|
+
const textParts = [];
|
|
556
|
+
const turnDurations = [];
|
|
557
|
+
const state = initialConvState();
|
|
558
|
+
const fileStream = (0, import_fs2.createReadStream)(filePath);
|
|
559
|
+
const rl = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
560
|
+
try {
|
|
561
|
+
for await (const line of rl) {
|
|
509
562
|
if (!line.trim()) continue;
|
|
563
|
+
let entry;
|
|
510
564
|
try {
|
|
511
|
-
|
|
565
|
+
entry = JSON.parse(line);
|
|
566
|
+
} catch {
|
|
567
|
+
badJsonLines++;
|
|
568
|
+
continue;
|
|
569
|
+
}
|
|
570
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
571
|
+
turnDurations.push({
|
|
572
|
+
durationMs: entry.durationMs,
|
|
573
|
+
messageCount: entry.messageCount || 0,
|
|
574
|
+
uuid: entry.uuid
|
|
575
|
+
});
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
const message = reduceConvLine(state, entry);
|
|
579
|
+
if (message) {
|
|
580
|
+
messages.push(message);
|
|
581
|
+
if (message.text) textParts.push(message.text);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
} catch (err) {
|
|
585
|
+
log.warn({ filePath, err }, "parseConversation: read failed");
|
|
586
|
+
return null;
|
|
587
|
+
}
|
|
588
|
+
if (badJsonLines > 0) {
|
|
589
|
+
log.warn({ filePath, badJsonLines }, "parseConversation: skipped malformed JSON lines");
|
|
590
|
+
}
|
|
591
|
+
if (messages.length === 0) {
|
|
592
|
+
log.trace({ filePath }, "parseConversation: no messages");
|
|
593
|
+
return null;
|
|
594
|
+
}
|
|
595
|
+
log.debug({ filePath, messageCount: messages.length }, "parseConversation: complete");
|
|
596
|
+
applyTeamInfo(messages, state);
|
|
597
|
+
return {
|
|
598
|
+
id: filePath,
|
|
599
|
+
filePath,
|
|
600
|
+
projectPath: state.cwd,
|
|
601
|
+
projectName: getShortProjectName2(state.cwd),
|
|
602
|
+
sessionId: state.sessionId || (0, import_path3.basename)(filePath, ".jsonl"),
|
|
603
|
+
sessionName: state.sessionName,
|
|
604
|
+
messages,
|
|
605
|
+
fullText: textParts.join(" "),
|
|
606
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
607
|
+
messageCount: messages.length,
|
|
608
|
+
account,
|
|
609
|
+
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
610
|
+
lastPrompt: state.lastPrompt || void 0
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
function extractTextContent(content) {
|
|
614
|
+
if (!content) return "";
|
|
615
|
+
if (typeof content === "string") return cleanSystemTags(content);
|
|
616
|
+
if (Array.isArray(content)) {
|
|
617
|
+
return content.map((item) => {
|
|
618
|
+
if (typeof item === "string") return item;
|
|
619
|
+
if (item?.type === "text" && item?.text) return item.text;
|
|
620
|
+
if (item?.type === "tool_result" && typeof item?.content === "string") return item.content;
|
|
621
|
+
return "";
|
|
622
|
+
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
623
|
+
}
|
|
624
|
+
return "";
|
|
625
|
+
}
|
|
626
|
+
function extractToolUseNames(content) {
|
|
627
|
+
if (!Array.isArray(content)) return [];
|
|
628
|
+
return content.filter((item) => item?.type === "tool_use" && item?.name).map((item) => item.name);
|
|
629
|
+
}
|
|
630
|
+
function extractToolUseBlocks(content) {
|
|
631
|
+
if (!Array.isArray(content)) return [];
|
|
632
|
+
return content.filter((item) => item?.type === "tool_use" && item?.name && item?.id).map((item) => ({
|
|
633
|
+
id: item.id,
|
|
634
|
+
name: item.name,
|
|
635
|
+
input: item.input || {}
|
|
636
|
+
}));
|
|
637
|
+
}
|
|
638
|
+
var TOOL_NAME_TO_TYPE = {
|
|
639
|
+
Edit: "edit",
|
|
640
|
+
Write: "write",
|
|
641
|
+
Read: "read",
|
|
642
|
+
Bash: "bash",
|
|
643
|
+
Grep: "grep",
|
|
644
|
+
Glob: "glob",
|
|
645
|
+
Agent: "taskAgent",
|
|
646
|
+
TaskCreate: "taskCreate",
|
|
647
|
+
TaskUpdate: "taskUpdate"
|
|
648
|
+
};
|
|
649
|
+
function extractToolResultBlocks(content, pendingToolUses) {
|
|
650
|
+
if (!Array.isArray(content)) return [];
|
|
651
|
+
return content.filter((item) => item?.type === "tool_result" && item?.tool_use_id).map((item) => {
|
|
652
|
+
const toolName = pendingToolUses.get(item.tool_use_id)?.name ?? "";
|
|
653
|
+
return {
|
|
654
|
+
toolUseId: item.tool_use_id,
|
|
655
|
+
type: TOOL_NAME_TO_TYPE[toolName] ?? "generic",
|
|
656
|
+
content: typeof item.content === "string" ? { text: item.content } : item.content ?? {},
|
|
657
|
+
isError: typeof item.is_error === "boolean" ? item.is_error : void 0
|
|
658
|
+
};
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
function collectToolNames(content, toolSet) {
|
|
662
|
+
if (!Array.isArray(content)) return;
|
|
663
|
+
for (const item of content) {
|
|
664
|
+
if (item?.type === "tool_use" && item?.name) {
|
|
665
|
+
toolSet.add(item.name);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
function isOnlyToolResultContent(content) {
|
|
670
|
+
if (!Array.isArray(content)) return false;
|
|
671
|
+
return content.length > 0 && content.every((item) => item?.type === "tool_result");
|
|
672
|
+
}
|
|
673
|
+
function isTeammateContent(content) {
|
|
674
|
+
const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
|
|
675
|
+
(item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
|
|
676
|
+
).join("") : "";
|
|
677
|
+
return raw.includes("<teammate-message");
|
|
678
|
+
}
|
|
679
|
+
function extractThinking(content) {
|
|
680
|
+
if (!Array.isArray(content)) return { content: "", signature: "" };
|
|
681
|
+
const blocks = content.filter((item) => item?.type === "thinking");
|
|
682
|
+
return {
|
|
683
|
+
content: blocks.map((b) => b.thinking).filter(Boolean).join("\n\n"),
|
|
684
|
+
signature: blocks.map((b) => b.signature).filter(Boolean).join("")
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
function hasImageBlocks(content) {
|
|
688
|
+
if (!Array.isArray(content)) return false;
|
|
689
|
+
return content.some(
|
|
690
|
+
(item) => item?.type === "image" && (item?.source?.type === "base64" || item?.file?.base64 !== void 0)
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
function parseTeammateMessageTag(content) {
|
|
694
|
+
const match = content.match(/<teammate-message\s+([^>]*)>/);
|
|
695
|
+
if (!match) return null;
|
|
696
|
+
const attrs = match[1];
|
|
697
|
+
const id = attrs.match(/teammate_id="([^"]*)"/)?.[1];
|
|
698
|
+
if (!id) return null;
|
|
699
|
+
const summary = attrs.match(/summary="([^"]*)"/)?.[1];
|
|
700
|
+
const color = attrs.match(/color="([^"]*)"/)?.[1];
|
|
701
|
+
return { teammateId: id, summary, color };
|
|
702
|
+
}
|
|
703
|
+
function getShortProjectName2(fullPath) {
|
|
704
|
+
const parts = fullPath.split("/").filter(Boolean);
|
|
705
|
+
return parts.slice(-3).join("/");
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
// src/persistent/conversation-reducer.ts
|
|
709
|
+
function initialConvState() {
|
|
710
|
+
return {
|
|
711
|
+
cwd: "",
|
|
712
|
+
sessionId: "",
|
|
713
|
+
sessionName: "",
|
|
714
|
+
latestTimestamp: "",
|
|
715
|
+
lastPrompt: "",
|
|
716
|
+
pendingToolUses: {},
|
|
717
|
+
teamInfo: {}
|
|
718
|
+
};
|
|
719
|
+
}
|
|
720
|
+
function reduceConvLine(state, entry) {
|
|
721
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
722
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
723
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
724
|
+
if (entry.timestamp) {
|
|
725
|
+
const ts = entry.timestamp;
|
|
726
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
727
|
+
}
|
|
728
|
+
const type = entry.type;
|
|
729
|
+
if (type === "last-prompt") {
|
|
730
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
731
|
+
return null;
|
|
732
|
+
}
|
|
733
|
+
if (type !== "user" && type !== "assistant") return null;
|
|
734
|
+
if (entry.isMeta) return null;
|
|
735
|
+
const msg = entry.message;
|
|
736
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
737
|
+
for (const block of toolUseBlocks) state.pendingToolUses[block.id] = block;
|
|
738
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
739
|
+
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
740
|
+
const content = extractTextContent(msg?.content);
|
|
741
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
742
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
743
|
+
if (!(content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking)) return null;
|
|
744
|
+
const metadata = {};
|
|
745
|
+
if (msg?.model) metadata.model = msg.model;
|
|
746
|
+
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
747
|
+
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
748
|
+
if (entry.version) metadata.version = entry.version;
|
|
749
|
+
const usage = msg?.usage;
|
|
750
|
+
if (usage) {
|
|
751
|
+
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
752
|
+
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
753
|
+
if (usage.cache_read_input_tokens) metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
754
|
+
if (usage.cache_creation_input_tokens)
|
|
755
|
+
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
756
|
+
}
|
|
757
|
+
const toolUseNames = extractToolUseNames(msg?.content);
|
|
758
|
+
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
759
|
+
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
760
|
+
if (isToolResultOnly) {
|
|
761
|
+
const pending = new Map(Object.entries(state.pendingToolUses));
|
|
762
|
+
const toolResultBlocks = extractToolResultBlocks(msg?.content, pending);
|
|
763
|
+
if (toolResultBlocks.length > 0) {
|
|
764
|
+
metadata.toolResults = toolResultBlocks;
|
|
765
|
+
for (const block of toolResultBlocks) delete state.pendingToolUses[block.toolUseId];
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
if (entry.teamName) {
|
|
769
|
+
metadata.teamName = entry.teamName;
|
|
770
|
+
if (!state.teamInfo[metadata.teamName] && content) {
|
|
771
|
+
const info = parseTeammateMessageTag(content);
|
|
772
|
+
if (info) state.teamInfo[metadata.teamName] = info;
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
const thinkingContent = thinking?.content || void 0;
|
|
776
|
+
const thinkingSignature = thinking?.signature || void 0;
|
|
777
|
+
const hasMetadata = Object.keys(metadata).length > 0;
|
|
778
|
+
return {
|
|
779
|
+
role: type,
|
|
780
|
+
text: content || "",
|
|
781
|
+
timestamp: entry.timestamp || "",
|
|
782
|
+
uuid: entry.uuid || void 0,
|
|
783
|
+
metadata: hasMetadata ? metadata : void 0,
|
|
784
|
+
isToolResult: isToolResultOnly || void 0,
|
|
785
|
+
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
786
|
+
thinkingContent,
|
|
787
|
+
thinkingSignature,
|
|
788
|
+
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
789
|
+
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
790
|
+
promptId: type === "user" ? entry.promptId : void 0,
|
|
791
|
+
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
792
|
+
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
793
|
+
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
794
|
+
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
795
|
+
};
|
|
796
|
+
}
|
|
797
|
+
function parseJsonlLine(line, state = initialConvState()) {
|
|
798
|
+
const text = line.trimEnd();
|
|
799
|
+
if (text.trim().length === 0) return null;
|
|
800
|
+
let entry;
|
|
801
|
+
try {
|
|
802
|
+
entry = JSON.parse(text);
|
|
803
|
+
} catch {
|
|
804
|
+
return null;
|
|
805
|
+
}
|
|
806
|
+
return reduceConvLine(state, entry);
|
|
807
|
+
}
|
|
808
|
+
function applyTeamInfo(messages, state) {
|
|
809
|
+
if (Object.keys(state.teamInfo).length === 0) return;
|
|
810
|
+
for (const m of messages) {
|
|
811
|
+
const name = m.metadata?.teamName;
|
|
812
|
+
if (name && state.teamInfo[name] && m.metadata) m.metadata.teamInfo = state.teamInfo[name];
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
// src/persistent/sidecar.ts
|
|
817
|
+
var import_fs3 = require("fs");
|
|
818
|
+
var SIDECAR_VERSION = 1;
|
|
819
|
+
function sidecarPath(jsonlPath) {
|
|
820
|
+
return `${jsonlPath}.idx.json`;
|
|
821
|
+
}
|
|
822
|
+
function buildSidecar(meta, cursor, updatedAt) {
|
|
823
|
+
return {
|
|
824
|
+
version: SIDECAR_VERSION,
|
|
825
|
+
sourcePath: meta.filePath,
|
|
826
|
+
sizeBytes: cursor.sizeBytes,
|
|
827
|
+
mtimeMs: cursor.mtimeMs,
|
|
828
|
+
lastIndexedOffset: cursor.offset,
|
|
829
|
+
lastIndexedLine: cursor.line,
|
|
830
|
+
messageCount: meta.messageCount,
|
|
831
|
+
projectPath: meta.projectPath,
|
|
832
|
+
projectName: meta.projectName,
|
|
833
|
+
branch: meta.gitBranch,
|
|
834
|
+
firstSentAt: meta.firstMessage?.timestamp ?? null,
|
|
835
|
+
firstSentText: meta.firstMessage?.text ?? null,
|
|
836
|
+
lastSentAt: meta.lastMessage?.timestamp ?? null,
|
|
837
|
+
lastSentText: meta.lastMessage?.text ?? null,
|
|
838
|
+
updatedAt
|
|
839
|
+
};
|
|
840
|
+
}
|
|
841
|
+
function writeSidecar(jsonlPath, sidecar) {
|
|
842
|
+
try {
|
|
843
|
+
(0, import_fs3.writeFileSync)(sidecarPath(jsonlPath), JSON.stringify(sidecar, null, 2));
|
|
844
|
+
} catch (err) {
|
|
845
|
+
getLogger().warn({ jsonlPath, err }, "sidecar: write failed");
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
function readSidecar(jsonlPath) {
|
|
849
|
+
try {
|
|
850
|
+
return JSON.parse((0, import_fs3.readFileSync)(sidecarPath(jsonlPath), "utf-8"));
|
|
851
|
+
} catch {
|
|
852
|
+
return null;
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
// src/profiles.ts
|
|
857
|
+
var import_promises = require("fs/promises");
|
|
858
|
+
var import_os = require("os");
|
|
859
|
+
var import_path4 = require("path");
|
|
860
|
+
var PROFILES_FILE = "profiles.json";
|
|
861
|
+
function resolveConfigDir(configDir) {
|
|
862
|
+
return configDir.replace(/^~/, (0, import_os.homedir)());
|
|
863
|
+
}
|
|
864
|
+
function getProjectsDir(profile) {
|
|
865
|
+
return (0, import_path4.join)(resolveConfigDir(profile.configDir), "projects");
|
|
866
|
+
}
|
|
867
|
+
async function detectDefaultProfile() {
|
|
868
|
+
return {
|
|
869
|
+
id: "default",
|
|
870
|
+
label: "Default",
|
|
871
|
+
configDir: (0, import_path4.join)((0, import_os.homedir)(), ".claude"),
|
|
872
|
+
enabled: true,
|
|
873
|
+
emoji: "\u{1F916}"
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
async function loadProfiles(configPath) {
|
|
877
|
+
const log = getLogger();
|
|
878
|
+
try {
|
|
879
|
+
const resolved = resolveConfigDir(configPath);
|
|
880
|
+
const data = await (0, import_promises.readFile)((0, import_path4.join)(resolved, PROFILES_FILE), "utf-8");
|
|
881
|
+
const profiles = JSON.parse(data);
|
|
882
|
+
log.debug({ configPath, count: profiles.length }, "profiles: loaded");
|
|
883
|
+
return profiles;
|
|
884
|
+
} catch (err) {
|
|
885
|
+
log.debug({ configPath, err }, "profiles: load failed, using default");
|
|
886
|
+
const defaultProfile = await detectDefaultProfile();
|
|
887
|
+
return [defaultProfile];
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
async function saveProfiles(profiles, configPath) {
|
|
891
|
+
const resolved = resolveConfigDir(configPath);
|
|
892
|
+
await (0, import_promises.mkdir)(resolved, { recursive: true });
|
|
893
|
+
await (0, import_promises.writeFile)((0, import_path4.join)(resolved, PROFILES_FILE), JSON.stringify(profiles, null, 2));
|
|
894
|
+
getLogger().debug({ configPath, count: profiles.length }, "profiles: saved");
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
// src/providers/codex-cli.ts
|
|
898
|
+
var import_fast_glob = __toESM(require("fast-glob"), 1);
|
|
899
|
+
var import_fs4 = require("fs");
|
|
900
|
+
var import_promises2 = require("fs/promises");
|
|
901
|
+
var import_path5 = require("path");
|
|
902
|
+
var import_readline2 = require("readline");
|
|
903
|
+
var CodexCliProvider = class {
|
|
904
|
+
name = CODEX_CLI_PROVIDER;
|
|
905
|
+
async discover(roots) {
|
|
906
|
+
const log = getLogger();
|
|
907
|
+
const results = [];
|
|
908
|
+
for (const root of roots) {
|
|
909
|
+
let paths;
|
|
910
|
+
try {
|
|
911
|
+
paths = await (0, import_fast_glob.default)(["**/rollout-*.jsonl", "**/*.jsonl"], {
|
|
912
|
+
cwd: root,
|
|
913
|
+
absolute: true,
|
|
914
|
+
dot: false,
|
|
915
|
+
unique: true
|
|
916
|
+
});
|
|
917
|
+
} catch (err) {
|
|
918
|
+
log.warn({ root, err }, "codex discovery: glob failed");
|
|
919
|
+
continue;
|
|
920
|
+
}
|
|
921
|
+
for (const filePath of paths) {
|
|
922
|
+
try {
|
|
923
|
+
const s = await (0, import_promises2.stat)(filePath);
|
|
924
|
+
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
925
|
+
} catch (err) {
|
|
926
|
+
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
return results;
|
|
931
|
+
}
|
|
932
|
+
// Codex rollout lines carry distinctive top-level types.
|
|
933
|
+
canParse(_filePath, sample) {
|
|
934
|
+
for (const line of sample.split("\n")) {
|
|
935
|
+
if (!line.trim()) continue;
|
|
936
|
+
try {
|
|
937
|
+
const e = JSON.parse(line);
|
|
512
938
|
if (e.type === "session_meta" || e.type === "response_item" || e.type === "event_msg") {
|
|
513
939
|
return true;
|
|
514
940
|
}
|
|
@@ -606,7 +1032,7 @@ function reduceCodexEntry(acc, entry, tier) {
|
|
|
606
1032
|
}
|
|
607
1033
|
function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
608
1034
|
if (acc.messageCount === 0) return null;
|
|
609
|
-
const sessionId = acc.sessionId || (0,
|
|
1035
|
+
const sessionId = acc.sessionId || (0, import_path5.basename)(filePath, ".jsonl");
|
|
610
1036
|
const projectPath = acc.cwd;
|
|
611
1037
|
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
612
1038
|
return {
|
|
@@ -618,7 +1044,7 @@ function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
|
618
1044
|
sessionId,
|
|
619
1045
|
sessionName: "",
|
|
620
1046
|
projectPath,
|
|
621
|
-
projectName:
|
|
1047
|
+
projectName: getShortProjectName3(projectPath),
|
|
622
1048
|
account,
|
|
623
1049
|
timestamp: acc.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
624
1050
|
messageCount: acc.messageCount,
|
|
@@ -637,7 +1063,7 @@ function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
|
637
1063
|
lastPrompt: acc.lastUser?.text || void 0
|
|
638
1064
|
};
|
|
639
1065
|
}
|
|
640
|
-
function
|
|
1066
|
+
function getShortProjectName3(fullPath) {
|
|
641
1067
|
return fullPath.split("/").filter(Boolean).slice(-3).join("/");
|
|
642
1068
|
}
|
|
643
1069
|
async function parseCodexConversation(filePath, account) {
|
|
@@ -648,7 +1074,7 @@ async function parseCodexConversation(filePath, account) {
|
|
|
648
1074
|
let cwd = "";
|
|
649
1075
|
let latestTimestamp = "";
|
|
650
1076
|
let lastUserText = "";
|
|
651
|
-
const rl = (0,
|
|
1077
|
+
const rl = (0, import_readline2.createInterface)({ input: (0, import_fs4.createReadStream)(filePath), crlfDelay: Infinity });
|
|
652
1078
|
try {
|
|
653
1079
|
for await (const line of rl) {
|
|
654
1080
|
if (!line.trim()) continue;
|
|
@@ -685,8 +1111,8 @@ async function parseCodexConversation(filePath, account) {
|
|
|
685
1111
|
id: filePath,
|
|
686
1112
|
filePath,
|
|
687
1113
|
projectPath: cwd,
|
|
688
|
-
projectName:
|
|
689
|
-
sessionId: sessionId || (0,
|
|
1114
|
+
projectName: getShortProjectName3(cwd),
|
|
1115
|
+
sessionId: sessionId || (0, import_path5.basename)(filePath, ".jsonl"),
|
|
690
1116
|
sessionName: "",
|
|
691
1117
|
messages,
|
|
692
1118
|
fullText: textParts.join(" "),
|
|
@@ -763,525 +1189,315 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
763
1189
|
return results;
|
|
764
1190
|
}
|
|
765
1191
|
|
|
766
|
-
// src/
|
|
767
|
-
var
|
|
1192
|
+
// src/providers/threadbase.ts
|
|
1193
|
+
var ThreadbaseProvider = class {
|
|
1194
|
+
name = CLAUDE_CODE_PROVIDER;
|
|
1195
|
+
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
1196
|
+
// per-root account through the shared interface. The scanner builds these.
|
|
1197
|
+
async discover(roots) {
|
|
1198
|
+
const dirs = roots.map((r) => {
|
|
1199
|
+
const [projectsDir, account = "default"] = r.split("\0");
|
|
1200
|
+
return { projectsDir, account };
|
|
1201
|
+
});
|
|
1202
|
+
return discoverJsonlFiles(dirs);
|
|
1203
|
+
}
|
|
1204
|
+
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
1205
|
+
canParse(_filePath, sample) {
|
|
1206
|
+
for (const line of sample.split("\n")) {
|
|
1207
|
+
if (!line.trim()) continue;
|
|
1208
|
+
try {
|
|
1209
|
+
const e = JSON.parse(line);
|
|
1210
|
+
if (e.type === "user" || e.type === "assistant") return true;
|
|
1211
|
+
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
1212
|
+
} catch {
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
return false;
|
|
1216
|
+
}
|
|
1217
|
+
createEmptyAccumulator() {
|
|
1218
|
+
return initialReducerState();
|
|
1219
|
+
}
|
|
1220
|
+
reduceEntry(acc, entry, tier) {
|
|
1221
|
+
reduceLine(acc, entry, tier);
|
|
1222
|
+
}
|
|
1223
|
+
finalize(acc, filePath, account, tier) {
|
|
1224
|
+
return finalizeMeta(acc, filePath, account, tier);
|
|
1225
|
+
}
|
|
1226
|
+
};
|
|
768
1227
|
|
|
769
|
-
// src/
|
|
770
|
-
var
|
|
771
|
-
var
|
|
772
|
-
var
|
|
1228
|
+
// src/scanner.ts
|
|
1229
|
+
var import_events = require("events");
|
|
1230
|
+
var import_fs11 = require("fs");
|
|
1231
|
+
var import_os2 = require("os");
|
|
1232
|
+
var import_path9 = require("path");
|
|
773
1233
|
|
|
774
|
-
// src/
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
latestTimestamp: "",
|
|
781
|
-
lastPrompt: "",
|
|
782
|
-
pendingToolUses: {},
|
|
783
|
-
teamInfo: {}
|
|
784
|
-
};
|
|
785
|
-
}
|
|
786
|
-
function reduceConvLine(state, entry) {
|
|
787
|
-
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
788
|
-
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
789
|
-
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
790
|
-
if (entry.timestamp) {
|
|
791
|
-
const ts = entry.timestamp;
|
|
792
|
-
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
1234
|
+
// src/cache.ts
|
|
1235
|
+
var LRUCache = class {
|
|
1236
|
+
map = /* @__PURE__ */ new Map();
|
|
1237
|
+
capacity;
|
|
1238
|
+
constructor(capacity) {
|
|
1239
|
+
this.capacity = capacity;
|
|
793
1240
|
}
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
if (
|
|
797
|
-
|
|
1241
|
+
get(key) {
|
|
1242
|
+
const value = this.map.get(key);
|
|
1243
|
+
if (value === void 0) return void 0;
|
|
1244
|
+
this.map.delete(key);
|
|
1245
|
+
this.map.set(key, value);
|
|
1246
|
+
return value;
|
|
798
1247
|
}
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
const content = extractTextContent(msg?.content);
|
|
807
|
-
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
808
|
-
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
809
|
-
if (!(content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking)) return null;
|
|
810
|
-
const metadata = {};
|
|
811
|
-
if (msg?.model) metadata.model = msg.model;
|
|
812
|
-
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
813
|
-
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
814
|
-
if (entry.version) metadata.version = entry.version;
|
|
815
|
-
const usage = msg?.usage;
|
|
816
|
-
if (usage) {
|
|
817
|
-
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
818
|
-
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
819
|
-
if (usage.cache_read_input_tokens) metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
820
|
-
if (usage.cache_creation_input_tokens)
|
|
821
|
-
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
1248
|
+
set(key, value) {
|
|
1249
|
+
this.map.delete(key);
|
|
1250
|
+
this.map.set(key, value);
|
|
1251
|
+
if (this.map.size > this.capacity) {
|
|
1252
|
+
const oldest = this.map.keys().next();
|
|
1253
|
+
if (!oldest.done) this.map.delete(oldest.value);
|
|
1254
|
+
}
|
|
822
1255
|
}
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
826
|
-
if (isToolResultOnly) {
|
|
827
|
-
const pending = new Map(Object.entries(state.pendingToolUses));
|
|
828
|
-
const toolResultBlocks = extractToolResultBlocks(msg?.content, pending);
|
|
829
|
-
if (toolResultBlocks.length > 0) metadata.toolResults = toolResultBlocks;
|
|
1256
|
+
has(key) {
|
|
1257
|
+
return this.map.has(key);
|
|
830
1258
|
}
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
if (!state.teamInfo[metadata.teamName] && content) {
|
|
834
|
-
const info = parseTeammateMessageTag(content);
|
|
835
|
-
if (info) state.teamInfo[metadata.teamName] = info;
|
|
836
|
-
}
|
|
1259
|
+
delete(key) {
|
|
1260
|
+
return this.map.delete(key);
|
|
837
1261
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
const hasMetadata = Object.keys(metadata).length > 0;
|
|
841
|
-
return {
|
|
842
|
-
role: type,
|
|
843
|
-
text: content || "",
|
|
844
|
-
timestamp: entry.timestamp || "",
|
|
845
|
-
uuid: entry.uuid || void 0,
|
|
846
|
-
metadata: hasMetadata ? metadata : void 0,
|
|
847
|
-
isToolResult: isToolResultOnly || void 0,
|
|
848
|
-
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
849
|
-
thinkingContent,
|
|
850
|
-
thinkingSignature,
|
|
851
|
-
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
852
|
-
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
853
|
-
promptId: type === "user" ? entry.promptId : void 0,
|
|
854
|
-
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
855
|
-
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
856
|
-
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
857
|
-
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
858
|
-
};
|
|
859
|
-
}
|
|
860
|
-
function applyTeamInfo(messages, state) {
|
|
861
|
-
if (Object.keys(state.teamInfo).length === 0) return;
|
|
862
|
-
for (const m of messages) {
|
|
863
|
-
const name = m.metadata?.teamName;
|
|
864
|
-
if (name && state.teamInfo[name] && m.metadata) m.metadata.teamInfo = state.teamInfo[name];
|
|
1262
|
+
clear() {
|
|
1263
|
+
this.map.clear();
|
|
865
1264
|
}
|
|
866
|
-
|
|
1265
|
+
get size() {
|
|
1266
|
+
return this.map.size;
|
|
1267
|
+
}
|
|
1268
|
+
};
|
|
867
1269
|
|
|
868
|
-
// src/
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
1270
|
+
// src/persistent/conversation-stream.ts
|
|
1271
|
+
var import_path6 = require("path");
|
|
1272
|
+
|
|
1273
|
+
// src/persistent/paged-reader.ts
|
|
1274
|
+
var import_fs6 = require("fs");
|
|
1275
|
+
var import_promises5 = require("timers/promises");
|
|
1276
|
+
|
|
1277
|
+
// src/persistent/jsonl-tail-reader.ts
|
|
1278
|
+
var import_fs5 = require("fs");
|
|
1279
|
+
var import_promises4 = require("timers/promises");
|
|
1280
|
+
var YIELD_EVERY_LINES = 500;
|
|
1281
|
+
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1282
|
+
const stream = (0, import_fs5.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1283
|
+
let buffer = "";
|
|
1284
|
+
let offset = startOffset;
|
|
1285
|
+
let line = startLine;
|
|
1286
|
+
let parsedLines = 0;
|
|
1287
|
+
let sinceYield = 0;
|
|
1288
|
+
for await (const chunk of stream) {
|
|
1289
|
+
buffer += chunk;
|
|
1290
|
+
let nl;
|
|
1291
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1292
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1293
|
+
const text = lineWithNewline.trimEnd();
|
|
1294
|
+
buffer = buffer.slice(nl + 1);
|
|
1295
|
+
if (text.length > 0) {
|
|
1296
|
+
try {
|
|
1297
|
+
reduceLine(state, JSON.parse(text), tier);
|
|
1298
|
+
} catch {
|
|
1299
|
+
state.badJsonLines++;
|
|
1300
|
+
}
|
|
1301
|
+
parsedLines++;
|
|
1302
|
+
}
|
|
1303
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1304
|
+
line++;
|
|
1305
|
+
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
1306
|
+
sinceYield = 0;
|
|
1307
|
+
await (0, import_promises4.setImmediate)();
|
|
884
1308
|
}
|
|
885
|
-
reduceLine(state, entry, tier);
|
|
886
1309
|
}
|
|
887
|
-
} catch (err) {
|
|
888
|
-
log.warn({ filePath, err }, "parseMeta: read failed");
|
|
889
|
-
return null;
|
|
890
|
-
}
|
|
891
|
-
if (state.badJsonLines > 0) {
|
|
892
|
-
log.warn(
|
|
893
|
-
{ filePath, badJsonLines: state.badJsonLines },
|
|
894
|
-
"parseMeta: skipped malformed JSON lines"
|
|
895
|
-
);
|
|
896
1310
|
}
|
|
897
|
-
|
|
898
|
-
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
899
|
-
return meta;
|
|
1311
|
+
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
900
1312
|
}
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
1313
|
+
|
|
1314
|
+
// src/persistent/paged-reader.ts
|
|
1315
|
+
var CHECKPOINT_INTERVAL = 500;
|
|
1316
|
+
async function streamMessages(filePath, startOffset, startLine, state, onMessage, onEntry) {
|
|
1317
|
+
const stream = (0, import_fs6.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1318
|
+
let buffer = "";
|
|
1319
|
+
let offset = startOffset;
|
|
1320
|
+
let line = startLine;
|
|
1321
|
+
let sinceYield = 0;
|
|
1322
|
+
for await (const chunk of stream) {
|
|
1323
|
+
buffer += chunk;
|
|
1324
|
+
let nl;
|
|
1325
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1326
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1327
|
+
const text = lineWithNewline.trimEnd();
|
|
1328
|
+
buffer = buffer.slice(nl + 1);
|
|
1329
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1330
|
+
line += 1;
|
|
1331
|
+
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
1332
|
+
sinceYield = 0;
|
|
1333
|
+
await (0, import_promises5.setImmediate)();
|
|
1334
|
+
}
|
|
1335
|
+
if (text.length === 0) continue;
|
|
914
1336
|
let entry;
|
|
915
1337
|
try {
|
|
916
|
-
entry = JSON.parse(
|
|
1338
|
+
entry = JSON.parse(text);
|
|
917
1339
|
} catch {
|
|
918
|
-
badJsonLines++;
|
|
919
|
-
continue;
|
|
920
|
-
}
|
|
921
|
-
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
922
|
-
turnDurations.push({
|
|
923
|
-
durationMs: entry.durationMs,
|
|
924
|
-
messageCount: entry.messageCount || 0,
|
|
925
|
-
uuid: entry.uuid
|
|
926
|
-
});
|
|
927
1340
|
continue;
|
|
928
1341
|
}
|
|
1342
|
+
if (onEntry?.(entry)) continue;
|
|
929
1343
|
const message = reduceConvLine(state, entry);
|
|
930
|
-
if (message) {
|
|
931
|
-
|
|
932
|
-
|
|
1344
|
+
if (message && onMessage(message, offset, line)) {
|
|
1345
|
+
stream.destroy();
|
|
1346
|
+
return { offset, line };
|
|
933
1347
|
}
|
|
934
1348
|
}
|
|
935
|
-
} catch (err) {
|
|
936
|
-
log.warn({ filePath, err }, "parseConversation: read failed");
|
|
937
|
-
return null;
|
|
938
|
-
}
|
|
939
|
-
if (badJsonLines > 0) {
|
|
940
|
-
log.warn({ filePath, badJsonLines }, "parseConversation: skipped malformed JSON lines");
|
|
941
|
-
}
|
|
942
|
-
if (messages.length === 0) {
|
|
943
|
-
log.trace({ filePath }, "parseConversation: no messages");
|
|
944
|
-
return null;
|
|
945
|
-
}
|
|
946
|
-
log.debug({ filePath, messageCount: messages.length }, "parseConversation: complete");
|
|
947
|
-
applyTeamInfo(messages, state);
|
|
948
|
-
return {
|
|
949
|
-
id: filePath,
|
|
950
|
-
filePath,
|
|
951
|
-
projectPath: state.cwd,
|
|
952
|
-
projectName: getShortProjectName2(state.cwd),
|
|
953
|
-
sessionId: state.sessionId || (0, import_path4.basename)(filePath, ".jsonl"),
|
|
954
|
-
sessionName: state.sessionName,
|
|
955
|
-
messages,
|
|
956
|
-
fullText: textParts.join(" "),
|
|
957
|
-
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
958
|
-
messageCount: messages.length,
|
|
959
|
-
account,
|
|
960
|
-
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
961
|
-
lastPrompt: state.lastPrompt || void 0
|
|
962
|
-
};
|
|
963
|
-
}
|
|
964
|
-
function extractTextContent(content) {
|
|
965
|
-
if (!content) return "";
|
|
966
|
-
if (typeof content === "string") return cleanSystemTags(content);
|
|
967
|
-
if (Array.isArray(content)) {
|
|
968
|
-
return content.map((item) => {
|
|
969
|
-
if (typeof item === "string") return item;
|
|
970
|
-
if (item?.type === "text" && item?.text) return item.text;
|
|
971
|
-
if (item?.type === "tool_result" && typeof item?.content === "string") return item.content;
|
|
972
|
-
return "";
|
|
973
|
-
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
974
1349
|
}
|
|
975
|
-
return
|
|
976
|
-
}
|
|
977
|
-
function extractToolUseNames(content) {
|
|
978
|
-
if (!Array.isArray(content)) return [];
|
|
979
|
-
return content.filter((item) => item?.type === "tool_use" && item?.name).map((item) => item.name);
|
|
980
|
-
}
|
|
981
|
-
function extractToolUseBlocks(content) {
|
|
982
|
-
if (!Array.isArray(content)) return [];
|
|
983
|
-
return content.filter((item) => item?.type === "tool_use" && item?.name && item?.id).map((item) => ({
|
|
984
|
-
id: item.id,
|
|
985
|
-
name: item.name,
|
|
986
|
-
input: item.input || {}
|
|
987
|
-
}));
|
|
988
|
-
}
|
|
989
|
-
var TOOL_NAME_TO_TYPE = {
|
|
990
|
-
Edit: "edit",
|
|
991
|
-
Write: "write",
|
|
992
|
-
Read: "read",
|
|
993
|
-
Bash: "bash",
|
|
994
|
-
Grep: "grep",
|
|
995
|
-
Glob: "glob",
|
|
996
|
-
Agent: "taskAgent",
|
|
997
|
-
TaskCreate: "taskCreate",
|
|
998
|
-
TaskUpdate: "taskUpdate"
|
|
999
|
-
};
|
|
1000
|
-
function extractToolResultBlocks(content, pendingToolUses) {
|
|
1001
|
-
if (!Array.isArray(content)) return [];
|
|
1002
|
-
return content.filter((item) => item?.type === "tool_result" && item?.tool_use_id).map((item) => {
|
|
1003
|
-
const toolName = pendingToolUses.get(item.tool_use_id)?.name ?? "";
|
|
1004
|
-
return {
|
|
1005
|
-
toolUseId: item.tool_use_id,
|
|
1006
|
-
type: TOOL_NAME_TO_TYPE[toolName] ?? "generic",
|
|
1007
|
-
content: typeof item.content === "string" ? { text: item.content } : item.content ?? {},
|
|
1008
|
-
isError: typeof item.is_error === "boolean" ? item.is_error : void 0
|
|
1009
|
-
};
|
|
1010
|
-
});
|
|
1350
|
+
return { offset, line };
|
|
1011
1351
|
}
|
|
1012
|
-
function
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1352
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
1353
|
+
const checkpoints = [];
|
|
1354
|
+
const state = from ? from.state : initialConvState();
|
|
1355
|
+
let index = from ? from.messageIndex : 0;
|
|
1356
|
+
await streamMessages(
|
|
1357
|
+
filePath,
|
|
1358
|
+
from?.byteOffset ?? 0,
|
|
1359
|
+
from?.lineNumber ?? 0,
|
|
1360
|
+
state,
|
|
1361
|
+
(_msg, nextOffset, nextLine) => {
|
|
1362
|
+
index += 1;
|
|
1363
|
+
if (index % interval === 0) {
|
|
1364
|
+
checkpoints.push({
|
|
1365
|
+
messageIndex: index,
|
|
1366
|
+
byteOffset: nextOffset,
|
|
1367
|
+
lineNumber: nextLine,
|
|
1368
|
+
state: structuredClone(state)
|
|
1369
|
+
});
|
|
1370
|
+
}
|
|
1371
|
+
return false;
|
|
1017
1372
|
}
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
function isOnlyToolResultContent(content) {
|
|
1021
|
-
if (!Array.isArray(content)) return false;
|
|
1022
|
-
return content.length > 0 && content.every((item) => item?.type === "tool_result");
|
|
1023
|
-
}
|
|
1024
|
-
function isTeammateContent(content) {
|
|
1025
|
-
const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
|
|
1026
|
-
(item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
|
|
1027
|
-
).join("") : "";
|
|
1028
|
-
return raw.includes("<teammate-message");
|
|
1029
|
-
}
|
|
1030
|
-
function extractThinking(content) {
|
|
1031
|
-
if (!Array.isArray(content)) return { content: "", signature: "" };
|
|
1032
|
-
const blocks = content.filter((item) => item?.type === "thinking");
|
|
1033
|
-
return {
|
|
1034
|
-
content: blocks.map((b) => b.thinking).filter(Boolean).join("\n\n"),
|
|
1035
|
-
signature: blocks.map((b) => b.signature).filter(Boolean).join("")
|
|
1036
|
-
};
|
|
1037
|
-
}
|
|
1038
|
-
function hasImageBlocks(content) {
|
|
1039
|
-
if (!Array.isArray(content)) return false;
|
|
1040
|
-
return content.some(
|
|
1041
|
-
(item) => item?.type === "image" && (item?.source?.type === "base64" || item?.file?.base64 !== void 0)
|
|
1042
1373
|
);
|
|
1374
|
+
return checkpoints;
|
|
1043
1375
|
}
|
|
1044
|
-
function
|
|
1045
|
-
const
|
|
1046
|
-
|
|
1047
|
-
const
|
|
1048
|
-
const
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
const
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1376
|
+
async function readPage(filePath, total, options, floor) {
|
|
1377
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
1378
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1379
|
+
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1380
|
+
const startOffset = floor ? floor.byteOffset : 0;
|
|
1381
|
+
const startLine = floor ? floor.lineNumber : 0;
|
|
1382
|
+
let index = floor ? floor.messageIndex : 0;
|
|
1383
|
+
const window = [];
|
|
1384
|
+
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1385
|
+
const current = index;
|
|
1386
|
+
index += 1;
|
|
1387
|
+
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1388
|
+
return index >= beforeIndex;
|
|
1389
|
+
});
|
|
1390
|
+
applyTeamInfo(window, state);
|
|
1391
|
+
return { messages: window, total, fromIndex };
|
|
1057
1392
|
}
|
|
1058
1393
|
|
|
1059
|
-
// src/persistent/
|
|
1060
|
-
function
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
};
|
|
1083
|
-
}
|
|
1084
|
-
function reduceLine(state, entry, tier) {
|
|
1085
|
-
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
1086
|
-
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
1087
|
-
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
1088
|
-
if (entry.teamName && !state.teamName) state.teamName = entry.teamName;
|
|
1089
|
-
if (entry.timestamp) {
|
|
1090
|
-
const ts = entry.timestamp;
|
|
1091
|
-
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
1092
|
-
}
|
|
1093
|
-
const type = entry.type;
|
|
1094
|
-
if (type === "last-prompt") {
|
|
1095
|
-
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
1096
|
-
return;
|
|
1097
|
-
}
|
|
1098
|
-
if (type !== "user" && type !== "assistant") return;
|
|
1099
|
-
if (entry.isMeta) return;
|
|
1100
|
-
const msg = entry.message;
|
|
1101
|
-
if (state.model === null && msg?.model) state.model = msg.model;
|
|
1102
|
-
if (type === "user" && !state.firstUserSeen) {
|
|
1103
|
-
state.firstUserSeen = true;
|
|
1104
|
-
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
1105
|
-
}
|
|
1106
|
-
const content = extractTextContent(msg?.content);
|
|
1107
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
1108
|
-
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
1109
|
-
const toolSet = new Set(state.toolNames);
|
|
1110
|
-
collectToolNames(msg?.content, toolSet);
|
|
1111
|
-
state.toolNames = Array.from(toolSet);
|
|
1112
|
-
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
1113
|
-
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
1114
|
-
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
1115
|
-
if (content || isOnlyToolResult || toolUseBlocks.length > 0 || hasThinking) {
|
|
1116
|
-
state.pageMessageCount++;
|
|
1117
|
-
}
|
|
1118
|
-
if (content || isOnlyToolResult) {
|
|
1119
|
-
state.messageCount++;
|
|
1120
|
-
state.lastMessageSender = type;
|
|
1121
|
-
if (content) {
|
|
1122
|
-
const ts = entry.timestamp || "";
|
|
1123
|
-
if (!state.firstMessage) state.firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
1124
|
-
state.lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
1125
|
-
if (state.previewLength < tier.previewMax) {
|
|
1126
|
-
state.previewParts.push(content);
|
|
1127
|
-
state.previewLength += content.length;
|
|
1128
|
-
}
|
|
1129
|
-
if (state.snippetLength < tier.snippetMax) {
|
|
1130
|
-
const remaining = tier.snippetMax - state.snippetLength;
|
|
1131
|
-
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
1132
|
-
state.snippetParts.push(chunk);
|
|
1133
|
-
state.snippetLength += chunk.length;
|
|
1394
|
+
// src/persistent/conversation-stream.ts
|
|
1395
|
+
async function foldTail(filePath, resume) {
|
|
1396
|
+
const messages = [];
|
|
1397
|
+
const textParts = [];
|
|
1398
|
+
const turnDurations = [];
|
|
1399
|
+
const end = await streamMessages(
|
|
1400
|
+
filePath,
|
|
1401
|
+
resume.offset,
|
|
1402
|
+
resume.line,
|
|
1403
|
+
resume.state,
|
|
1404
|
+
(message) => {
|
|
1405
|
+
messages.push(message);
|
|
1406
|
+
if (message.text) textParts.push(message.text);
|
|
1407
|
+
return false;
|
|
1408
|
+
},
|
|
1409
|
+
(entry) => {
|
|
1410
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
1411
|
+
turnDurations.push({
|
|
1412
|
+
durationMs: entry.durationMs,
|
|
1413
|
+
messageCount: entry.messageCount || 0,
|
|
1414
|
+
uuid: entry.uuid
|
|
1415
|
+
});
|
|
1416
|
+
return true;
|
|
1134
1417
|
}
|
|
1418
|
+
return false;
|
|
1135
1419
|
}
|
|
1136
|
-
|
|
1420
|
+
);
|
|
1421
|
+
return { messages, textParts, turnDurations, end };
|
|
1137
1422
|
}
|
|
1138
|
-
function
|
|
1139
|
-
if (state.messageCount === 0) return null;
|
|
1140
|
-
const isSubagent = filePath.includes("/subagents/");
|
|
1141
|
-
let parentSessionId = null;
|
|
1142
|
-
if (isSubagent) {
|
|
1143
|
-
const uuidDir = (0, import_path5.dirname)((0, import_path5.dirname)(filePath));
|
|
1144
|
-
parentSessionId = (0, import_path5.join)((0, import_path5.dirname)(uuidDir), `${(0, import_path5.basename)(uuidDir)}.jsonl`);
|
|
1145
|
-
}
|
|
1146
|
-
const projectPath = state.cwd;
|
|
1423
|
+
function assemble(filePath, account, messages, fullText, turnDurations, state) {
|
|
1147
1424
|
return {
|
|
1148
1425
|
id: filePath,
|
|
1149
1426
|
filePath,
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1427
|
+
projectPath: state.cwd,
|
|
1428
|
+
projectName: getShortProjectName2(state.cwd),
|
|
1429
|
+
sessionId: state.sessionId || (0, import_path6.basename)(filePath, ".jsonl"),
|
|
1430
|
+
sessionName: state.sessionName,
|
|
1431
|
+
messages,
|
|
1432
|
+
fullText,
|
|
1156
1433
|
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1157
|
-
messageCount:
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
contentSnippet: state.snippetParts.join(" "),
|
|
1161
|
-
gitBranch: null,
|
|
1162
|
-
model: state.model,
|
|
1163
|
-
isSubagent,
|
|
1164
|
-
parentSessionId,
|
|
1165
|
-
isTeammate: state.isTeammate,
|
|
1166
|
-
teamName: state.teamName || null,
|
|
1167
|
-
toolNames: state.toolNames,
|
|
1168
|
-
firstMessage: state.firstMessage,
|
|
1169
|
-
lastMessage: state.lastMessage,
|
|
1434
|
+
messageCount: messages.length,
|
|
1435
|
+
account,
|
|
1436
|
+
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
1170
1437
|
lastPrompt: state.lastPrompt || void 0
|
|
1171
1438
|
};
|
|
1172
1439
|
}
|
|
1173
|
-
function
|
|
1174
|
-
const
|
|
1175
|
-
|
|
1440
|
+
async function parseConversationResumable(filePath, account) {
|
|
1441
|
+
const resume = { state: initialConvState(), offset: 0, line: 0 };
|
|
1442
|
+
const { messages, textParts, turnDurations, end } = await foldTail(filePath, resume);
|
|
1443
|
+
if (messages.length === 0) return null;
|
|
1444
|
+
applyTeamInfo(messages, resume.state);
|
|
1445
|
+
const conversation = assemble(
|
|
1446
|
+
filePath,
|
|
1447
|
+
account,
|
|
1448
|
+
messages,
|
|
1449
|
+
textParts.join(" "),
|
|
1450
|
+
turnDurations,
|
|
1451
|
+
resume.state
|
|
1452
|
+
);
|
|
1453
|
+
return { conversation, resume: { state: resume.state, offset: end.offset, line: end.line } };
|
|
1454
|
+
}
|
|
1455
|
+
async function extendConversation(previous, resume, filePath, account) {
|
|
1456
|
+
const { messages: fresh, textParts, turnDurations, end } = await foldTail(filePath, resume);
|
|
1457
|
+
const messages = fresh.length > 0 ? previous.messages.concat(fresh) : previous.messages;
|
|
1458
|
+
applyTeamInfo(messages, resume.state);
|
|
1459
|
+
const fullText = textParts.length === 0 ? previous.fullText : previous.fullText ? `${previous.fullText} ${textParts.join(" ")}` : textParts.join(" ");
|
|
1460
|
+
const allTurnDurations = (previous.turnDurations ?? []).concat(turnDurations);
|
|
1461
|
+
const conversation = assemble(
|
|
1462
|
+
filePath,
|
|
1463
|
+
account,
|
|
1464
|
+
messages,
|
|
1465
|
+
fullText,
|
|
1466
|
+
allTurnDurations,
|
|
1467
|
+
resume.state
|
|
1468
|
+
);
|
|
1469
|
+
return { conversation, resume: { state: resume.state, offset: end.offset, line: end.line } };
|
|
1176
1470
|
}
|
|
1177
1471
|
|
|
1178
|
-
// src/providers/threadbase.ts
|
|
1179
|
-
var ThreadbaseProvider = class {
|
|
1180
|
-
name = CLAUDE_CODE_PROVIDER;
|
|
1181
|
-
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
1182
|
-
// per-root account through the shared interface. The scanner builds these.
|
|
1183
|
-
async discover(roots) {
|
|
1184
|
-
const dirs = roots.map((r) => {
|
|
1185
|
-
const [projectsDir, account = "default"] = r.split("\0");
|
|
1186
|
-
return { projectsDir, account };
|
|
1187
|
-
});
|
|
1188
|
-
return discoverJsonlFiles(dirs);
|
|
1189
|
-
}
|
|
1190
|
-
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
1191
|
-
canParse(_filePath, sample) {
|
|
1192
|
-
for (const line of sample.split("\n")) {
|
|
1193
|
-
if (!line.trim()) continue;
|
|
1194
|
-
try {
|
|
1195
|
-
const e = JSON.parse(line);
|
|
1196
|
-
if (e.type === "user" || e.type === "assistant") return true;
|
|
1197
|
-
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
1198
|
-
} catch {
|
|
1199
|
-
}
|
|
1200
|
-
}
|
|
1201
|
-
return false;
|
|
1202
|
-
}
|
|
1203
|
-
createEmptyAccumulator() {
|
|
1204
|
-
return initialReducerState();
|
|
1205
|
-
}
|
|
1206
|
-
reduceEntry(acc, entry, tier) {
|
|
1207
|
-
reduceLine(acc, entry, tier);
|
|
1208
|
-
}
|
|
1209
|
-
finalize(acc, filePath, account, tier) {
|
|
1210
|
-
return finalizeMeta(acc, filePath, account, tier);
|
|
1211
|
-
}
|
|
1212
|
-
};
|
|
1213
|
-
|
|
1214
|
-
// src/scanner.ts
|
|
1215
|
-
var import_events = require("events");
|
|
1216
|
-
var import_fs10 = require("fs");
|
|
1217
|
-
var import_os2 = require("os");
|
|
1218
|
-
var import_path8 = require("path");
|
|
1219
|
-
|
|
1220
|
-
// src/cache.ts
|
|
1221
|
-
var LRUCache = class {
|
|
1222
|
-
map = /* @__PURE__ */ new Map();
|
|
1223
|
-
capacity;
|
|
1224
|
-
constructor(capacity) {
|
|
1225
|
-
this.capacity = capacity;
|
|
1226
|
-
}
|
|
1227
|
-
get(key) {
|
|
1228
|
-
const value = this.map.get(key);
|
|
1229
|
-
if (value === void 0) return void 0;
|
|
1230
|
-
this.map.delete(key);
|
|
1231
|
-
this.map.set(key, value);
|
|
1232
|
-
return value;
|
|
1233
|
-
}
|
|
1234
|
-
set(key, value) {
|
|
1235
|
-
this.map.delete(key);
|
|
1236
|
-
this.map.set(key, value);
|
|
1237
|
-
if (this.map.size > this.capacity) {
|
|
1238
|
-
const oldest = this.map.keys().next();
|
|
1239
|
-
if (!oldest.done) this.map.delete(oldest.value);
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
has(key) {
|
|
1243
|
-
return this.map.has(key);
|
|
1244
|
-
}
|
|
1245
|
-
delete(key) {
|
|
1246
|
-
return this.map.delete(key);
|
|
1247
|
-
}
|
|
1248
|
-
clear() {
|
|
1249
|
-
this.map.clear();
|
|
1250
|
-
}
|
|
1251
|
-
get size() {
|
|
1252
|
-
return this.map.size;
|
|
1253
|
-
}
|
|
1254
|
-
};
|
|
1255
|
-
|
|
1256
1472
|
// src/persistent/cursor.ts
|
|
1257
1473
|
var import_crypto = require("crypto");
|
|
1258
|
-
var
|
|
1474
|
+
var import_fs7 = require("fs");
|
|
1259
1475
|
var FP_BYTES = 4096;
|
|
1260
1476
|
function fingerprint(filePath, size) {
|
|
1261
1477
|
const hash = (0, import_crypto.createHash)("sha1");
|
|
1262
1478
|
hash.update(String(size));
|
|
1263
|
-
const fd = (0,
|
|
1479
|
+
const fd = (0, import_fs7.openSync)(filePath, "r");
|
|
1264
1480
|
try {
|
|
1265
1481
|
const head = Buffer.alloc(Math.min(FP_BYTES, size));
|
|
1266
1482
|
if (head.length > 0) {
|
|
1267
|
-
(0,
|
|
1483
|
+
(0, import_fs7.readSync)(fd, head, 0, head.length, 0);
|
|
1268
1484
|
hash.update(head);
|
|
1269
1485
|
}
|
|
1270
1486
|
if (size > FP_BYTES) {
|
|
1271
1487
|
const tailLen = Math.min(FP_BYTES, size);
|
|
1272
1488
|
const tail = Buffer.alloc(tailLen);
|
|
1273
|
-
(0,
|
|
1489
|
+
(0, import_fs7.readSync)(fd, tail, 0, tailLen, size - tailLen);
|
|
1274
1490
|
hash.update(tail);
|
|
1275
1491
|
}
|
|
1276
1492
|
} finally {
|
|
1277
|
-
(0,
|
|
1493
|
+
(0, import_fs7.closeSync)(fd);
|
|
1278
1494
|
}
|
|
1279
1495
|
return hash.digest("hex");
|
|
1280
1496
|
}
|
|
1281
1497
|
function classify(filePath, existing) {
|
|
1282
1498
|
let stat4;
|
|
1283
1499
|
try {
|
|
1284
|
-
const s = (0,
|
|
1500
|
+
const s = (0, import_fs7.statSync)(filePath);
|
|
1285
1501
|
stat4 = { size: s.size, mtimeMs: s.mtimeMs };
|
|
1286
1502
|
} catch {
|
|
1287
1503
|
return { change: "vanished" };
|
|
@@ -1308,14 +1524,17 @@ function classify(filePath, existing) {
|
|
|
1308
1524
|
return { change: "reindex", stat: stat4 };
|
|
1309
1525
|
}
|
|
1310
1526
|
|
|
1527
|
+
// src/persistent/index-engine.ts
|
|
1528
|
+
var import_fs10 = require("fs");
|
|
1529
|
+
|
|
1311
1530
|
// src/providers/parse.ts
|
|
1312
|
-
var
|
|
1531
|
+
var import_fs8 = require("fs");
|
|
1313
1532
|
var import_readline3 = require("readline");
|
|
1314
1533
|
async function parseMetaWithProvider(provider, filePath, account, tier) {
|
|
1315
1534
|
const log = getLogger();
|
|
1316
1535
|
const acc = provider.createEmptyAccumulator();
|
|
1317
1536
|
const rl = (0, import_readline3.createInterface)({
|
|
1318
|
-
input: (0,
|
|
1537
|
+
input: (0, import_fs8.createReadStream)(filePath),
|
|
1319
1538
|
crlfDelay: Infinity
|
|
1320
1539
|
});
|
|
1321
1540
|
try {
|
|
@@ -1357,8 +1576,8 @@ function resolveTier(tierName, customTiers) {
|
|
|
1357
1576
|
|
|
1358
1577
|
// src/persistent/db.ts
|
|
1359
1578
|
var import_better_sqlite3 = __toESM(require("better-sqlite3"), 1);
|
|
1360
|
-
var
|
|
1361
|
-
var
|
|
1579
|
+
var import_fs9 = require("fs");
|
|
1580
|
+
var import_path7 = require("path");
|
|
1362
1581
|
|
|
1363
1582
|
// src/persistent/schema.ts
|
|
1364
1583
|
var SCHEMA_VERSION = 4;
|
|
@@ -1551,7 +1770,7 @@ function hasColumn(db, table, column) {
|
|
|
1551
1770
|
// src/persistent/db.ts
|
|
1552
1771
|
function openDatabase(dbPath) {
|
|
1553
1772
|
if (dbPath !== ":memory:") {
|
|
1554
|
-
(0,
|
|
1773
|
+
(0, import_fs9.mkdirSync)((0, import_path7.dirname)(dbPath), { recursive: true });
|
|
1555
1774
|
}
|
|
1556
1775
|
const db = new import_better_sqlite3.default(dbPath);
|
|
1557
1776
|
db.pragma("journal_mode = WAL");
|
|
@@ -1564,7 +1783,7 @@ function openDatabase(dbPath) {
|
|
|
1564
1783
|
}
|
|
1565
1784
|
|
|
1566
1785
|
// src/persistent/dir-watermark.ts
|
|
1567
|
-
var
|
|
1786
|
+
var import_promises6 = require("fs/promises");
|
|
1568
1787
|
var FULL_RECONCILE_EVERY_N_SCANS = 20;
|
|
1569
1788
|
async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
1570
1789
|
const log = getLogger();
|
|
@@ -1580,7 +1799,7 @@ async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
|
1580
1799
|
for (const projectDir of resolved.entries) {
|
|
1581
1800
|
let dirStat;
|
|
1582
1801
|
try {
|
|
1583
|
-
dirStat = await (0,
|
|
1802
|
+
dirStat = await (0, import_promises6.stat)(projectDir);
|
|
1584
1803
|
} catch {
|
|
1585
1804
|
scannedDirs.remove(projectDir);
|
|
1586
1805
|
continue;
|
|
@@ -1618,7 +1837,7 @@ async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
|
1618
1837
|
async function resolveProjectDirs(projectsDir, scannedDirs) {
|
|
1619
1838
|
let rootStat;
|
|
1620
1839
|
try {
|
|
1621
|
-
rootStat = await (0,
|
|
1840
|
+
rootStat = await (0, import_promises6.stat)(projectsDir);
|
|
1622
1841
|
} catch {
|
|
1623
1842
|
return null;
|
|
1624
1843
|
}
|
|
@@ -1628,7 +1847,7 @@ async function resolveProjectDirs(projectsDir, scannedDirs) {
|
|
|
1628
1847
|
}
|
|
1629
1848
|
let entries;
|
|
1630
1849
|
try {
|
|
1631
|
-
entries = (await (0,
|
|
1850
|
+
entries = (await (0, import_promises6.readdir)(projectsDir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => joinPath(projectsDir, e.name));
|
|
1632
1851
|
} catch {
|
|
1633
1852
|
return null;
|
|
1634
1853
|
}
|
|
@@ -1642,104 +1861,6 @@ function joinPath(dir, name) {
|
|
|
1642
1861
|
return dir.endsWith("/") ? `${dir}${name}` : `${dir}/${name}`;
|
|
1643
1862
|
}
|
|
1644
1863
|
|
|
1645
|
-
// src/persistent/jsonl-tail-reader.ts
|
|
1646
|
-
var import_fs8 = require("fs");
|
|
1647
|
-
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1648
|
-
const stream = (0, import_fs8.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1649
|
-
let buffer = "";
|
|
1650
|
-
let offset = startOffset;
|
|
1651
|
-
let line = startLine;
|
|
1652
|
-
let parsedLines = 0;
|
|
1653
|
-
for await (const chunk of stream) {
|
|
1654
|
-
buffer += chunk;
|
|
1655
|
-
let nl;
|
|
1656
|
-
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1657
|
-
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1658
|
-
const text = lineWithNewline.trimEnd();
|
|
1659
|
-
buffer = buffer.slice(nl + 1);
|
|
1660
|
-
if (text.length > 0) {
|
|
1661
|
-
try {
|
|
1662
|
-
reduceLine(state, JSON.parse(text), tier);
|
|
1663
|
-
} catch {
|
|
1664
|
-
state.badJsonLines++;
|
|
1665
|
-
}
|
|
1666
|
-
parsedLines++;
|
|
1667
|
-
}
|
|
1668
|
-
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1669
|
-
line++;
|
|
1670
|
-
}
|
|
1671
|
-
}
|
|
1672
|
-
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
|
-
// src/persistent/paged-reader.ts
|
|
1676
|
-
var import_fs9 = require("fs");
|
|
1677
|
-
var CHECKPOINT_INTERVAL = 500;
|
|
1678
|
-
async function streamMessages(filePath, startOffset, startLine, state, onMessage) {
|
|
1679
|
-
const stream = (0, import_fs9.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1680
|
-
let buffer = "";
|
|
1681
|
-
let offset = startOffset;
|
|
1682
|
-
let line = startLine;
|
|
1683
|
-
for await (const chunk of stream) {
|
|
1684
|
-
buffer += chunk;
|
|
1685
|
-
let nl;
|
|
1686
|
-
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1687
|
-
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1688
|
-
const text = lineWithNewline.trimEnd();
|
|
1689
|
-
buffer = buffer.slice(nl + 1);
|
|
1690
|
-
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1691
|
-
line += 1;
|
|
1692
|
-
if (text.length === 0) continue;
|
|
1693
|
-
let entry;
|
|
1694
|
-
try {
|
|
1695
|
-
entry = JSON.parse(text);
|
|
1696
|
-
} catch {
|
|
1697
|
-
continue;
|
|
1698
|
-
}
|
|
1699
|
-
const message = reduceConvLine(state, entry);
|
|
1700
|
-
if (message && onMessage(message, offset, line)) {
|
|
1701
|
-
stream.destroy();
|
|
1702
|
-
return;
|
|
1703
|
-
}
|
|
1704
|
-
}
|
|
1705
|
-
}
|
|
1706
|
-
}
|
|
1707
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL) {
|
|
1708
|
-
const checkpoints = [];
|
|
1709
|
-
const state = initialConvState();
|
|
1710
|
-
let index = 0;
|
|
1711
|
-
await streamMessages(filePath, 0, 0, state, (_msg, nextOffset, nextLine) => {
|
|
1712
|
-
index += 1;
|
|
1713
|
-
if (index % interval === 0) {
|
|
1714
|
-
checkpoints.push({
|
|
1715
|
-
messageIndex: index,
|
|
1716
|
-
byteOffset: nextOffset,
|
|
1717
|
-
lineNumber: nextLine,
|
|
1718
|
-
state: structuredClone(state)
|
|
1719
|
-
});
|
|
1720
|
-
}
|
|
1721
|
-
return false;
|
|
1722
|
-
});
|
|
1723
|
-
return checkpoints;
|
|
1724
|
-
}
|
|
1725
|
-
async function readPage(filePath, total, options, floor) {
|
|
1726
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
1727
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1728
|
-
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1729
|
-
const startOffset = floor ? floor.byteOffset : 0;
|
|
1730
|
-
const startLine = floor ? floor.lineNumber : 0;
|
|
1731
|
-
let index = floor ? floor.messageIndex : 0;
|
|
1732
|
-
const window = [];
|
|
1733
|
-
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1734
|
-
const current = index;
|
|
1735
|
-
index += 1;
|
|
1736
|
-
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1737
|
-
return index >= beforeIndex;
|
|
1738
|
-
});
|
|
1739
|
-
applyTeamInfo(window, state);
|
|
1740
|
-
return { messages: window, total, fromIndex };
|
|
1741
|
-
}
|
|
1742
|
-
|
|
1743
1864
|
// src/persistent/repositories/checkpoints.repo.ts
|
|
1744
1865
|
var CheckpointsRepo = class {
|
|
1745
1866
|
constructor(db) {
|
|
@@ -1760,6 +1881,22 @@ var CheckpointsRepo = class {
|
|
|
1760
1881
|
});
|
|
1761
1882
|
tx();
|
|
1762
1883
|
}
|
|
1884
|
+
// Insert checkpoints without touching existing rows. Appends never invalidate
|
|
1885
|
+
// the chain covering the immutable prefix (Kafka sparse-index style); rows are
|
|
1886
|
+
// only ever removed on truncation/replace or deletion.
|
|
1887
|
+
append(sourcePath, checkpoints) {
|
|
1888
|
+
const tx = this.db.transaction(() => {
|
|
1889
|
+
const insert = this.db.prepare(
|
|
1890
|
+
`INSERT INTO message_checkpoints
|
|
1891
|
+
(source_path, message_index, byte_offset, line_number, parser_state)
|
|
1892
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
1893
|
+
);
|
|
1894
|
+
for (const c of checkpoints) {
|
|
1895
|
+
insert.run(sourcePath, c.messageIndex, c.byteOffset, c.lineNumber, JSON.stringify(c.state));
|
|
1896
|
+
}
|
|
1897
|
+
});
|
|
1898
|
+
tx();
|
|
1899
|
+
}
|
|
1763
1900
|
// The latest checkpoint at or before `messageIndex`, or null if none (read
|
|
1764
1901
|
// from the file start). Lets a page seek to the nearest prior anchor.
|
|
1765
1902
|
floor(sourcePath, messageIndex) {
|
|
@@ -1771,6 +1908,17 @@ var CheckpointsRepo = class {
|
|
|
1771
1908
|
).get(sourcePath, messageIndex);
|
|
1772
1909
|
return row ? toCheckpoint(row) : null;
|
|
1773
1910
|
}
|
|
1911
|
+
// The highest-index checkpoint for a file, or null if none. The resume point
|
|
1912
|
+
// for extending the chain after an append.
|
|
1913
|
+
last(sourcePath) {
|
|
1914
|
+
const row = this.db.prepare(
|
|
1915
|
+
`SELECT message_index, byte_offset, line_number, parser_state
|
|
1916
|
+
FROM message_checkpoints
|
|
1917
|
+
WHERE source_path = ?
|
|
1918
|
+
ORDER BY message_index DESC LIMIT 1`
|
|
1919
|
+
).get(sourcePath);
|
|
1920
|
+
return row ? toCheckpoint(row) : null;
|
|
1921
|
+
}
|
|
1774
1922
|
count(sourcePath) {
|
|
1775
1923
|
return this.db.prepare("SELECT COUNT(*) AS n FROM message_checkpoints WHERE source_path = ?").get(sourcePath).n;
|
|
1776
1924
|
}
|
|
@@ -1788,7 +1936,7 @@ function toCheckpoint(row) {
|
|
|
1788
1936
|
}
|
|
1789
1937
|
|
|
1790
1938
|
// src/persistent/repositories/conversation-files.repo.ts
|
|
1791
|
-
var
|
|
1939
|
+
var import_path8 = require("path");
|
|
1792
1940
|
var ConversationFilesRepo = class {
|
|
1793
1941
|
constructor(db) {
|
|
1794
1942
|
this.db = db;
|
|
@@ -1805,7 +1953,7 @@ var ConversationFilesRepo = class {
|
|
|
1805
1953
|
const info = this.db.prepare(
|
|
1806
1954
|
`INSERT INTO conversation_files (absolute_path, parent_dir, file_name, account)
|
|
1807
1955
|
VALUES (?, ?, ?, ?)`
|
|
1808
|
-
).run(absolutePath, (0,
|
|
1956
|
+
).run(absolutePath, (0, import_path8.dirname)(absolutePath), (0, import_path8.basename)(absolutePath), account);
|
|
1809
1957
|
return Number(info.lastInsertRowid);
|
|
1810
1958
|
}
|
|
1811
1959
|
// Advance the cursor + persisted reducer state after a successful index pass.
|
|
@@ -2164,6 +2312,9 @@ var PersistentEngine = class {
|
|
|
2164
2312
|
// restart just means the first few post-restart scans don't force an early
|
|
2165
2313
|
// backstop pass, which is harmless (watermarks themselves persist in the DB).
|
|
2166
2314
|
scanCount = 0;
|
|
2315
|
+
// In-flight checkpoint build/extension per file, so concurrent getPage
|
|
2316
|
+
// callers share one stream instead of each walking the file.
|
|
2317
|
+
checkpointBuilds = /* @__PURE__ */ new Map();
|
|
2167
2318
|
constructor(dbPath, options = {}) {
|
|
2168
2319
|
this.db = openDatabase(dbPath);
|
|
2169
2320
|
this.files = new ConversationFilesRepo(this.db);
|
|
@@ -2216,7 +2367,7 @@ var PersistentEngine = class {
|
|
|
2216
2367
|
const batch = discovered.slice(i, i + BATCH_SIZE);
|
|
2217
2368
|
const results = await Promise.all(
|
|
2218
2369
|
batch.map(async ({ filePath, account, provider }) => {
|
|
2219
|
-
const meta = await this.indexFile(
|
|
2370
|
+
const { meta } = await this.indexFile(
|
|
2220
2371
|
filePath,
|
|
2221
2372
|
account,
|
|
2222
2373
|
tier.name,
|
|
@@ -2251,7 +2402,9 @@ var PersistentEngine = class {
|
|
|
2251
2402
|
// unchanged → return the stored summary; appended → resume the fold and read
|
|
2252
2403
|
// only new bytes; reindex/force → fold from offset 0. Writes the summary +
|
|
2253
2404
|
// cursor + reducer state in one transaction so a crash never leaves a
|
|
2254
|
-
// half-written row or an over-advanced cursor.
|
|
2405
|
+
// half-written row or an over-advanced cursor. Returns the classification
|
|
2406
|
+
// alongside the meta so callers (refreshFile) can keep, extend, or evict
|
|
2407
|
+
// their own per-file caches without re-stat'ing the file (racy) themselves.
|
|
2255
2408
|
async indexFile(filePath, account, tierName, customTiers, resolveGitBranch, force = false, provider) {
|
|
2256
2409
|
const log = getLogger();
|
|
2257
2410
|
const tier = resolveTier(tierName, customTiers);
|
|
@@ -2259,13 +2412,21 @@ var PersistentEngine = class {
|
|
|
2259
2412
|
const { change, stat: stat4 } = classify(filePath, existing);
|
|
2260
2413
|
if (change === "vanished" || !stat4) {
|
|
2261
2414
|
this.markDeleted(filePath);
|
|
2262
|
-
return null;
|
|
2415
|
+
return { meta: null, change: "vanished" };
|
|
2263
2416
|
}
|
|
2264
2417
|
if (change === "unchanged" && !force) {
|
|
2265
|
-
return this.conversations.getBySourcePath(filePath);
|
|
2418
|
+
return { meta: this.conversations.getBySourcePath(filePath), change };
|
|
2266
2419
|
}
|
|
2267
2420
|
if (provider && provider.name !== CLAUDE_CODE_PROVIDER) {
|
|
2268
|
-
|
|
2421
|
+
const meta2 = await this.indexFileWithProvider(
|
|
2422
|
+
provider,
|
|
2423
|
+
filePath,
|
|
2424
|
+
account,
|
|
2425
|
+
tier,
|
|
2426
|
+
stat4,
|
|
2427
|
+
resolveGitBranch
|
|
2428
|
+
);
|
|
2429
|
+
return { meta: meta2, change };
|
|
2269
2430
|
}
|
|
2270
2431
|
const resume = change === "appended" && !force && existing?.reducer_state;
|
|
2271
2432
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
@@ -2276,12 +2437,12 @@ var PersistentEngine = class {
|
|
|
2276
2437
|
result = await tailReduce(filePath, startOffset, startLine, state, tier);
|
|
2277
2438
|
} catch (err) {
|
|
2278
2439
|
log.warn({ filePath, err }, "persistent: tail read failed");
|
|
2279
|
-
return null;
|
|
2440
|
+
return { meta: null, change };
|
|
2280
2441
|
}
|
|
2281
2442
|
const meta = finalizeMeta(state, filePath, account, tier);
|
|
2282
2443
|
if (!meta) {
|
|
2283
2444
|
this.markDeleted(filePath);
|
|
2284
|
-
return null;
|
|
2445
|
+
return { meta: null, change };
|
|
2285
2446
|
}
|
|
2286
2447
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2287
2448
|
const fp = stat4.size > 0 ? fingerprint(filePath, stat4.size) : null;
|
|
@@ -2289,7 +2450,7 @@ var PersistentEngine = class {
|
|
|
2289
2450
|
const upsert = this.db.transaction(() => {
|
|
2290
2451
|
this.conversations.upsert(fileId, meta, state.pageMessageCount);
|
|
2291
2452
|
this.fts.upsert(meta);
|
|
2292
|
-
this.checkpoints.remove(filePath);
|
|
2453
|
+
if (!resume) this.checkpoints.remove(filePath);
|
|
2293
2454
|
this.files.updateCursor(fileId, {
|
|
2294
2455
|
sizeBytes: stat4.size,
|
|
2295
2456
|
mtimeMs: stat4.mtimeMs,
|
|
@@ -2322,7 +2483,7 @@ var PersistentEngine = class {
|
|
|
2322
2483
|
{ filePath, change, bytesRead: result.newOffset - startOffset, msgs: meta.messageCount },
|
|
2323
2484
|
"persistent: indexed file"
|
|
2324
2485
|
);
|
|
2325
|
-
return meta;
|
|
2486
|
+
return { meta, change };
|
|
2326
2487
|
}
|
|
2327
2488
|
// Index a non-Threadbase provider file: full reparse from offset 0 through the
|
|
2328
2489
|
// provider's reducer/finalize, then the same upsert + FTS write + cursor bump
|
|
@@ -2422,15 +2583,40 @@ var PersistentEngine = class {
|
|
|
2422
2583
|
return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
|
|
2423
2584
|
}
|
|
2424
2585
|
const total = this.conversations.pageMessageCount(filePath);
|
|
2425
|
-
|
|
2426
|
-
const built = await buildCheckpoints(filePath);
|
|
2427
|
-
if (built.length > 0) this.checkpoints.replaceAll(filePath, built);
|
|
2428
|
-
}
|
|
2586
|
+
await this.ensureCheckpoints(filePath, total);
|
|
2429
2587
|
const beforeIndex = options.beforeIndex ?? total;
|
|
2430
2588
|
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2431
|
-
|
|
2589
|
+
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
2590
|
+
if (floor) {
|
|
2591
|
+
try {
|
|
2592
|
+
if (floor.byteOffset > (0, import_fs10.statSync)(filePath).size) floor = null;
|
|
2593
|
+
} catch {
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2432
2596
|
return readPage(filePath, total, options, floor);
|
|
2433
2597
|
}
|
|
2598
|
+
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
2599
|
+
// file → full build; a file that grew → extend from the last persisted
|
|
2600
|
+
// checkpoint (reads only past its offset, never the prefix). Single-flighted
|
|
2601
|
+
// per path: concurrent getPage callers await the same build instead of
|
|
2602
|
+
// streaming the file in parallel.
|
|
2603
|
+
ensureCheckpoints(filePath, total) {
|
|
2604
|
+
if (total <= CHECKPOINT_INTERVAL) return Promise.resolve();
|
|
2605
|
+
const inFlight = this.checkpointBuilds.get(filePath);
|
|
2606
|
+
if (inFlight) return inFlight;
|
|
2607
|
+
const build = (async () => {
|
|
2608
|
+
const last = this.checkpoints.last(filePath);
|
|
2609
|
+
if (last && total < last.messageIndex + CHECKPOINT_INTERVAL) return;
|
|
2610
|
+
const fresh = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, last);
|
|
2611
|
+
if (fresh.length === 0) return;
|
|
2612
|
+
if (last) this.checkpoints.append(filePath, fresh);
|
|
2613
|
+
else this.checkpoints.replaceAll(filePath, fresh);
|
|
2614
|
+
})().finally(() => {
|
|
2615
|
+
if (this.checkpointBuilds.get(filePath) === build) this.checkpointBuilds.delete(filePath);
|
|
2616
|
+
});
|
|
2617
|
+
this.checkpointBuilds.set(filePath, build);
|
|
2618
|
+
return build;
|
|
2619
|
+
}
|
|
2434
2620
|
};
|
|
2435
2621
|
|
|
2436
2622
|
// src/watcher/file-watcher.ts
|
|
@@ -2532,10 +2718,12 @@ var IndexQueue = class {
|
|
|
2532
2718
|
var BATCH_SIZE2 = 12;
|
|
2533
2719
|
var DEFAULT_CONFIG_PATH = "~/.config/threadbase-scanner";
|
|
2534
2720
|
function defaultDbPath() {
|
|
2535
|
-
return process.env.TB_SCANNER_DB ?? (0,
|
|
2721
|
+
return process.env.TB_SCANNER_DB ?? (0, import_path9.join)((0, import_os2.homedir)(), ".config", "threadbase-scanner", "index.db");
|
|
2536
2722
|
}
|
|
2537
2723
|
var ConversationScanner = class {
|
|
2538
2724
|
metadataCache = /* @__PURE__ */ new Map();
|
|
2725
|
+
// Parsed conversations plus (persistent claude-code entries only) the resume
|
|
2726
|
+
// point that lets refreshFile extend them in place when the file grows.
|
|
2539
2727
|
conversationLRU;
|
|
2540
2728
|
// session_id is NOT unique, so this maps a sessionId to every active meta that
|
|
2541
2729
|
// carries it. Resolution picks deterministically (newest timestamp, then path
|
|
@@ -2567,7 +2755,9 @@ var ConversationScanner = class {
|
|
|
2567
2755
|
// can't hit a closed DB (the watch-mode half of Bug #4).
|
|
2568
2756
|
inFlightReconcile = null;
|
|
2569
2757
|
constructor(options) {
|
|
2570
|
-
this.conversationLRU = new LRUCache(
|
|
2758
|
+
this.conversationLRU = new LRUCache(
|
|
2759
|
+
options?.conversationCacheSize ?? 5
|
|
2760
|
+
);
|
|
2571
2761
|
if (options?.persistent === false) {
|
|
2572
2762
|
this.dbPath = null;
|
|
2573
2763
|
this.sidecarEnabled = false;
|
|
@@ -2699,7 +2889,7 @@ var ConversationScanner = class {
|
|
|
2699
2889
|
const cached = statCache.get(filePath);
|
|
2700
2890
|
if (cached) {
|
|
2701
2891
|
try {
|
|
2702
|
-
const s = (0,
|
|
2892
|
+
const s = (0, import_fs11.statSync)(filePath);
|
|
2703
2893
|
if (s.mtimeMs === cached.stat.mtimeMs && s.size === cached.stat.size) {
|
|
2704
2894
|
return cached.meta;
|
|
2705
2895
|
}
|
|
@@ -2824,7 +3014,7 @@ var ConversationScanner = class {
|
|
|
2824
3014
|
const cached = this.conversationLRU.get(id);
|
|
2825
3015
|
if (cached) {
|
|
2826
3016
|
log.debug({ id }, "getConversation: cache hit");
|
|
2827
|
-
return cached;
|
|
3017
|
+
return cached.conversation;
|
|
2828
3018
|
}
|
|
2829
3019
|
const meta = this.persistent ? this.engine().getByIdOrSession(id) : this.metadataCache.get(id) ?? this.resolveSessionId(id);
|
|
2830
3020
|
if (!meta) {
|
|
@@ -2833,9 +3023,14 @@ var ConversationScanner = class {
|
|
|
2833
3023
|
}
|
|
2834
3024
|
log.debug({ id, filePath: meta.filePath }, "getConversation: cache miss, parsing");
|
|
2835
3025
|
try {
|
|
3026
|
+
if (this.persistent && meta.provider !== CODEX_CLI_PROVIDER) {
|
|
3027
|
+
const parsed = await parseConversationResumable(meta.filePath, meta.account);
|
|
3028
|
+
if (parsed) this.conversationLRU.set(id, parsed);
|
|
3029
|
+
return parsed?.conversation ?? null;
|
|
3030
|
+
}
|
|
2836
3031
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
2837
3032
|
if (conversation) {
|
|
2838
|
-
this.conversationLRU.set(id, conversation);
|
|
3033
|
+
this.conversationLRU.set(id, { conversation });
|
|
2839
3034
|
}
|
|
2840
3035
|
return conversation;
|
|
2841
3036
|
} catch (err) {
|
|
@@ -2907,20 +3102,30 @@ var ConversationScanner = class {
|
|
|
2907
3102
|
// not seen before. Returns the fresh ConversationMeta, or null when the file
|
|
2908
3103
|
// no longer parses (missing/empty) — in which case any prior entry for it is
|
|
2909
3104
|
// dropped from all indexes.
|
|
2910
|
-
|
|
3105
|
+
//
|
|
3106
|
+
// Single-flighted per path: concurrent callers (stacked client retries, a
|
|
3107
|
+
// watcher tick racing a caller) await the one in-flight refresh instead of
|
|
3108
|
+
// each re-reading the file.
|
|
3109
|
+
refreshesInFlight = /* @__PURE__ */ new Map();
|
|
3110
|
+
refreshFile(filePath, account) {
|
|
3111
|
+
const inFlight = this.refreshesInFlight.get(filePath);
|
|
3112
|
+
if (inFlight) return inFlight;
|
|
3113
|
+
const refresh = this.doRefreshFile(filePath, account).finally(() => {
|
|
3114
|
+
if (this.refreshesInFlight.get(filePath) === refresh) {
|
|
3115
|
+
this.refreshesInFlight.delete(filePath);
|
|
3116
|
+
}
|
|
3117
|
+
});
|
|
3118
|
+
this.refreshesInFlight.set(filePath, refresh);
|
|
3119
|
+
return refresh;
|
|
3120
|
+
}
|
|
3121
|
+
async doRefreshFile(filePath, account) {
|
|
2911
3122
|
const log = getLogger();
|
|
2912
3123
|
if (this.persistent) {
|
|
2913
3124
|
const engine = this.engine();
|
|
2914
3125
|
const previous2 = engine.getByIdOrSession(filePath);
|
|
2915
3126
|
const resolvedAccount2 = account ?? previous2?.account ?? "default";
|
|
2916
|
-
const evict2 = (m) => {
|
|
2917
|
-
if (!m) return;
|
|
2918
|
-
this.conversationLRU.delete(m.id);
|
|
2919
|
-
this.conversationLRU.delete(m.sessionId);
|
|
2920
|
-
};
|
|
2921
|
-
evict2(previous2);
|
|
2922
3127
|
const provider = await this.resolveProviderForFile(filePath, previous2);
|
|
2923
|
-
const meta2 = await engine.indexFile(
|
|
3128
|
+
const { meta: meta2, change } = await engine.indexFile(
|
|
2924
3129
|
filePath,
|
|
2925
3130
|
resolvedAccount2,
|
|
2926
3131
|
this.lastTier.name,
|
|
@@ -2929,8 +3134,19 @@ var ConversationScanner = class {
|
|
|
2929
3134
|
false,
|
|
2930
3135
|
provider
|
|
2931
3136
|
);
|
|
2932
|
-
|
|
2933
|
-
|
|
3137
|
+
const cacheKeys = /* @__PURE__ */ new Set();
|
|
3138
|
+
for (const m of [previous2, meta2]) {
|
|
3139
|
+
if (m) {
|
|
3140
|
+
cacheKeys.add(m.id);
|
|
3141
|
+
cacheKeys.add(m.sessionId);
|
|
3142
|
+
}
|
|
3143
|
+
}
|
|
3144
|
+
if (!meta2 || change === "reindex" || change === "vanished") {
|
|
3145
|
+
for (const key of cacheKeys) this.conversationLRU.delete(key);
|
|
3146
|
+
} else if (change === "appended") {
|
|
3147
|
+
await this.extendCachedConversations(cacheKeys, filePath, meta2.account);
|
|
3148
|
+
}
|
|
3149
|
+
log.debug({ filePath, change, kept: !!meta2 }, "refreshFile: updated persistent index");
|
|
2934
3150
|
return meta2;
|
|
2935
3151
|
}
|
|
2936
3152
|
const previous = this.metadataCache.get(filePath);
|
|
@@ -2974,6 +3190,39 @@ var ConversationScanner = class {
|
|
|
2974
3190
|
);
|
|
2975
3191
|
return meta;
|
|
2976
3192
|
}
|
|
3193
|
+
// Advance every cached parse of an appended file by folding only the new
|
|
3194
|
+
// bytes through the conversation reducer — the in-memory analogue of the
|
|
3195
|
+
// persisted metadata fold. Entries without resume state (Codex) and entries
|
|
3196
|
+
// whose extension fails are evicted so the next read re-parses from scratch.
|
|
3197
|
+
async extendCachedConversations(cacheKeys, filePath, account) {
|
|
3198
|
+
const wrappers = /* @__PURE__ */ new Map();
|
|
3199
|
+
for (const key of cacheKeys) {
|
|
3200
|
+
const wrapper = this.conversationLRU.get(key);
|
|
3201
|
+
if (!wrapper) continue;
|
|
3202
|
+
const keys = wrappers.get(wrapper) ?? [];
|
|
3203
|
+
keys.push(key);
|
|
3204
|
+
wrappers.set(wrapper, keys);
|
|
3205
|
+
}
|
|
3206
|
+
for (const [wrapper, keys] of wrappers) {
|
|
3207
|
+
if (!wrapper.resume) {
|
|
3208
|
+
for (const key of keys) this.conversationLRU.delete(key);
|
|
3209
|
+
continue;
|
|
3210
|
+
}
|
|
3211
|
+
try {
|
|
3212
|
+
const extended = await extendConversation(
|
|
3213
|
+
wrapper.conversation,
|
|
3214
|
+
wrapper.resume,
|
|
3215
|
+
filePath,
|
|
3216
|
+
account
|
|
3217
|
+
);
|
|
3218
|
+
wrapper.conversation = extended.conversation;
|
|
3219
|
+
wrapper.resume = extended.resume;
|
|
3220
|
+
} catch (err) {
|
|
3221
|
+
getLogger().warn({ filePath, err }, "refreshFile: cache extension failed, evicting");
|
|
3222
|
+
for (const key of keys) this.conversationLRU.delete(key);
|
|
3223
|
+
}
|
|
3224
|
+
}
|
|
3225
|
+
}
|
|
2977
3226
|
getMetadataCache() {
|
|
2978
3227
|
if (this.persistent) {
|
|
2979
3228
|
const map = /* @__PURE__ */ new Map();
|
|
@@ -3138,13 +3387,13 @@ var ConversationScanner = class {
|
|
|
3138
3387
|
if (previous?.provider) return void 0;
|
|
3139
3388
|
let sample = "";
|
|
3140
3389
|
try {
|
|
3141
|
-
const fd = (0,
|
|
3390
|
+
const fd = (0, import_fs11.openSync)(filePath, "r");
|
|
3142
3391
|
try {
|
|
3143
3392
|
const buf = Buffer.alloc(8192);
|
|
3144
|
-
const n = (0,
|
|
3393
|
+
const n = (0, import_fs11.readSync)(fd, buf, 0, buf.length, 0);
|
|
3145
3394
|
sample = buf.subarray(0, n).toString("utf8");
|
|
3146
3395
|
} finally {
|
|
3147
|
-
(0,
|
|
3396
|
+
(0, import_fs11.closeSync)(fd);
|
|
3148
3397
|
}
|
|
3149
3398
|
} catch {
|
|
3150
3399
|
return void 0;
|
|
@@ -3271,12 +3520,14 @@ async function getConversation(id, options, scanner) {
|
|
|
3271
3520
|
applySinceFilter,
|
|
3272
3521
|
applySort,
|
|
3273
3522
|
cleanSystemTags,
|
|
3523
|
+
createJsonlParseState,
|
|
3274
3524
|
createLogger,
|
|
3275
3525
|
detectDefaultProfile,
|
|
3276
3526
|
getConversation,
|
|
3277
3527
|
getLogger,
|
|
3278
3528
|
getProjectsDir,
|
|
3279
3529
|
loadProfiles,
|
|
3530
|
+
parseJsonlLine,
|
|
3280
3531
|
readGitBranch,
|
|
3281
3532
|
readSidecar,
|
|
3282
3533
|
resetDefaultScanner,
|