@threadbase-sh/scanner 0.9.4 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +364 -135
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +1086 -844
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +16 -1
- package/dist/index.d.ts +16 -1
- package/dist/index.js +1049 -809
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -43,12 +43,14 @@ __export(index_exports, {
|
|
|
43
43
|
applySinceFilter: () => applySinceFilter,
|
|
44
44
|
applySort: () => applySort,
|
|
45
45
|
cleanSystemTags: () => cleanSystemTags,
|
|
46
|
+
createJsonlParseState: () => initialConvState,
|
|
46
47
|
createLogger: () => createLogger,
|
|
47
48
|
detectDefaultProfile: () => detectDefaultProfile,
|
|
48
49
|
getConversation: () => getConversation,
|
|
49
50
|
getLogger: () => getLogger,
|
|
50
51
|
getProjectsDir: () => getProjectsDir,
|
|
51
52
|
loadProfiles: () => loadProfiles,
|
|
53
|
+
parseJsonlLine: () => parseJsonlLine,
|
|
52
54
|
readGitBranch: () => readGitBranch,
|
|
53
55
|
readSidecar: () => readSidecar,
|
|
54
56
|
resetDefaultScanner: () => resetDefaultScanner,
|
|
@@ -353,94 +355,137 @@ var SearchIndexer = class {
|
|
|
353
355
|
}
|
|
354
356
|
};
|
|
355
357
|
|
|
356
|
-
// src/
|
|
358
|
+
// src/parser.ts
|
|
357
359
|
var import_fs2 = require("fs");
|
|
358
|
-
var
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
360
|
+
var import_path3 = require("path");
|
|
361
|
+
var import_readline = require("readline");
|
|
362
|
+
|
|
363
|
+
// src/persistent/metadata-reducer.ts
|
|
364
|
+
var import_path2 = require("path");
|
|
365
|
+
|
|
366
|
+
// src/providers/provider.ts
|
|
367
|
+
var CLAUDE_CODE_PROVIDER = "claude-code";
|
|
368
|
+
var CODEX_CLI_PROVIDER = "codex-cli";
|
|
369
|
+
|
|
370
|
+
// src/persistent/metadata-reducer.ts
|
|
371
|
+
function initialReducerState() {
|
|
363
372
|
return {
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
messageCount:
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
373
|
+
sessionId: "",
|
|
374
|
+
sessionName: "",
|
|
375
|
+
latestTimestamp: "",
|
|
376
|
+
cwd: "",
|
|
377
|
+
teamName: "",
|
|
378
|
+
model: null,
|
|
379
|
+
messageCount: 0,
|
|
380
|
+
lastMessageSender: "user",
|
|
381
|
+
isTeammate: false,
|
|
382
|
+
firstUserSeen: false,
|
|
383
|
+
firstMessage: null,
|
|
384
|
+
lastMessage: null,
|
|
385
|
+
lastPrompt: "",
|
|
386
|
+
pageMessageCount: 0,
|
|
387
|
+
toolNames: [],
|
|
388
|
+
previewParts: [],
|
|
389
|
+
snippetParts: [],
|
|
390
|
+
previewLength: 0,
|
|
391
|
+
snippetLength: 0,
|
|
392
|
+
badJsonLines: 0
|
|
379
393
|
};
|
|
380
394
|
}
|
|
381
|
-
function
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
395
|
+
function reduceLine(state, entry, tier) {
|
|
396
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
397
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
398
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
399
|
+
if (entry.teamName && !state.teamName) state.teamName = entry.teamName;
|
|
400
|
+
if (entry.timestamp) {
|
|
401
|
+
const ts = entry.timestamp;
|
|
402
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
386
403
|
}
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
return
|
|
391
|
-
}
|
|
392
|
-
|
|
404
|
+
const type = entry.type;
|
|
405
|
+
if (type === "last-prompt") {
|
|
406
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
if (type !== "user" && type !== "assistant") return;
|
|
410
|
+
if (entry.isMeta) return;
|
|
411
|
+
const msg = entry.message;
|
|
412
|
+
if (state.model === null && msg?.model) state.model = msg.model;
|
|
413
|
+
if (type === "user" && !state.firstUserSeen) {
|
|
414
|
+
state.firstUserSeen = true;
|
|
415
|
+
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
416
|
+
}
|
|
417
|
+
const content = extractTextContent(msg?.content);
|
|
418
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
419
|
+
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
420
|
+
const toolSet = new Set(state.toolNames);
|
|
421
|
+
collectToolNames(msg?.content, toolSet);
|
|
422
|
+
state.toolNames = Array.from(toolSet);
|
|
423
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
424
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
425
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
426
|
+
if (content || isOnlyToolResult || toolUseBlocks.length > 0 || hasThinking) {
|
|
427
|
+
state.pageMessageCount++;
|
|
428
|
+
}
|
|
429
|
+
if (content || isOnlyToolResult) {
|
|
430
|
+
state.messageCount++;
|
|
431
|
+
state.lastMessageSender = type;
|
|
432
|
+
if (content) {
|
|
433
|
+
const ts = entry.timestamp || "";
|
|
434
|
+
if (!state.firstMessage) state.firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
435
|
+
state.lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
436
|
+
if (state.previewLength < tier.previewMax) {
|
|
437
|
+
state.previewParts.push(content);
|
|
438
|
+
state.previewLength += content.length;
|
|
439
|
+
}
|
|
440
|
+
if (state.snippetLength < tier.snippetMax) {
|
|
441
|
+
const remaining = tier.snippetMax - state.snippetLength;
|
|
442
|
+
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
443
|
+
state.snippetParts.push(chunk);
|
|
444
|
+
state.snippetLength += chunk.length;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
393
447
|
}
|
|
394
448
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
function getProjectsDir(profile) {
|
|
405
|
-
return (0, import_path2.join)(resolveConfigDir(profile.configDir), "projects");
|
|
406
|
-
}
|
|
407
|
-
async function detectDefaultProfile() {
|
|
449
|
+
function finalizeMeta(state, filePath, account, tier) {
|
|
450
|
+
if (state.messageCount === 0) return null;
|
|
451
|
+
const isSubagent = filePath.includes("/subagents/");
|
|
452
|
+
let parentSessionId = null;
|
|
453
|
+
if (isSubagent) {
|
|
454
|
+
const uuidDir = (0, import_path2.dirname)((0, import_path2.dirname)(filePath));
|
|
455
|
+
parentSessionId = (0, import_path2.join)((0, import_path2.dirname)(uuidDir), `${(0, import_path2.basename)(uuidDir)}.jsonl`);
|
|
456
|
+
}
|
|
457
|
+
const projectPath = state.cwd;
|
|
408
458
|
return {
|
|
409
|
-
id:
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
459
|
+
id: filePath,
|
|
460
|
+
filePath,
|
|
461
|
+
provider: CLAUDE_CODE_PROVIDER,
|
|
462
|
+
sessionId: state.sessionId || (0, import_path2.basename)(filePath, ".jsonl"),
|
|
463
|
+
sessionName: state.sessionName,
|
|
464
|
+
projectPath,
|
|
465
|
+
projectName: getShortProjectName(projectPath),
|
|
466
|
+
account,
|
|
467
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
468
|
+
messageCount: state.messageCount,
|
|
469
|
+
lastMessageSender: state.lastMessageSender,
|
|
470
|
+
preview: state.previewParts.join(" ").slice(0, tier.previewMax),
|
|
471
|
+
contentSnippet: state.snippetParts.join(" "),
|
|
472
|
+
gitBranch: null,
|
|
473
|
+
model: state.model,
|
|
474
|
+
isSubagent,
|
|
475
|
+
parentSessionId,
|
|
476
|
+
isTeammate: state.isTeammate,
|
|
477
|
+
teamName: state.teamName || null,
|
|
478
|
+
toolNames: state.toolNames,
|
|
479
|
+
firstMessage: state.firstMessage,
|
|
480
|
+
lastMessage: state.lastMessage,
|
|
481
|
+
lastPrompt: state.lastPrompt || void 0
|
|
414
482
|
};
|
|
415
483
|
}
|
|
416
|
-
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
const resolved = resolveConfigDir(configPath);
|
|
420
|
-
const data = await (0, import_promises.readFile)((0, import_path2.join)(resolved, PROFILES_FILE), "utf-8");
|
|
421
|
-
const profiles = JSON.parse(data);
|
|
422
|
-
log.debug({ configPath, count: profiles.length }, "profiles: loaded");
|
|
423
|
-
return profiles;
|
|
424
|
-
} catch (err) {
|
|
425
|
-
log.debug({ configPath, err }, "profiles: load failed, using default");
|
|
426
|
-
const defaultProfile = await detectDefaultProfile();
|
|
427
|
-
return [defaultProfile];
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
async function saveProfiles(profiles, configPath) {
|
|
431
|
-
const resolved = resolveConfigDir(configPath);
|
|
432
|
-
await (0, import_promises.mkdir)(resolved, { recursive: true });
|
|
433
|
-
await (0, import_promises.writeFile)((0, import_path2.join)(resolved, PROFILES_FILE), JSON.stringify(profiles, null, 2));
|
|
434
|
-
getLogger().debug({ configPath, count: profiles.length }, "profiles: saved");
|
|
484
|
+
function getShortProjectName(fullPath) {
|
|
485
|
+
const parts = fullPath.split("/").filter(Boolean);
|
|
486
|
+
return parts.slice(-3).join("/");
|
|
435
487
|
}
|
|
436
488
|
|
|
437
|
-
// src/providers/codex-cli.ts
|
|
438
|
-
var import_fast_glob = __toESM(require("fast-glob"), 1);
|
|
439
|
-
var import_fs3 = require("fs");
|
|
440
|
-
var import_promises2 = require("fs/promises");
|
|
441
|
-
var import_path3 = require("path");
|
|
442
|
-
var import_readline = require("readline");
|
|
443
|
-
|
|
444
489
|
// src/tags.ts
|
|
445
490
|
var SYSTEM_TAGS = [
|
|
446
491
|
"system-reminder",
|
|
@@ -469,496 +514,100 @@ function cleanSystemTags(text) {
|
|
|
469
514
|
return text.replace(SYSTEM_TAG_RE, "").replace(/[^\S\n]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
|
|
470
515
|
}
|
|
471
516
|
|
|
472
|
-
// src/
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
const
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
let paths;
|
|
517
|
+
// src/parser.ts
|
|
518
|
+
async function parseMeta(filePath, account, tier) {
|
|
519
|
+
const log = getLogger();
|
|
520
|
+
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
521
|
+
const state = initialReducerState();
|
|
522
|
+
const fileStream = (0, import_fs2.createReadStream)(filePath);
|
|
523
|
+
const rl = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
524
|
+
try {
|
|
525
|
+
for await (const line of rl) {
|
|
526
|
+
if (!line.trim()) continue;
|
|
527
|
+
let entry;
|
|
484
528
|
try {
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
dot: false,
|
|
489
|
-
unique: true
|
|
490
|
-
});
|
|
491
|
-
} catch (err) {
|
|
492
|
-
log.warn({ root, err }, "codex discovery: glob failed");
|
|
529
|
+
entry = JSON.parse(line);
|
|
530
|
+
} catch {
|
|
531
|
+
state.badJsonLines++;
|
|
493
532
|
continue;
|
|
494
533
|
}
|
|
495
|
-
|
|
496
|
-
try {
|
|
497
|
-
const s = await (0, import_promises2.stat)(filePath);
|
|
498
|
-
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
499
|
-
} catch (err) {
|
|
500
|
-
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
501
|
-
}
|
|
502
|
-
}
|
|
534
|
+
reduceLine(state, entry, tier);
|
|
503
535
|
}
|
|
504
|
-
|
|
536
|
+
} catch (err) {
|
|
537
|
+
log.warn({ filePath, err }, "parseMeta: read failed");
|
|
538
|
+
return null;
|
|
505
539
|
}
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
540
|
+
if (state.badJsonLines > 0) {
|
|
541
|
+
log.warn(
|
|
542
|
+
{ filePath, badJsonLines: state.badJsonLines },
|
|
543
|
+
"parseMeta: skipped malformed JSON lines"
|
|
544
|
+
);
|
|
545
|
+
}
|
|
546
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
547
|
+
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
548
|
+
return meta;
|
|
549
|
+
}
|
|
550
|
+
async function parseConversation(filePath, account) {
|
|
551
|
+
const log = getLogger();
|
|
552
|
+
log.trace({ filePath, account }, "parseConversation: start");
|
|
553
|
+
const messages = [];
|
|
554
|
+
let badJsonLines = 0;
|
|
555
|
+
const textParts = [];
|
|
556
|
+
const turnDurations = [];
|
|
557
|
+
const state = initialConvState();
|
|
558
|
+
const fileStream = (0, import_fs2.createReadStream)(filePath);
|
|
559
|
+
const rl = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
560
|
+
try {
|
|
561
|
+
for await (const line of rl) {
|
|
509
562
|
if (!line.trim()) continue;
|
|
510
|
-
|
|
511
|
-
const e = JSON.parse(line);
|
|
512
|
-
if (e.type === "session_meta" || e.type === "response_item" || e.type === "event_msg") {
|
|
513
|
-
return true;
|
|
514
|
-
}
|
|
515
|
-
if (e.type === "user" || e.type === "assistant") return false;
|
|
516
|
-
} catch {
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
return false;
|
|
520
|
-
}
|
|
521
|
-
createEmptyAccumulator() {
|
|
522
|
-
return {
|
|
523
|
-
sessionId: "",
|
|
524
|
-
cwd: "",
|
|
525
|
-
gitBranch: null,
|
|
526
|
-
model: null,
|
|
527
|
-
latestTimestamp: "",
|
|
528
|
-
messageCount: 0,
|
|
529
|
-
lastMessageSender: "user",
|
|
530
|
-
firstUser: null,
|
|
531
|
-
lastUser: null,
|
|
532
|
-
lastAssistant: null,
|
|
533
|
-
toolNames: [],
|
|
534
|
-
previewParts: [],
|
|
535
|
-
previewLength: 0,
|
|
536
|
-
snippetParts: [],
|
|
537
|
-
snippetLength: 0
|
|
538
|
-
};
|
|
539
|
-
}
|
|
540
|
-
reduceEntry(acc, entry, tier) {
|
|
541
|
-
reduceCodexEntry(acc, entry, tier);
|
|
542
|
-
}
|
|
543
|
-
finalize(acc, filePath, account, tier) {
|
|
544
|
-
return finalizeCodexMeta(acc, filePath, account, tier);
|
|
545
|
-
}
|
|
546
|
-
};
|
|
547
|
-
var asString = (v) => typeof v === "string" ? v : "";
|
|
548
|
-
function extractCodexText(content) {
|
|
549
|
-
if (typeof content === "string") return cleanSystemTags(content);
|
|
550
|
-
if (!Array.isArray(content)) return "";
|
|
551
|
-
return content.map((item) => {
|
|
552
|
-
if (typeof item === "string") return item;
|
|
553
|
-
const t = item?.type;
|
|
554
|
-
if ((t === "input_text" || t === "output_text" || t === "text") && item?.text) {
|
|
555
|
-
return item.text;
|
|
556
|
-
}
|
|
557
|
-
return "";
|
|
558
|
-
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
559
|
-
}
|
|
560
|
-
function reduceCodexEntry(acc, entry, tier) {
|
|
561
|
-
const ts = asString(entry.timestamp);
|
|
562
|
-
if (ts && (!acc.latestTimestamp || ts > acc.latestTimestamp)) acc.latestTimestamp = ts;
|
|
563
|
-
const payload = entry.payload;
|
|
564
|
-
if (!payload || typeof payload !== "object") return;
|
|
565
|
-
const type = entry.type;
|
|
566
|
-
if (type === "session_meta") {
|
|
567
|
-
if (!acc.sessionId) acc.sessionId = asString(payload.id);
|
|
568
|
-
if (!acc.cwd) acc.cwd = asString(payload.cwd);
|
|
569
|
-
const git = payload.git;
|
|
570
|
-
if (acc.gitBranch === null && git?.branch) acc.gitBranch = asString(git.branch) || null;
|
|
571
|
-
return;
|
|
572
|
-
}
|
|
573
|
-
if (acc.model === null && payload.model) acc.model = asString(payload.model) || null;
|
|
574
|
-
if (type !== "response_item") return;
|
|
575
|
-
const ptype = payload.type;
|
|
576
|
-
if (ptype === "function_call" || ptype === "custom_tool_call") {
|
|
577
|
-
const name = asString(payload.name);
|
|
578
|
-
if (name && !acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
579
|
-
return;
|
|
580
|
-
}
|
|
581
|
-
if (ptype !== "message") return;
|
|
582
|
-
const role = payload.role;
|
|
583
|
-
if (role !== "user" && role !== "assistant") return;
|
|
584
|
-
const text = extractCodexText(payload.content);
|
|
585
|
-
if (!text) return;
|
|
586
|
-
const sender = role;
|
|
587
|
-
acc.messageCount++;
|
|
588
|
-
acc.lastMessageSender = sender;
|
|
589
|
-
const snapshot = { text: text.slice(0, 200), timestamp: ts };
|
|
590
|
-
if (sender === "user") {
|
|
591
|
-
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
592
|
-
acc.lastUser = snapshot;
|
|
593
|
-
} else {
|
|
594
|
-
acc.lastAssistant = snapshot;
|
|
595
|
-
}
|
|
596
|
-
if (acc.previewLength < tier.previewMax) {
|
|
597
|
-
acc.previewParts.push(text);
|
|
598
|
-
acc.previewLength += text.length;
|
|
599
|
-
}
|
|
600
|
-
if (acc.snippetLength < tier.snippetMax) {
|
|
601
|
-
const remaining = tier.snippetMax - acc.snippetLength;
|
|
602
|
-
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
603
|
-
acc.snippetParts.push(chunk);
|
|
604
|
-
acc.snippetLength += chunk.length;
|
|
605
|
-
}
|
|
606
|
-
}
|
|
607
|
-
function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
608
|
-
if (acc.messageCount === 0) return null;
|
|
609
|
-
const sessionId = acc.sessionId || (0, import_path3.basename)(filePath, ".jsonl");
|
|
610
|
-
const projectPath = acc.cwd;
|
|
611
|
-
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
612
|
-
return {
|
|
613
|
-
id: filePath,
|
|
614
|
-
filePath,
|
|
615
|
-
provider: CODEX_CLI_PROVIDER,
|
|
616
|
-
kind,
|
|
617
|
-
externalSessionId: acc.sessionId || void 0,
|
|
618
|
-
sessionId,
|
|
619
|
-
sessionName: "",
|
|
620
|
-
projectPath,
|
|
621
|
-
projectName: getShortProjectName(projectPath),
|
|
622
|
-
account,
|
|
623
|
-
timestamp: acc.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
624
|
-
messageCount: acc.messageCount,
|
|
625
|
-
lastMessageSender: acc.lastMessageSender,
|
|
626
|
-
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
627
|
-
contentSnippet: acc.snippetParts.join(" "),
|
|
628
|
-
gitBranch: acc.gitBranch,
|
|
629
|
-
model: acc.model,
|
|
630
|
-
isSubagent: false,
|
|
631
|
-
parentSessionId: null,
|
|
632
|
-
isTeammate: false,
|
|
633
|
-
teamName: null,
|
|
634
|
-
toolNames: acc.toolNames,
|
|
635
|
-
firstMessage: acc.firstUser,
|
|
636
|
-
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
637
|
-
lastPrompt: acc.lastUser?.text || void 0
|
|
638
|
-
};
|
|
639
|
-
}
|
|
640
|
-
function getShortProjectName(fullPath) {
|
|
641
|
-
return fullPath.split("/").filter(Boolean).slice(-3).join("/");
|
|
642
|
-
}
|
|
643
|
-
async function parseCodexConversation(filePath, account) {
|
|
644
|
-
const log = getLogger();
|
|
645
|
-
const messages = [];
|
|
646
|
-
const textParts = [];
|
|
647
|
-
let sessionId = "";
|
|
648
|
-
let cwd = "";
|
|
649
|
-
let latestTimestamp = "";
|
|
650
|
-
let lastUserText = "";
|
|
651
|
-
const rl = (0, import_readline.createInterface)({ input: (0, import_fs3.createReadStream)(filePath), crlfDelay: Infinity });
|
|
652
|
-
try {
|
|
653
|
-
for await (const line of rl) {
|
|
654
|
-
if (!line.trim()) continue;
|
|
655
|
-
let entry;
|
|
563
|
+
let entry;
|
|
656
564
|
try {
|
|
657
565
|
entry = JSON.parse(line);
|
|
658
566
|
} catch {
|
|
567
|
+
badJsonLines++;
|
|
659
568
|
continue;
|
|
660
569
|
}
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
if (!cwd) cwd = asString(payload.cwd);
|
|
570
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
571
|
+
turnDurations.push({
|
|
572
|
+
durationMs: entry.durationMs,
|
|
573
|
+
messageCount: entry.messageCount || 0,
|
|
574
|
+
uuid: entry.uuid
|
|
575
|
+
});
|
|
668
576
|
continue;
|
|
669
577
|
}
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
messages.push({ role, text, timestamp: ts });
|
|
676
|
-
textParts.push(text);
|
|
677
|
-
if (role === "user") lastUserText = text;
|
|
578
|
+
const message = reduceConvLine(state, entry);
|
|
579
|
+
if (message) {
|
|
580
|
+
messages.push(message);
|
|
581
|
+
if (message.text) textParts.push(message.text);
|
|
582
|
+
}
|
|
678
583
|
}
|
|
679
584
|
} catch (err) {
|
|
680
|
-
log.warn({ filePath, err }, "
|
|
585
|
+
log.warn({ filePath, err }, "parseConversation: read failed");
|
|
681
586
|
return null;
|
|
682
587
|
}
|
|
683
|
-
if (
|
|
588
|
+
if (badJsonLines > 0) {
|
|
589
|
+
log.warn({ filePath, badJsonLines }, "parseConversation: skipped malformed JSON lines");
|
|
590
|
+
}
|
|
591
|
+
if (messages.length === 0) {
|
|
592
|
+
log.trace({ filePath }, "parseConversation: no messages");
|
|
593
|
+
return null;
|
|
594
|
+
}
|
|
595
|
+
log.debug({ filePath, messageCount: messages.length }, "parseConversation: complete");
|
|
596
|
+
applyTeamInfo(messages, state);
|
|
684
597
|
return {
|
|
685
598
|
id: filePath,
|
|
686
599
|
filePath,
|
|
687
|
-
projectPath: cwd,
|
|
688
|
-
projectName:
|
|
689
|
-
sessionId: sessionId || (0, import_path3.basename)(filePath, ".jsonl"),
|
|
690
|
-
sessionName:
|
|
600
|
+
projectPath: state.cwd,
|
|
601
|
+
projectName: getShortProjectName2(state.cwd),
|
|
602
|
+
sessionId: state.sessionId || (0, import_path3.basename)(filePath, ".jsonl"),
|
|
603
|
+
sessionName: state.sessionName,
|
|
691
604
|
messages,
|
|
692
605
|
fullText: textParts.join(" "),
|
|
693
|
-
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
606
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
694
607
|
messageCount: messages.length,
|
|
695
608
|
account,
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
// src/discovery.ts
|
|
701
|
-
var import_fast_glob2 = __toESM(require("fast-glob"), 1);
|
|
702
|
-
var import_promises3 = require("fs/promises");
|
|
703
|
-
var EXCLUDED_SEGMENTS = ["/memory/", "/tool-results/"];
|
|
704
|
-
var STAT_CONCURRENCY = 32;
|
|
705
|
-
async function discoverJsonlFiles(dirs, onProgress) {
|
|
706
|
-
const log = getLogger();
|
|
707
|
-
const results = [];
|
|
708
|
-
for (const { projectsDir, account } of dirs) {
|
|
709
|
-
let filePaths;
|
|
710
|
-
try {
|
|
711
|
-
filePaths = await (0, import_fast_glob2.default)("**/*.jsonl", {
|
|
712
|
-
cwd: projectsDir,
|
|
713
|
-
absolute: true,
|
|
714
|
-
dot: false
|
|
715
|
-
});
|
|
716
|
-
} catch (err) {
|
|
717
|
-
log.warn({ projectsDir, account, err }, "discovery: glob failed");
|
|
718
|
-
continue;
|
|
719
|
-
}
|
|
720
|
-
const filtered = filePaths.filter((fp) => !EXCLUDED_SEGMENTS.some((seg) => fp.includes(seg)));
|
|
721
|
-
let kept = 0;
|
|
722
|
-
let skippedEmpty = 0;
|
|
723
|
-
let skippedInaccessible = 0;
|
|
724
|
-
for (let i = 0; i < filtered.length; i += STAT_CONCURRENCY) {
|
|
725
|
-
const chunk = filtered.slice(i, i + STAT_CONCURRENCY);
|
|
726
|
-
const statted = await Promise.all(
|
|
727
|
-
chunk.map(async (filePath) => {
|
|
728
|
-
try {
|
|
729
|
-
const s = await (0, import_promises3.stat)(filePath);
|
|
730
|
-
return { filePath, size: s.size };
|
|
731
|
-
} catch (err) {
|
|
732
|
-
log.warn({ filePath, err }, "discovery: stat failed");
|
|
733
|
-
return { filePath, size: -1 };
|
|
734
|
-
}
|
|
735
|
-
})
|
|
736
|
-
);
|
|
737
|
-
for (const { filePath, size } of statted) {
|
|
738
|
-
if (size < 0) {
|
|
739
|
-
skippedInaccessible++;
|
|
740
|
-
} else if (size > 0) {
|
|
741
|
-
results.push({ filePath, account });
|
|
742
|
-
kept++;
|
|
743
|
-
} else {
|
|
744
|
-
skippedEmpty++;
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
}
|
|
748
|
-
log.debug(
|
|
749
|
-
{
|
|
750
|
-
projectsDir,
|
|
751
|
-
account,
|
|
752
|
-
globMatches: filePaths.length,
|
|
753
|
-
afterExclusions: filtered.length,
|
|
754
|
-
kept,
|
|
755
|
-
skippedEmpty,
|
|
756
|
-
skippedInaccessible
|
|
757
|
-
},
|
|
758
|
-
"discovery: directory scanned"
|
|
759
|
-
);
|
|
760
|
-
onProgress?.(results.length);
|
|
761
|
-
}
|
|
762
|
-
log.debug({ totalFiles: results.length, dirs: dirs.length }, "discovery: complete");
|
|
763
|
-
return results;
|
|
764
|
-
}
|
|
765
|
-
|
|
766
|
-
// src/persistent/metadata-reducer.ts
|
|
767
|
-
var import_path5 = require("path");
|
|
768
|
-
|
|
769
|
-
// src/parser.ts
|
|
770
|
-
var import_fs4 = require("fs");
|
|
771
|
-
var import_path4 = require("path");
|
|
772
|
-
var import_readline2 = require("readline");
|
|
773
|
-
|
|
774
|
-
// src/persistent/conversation-reducer.ts
|
|
775
|
-
function initialConvState() {
|
|
776
|
-
return {
|
|
777
|
-
cwd: "",
|
|
778
|
-
sessionId: "",
|
|
779
|
-
sessionName: "",
|
|
780
|
-
latestTimestamp: "",
|
|
781
|
-
lastPrompt: "",
|
|
782
|
-
pendingToolUses: {},
|
|
783
|
-
teamInfo: {}
|
|
784
|
-
};
|
|
785
|
-
}
|
|
786
|
-
function reduceConvLine(state, entry) {
|
|
787
|
-
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
788
|
-
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
789
|
-
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
790
|
-
if (entry.timestamp) {
|
|
791
|
-
const ts = entry.timestamp;
|
|
792
|
-
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
793
|
-
}
|
|
794
|
-
const type = entry.type;
|
|
795
|
-
if (type === "last-prompt") {
|
|
796
|
-
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
797
|
-
return null;
|
|
798
|
-
}
|
|
799
|
-
if (type !== "user" && type !== "assistant") return null;
|
|
800
|
-
if (entry.isMeta) return null;
|
|
801
|
-
const msg = entry.message;
|
|
802
|
-
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
803
|
-
for (const block of toolUseBlocks) state.pendingToolUses[block.id] = block;
|
|
804
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
805
|
-
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
806
|
-
const content = extractTextContent(msg?.content);
|
|
807
|
-
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
808
|
-
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
809
|
-
if (!(content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking)) return null;
|
|
810
|
-
const metadata = {};
|
|
811
|
-
if (msg?.model) metadata.model = msg.model;
|
|
812
|
-
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
813
|
-
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
814
|
-
if (entry.version) metadata.version = entry.version;
|
|
815
|
-
const usage = msg?.usage;
|
|
816
|
-
if (usage) {
|
|
817
|
-
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
818
|
-
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
819
|
-
if (usage.cache_read_input_tokens) metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
820
|
-
if (usage.cache_creation_input_tokens)
|
|
821
|
-
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
822
|
-
}
|
|
823
|
-
const toolUseNames = extractToolUseNames(msg?.content);
|
|
824
|
-
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
825
|
-
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
826
|
-
if (isToolResultOnly) {
|
|
827
|
-
const pending = new Map(Object.entries(state.pendingToolUses));
|
|
828
|
-
const toolResultBlocks = extractToolResultBlocks(msg?.content, pending);
|
|
829
|
-
if (toolResultBlocks.length > 0) metadata.toolResults = toolResultBlocks;
|
|
830
|
-
}
|
|
831
|
-
if (entry.teamName) {
|
|
832
|
-
metadata.teamName = entry.teamName;
|
|
833
|
-
if (!state.teamInfo[metadata.teamName] && content) {
|
|
834
|
-
const info = parseTeammateMessageTag(content);
|
|
835
|
-
if (info) state.teamInfo[metadata.teamName] = info;
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
const thinkingContent = thinking?.content || void 0;
|
|
839
|
-
const thinkingSignature = thinking?.signature || void 0;
|
|
840
|
-
const hasMetadata = Object.keys(metadata).length > 0;
|
|
841
|
-
return {
|
|
842
|
-
role: type,
|
|
843
|
-
text: content || "",
|
|
844
|
-
timestamp: entry.timestamp || "",
|
|
845
|
-
uuid: entry.uuid || void 0,
|
|
846
|
-
metadata: hasMetadata ? metadata : void 0,
|
|
847
|
-
isToolResult: isToolResultOnly || void 0,
|
|
848
|
-
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
849
|
-
thinkingContent,
|
|
850
|
-
thinkingSignature,
|
|
851
|
-
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
852
|
-
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
853
|
-
promptId: type === "user" ? entry.promptId : void 0,
|
|
854
|
-
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
855
|
-
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
856
|
-
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
857
|
-
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
858
|
-
};
|
|
859
|
-
}
|
|
860
|
-
function applyTeamInfo(messages, state) {
|
|
861
|
-
if (Object.keys(state.teamInfo).length === 0) return;
|
|
862
|
-
for (const m of messages) {
|
|
863
|
-
const name = m.metadata?.teamName;
|
|
864
|
-
if (name && state.teamInfo[name] && m.metadata) m.metadata.teamInfo = state.teamInfo[name];
|
|
865
|
-
}
|
|
866
|
-
}
|
|
867
|
-
|
|
868
|
-
// src/parser.ts
|
|
869
|
-
async function parseMeta(filePath, account, tier) {
|
|
870
|
-
const log = getLogger();
|
|
871
|
-
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
872
|
-
const state = initialReducerState();
|
|
873
|
-
const fileStream = (0, import_fs4.createReadStream)(filePath);
|
|
874
|
-
const rl = (0, import_readline2.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
875
|
-
try {
|
|
876
|
-
for await (const line of rl) {
|
|
877
|
-
if (!line.trim()) continue;
|
|
878
|
-
let entry;
|
|
879
|
-
try {
|
|
880
|
-
entry = JSON.parse(line);
|
|
881
|
-
} catch {
|
|
882
|
-
state.badJsonLines++;
|
|
883
|
-
continue;
|
|
884
|
-
}
|
|
885
|
-
reduceLine(state, entry, tier);
|
|
886
|
-
}
|
|
887
|
-
} catch (err) {
|
|
888
|
-
log.warn({ filePath, err }, "parseMeta: read failed");
|
|
889
|
-
return null;
|
|
890
|
-
}
|
|
891
|
-
if (state.badJsonLines > 0) {
|
|
892
|
-
log.warn(
|
|
893
|
-
{ filePath, badJsonLines: state.badJsonLines },
|
|
894
|
-
"parseMeta: skipped malformed JSON lines"
|
|
895
|
-
);
|
|
896
|
-
}
|
|
897
|
-
const meta = finalizeMeta(state, filePath, account, tier);
|
|
898
|
-
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
899
|
-
return meta;
|
|
900
|
-
}
|
|
901
|
-
async function parseConversation(filePath, account) {
|
|
902
|
-
const log = getLogger();
|
|
903
|
-
log.trace({ filePath, account }, "parseConversation: start");
|
|
904
|
-
const messages = [];
|
|
905
|
-
let badJsonLines = 0;
|
|
906
|
-
const textParts = [];
|
|
907
|
-
const turnDurations = [];
|
|
908
|
-
const state = initialConvState();
|
|
909
|
-
const fileStream = (0, import_fs4.createReadStream)(filePath);
|
|
910
|
-
const rl = (0, import_readline2.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
911
|
-
try {
|
|
912
|
-
for await (const line of rl) {
|
|
913
|
-
if (!line.trim()) continue;
|
|
914
|
-
let entry;
|
|
915
|
-
try {
|
|
916
|
-
entry = JSON.parse(line);
|
|
917
|
-
} catch {
|
|
918
|
-
badJsonLines++;
|
|
919
|
-
continue;
|
|
920
|
-
}
|
|
921
|
-
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
922
|
-
turnDurations.push({
|
|
923
|
-
durationMs: entry.durationMs,
|
|
924
|
-
messageCount: entry.messageCount || 0,
|
|
925
|
-
uuid: entry.uuid
|
|
926
|
-
});
|
|
927
|
-
continue;
|
|
928
|
-
}
|
|
929
|
-
const message = reduceConvLine(state, entry);
|
|
930
|
-
if (message) {
|
|
931
|
-
messages.push(message);
|
|
932
|
-
if (message.text) textParts.push(message.text);
|
|
933
|
-
}
|
|
934
|
-
}
|
|
935
|
-
} catch (err) {
|
|
936
|
-
log.warn({ filePath, err }, "parseConversation: read failed");
|
|
937
|
-
return null;
|
|
938
|
-
}
|
|
939
|
-
if (badJsonLines > 0) {
|
|
940
|
-
log.warn({ filePath, badJsonLines }, "parseConversation: skipped malformed JSON lines");
|
|
941
|
-
}
|
|
942
|
-
if (messages.length === 0) {
|
|
943
|
-
log.trace({ filePath }, "parseConversation: no messages");
|
|
944
|
-
return null;
|
|
945
|
-
}
|
|
946
|
-
log.debug({ filePath, messageCount: messages.length }, "parseConversation: complete");
|
|
947
|
-
applyTeamInfo(messages, state);
|
|
948
|
-
return {
|
|
949
|
-
id: filePath,
|
|
950
|
-
filePath,
|
|
951
|
-
projectPath: state.cwd,
|
|
952
|
-
projectName: getShortProjectName2(state.cwd),
|
|
953
|
-
sessionId: state.sessionId || (0, import_path4.basename)(filePath, ".jsonl"),
|
|
954
|
-
sessionName: state.sessionName,
|
|
955
|
-
messages,
|
|
956
|
-
fullText: textParts.join(" "),
|
|
957
|
-
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
958
|
-
messageCount: messages.length,
|
|
959
|
-
account,
|
|
960
|
-
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
961
|
-
lastPrompt: state.lastPrompt || void 0
|
|
609
|
+
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
610
|
+
lastPrompt: state.lastPrompt || void 0
|
|
962
611
|
};
|
|
963
612
|
}
|
|
964
613
|
function extractTextContent(content) {
|
|
@@ -1056,36 +705,22 @@ function getShortProjectName2(fullPath) {
|
|
|
1056
705
|
return parts.slice(-3).join("/");
|
|
1057
706
|
}
|
|
1058
707
|
|
|
1059
|
-
// src/persistent/
|
|
1060
|
-
function
|
|
708
|
+
// src/persistent/conversation-reducer.ts
|
|
709
|
+
function initialConvState() {
|
|
1061
710
|
return {
|
|
711
|
+
cwd: "",
|
|
1062
712
|
sessionId: "",
|
|
1063
713
|
sessionName: "",
|
|
1064
714
|
latestTimestamp: "",
|
|
1065
|
-
cwd: "",
|
|
1066
|
-
teamName: "",
|
|
1067
|
-
model: null,
|
|
1068
|
-
messageCount: 0,
|
|
1069
|
-
lastMessageSender: "user",
|
|
1070
|
-
isTeammate: false,
|
|
1071
|
-
firstUserSeen: false,
|
|
1072
|
-
firstMessage: null,
|
|
1073
|
-
lastMessage: null,
|
|
1074
715
|
lastPrompt: "",
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
previewParts: [],
|
|
1078
|
-
snippetParts: [],
|
|
1079
|
-
previewLength: 0,
|
|
1080
|
-
snippetLength: 0,
|
|
1081
|
-
badJsonLines: 0
|
|
716
|
+
pendingToolUses: {},
|
|
717
|
+
teamInfo: {}
|
|
1082
718
|
};
|
|
1083
719
|
}
|
|
1084
|
-
function
|
|
720
|
+
function reduceConvLine(state, entry) {
|
|
1085
721
|
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
1086
722
|
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
1087
723
|
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
1088
|
-
if (entry.teamName && !state.teamName) state.teamName = entry.teamName;
|
|
1089
724
|
if (entry.timestamp) {
|
|
1090
725
|
const ts = entry.timestamp;
|
|
1091
726
|
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
@@ -1093,195 +728,776 @@ function reduceLine(state, entry, tier) {
|
|
|
1093
728
|
const type = entry.type;
|
|
1094
729
|
if (type === "last-prompt") {
|
|
1095
730
|
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
1096
|
-
return;
|
|
731
|
+
return null;
|
|
1097
732
|
}
|
|
1098
|
-
if (type !== "user" && type !== "assistant") return;
|
|
1099
|
-
if (entry.isMeta) return;
|
|
733
|
+
if (type !== "user" && type !== "assistant") return null;
|
|
734
|
+
if (entry.isMeta) return null;
|
|
1100
735
|
const msg = entry.message;
|
|
1101
|
-
if (state.model === null && msg?.model) state.model = msg.model;
|
|
1102
|
-
if (type === "user" && !state.firstUserSeen) {
|
|
1103
|
-
state.firstUserSeen = true;
|
|
1104
|
-
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
1105
|
-
}
|
|
1106
|
-
const content = extractTextContent(msg?.content);
|
|
1107
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
1108
|
-
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
1109
|
-
const toolSet = new Set(state.toolNames);
|
|
1110
|
-
collectToolNames(msg?.content, toolSet);
|
|
1111
|
-
state.toolNames = Array.from(toolSet);
|
|
1112
736
|
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
737
|
+
for (const block of toolUseBlocks) state.pendingToolUses[block.id] = block;
|
|
738
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
739
|
+
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
740
|
+
const content = extractTextContent(msg?.content);
|
|
1113
741
|
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
1114
742
|
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
1115
|
-
if (content ||
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
if (
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
743
|
+
if (!(content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking)) return null;
|
|
744
|
+
const metadata = {};
|
|
745
|
+
if (msg?.model) metadata.model = msg.model;
|
|
746
|
+
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
747
|
+
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
748
|
+
if (entry.version) metadata.version = entry.version;
|
|
749
|
+
const usage = msg?.usage;
|
|
750
|
+
if (usage) {
|
|
751
|
+
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
752
|
+
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
753
|
+
if (usage.cache_read_input_tokens) metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
754
|
+
if (usage.cache_creation_input_tokens)
|
|
755
|
+
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
756
|
+
}
|
|
757
|
+
const toolUseNames = extractToolUseNames(msg?.content);
|
|
758
|
+
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
759
|
+
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
760
|
+
if (isToolResultOnly) {
|
|
761
|
+
const pending = new Map(Object.entries(state.pendingToolUses));
|
|
762
|
+
const toolResultBlocks = extractToolResultBlocks(msg?.content, pending);
|
|
763
|
+
if (toolResultBlocks.length > 0) {
|
|
764
|
+
metadata.toolResults = toolResultBlocks;
|
|
765
|
+
for (const block of toolResultBlocks) delete state.pendingToolUses[block.toolUseId];
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
if (entry.teamName) {
|
|
769
|
+
metadata.teamName = entry.teamName;
|
|
770
|
+
if (!state.teamInfo[metadata.teamName] && content) {
|
|
771
|
+
const info = parseTeammateMessageTag(content);
|
|
772
|
+
if (info) state.teamInfo[metadata.teamName] = info;
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
const thinkingContent = thinking?.content || void 0;
|
|
776
|
+
const thinkingSignature = thinking?.signature || void 0;
|
|
777
|
+
const hasMetadata = Object.keys(metadata).length > 0;
|
|
778
|
+
return {
|
|
779
|
+
role: type,
|
|
780
|
+
text: content || "",
|
|
781
|
+
timestamp: entry.timestamp || "",
|
|
782
|
+
uuid: entry.uuid || void 0,
|
|
783
|
+
metadata: hasMetadata ? metadata : void 0,
|
|
784
|
+
isToolResult: isToolResultOnly || void 0,
|
|
785
|
+
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
786
|
+
thinkingContent,
|
|
787
|
+
thinkingSignature,
|
|
788
|
+
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
789
|
+
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
790
|
+
promptId: type === "user" ? entry.promptId : void 0,
|
|
791
|
+
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
792
|
+
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
793
|
+
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
794
|
+
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
795
|
+
};
|
|
796
|
+
}
|
|
797
|
+
function parseJsonlLine(line, state = initialConvState()) {
|
|
798
|
+
const text = line.trimEnd();
|
|
799
|
+
if (text.trim().length === 0) return null;
|
|
800
|
+
let entry;
|
|
801
|
+
try {
|
|
802
|
+
entry = JSON.parse(text);
|
|
803
|
+
} catch {
|
|
804
|
+
return null;
|
|
805
|
+
}
|
|
806
|
+
return reduceConvLine(state, entry);
|
|
807
|
+
}
|
|
808
|
+
function applyTeamInfo(messages, state) {
|
|
809
|
+
if (Object.keys(state.teamInfo).length === 0) return;
|
|
810
|
+
for (const m of messages) {
|
|
811
|
+
const name = m.metadata?.teamName;
|
|
812
|
+
if (name && state.teamInfo[name] && m.metadata) m.metadata.teamInfo = state.teamInfo[name];
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
// src/persistent/sidecar.ts
|
|
817
|
+
var import_fs3 = require("fs");
|
|
818
|
+
var SIDECAR_VERSION = 1;
|
|
819
|
+
function sidecarPath(jsonlPath) {
|
|
820
|
+
return `${jsonlPath}.idx.json`;
|
|
821
|
+
}
|
|
822
|
+
function buildSidecar(meta, cursor, updatedAt) {
|
|
823
|
+
return {
|
|
824
|
+
version: SIDECAR_VERSION,
|
|
825
|
+
sourcePath: meta.filePath,
|
|
826
|
+
sizeBytes: cursor.sizeBytes,
|
|
827
|
+
mtimeMs: cursor.mtimeMs,
|
|
828
|
+
lastIndexedOffset: cursor.offset,
|
|
829
|
+
lastIndexedLine: cursor.line,
|
|
830
|
+
messageCount: meta.messageCount,
|
|
831
|
+
projectPath: meta.projectPath,
|
|
832
|
+
projectName: meta.projectName,
|
|
833
|
+
branch: meta.gitBranch,
|
|
834
|
+
firstSentAt: meta.firstMessage?.timestamp ?? null,
|
|
835
|
+
firstSentText: meta.firstMessage?.text ?? null,
|
|
836
|
+
lastSentAt: meta.lastMessage?.timestamp ?? null,
|
|
837
|
+
lastSentText: meta.lastMessage?.text ?? null,
|
|
838
|
+
updatedAt
|
|
839
|
+
};
|
|
840
|
+
}
|
|
841
|
+
function writeSidecar(jsonlPath, sidecar) {
|
|
842
|
+
try {
|
|
843
|
+
(0, import_fs3.writeFileSync)(sidecarPath(jsonlPath), JSON.stringify(sidecar, null, 2));
|
|
844
|
+
} catch (err) {
|
|
845
|
+
getLogger().warn({ jsonlPath, err }, "sidecar: write failed");
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
function readSidecar(jsonlPath) {
|
|
849
|
+
try {
|
|
850
|
+
return JSON.parse((0, import_fs3.readFileSync)(sidecarPath(jsonlPath), "utf-8"));
|
|
851
|
+
} catch {
|
|
852
|
+
return null;
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
// src/profiles.ts
|
|
857
|
+
var import_promises = require("fs/promises");
|
|
858
|
+
var import_os = require("os");
|
|
859
|
+
var import_path4 = require("path");
|
|
860
|
+
var PROFILES_FILE = "profiles.json";
|
|
861
|
+
function resolveConfigDir(configDir) {
|
|
862
|
+
return configDir.replace(/^~/, (0, import_os.homedir)());
|
|
863
|
+
}
|
|
864
|
+
function getProjectsDir(profile) {
|
|
865
|
+
return (0, import_path4.join)(resolveConfigDir(profile.configDir), "projects");
|
|
866
|
+
}
|
|
867
|
+
async function detectDefaultProfile() {
|
|
868
|
+
return {
|
|
869
|
+
id: "default",
|
|
870
|
+
label: "Default",
|
|
871
|
+
configDir: (0, import_path4.join)((0, import_os.homedir)(), ".claude"),
|
|
872
|
+
enabled: true,
|
|
873
|
+
emoji: "\u{1F916}"
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
async function loadProfiles(configPath) {
|
|
877
|
+
const log = getLogger();
|
|
878
|
+
try {
|
|
879
|
+
const resolved = resolveConfigDir(configPath);
|
|
880
|
+
const data = await (0, import_promises.readFile)((0, import_path4.join)(resolved, PROFILES_FILE), "utf-8");
|
|
881
|
+
const profiles = JSON.parse(data);
|
|
882
|
+
log.debug({ configPath, count: profiles.length }, "profiles: loaded");
|
|
883
|
+
return profiles;
|
|
884
|
+
} catch (err) {
|
|
885
|
+
log.debug({ configPath, err }, "profiles: load failed, using default");
|
|
886
|
+
const defaultProfile = await detectDefaultProfile();
|
|
887
|
+
return [defaultProfile];
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
async function saveProfiles(profiles, configPath) {
|
|
891
|
+
const resolved = resolveConfigDir(configPath);
|
|
892
|
+
await (0, import_promises.mkdir)(resolved, { recursive: true });
|
|
893
|
+
await (0, import_promises.writeFile)((0, import_path4.join)(resolved, PROFILES_FILE), JSON.stringify(profiles, null, 2));
|
|
894
|
+
getLogger().debug({ configPath, count: profiles.length }, "profiles: saved");
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
// src/providers/codex-cli.ts
|
|
898
|
+
var import_fast_glob = __toESM(require("fast-glob"), 1);
|
|
899
|
+
var import_fs4 = require("fs");
|
|
900
|
+
var import_promises2 = require("fs/promises");
|
|
901
|
+
var import_path5 = require("path");
|
|
902
|
+
var import_readline2 = require("readline");
|
|
903
|
+
var CodexCliProvider = class {
|
|
904
|
+
name = CODEX_CLI_PROVIDER;
|
|
905
|
+
async discover(roots) {
|
|
906
|
+
const log = getLogger();
|
|
907
|
+
const results = [];
|
|
908
|
+
for (const root of roots) {
|
|
909
|
+
let paths;
|
|
910
|
+
try {
|
|
911
|
+
paths = await (0, import_fast_glob.default)(["**/rollout-*.jsonl", "**/*.jsonl"], {
|
|
912
|
+
cwd: root,
|
|
913
|
+
absolute: true,
|
|
914
|
+
dot: false,
|
|
915
|
+
unique: true
|
|
916
|
+
});
|
|
917
|
+
} catch (err) {
|
|
918
|
+
log.warn({ root, err }, "codex discovery: glob failed");
|
|
919
|
+
continue;
|
|
920
|
+
}
|
|
921
|
+
for (const filePath of paths) {
|
|
922
|
+
try {
|
|
923
|
+
const s = await (0, import_promises2.stat)(filePath);
|
|
924
|
+
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
925
|
+
} catch (err) {
|
|
926
|
+
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
return results;
|
|
931
|
+
}
|
|
932
|
+
// Codex rollout lines carry distinctive top-level types.
|
|
933
|
+
canParse(_filePath, sample) {
|
|
934
|
+
for (const line of sample.split("\n")) {
|
|
935
|
+
if (!line.trim()) continue;
|
|
936
|
+
try {
|
|
937
|
+
const e = JSON.parse(line);
|
|
938
|
+
if (e.type === "session_meta" || e.type === "response_item" || e.type === "event_msg") {
|
|
939
|
+
return true;
|
|
940
|
+
}
|
|
941
|
+
if (e.type === "user" || e.type === "assistant") return false;
|
|
942
|
+
} catch {
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
return false;
|
|
946
|
+
}
|
|
947
|
+
createEmptyAccumulator() {
|
|
948
|
+
return {
|
|
949
|
+
sessionId: "",
|
|
950
|
+
cwd: "",
|
|
951
|
+
gitBranch: null,
|
|
952
|
+
model: null,
|
|
953
|
+
latestTimestamp: "",
|
|
954
|
+
messageCount: 0,
|
|
955
|
+
lastMessageSender: "user",
|
|
956
|
+
firstUser: null,
|
|
957
|
+
lastUser: null,
|
|
958
|
+
lastAssistant: null,
|
|
959
|
+
toolNames: [],
|
|
960
|
+
previewParts: [],
|
|
961
|
+
previewLength: 0,
|
|
962
|
+
snippetParts: [],
|
|
963
|
+
snippetLength: 0
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
reduceEntry(acc, entry, tier) {
|
|
967
|
+
reduceCodexEntry(acc, entry, tier);
|
|
968
|
+
}
|
|
969
|
+
finalize(acc, filePath, account, tier) {
|
|
970
|
+
return finalizeCodexMeta(acc, filePath, account, tier);
|
|
971
|
+
}
|
|
972
|
+
};
|
|
973
|
+
var asString = (v) => typeof v === "string" ? v : "";
|
|
974
|
+
function extractCodexText(content) {
|
|
975
|
+
if (typeof content === "string") return cleanSystemTags(content);
|
|
976
|
+
if (!Array.isArray(content)) return "";
|
|
977
|
+
return content.map((item) => {
|
|
978
|
+
if (typeof item === "string") return item;
|
|
979
|
+
const t = item?.type;
|
|
980
|
+
if ((t === "input_text" || t === "output_text" || t === "text") && item?.text) {
|
|
981
|
+
return item.text;
|
|
982
|
+
}
|
|
983
|
+
return "";
|
|
984
|
+
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
985
|
+
}
|
|
986
|
+
function reduceCodexEntry(acc, entry, tier) {
|
|
987
|
+
const ts = asString(entry.timestamp);
|
|
988
|
+
if (ts && (!acc.latestTimestamp || ts > acc.latestTimestamp)) acc.latestTimestamp = ts;
|
|
989
|
+
const payload = entry.payload;
|
|
990
|
+
if (!payload || typeof payload !== "object") return;
|
|
991
|
+
const type = entry.type;
|
|
992
|
+
if (type === "session_meta") {
|
|
993
|
+
if (!acc.sessionId) acc.sessionId = asString(payload.id);
|
|
994
|
+
if (!acc.cwd) acc.cwd = asString(payload.cwd);
|
|
995
|
+
const git = payload.git;
|
|
996
|
+
if (acc.gitBranch === null && git?.branch) acc.gitBranch = asString(git.branch) || null;
|
|
997
|
+
return;
|
|
998
|
+
}
|
|
999
|
+
if (acc.model === null && payload.model) acc.model = asString(payload.model) || null;
|
|
1000
|
+
if (type !== "response_item") return;
|
|
1001
|
+
const ptype = payload.type;
|
|
1002
|
+
if (ptype === "function_call" || ptype === "custom_tool_call") {
|
|
1003
|
+
const name = asString(payload.name);
|
|
1004
|
+
if (name && !acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
1007
|
+
if (ptype !== "message") return;
|
|
1008
|
+
const role = payload.role;
|
|
1009
|
+
if (role !== "user" && role !== "assistant") return;
|
|
1010
|
+
const text = extractCodexText(payload.content);
|
|
1011
|
+
if (!text) return;
|
|
1012
|
+
const sender = role;
|
|
1013
|
+
acc.messageCount++;
|
|
1014
|
+
acc.lastMessageSender = sender;
|
|
1015
|
+
const snapshot = { text: text.slice(0, 200), timestamp: ts };
|
|
1016
|
+
if (sender === "user") {
|
|
1017
|
+
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
1018
|
+
acc.lastUser = snapshot;
|
|
1019
|
+
} else {
|
|
1020
|
+
acc.lastAssistant = snapshot;
|
|
1021
|
+
}
|
|
1022
|
+
if (acc.previewLength < tier.previewMax) {
|
|
1023
|
+
acc.previewParts.push(text);
|
|
1024
|
+
acc.previewLength += text.length;
|
|
1025
|
+
}
|
|
1026
|
+
if (acc.snippetLength < tier.snippetMax) {
|
|
1027
|
+
const remaining = tier.snippetMax - acc.snippetLength;
|
|
1028
|
+
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
1029
|
+
acc.snippetParts.push(chunk);
|
|
1030
|
+
acc.snippetLength += chunk.length;
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
1034
|
+
if (acc.messageCount === 0) return null;
|
|
1035
|
+
const sessionId = acc.sessionId || (0, import_path5.basename)(filePath, ".jsonl");
|
|
1036
|
+
const projectPath = acc.cwd;
|
|
1037
|
+
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
1038
|
+
return {
|
|
1039
|
+
id: filePath,
|
|
1040
|
+
filePath,
|
|
1041
|
+
provider: CODEX_CLI_PROVIDER,
|
|
1042
|
+
kind,
|
|
1043
|
+
externalSessionId: acc.sessionId || void 0,
|
|
1044
|
+
sessionId,
|
|
1045
|
+
sessionName: "",
|
|
1046
|
+
projectPath,
|
|
1047
|
+
projectName: getShortProjectName3(projectPath),
|
|
1048
|
+
account,
|
|
1049
|
+
timestamp: acc.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1050
|
+
messageCount: acc.messageCount,
|
|
1051
|
+
lastMessageSender: acc.lastMessageSender,
|
|
1052
|
+
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
1053
|
+
contentSnippet: acc.snippetParts.join(" "),
|
|
1054
|
+
gitBranch: acc.gitBranch,
|
|
1055
|
+
model: acc.model,
|
|
1056
|
+
isSubagent: false,
|
|
1057
|
+
parentSessionId: null,
|
|
1058
|
+
isTeammate: false,
|
|
1059
|
+
teamName: null,
|
|
1060
|
+
toolNames: acc.toolNames,
|
|
1061
|
+
firstMessage: acc.firstUser,
|
|
1062
|
+
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
1063
|
+
lastPrompt: acc.lastUser?.text || void 0
|
|
1064
|
+
};
|
|
1065
|
+
}
|
|
1066
|
+
function getShortProjectName3(fullPath) {
|
|
1067
|
+
return fullPath.split("/").filter(Boolean).slice(-3).join("/");
|
|
1068
|
+
}
|
|
1069
|
+
async function parseCodexConversation(filePath, account) {
|
|
1070
|
+
const log = getLogger();
|
|
1071
|
+
const messages = [];
|
|
1072
|
+
const textParts = [];
|
|
1073
|
+
let sessionId = "";
|
|
1074
|
+
let cwd = "";
|
|
1075
|
+
let latestTimestamp = "";
|
|
1076
|
+
let lastUserText = "";
|
|
1077
|
+
const rl = (0, import_readline2.createInterface)({ input: (0, import_fs4.createReadStream)(filePath), crlfDelay: Infinity });
|
|
1078
|
+
try {
|
|
1079
|
+
for await (const line of rl) {
|
|
1080
|
+
if (!line.trim()) continue;
|
|
1081
|
+
let entry;
|
|
1082
|
+
try {
|
|
1083
|
+
entry = JSON.parse(line);
|
|
1084
|
+
} catch {
|
|
1085
|
+
continue;
|
|
1086
|
+
}
|
|
1087
|
+
const ts = asString(entry.timestamp);
|
|
1088
|
+
if (ts && (!latestTimestamp || ts > latestTimestamp)) latestTimestamp = ts;
|
|
1089
|
+
const payload = entry.payload;
|
|
1090
|
+
if (!payload || typeof payload !== "object") continue;
|
|
1091
|
+
if (entry.type === "session_meta") {
|
|
1092
|
+
if (!sessionId) sessionId = asString(payload.id);
|
|
1093
|
+
if (!cwd) cwd = asString(payload.cwd);
|
|
1094
|
+
continue;
|
|
1095
|
+
}
|
|
1096
|
+
if (entry.type !== "response_item" || payload.type !== "message") continue;
|
|
1097
|
+
const role = payload.role;
|
|
1098
|
+
if (role !== "user" && role !== "assistant") continue;
|
|
1099
|
+
const text = extractCodexText(payload.content);
|
|
1100
|
+
if (!text) continue;
|
|
1101
|
+
messages.push({ role, text, timestamp: ts });
|
|
1102
|
+
textParts.push(text);
|
|
1103
|
+
if (role === "user") lastUserText = text;
|
|
1104
|
+
}
|
|
1105
|
+
} catch (err) {
|
|
1106
|
+
log.warn({ filePath, err }, "parseCodexConversation: read failed");
|
|
1107
|
+
return null;
|
|
1108
|
+
}
|
|
1109
|
+
if (messages.length === 0) return null;
|
|
1110
|
+
return {
|
|
1111
|
+
id: filePath,
|
|
1112
|
+
filePath,
|
|
1113
|
+
projectPath: cwd,
|
|
1114
|
+
projectName: getShortProjectName3(cwd),
|
|
1115
|
+
sessionId: sessionId || (0, import_path5.basename)(filePath, ".jsonl"),
|
|
1116
|
+
sessionName: "",
|
|
1117
|
+
messages,
|
|
1118
|
+
fullText: textParts.join(" "),
|
|
1119
|
+
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1120
|
+
messageCount: messages.length,
|
|
1121
|
+
account,
|
|
1122
|
+
lastPrompt: lastUserText || void 0
|
|
1123
|
+
};
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
// src/discovery.ts
|
|
1127
|
+
var import_fast_glob2 = __toESM(require("fast-glob"), 1);
|
|
1128
|
+
var import_promises3 = require("fs/promises");
|
|
1129
|
+
var EXCLUDED_SEGMENTS = ["/memory/", "/tool-results/"];
|
|
1130
|
+
var STAT_CONCURRENCY = 32;
|
|
1131
|
+
async function discoverJsonlFiles(dirs, onProgress) {
|
|
1132
|
+
const log = getLogger();
|
|
1133
|
+
const results = [];
|
|
1134
|
+
for (const { projectsDir, account } of dirs) {
|
|
1135
|
+
let filePaths;
|
|
1136
|
+
try {
|
|
1137
|
+
filePaths = await (0, import_fast_glob2.default)("**/*.jsonl", {
|
|
1138
|
+
cwd: projectsDir,
|
|
1139
|
+
absolute: true,
|
|
1140
|
+
dot: false
|
|
1141
|
+
});
|
|
1142
|
+
} catch (err) {
|
|
1143
|
+
log.warn({ projectsDir, account, err }, "discovery: glob failed");
|
|
1144
|
+
continue;
|
|
1145
|
+
}
|
|
1146
|
+
const filtered = filePaths.filter((fp) => !EXCLUDED_SEGMENTS.some((seg) => fp.includes(seg)));
|
|
1147
|
+
let kept = 0;
|
|
1148
|
+
let skippedEmpty = 0;
|
|
1149
|
+
let skippedInaccessible = 0;
|
|
1150
|
+
for (let i = 0; i < filtered.length; i += STAT_CONCURRENCY) {
|
|
1151
|
+
const chunk = filtered.slice(i, i + STAT_CONCURRENCY);
|
|
1152
|
+
const statted = await Promise.all(
|
|
1153
|
+
chunk.map(async (filePath) => {
|
|
1154
|
+
try {
|
|
1155
|
+
const s = await (0, import_promises3.stat)(filePath);
|
|
1156
|
+
return { filePath, size: s.size };
|
|
1157
|
+
} catch (err) {
|
|
1158
|
+
log.warn({ filePath, err }, "discovery: stat failed");
|
|
1159
|
+
return { filePath, size: -1 };
|
|
1160
|
+
}
|
|
1161
|
+
})
|
|
1162
|
+
);
|
|
1163
|
+
for (const { filePath, size } of statted) {
|
|
1164
|
+
if (size < 0) {
|
|
1165
|
+
skippedInaccessible++;
|
|
1166
|
+
} else if (size > 0) {
|
|
1167
|
+
results.push({ filePath, account });
|
|
1168
|
+
kept++;
|
|
1169
|
+
} else {
|
|
1170
|
+
skippedEmpty++;
|
|
1171
|
+
}
|
|
1134
1172
|
}
|
|
1135
1173
|
}
|
|
1174
|
+
log.debug(
|
|
1175
|
+
{
|
|
1176
|
+
projectsDir,
|
|
1177
|
+
account,
|
|
1178
|
+
globMatches: filePaths.length,
|
|
1179
|
+
afterExclusions: filtered.length,
|
|
1180
|
+
kept,
|
|
1181
|
+
skippedEmpty,
|
|
1182
|
+
skippedInaccessible
|
|
1183
|
+
},
|
|
1184
|
+
"discovery: directory scanned"
|
|
1185
|
+
);
|
|
1186
|
+
onProgress?.(results.length);
|
|
1136
1187
|
}
|
|
1188
|
+
log.debug({ totalFiles: results.length, dirs: dirs.length }, "discovery: complete");
|
|
1189
|
+
return results;
|
|
1137
1190
|
}
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1191
|
+
|
|
1192
|
+
// src/providers/threadbase.ts
|
|
1193
|
+
var ThreadbaseProvider = class {
|
|
1194
|
+
name = CLAUDE_CODE_PROVIDER;
|
|
1195
|
+
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
1196
|
+
// per-root account through the shared interface. The scanner builds these.
|
|
1197
|
+
async discover(roots) {
|
|
1198
|
+
const dirs = roots.map((r) => {
|
|
1199
|
+
const [projectsDir, account = "default"] = r.split("\0");
|
|
1200
|
+
return { projectsDir, account };
|
|
1201
|
+
});
|
|
1202
|
+
return discoverJsonlFiles(dirs);
|
|
1145
1203
|
}
|
|
1146
|
-
|
|
1204
|
+
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
1205
|
+
canParse(_filePath, sample) {
|
|
1206
|
+
for (const line of sample.split("\n")) {
|
|
1207
|
+
if (!line.trim()) continue;
|
|
1208
|
+
try {
|
|
1209
|
+
const e = JSON.parse(line);
|
|
1210
|
+
if (e.type === "user" || e.type === "assistant") return true;
|
|
1211
|
+
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
1212
|
+
} catch {
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
return false;
|
|
1216
|
+
}
|
|
1217
|
+
createEmptyAccumulator() {
|
|
1218
|
+
return initialReducerState();
|
|
1219
|
+
}
|
|
1220
|
+
reduceEntry(acc, entry, tier) {
|
|
1221
|
+
reduceLine(acc, entry, tier);
|
|
1222
|
+
}
|
|
1223
|
+
finalize(acc, filePath, account, tier) {
|
|
1224
|
+
return finalizeMeta(acc, filePath, account, tier);
|
|
1225
|
+
}
|
|
1226
|
+
};
|
|
1227
|
+
|
|
1228
|
+
// src/scanner.ts
|
|
1229
|
+
var import_events = require("events");
|
|
1230
|
+
var import_fs10 = require("fs");
|
|
1231
|
+
var import_os2 = require("os");
|
|
1232
|
+
var import_path9 = require("path");
|
|
1233
|
+
|
|
1234
|
+
// src/cache.ts
|
|
1235
|
+
var LRUCache = class {
|
|
1236
|
+
map = /* @__PURE__ */ new Map();
|
|
1237
|
+
capacity;
|
|
1238
|
+
constructor(capacity) {
|
|
1239
|
+
this.capacity = capacity;
|
|
1240
|
+
}
|
|
1241
|
+
get(key) {
|
|
1242
|
+
const value = this.map.get(key);
|
|
1243
|
+
if (value === void 0) return void 0;
|
|
1244
|
+
this.map.delete(key);
|
|
1245
|
+
this.map.set(key, value);
|
|
1246
|
+
return value;
|
|
1247
|
+
}
|
|
1248
|
+
set(key, value) {
|
|
1249
|
+
this.map.delete(key);
|
|
1250
|
+
this.map.set(key, value);
|
|
1251
|
+
if (this.map.size > this.capacity) {
|
|
1252
|
+
const oldest = this.map.keys().next();
|
|
1253
|
+
if (!oldest.done) this.map.delete(oldest.value);
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
has(key) {
|
|
1257
|
+
return this.map.has(key);
|
|
1258
|
+
}
|
|
1259
|
+
delete(key) {
|
|
1260
|
+
return this.map.delete(key);
|
|
1261
|
+
}
|
|
1262
|
+
clear() {
|
|
1263
|
+
this.map.clear();
|
|
1264
|
+
}
|
|
1265
|
+
get size() {
|
|
1266
|
+
return this.map.size;
|
|
1267
|
+
}
|
|
1268
|
+
};
|
|
1269
|
+
|
|
1270
|
+
// src/persistent/conversation-stream.ts
|
|
1271
|
+
var import_path6 = require("path");
|
|
1272
|
+
|
|
1273
|
+
// src/persistent/paged-reader.ts
|
|
1274
|
+
var import_fs6 = require("fs");
|
|
1275
|
+
var import_promises5 = require("timers/promises");
|
|
1276
|
+
|
|
1277
|
+
// src/persistent/jsonl-tail-reader.ts
|
|
1278
|
+
var import_fs5 = require("fs");
|
|
1279
|
+
var import_promises4 = require("timers/promises");
|
|
1280
|
+
var YIELD_EVERY_LINES = 500;
|
|
1281
|
+
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1282
|
+
const stream = (0, import_fs5.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1283
|
+
let buffer = "";
|
|
1284
|
+
let offset = startOffset;
|
|
1285
|
+
let line = startLine;
|
|
1286
|
+
let parsedLines = 0;
|
|
1287
|
+
let sinceYield = 0;
|
|
1288
|
+
for await (const chunk of stream) {
|
|
1289
|
+
buffer += chunk;
|
|
1290
|
+
let nl;
|
|
1291
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1292
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1293
|
+
const text = lineWithNewline.trimEnd();
|
|
1294
|
+
buffer = buffer.slice(nl + 1);
|
|
1295
|
+
if (text.length > 0) {
|
|
1296
|
+
try {
|
|
1297
|
+
reduceLine(state, JSON.parse(text), tier);
|
|
1298
|
+
} catch {
|
|
1299
|
+
state.badJsonLines++;
|
|
1300
|
+
}
|
|
1301
|
+
parsedLines++;
|
|
1302
|
+
}
|
|
1303
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1304
|
+
line++;
|
|
1305
|
+
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
1306
|
+
sinceYield = 0;
|
|
1307
|
+
await (0, import_promises4.setImmediate)();
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
// src/persistent/paged-reader.ts
|
|
1315
|
+
var CHECKPOINT_INTERVAL = 500;
|
|
1316
|
+
async function streamMessages(filePath, startOffset, startLine, state, onMessage, onEntry) {
|
|
1317
|
+
const stream = (0, import_fs6.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1318
|
+
let buffer = "";
|
|
1319
|
+
let offset = startOffset;
|
|
1320
|
+
let line = startLine;
|
|
1321
|
+
let sinceYield = 0;
|
|
1322
|
+
for await (const chunk of stream) {
|
|
1323
|
+
buffer += chunk;
|
|
1324
|
+
let nl;
|
|
1325
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1326
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1327
|
+
const text = lineWithNewline.trimEnd();
|
|
1328
|
+
buffer = buffer.slice(nl + 1);
|
|
1329
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1330
|
+
line += 1;
|
|
1331
|
+
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
1332
|
+
sinceYield = 0;
|
|
1333
|
+
await (0, import_promises5.setImmediate)();
|
|
1334
|
+
}
|
|
1335
|
+
if (text.length === 0) continue;
|
|
1336
|
+
let entry;
|
|
1337
|
+
try {
|
|
1338
|
+
entry = JSON.parse(text);
|
|
1339
|
+
} catch {
|
|
1340
|
+
continue;
|
|
1341
|
+
}
|
|
1342
|
+
if (onEntry?.(entry)) continue;
|
|
1343
|
+
const message = reduceConvLine(state, entry);
|
|
1344
|
+
if (message && onMessage(message, offset, line)) {
|
|
1345
|
+
stream.destroy();
|
|
1346
|
+
return { offset, line };
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
return { offset, line };
|
|
1351
|
+
}
|
|
1352
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
1353
|
+
const checkpoints = [];
|
|
1354
|
+
const state = from ? from.state : initialConvState();
|
|
1355
|
+
let index = from ? from.messageIndex : 0;
|
|
1356
|
+
await streamMessages(
|
|
1357
|
+
filePath,
|
|
1358
|
+
from?.byteOffset ?? 0,
|
|
1359
|
+
from?.lineNumber ?? 0,
|
|
1360
|
+
state,
|
|
1361
|
+
(_msg, nextOffset, nextLine) => {
|
|
1362
|
+
index += 1;
|
|
1363
|
+
if (index % interval === 0) {
|
|
1364
|
+
checkpoints.push({
|
|
1365
|
+
messageIndex: index,
|
|
1366
|
+
byteOffset: nextOffset,
|
|
1367
|
+
lineNumber: nextLine,
|
|
1368
|
+
state: structuredClone(state)
|
|
1369
|
+
});
|
|
1370
|
+
}
|
|
1371
|
+
return false;
|
|
1372
|
+
}
|
|
1373
|
+
);
|
|
1374
|
+
return checkpoints;
|
|
1375
|
+
}
|
|
1376
|
+
async function readPage(filePath, total, options, floor) {
|
|
1377
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
1378
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1379
|
+
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1380
|
+
const startOffset = floor ? floor.byteOffset : 0;
|
|
1381
|
+
const startLine = floor ? floor.lineNumber : 0;
|
|
1382
|
+
let index = floor ? floor.messageIndex : 0;
|
|
1383
|
+
const window = [];
|
|
1384
|
+
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1385
|
+
const current = index;
|
|
1386
|
+
index += 1;
|
|
1387
|
+
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1388
|
+
return index >= beforeIndex;
|
|
1389
|
+
});
|
|
1390
|
+
applyTeamInfo(window, state);
|
|
1391
|
+
return { messages: window, total, fromIndex };
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
// src/persistent/conversation-stream.ts
|
|
1395
|
+
async function foldTail(filePath, resume) {
|
|
1396
|
+
const messages = [];
|
|
1397
|
+
const textParts = [];
|
|
1398
|
+
const turnDurations = [];
|
|
1399
|
+
const end = await streamMessages(
|
|
1400
|
+
filePath,
|
|
1401
|
+
resume.offset,
|
|
1402
|
+
resume.line,
|
|
1403
|
+
resume.state,
|
|
1404
|
+
(message) => {
|
|
1405
|
+
messages.push(message);
|
|
1406
|
+
if (message.text) textParts.push(message.text);
|
|
1407
|
+
return false;
|
|
1408
|
+
},
|
|
1409
|
+
(entry) => {
|
|
1410
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
1411
|
+
turnDurations.push({
|
|
1412
|
+
durationMs: entry.durationMs,
|
|
1413
|
+
messageCount: entry.messageCount || 0,
|
|
1414
|
+
uuid: entry.uuid
|
|
1415
|
+
});
|
|
1416
|
+
return true;
|
|
1417
|
+
}
|
|
1418
|
+
return false;
|
|
1419
|
+
}
|
|
1420
|
+
);
|
|
1421
|
+
return { messages, textParts, turnDurations, end };
|
|
1422
|
+
}
|
|
1423
|
+
function assemble(filePath, account, messages, fullText, turnDurations, state) {
|
|
1147
1424
|
return {
|
|
1148
1425
|
id: filePath,
|
|
1149
1426
|
filePath,
|
|
1150
|
-
|
|
1151
|
-
|
|
1427
|
+
projectPath: state.cwd,
|
|
1428
|
+
projectName: getShortProjectName2(state.cwd),
|
|
1429
|
+
sessionId: state.sessionId || (0, import_path6.basename)(filePath, ".jsonl"),
|
|
1152
1430
|
sessionName: state.sessionName,
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
account,
|
|
1431
|
+
messages,
|
|
1432
|
+
fullText,
|
|
1156
1433
|
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1157
|
-
messageCount:
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
contentSnippet: state.snippetParts.join(" "),
|
|
1161
|
-
gitBranch: null,
|
|
1162
|
-
model: state.model,
|
|
1163
|
-
isSubagent,
|
|
1164
|
-
parentSessionId,
|
|
1165
|
-
isTeammate: state.isTeammate,
|
|
1166
|
-
teamName: state.teamName || null,
|
|
1167
|
-
toolNames: state.toolNames,
|
|
1168
|
-
firstMessage: state.firstMessage,
|
|
1169
|
-
lastMessage: state.lastMessage,
|
|
1434
|
+
messageCount: messages.length,
|
|
1435
|
+
account,
|
|
1436
|
+
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
1170
1437
|
lastPrompt: state.lastPrompt || void 0
|
|
1171
1438
|
};
|
|
1172
1439
|
}
|
|
1173
|
-
function
|
|
1174
|
-
const
|
|
1175
|
-
|
|
1440
|
+
async function parseConversationResumable(filePath, account) {
|
|
1441
|
+
const resume = { state: initialConvState(), offset: 0, line: 0 };
|
|
1442
|
+
const { messages, textParts, turnDurations, end } = await foldTail(filePath, resume);
|
|
1443
|
+
if (messages.length === 0) return null;
|
|
1444
|
+
applyTeamInfo(messages, resume.state);
|
|
1445
|
+
const conversation = assemble(
|
|
1446
|
+
filePath,
|
|
1447
|
+
account,
|
|
1448
|
+
messages,
|
|
1449
|
+
textParts.join(" "),
|
|
1450
|
+
turnDurations,
|
|
1451
|
+
resume.state
|
|
1452
|
+
);
|
|
1453
|
+
return { conversation, resume: { state: resume.state, offset: end.offset, line: end.line } };
|
|
1454
|
+
}
|
|
1455
|
+
async function extendConversation(previous, resume, filePath, account) {
|
|
1456
|
+
const { messages: fresh, textParts, turnDurations, end } = await foldTail(filePath, resume);
|
|
1457
|
+
const messages = fresh.length > 0 ? previous.messages.concat(fresh) : previous.messages;
|
|
1458
|
+
applyTeamInfo(messages, resume.state);
|
|
1459
|
+
const fullText = textParts.length === 0 ? previous.fullText : previous.fullText ? `${previous.fullText} ${textParts.join(" ")}` : textParts.join(" ");
|
|
1460
|
+
const allTurnDurations = (previous.turnDurations ?? []).concat(turnDurations);
|
|
1461
|
+
const conversation = assemble(
|
|
1462
|
+
filePath,
|
|
1463
|
+
account,
|
|
1464
|
+
messages,
|
|
1465
|
+
fullText,
|
|
1466
|
+
allTurnDurations,
|
|
1467
|
+
resume.state
|
|
1468
|
+
);
|
|
1469
|
+
return { conversation, resume: { state: resume.state, offset: end.offset, line: end.line } };
|
|
1176
1470
|
}
|
|
1177
1471
|
|
|
1178
|
-
// src/providers/threadbase.ts
|
|
1179
|
-
var ThreadbaseProvider = class {
|
|
1180
|
-
name = CLAUDE_CODE_PROVIDER;
|
|
1181
|
-
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
1182
|
-
// per-root account through the shared interface. The scanner builds these.
|
|
1183
|
-
async discover(roots) {
|
|
1184
|
-
const dirs = roots.map((r) => {
|
|
1185
|
-
const [projectsDir, account = "default"] = r.split("\0");
|
|
1186
|
-
return { projectsDir, account };
|
|
1187
|
-
});
|
|
1188
|
-
return discoverJsonlFiles(dirs);
|
|
1189
|
-
}
|
|
1190
|
-
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
1191
|
-
canParse(_filePath, sample) {
|
|
1192
|
-
for (const line of sample.split("\n")) {
|
|
1193
|
-
if (!line.trim()) continue;
|
|
1194
|
-
try {
|
|
1195
|
-
const e = JSON.parse(line);
|
|
1196
|
-
if (e.type === "user" || e.type === "assistant") return true;
|
|
1197
|
-
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
1198
|
-
} catch {
|
|
1199
|
-
}
|
|
1200
|
-
}
|
|
1201
|
-
return false;
|
|
1202
|
-
}
|
|
1203
|
-
createEmptyAccumulator() {
|
|
1204
|
-
return initialReducerState();
|
|
1205
|
-
}
|
|
1206
|
-
reduceEntry(acc, entry, tier) {
|
|
1207
|
-
reduceLine(acc, entry, tier);
|
|
1208
|
-
}
|
|
1209
|
-
finalize(acc, filePath, account, tier) {
|
|
1210
|
-
return finalizeMeta(acc, filePath, account, tier);
|
|
1211
|
-
}
|
|
1212
|
-
};
|
|
1213
|
-
|
|
1214
|
-
// src/scanner.ts
|
|
1215
|
-
var import_events = require("events");
|
|
1216
|
-
var import_fs10 = require("fs");
|
|
1217
|
-
var import_os2 = require("os");
|
|
1218
|
-
var import_path8 = require("path");
|
|
1219
|
-
|
|
1220
|
-
// src/cache.ts
|
|
1221
|
-
var LRUCache = class {
|
|
1222
|
-
map = /* @__PURE__ */ new Map();
|
|
1223
|
-
capacity;
|
|
1224
|
-
constructor(capacity) {
|
|
1225
|
-
this.capacity = capacity;
|
|
1226
|
-
}
|
|
1227
|
-
get(key) {
|
|
1228
|
-
const value = this.map.get(key);
|
|
1229
|
-
if (value === void 0) return void 0;
|
|
1230
|
-
this.map.delete(key);
|
|
1231
|
-
this.map.set(key, value);
|
|
1232
|
-
return value;
|
|
1233
|
-
}
|
|
1234
|
-
set(key, value) {
|
|
1235
|
-
this.map.delete(key);
|
|
1236
|
-
this.map.set(key, value);
|
|
1237
|
-
if (this.map.size > this.capacity) {
|
|
1238
|
-
const oldest = this.map.keys().next();
|
|
1239
|
-
if (!oldest.done) this.map.delete(oldest.value);
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
has(key) {
|
|
1243
|
-
return this.map.has(key);
|
|
1244
|
-
}
|
|
1245
|
-
delete(key) {
|
|
1246
|
-
return this.map.delete(key);
|
|
1247
|
-
}
|
|
1248
|
-
clear() {
|
|
1249
|
-
this.map.clear();
|
|
1250
|
-
}
|
|
1251
|
-
get size() {
|
|
1252
|
-
return this.map.size;
|
|
1253
|
-
}
|
|
1254
|
-
};
|
|
1255
|
-
|
|
1256
1472
|
// src/persistent/cursor.ts
|
|
1257
1473
|
var import_crypto = require("crypto");
|
|
1258
|
-
var
|
|
1474
|
+
var import_fs7 = require("fs");
|
|
1259
1475
|
var FP_BYTES = 4096;
|
|
1260
1476
|
function fingerprint(filePath, size) {
|
|
1261
1477
|
const hash = (0, import_crypto.createHash)("sha1");
|
|
1262
1478
|
hash.update(String(size));
|
|
1263
|
-
const fd = (0,
|
|
1479
|
+
const fd = (0, import_fs7.openSync)(filePath, "r");
|
|
1264
1480
|
try {
|
|
1265
1481
|
const head = Buffer.alloc(Math.min(FP_BYTES, size));
|
|
1266
1482
|
if (head.length > 0) {
|
|
1267
|
-
(0,
|
|
1483
|
+
(0, import_fs7.readSync)(fd, head, 0, head.length, 0);
|
|
1268
1484
|
hash.update(head);
|
|
1269
1485
|
}
|
|
1270
1486
|
if (size > FP_BYTES) {
|
|
1271
1487
|
const tailLen = Math.min(FP_BYTES, size);
|
|
1272
1488
|
const tail = Buffer.alloc(tailLen);
|
|
1273
|
-
(0,
|
|
1489
|
+
(0, import_fs7.readSync)(fd, tail, 0, tailLen, size - tailLen);
|
|
1274
1490
|
hash.update(tail);
|
|
1275
1491
|
}
|
|
1276
1492
|
} finally {
|
|
1277
|
-
(0,
|
|
1493
|
+
(0, import_fs7.closeSync)(fd);
|
|
1278
1494
|
}
|
|
1279
1495
|
return hash.digest("hex");
|
|
1280
1496
|
}
|
|
1281
1497
|
function classify(filePath, existing) {
|
|
1282
1498
|
let stat4;
|
|
1283
1499
|
try {
|
|
1284
|
-
const s = (0,
|
|
1500
|
+
const s = (0, import_fs7.statSync)(filePath);
|
|
1285
1501
|
stat4 = { size: s.size, mtimeMs: s.mtimeMs };
|
|
1286
1502
|
} catch {
|
|
1287
1503
|
return { change: "vanished" };
|
|
@@ -1309,13 +1525,13 @@ function classify(filePath, existing) {
|
|
|
1309
1525
|
}
|
|
1310
1526
|
|
|
1311
1527
|
// src/providers/parse.ts
|
|
1312
|
-
var
|
|
1528
|
+
var import_fs8 = require("fs");
|
|
1313
1529
|
var import_readline3 = require("readline");
|
|
1314
1530
|
async function parseMetaWithProvider(provider, filePath, account, tier) {
|
|
1315
1531
|
const log = getLogger();
|
|
1316
1532
|
const acc = provider.createEmptyAccumulator();
|
|
1317
1533
|
const rl = (0, import_readline3.createInterface)({
|
|
1318
|
-
input: (0,
|
|
1534
|
+
input: (0, import_fs8.createReadStream)(filePath),
|
|
1319
1535
|
crlfDelay: Infinity
|
|
1320
1536
|
});
|
|
1321
1537
|
try {
|
|
@@ -1357,8 +1573,8 @@ function resolveTier(tierName, customTiers) {
|
|
|
1357
1573
|
|
|
1358
1574
|
// src/persistent/db.ts
|
|
1359
1575
|
var import_better_sqlite3 = __toESM(require("better-sqlite3"), 1);
|
|
1360
|
-
var
|
|
1361
|
-
var
|
|
1576
|
+
var import_fs9 = require("fs");
|
|
1577
|
+
var import_path7 = require("path");
|
|
1362
1578
|
|
|
1363
1579
|
// src/persistent/schema.ts
|
|
1364
1580
|
var SCHEMA_VERSION = 4;
|
|
@@ -1551,7 +1767,7 @@ function hasColumn(db, table, column) {
|
|
|
1551
1767
|
// src/persistent/db.ts
|
|
1552
1768
|
function openDatabase(dbPath) {
|
|
1553
1769
|
if (dbPath !== ":memory:") {
|
|
1554
|
-
(0,
|
|
1770
|
+
(0, import_fs9.mkdirSync)((0, import_path7.dirname)(dbPath), { recursive: true });
|
|
1555
1771
|
}
|
|
1556
1772
|
const db = new import_better_sqlite3.default(dbPath);
|
|
1557
1773
|
db.pragma("journal_mode = WAL");
|
|
@@ -1564,7 +1780,7 @@ function openDatabase(dbPath) {
|
|
|
1564
1780
|
}
|
|
1565
1781
|
|
|
1566
1782
|
// src/persistent/dir-watermark.ts
|
|
1567
|
-
var
|
|
1783
|
+
var import_promises6 = require("fs/promises");
|
|
1568
1784
|
var FULL_RECONCILE_EVERY_N_SCANS = 20;
|
|
1569
1785
|
async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
1570
1786
|
const log = getLogger();
|
|
@@ -1580,7 +1796,7 @@ async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
|
1580
1796
|
for (const projectDir of resolved.entries) {
|
|
1581
1797
|
let dirStat;
|
|
1582
1798
|
try {
|
|
1583
|
-
dirStat = await (0,
|
|
1799
|
+
dirStat = await (0, import_promises6.stat)(projectDir);
|
|
1584
1800
|
} catch {
|
|
1585
1801
|
scannedDirs.remove(projectDir);
|
|
1586
1802
|
continue;
|
|
@@ -1618,7 +1834,7 @@ async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
|
1618
1834
|
async function resolveProjectDirs(projectsDir, scannedDirs) {
|
|
1619
1835
|
let rootStat;
|
|
1620
1836
|
try {
|
|
1621
|
-
rootStat = await (0,
|
|
1837
|
+
rootStat = await (0, import_promises6.stat)(projectsDir);
|
|
1622
1838
|
} catch {
|
|
1623
1839
|
return null;
|
|
1624
1840
|
}
|
|
@@ -1628,7 +1844,7 @@ async function resolveProjectDirs(projectsDir, scannedDirs) {
|
|
|
1628
1844
|
}
|
|
1629
1845
|
let entries;
|
|
1630
1846
|
try {
|
|
1631
|
-
entries = (await (0,
|
|
1847
|
+
entries = (await (0, import_promises6.readdir)(projectsDir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => joinPath(projectsDir, e.name));
|
|
1632
1848
|
} catch {
|
|
1633
1849
|
return null;
|
|
1634
1850
|
}
|
|
@@ -1642,104 +1858,6 @@ function joinPath(dir, name) {
|
|
|
1642
1858
|
return dir.endsWith("/") ? `${dir}${name}` : `${dir}/${name}`;
|
|
1643
1859
|
}
|
|
1644
1860
|
|
|
1645
|
-
// src/persistent/jsonl-tail-reader.ts
|
|
1646
|
-
var import_fs8 = require("fs");
|
|
1647
|
-
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1648
|
-
const stream = (0, import_fs8.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1649
|
-
let buffer = "";
|
|
1650
|
-
let offset = startOffset;
|
|
1651
|
-
let line = startLine;
|
|
1652
|
-
let parsedLines = 0;
|
|
1653
|
-
for await (const chunk of stream) {
|
|
1654
|
-
buffer += chunk;
|
|
1655
|
-
let nl;
|
|
1656
|
-
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1657
|
-
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1658
|
-
const text = lineWithNewline.trimEnd();
|
|
1659
|
-
buffer = buffer.slice(nl + 1);
|
|
1660
|
-
if (text.length > 0) {
|
|
1661
|
-
try {
|
|
1662
|
-
reduceLine(state, JSON.parse(text), tier);
|
|
1663
|
-
} catch {
|
|
1664
|
-
state.badJsonLines++;
|
|
1665
|
-
}
|
|
1666
|
-
parsedLines++;
|
|
1667
|
-
}
|
|
1668
|
-
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1669
|
-
line++;
|
|
1670
|
-
}
|
|
1671
|
-
}
|
|
1672
|
-
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
|
-
// src/persistent/paged-reader.ts
|
|
1676
|
-
var import_fs9 = require("fs");
|
|
1677
|
-
var CHECKPOINT_INTERVAL = 500;
|
|
1678
|
-
async function streamMessages(filePath, startOffset, startLine, state, onMessage) {
|
|
1679
|
-
const stream = (0, import_fs9.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1680
|
-
let buffer = "";
|
|
1681
|
-
let offset = startOffset;
|
|
1682
|
-
let line = startLine;
|
|
1683
|
-
for await (const chunk of stream) {
|
|
1684
|
-
buffer += chunk;
|
|
1685
|
-
let nl;
|
|
1686
|
-
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1687
|
-
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1688
|
-
const text = lineWithNewline.trimEnd();
|
|
1689
|
-
buffer = buffer.slice(nl + 1);
|
|
1690
|
-
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1691
|
-
line += 1;
|
|
1692
|
-
if (text.length === 0) continue;
|
|
1693
|
-
let entry;
|
|
1694
|
-
try {
|
|
1695
|
-
entry = JSON.parse(text);
|
|
1696
|
-
} catch {
|
|
1697
|
-
continue;
|
|
1698
|
-
}
|
|
1699
|
-
const message = reduceConvLine(state, entry);
|
|
1700
|
-
if (message && onMessage(message, offset, line)) {
|
|
1701
|
-
stream.destroy();
|
|
1702
|
-
return;
|
|
1703
|
-
}
|
|
1704
|
-
}
|
|
1705
|
-
}
|
|
1706
|
-
}
|
|
1707
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL) {
|
|
1708
|
-
const checkpoints = [];
|
|
1709
|
-
const state = initialConvState();
|
|
1710
|
-
let index = 0;
|
|
1711
|
-
await streamMessages(filePath, 0, 0, state, (_msg, nextOffset, nextLine) => {
|
|
1712
|
-
index += 1;
|
|
1713
|
-
if (index % interval === 0) {
|
|
1714
|
-
checkpoints.push({
|
|
1715
|
-
messageIndex: index,
|
|
1716
|
-
byteOffset: nextOffset,
|
|
1717
|
-
lineNumber: nextLine,
|
|
1718
|
-
state: structuredClone(state)
|
|
1719
|
-
});
|
|
1720
|
-
}
|
|
1721
|
-
return false;
|
|
1722
|
-
});
|
|
1723
|
-
return checkpoints;
|
|
1724
|
-
}
|
|
1725
|
-
async function readPage(filePath, total, options, floor) {
|
|
1726
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
1727
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1728
|
-
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1729
|
-
const startOffset = floor ? floor.byteOffset : 0;
|
|
1730
|
-
const startLine = floor ? floor.lineNumber : 0;
|
|
1731
|
-
let index = floor ? floor.messageIndex : 0;
|
|
1732
|
-
const window = [];
|
|
1733
|
-
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1734
|
-
const current = index;
|
|
1735
|
-
index += 1;
|
|
1736
|
-
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1737
|
-
return index >= beforeIndex;
|
|
1738
|
-
});
|
|
1739
|
-
applyTeamInfo(window, state);
|
|
1740
|
-
return { messages: window, total, fromIndex };
|
|
1741
|
-
}
|
|
1742
|
-
|
|
1743
1861
|
// src/persistent/repositories/checkpoints.repo.ts
|
|
1744
1862
|
var CheckpointsRepo = class {
|
|
1745
1863
|
constructor(db) {
|
|
@@ -1760,6 +1878,22 @@ var CheckpointsRepo = class {
|
|
|
1760
1878
|
});
|
|
1761
1879
|
tx();
|
|
1762
1880
|
}
|
|
1881
|
+
// Insert checkpoints without touching existing rows. Appends never invalidate
|
|
1882
|
+
// the chain covering the immutable prefix (Kafka sparse-index style); rows are
|
|
1883
|
+
// only ever removed on truncation/replace or deletion.
|
|
1884
|
+
append(sourcePath, checkpoints) {
|
|
1885
|
+
const tx = this.db.transaction(() => {
|
|
1886
|
+
const insert = this.db.prepare(
|
|
1887
|
+
`INSERT INTO message_checkpoints
|
|
1888
|
+
(source_path, message_index, byte_offset, line_number, parser_state)
|
|
1889
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
1890
|
+
);
|
|
1891
|
+
for (const c of checkpoints) {
|
|
1892
|
+
insert.run(sourcePath, c.messageIndex, c.byteOffset, c.lineNumber, JSON.stringify(c.state));
|
|
1893
|
+
}
|
|
1894
|
+
});
|
|
1895
|
+
tx();
|
|
1896
|
+
}
|
|
1763
1897
|
// The latest checkpoint at or before `messageIndex`, or null if none (read
|
|
1764
1898
|
// from the file start). Lets a page seek to the nearest prior anchor.
|
|
1765
1899
|
floor(sourcePath, messageIndex) {
|
|
@@ -1771,6 +1905,17 @@ var CheckpointsRepo = class {
|
|
|
1771
1905
|
).get(sourcePath, messageIndex);
|
|
1772
1906
|
return row ? toCheckpoint(row) : null;
|
|
1773
1907
|
}
|
|
1908
|
+
// The highest-index checkpoint for a file, or null if none. The resume point
|
|
1909
|
+
// for extending the chain after an append.
|
|
1910
|
+
last(sourcePath) {
|
|
1911
|
+
const row = this.db.prepare(
|
|
1912
|
+
`SELECT message_index, byte_offset, line_number, parser_state
|
|
1913
|
+
FROM message_checkpoints
|
|
1914
|
+
WHERE source_path = ?
|
|
1915
|
+
ORDER BY message_index DESC LIMIT 1`
|
|
1916
|
+
).get(sourcePath);
|
|
1917
|
+
return row ? toCheckpoint(row) : null;
|
|
1918
|
+
}
|
|
1774
1919
|
count(sourcePath) {
|
|
1775
1920
|
return this.db.prepare("SELECT COUNT(*) AS n FROM message_checkpoints WHERE source_path = ?").get(sourcePath).n;
|
|
1776
1921
|
}
|
|
@@ -1788,7 +1933,7 @@ function toCheckpoint(row) {
|
|
|
1788
1933
|
}
|
|
1789
1934
|
|
|
1790
1935
|
// src/persistent/repositories/conversation-files.repo.ts
|
|
1791
|
-
var
|
|
1936
|
+
var import_path8 = require("path");
|
|
1792
1937
|
var ConversationFilesRepo = class {
|
|
1793
1938
|
constructor(db) {
|
|
1794
1939
|
this.db = db;
|
|
@@ -1805,7 +1950,7 @@ var ConversationFilesRepo = class {
|
|
|
1805
1950
|
const info = this.db.prepare(
|
|
1806
1951
|
`INSERT INTO conversation_files (absolute_path, parent_dir, file_name, account)
|
|
1807
1952
|
VALUES (?, ?, ?, ?)`
|
|
1808
|
-
).run(absolutePath, (0,
|
|
1953
|
+
).run(absolutePath, (0, import_path8.dirname)(absolutePath), (0, import_path8.basename)(absolutePath), account);
|
|
1809
1954
|
return Number(info.lastInsertRowid);
|
|
1810
1955
|
}
|
|
1811
1956
|
// Advance the cursor + persisted reducer state after a successful index pass.
|
|
@@ -2164,6 +2309,9 @@ var PersistentEngine = class {
|
|
|
2164
2309
|
// restart just means the first few post-restart scans don't force an early
|
|
2165
2310
|
// backstop pass, which is harmless (watermarks themselves persist in the DB).
|
|
2166
2311
|
scanCount = 0;
|
|
2312
|
+
// In-flight checkpoint build/extension per file, so concurrent getPage
|
|
2313
|
+
// callers share one stream instead of each walking the file.
|
|
2314
|
+
checkpointBuilds = /* @__PURE__ */ new Map();
|
|
2167
2315
|
constructor(dbPath, options = {}) {
|
|
2168
2316
|
this.db = openDatabase(dbPath);
|
|
2169
2317
|
this.files = new ConversationFilesRepo(this.db);
|
|
@@ -2216,7 +2364,7 @@ var PersistentEngine = class {
|
|
|
2216
2364
|
const batch = discovered.slice(i, i + BATCH_SIZE);
|
|
2217
2365
|
const results = await Promise.all(
|
|
2218
2366
|
batch.map(async ({ filePath, account, provider }) => {
|
|
2219
|
-
const meta = await this.indexFile(
|
|
2367
|
+
const { meta } = await this.indexFile(
|
|
2220
2368
|
filePath,
|
|
2221
2369
|
account,
|
|
2222
2370
|
tier.name,
|
|
@@ -2251,7 +2399,9 @@ var PersistentEngine = class {
|
|
|
2251
2399
|
// unchanged → return the stored summary; appended → resume the fold and read
|
|
2252
2400
|
// only new bytes; reindex/force → fold from offset 0. Writes the summary +
|
|
2253
2401
|
// cursor + reducer state in one transaction so a crash never leaves a
|
|
2254
|
-
// half-written row or an over-advanced cursor.
|
|
2402
|
+
// half-written row or an over-advanced cursor. Returns the classification
|
|
2403
|
+
// alongside the meta so callers (refreshFile) can keep, extend, or evict
|
|
2404
|
+
// their own per-file caches without re-stat'ing the file (racy) themselves.
|
|
2255
2405
|
async indexFile(filePath, account, tierName, customTiers, resolveGitBranch, force = false, provider) {
|
|
2256
2406
|
const log = getLogger();
|
|
2257
2407
|
const tier = resolveTier(tierName, customTiers);
|
|
@@ -2259,13 +2409,21 @@ var PersistentEngine = class {
|
|
|
2259
2409
|
const { change, stat: stat4 } = classify(filePath, existing);
|
|
2260
2410
|
if (change === "vanished" || !stat4) {
|
|
2261
2411
|
this.markDeleted(filePath);
|
|
2262
|
-
return null;
|
|
2412
|
+
return { meta: null, change: "vanished" };
|
|
2263
2413
|
}
|
|
2264
2414
|
if (change === "unchanged" && !force) {
|
|
2265
|
-
return this.conversations.getBySourcePath(filePath);
|
|
2415
|
+
return { meta: this.conversations.getBySourcePath(filePath), change };
|
|
2266
2416
|
}
|
|
2267
2417
|
if (provider && provider.name !== CLAUDE_CODE_PROVIDER) {
|
|
2268
|
-
|
|
2418
|
+
const meta2 = await this.indexFileWithProvider(
|
|
2419
|
+
provider,
|
|
2420
|
+
filePath,
|
|
2421
|
+
account,
|
|
2422
|
+
tier,
|
|
2423
|
+
stat4,
|
|
2424
|
+
resolveGitBranch
|
|
2425
|
+
);
|
|
2426
|
+
return { meta: meta2, change };
|
|
2269
2427
|
}
|
|
2270
2428
|
const resume = change === "appended" && !force && existing?.reducer_state;
|
|
2271
2429
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
@@ -2276,12 +2434,12 @@ var PersistentEngine = class {
|
|
|
2276
2434
|
result = await tailReduce(filePath, startOffset, startLine, state, tier);
|
|
2277
2435
|
} catch (err) {
|
|
2278
2436
|
log.warn({ filePath, err }, "persistent: tail read failed");
|
|
2279
|
-
return null;
|
|
2437
|
+
return { meta: null, change };
|
|
2280
2438
|
}
|
|
2281
2439
|
const meta = finalizeMeta(state, filePath, account, tier);
|
|
2282
2440
|
if (!meta) {
|
|
2283
2441
|
this.markDeleted(filePath);
|
|
2284
|
-
return null;
|
|
2442
|
+
return { meta: null, change };
|
|
2285
2443
|
}
|
|
2286
2444
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2287
2445
|
const fp = stat4.size > 0 ? fingerprint(filePath, stat4.size) : null;
|
|
@@ -2289,7 +2447,7 @@ var PersistentEngine = class {
|
|
|
2289
2447
|
const upsert = this.db.transaction(() => {
|
|
2290
2448
|
this.conversations.upsert(fileId, meta, state.pageMessageCount);
|
|
2291
2449
|
this.fts.upsert(meta);
|
|
2292
|
-
this.checkpoints.remove(filePath);
|
|
2450
|
+
if (!resume) this.checkpoints.remove(filePath);
|
|
2293
2451
|
this.files.updateCursor(fileId, {
|
|
2294
2452
|
sizeBytes: stat4.size,
|
|
2295
2453
|
mtimeMs: stat4.mtimeMs,
|
|
@@ -2322,7 +2480,7 @@ var PersistentEngine = class {
|
|
|
2322
2480
|
{ filePath, change, bytesRead: result.newOffset - startOffset, msgs: meta.messageCount },
|
|
2323
2481
|
"persistent: indexed file"
|
|
2324
2482
|
);
|
|
2325
|
-
return meta;
|
|
2483
|
+
return { meta, change };
|
|
2326
2484
|
}
|
|
2327
2485
|
// Index a non-Threadbase provider file: full reparse from offset 0 through the
|
|
2328
2486
|
// provider's reducer/finalize, then the same upsert + FTS write + cursor bump
|
|
@@ -2422,15 +2580,34 @@ var PersistentEngine = class {
|
|
|
2422
2580
|
return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
|
|
2423
2581
|
}
|
|
2424
2582
|
const total = this.conversations.pageMessageCount(filePath);
|
|
2425
|
-
|
|
2426
|
-
const built = await buildCheckpoints(filePath);
|
|
2427
|
-
if (built.length > 0) this.checkpoints.replaceAll(filePath, built);
|
|
2428
|
-
}
|
|
2583
|
+
await this.ensureCheckpoints(filePath, total);
|
|
2429
2584
|
const beforeIndex = options.beforeIndex ?? total;
|
|
2430
2585
|
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2431
2586
|
const floor = this.checkpoints.floor(filePath, fromIndex);
|
|
2432
2587
|
return readPage(filePath, total, options, floor);
|
|
2433
2588
|
}
|
|
2589
|
+
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
2590
|
+
// file → full build; a file that grew → extend from the last persisted
|
|
2591
|
+
// checkpoint (reads only past its offset, never the prefix). Single-flighted
|
|
2592
|
+
// per path: concurrent getPage callers await the same build instead of
|
|
2593
|
+
// streaming the file in parallel.
|
|
2594
|
+
ensureCheckpoints(filePath, total) {
|
|
2595
|
+
if (total <= CHECKPOINT_INTERVAL) return Promise.resolve();
|
|
2596
|
+
const inFlight = this.checkpointBuilds.get(filePath);
|
|
2597
|
+
if (inFlight) return inFlight;
|
|
2598
|
+
const build = (async () => {
|
|
2599
|
+
const last = this.checkpoints.last(filePath);
|
|
2600
|
+
if (last && total < last.messageIndex + CHECKPOINT_INTERVAL) return;
|
|
2601
|
+
const fresh = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, last);
|
|
2602
|
+
if (fresh.length === 0) return;
|
|
2603
|
+
if (last) this.checkpoints.append(filePath, fresh);
|
|
2604
|
+
else this.checkpoints.replaceAll(filePath, fresh);
|
|
2605
|
+
})().finally(() => {
|
|
2606
|
+
if (this.checkpointBuilds.get(filePath) === build) this.checkpointBuilds.delete(filePath);
|
|
2607
|
+
});
|
|
2608
|
+
this.checkpointBuilds.set(filePath, build);
|
|
2609
|
+
return build;
|
|
2610
|
+
}
|
|
2434
2611
|
};
|
|
2435
2612
|
|
|
2436
2613
|
// src/watcher/file-watcher.ts
|
|
@@ -2532,10 +2709,12 @@ var IndexQueue = class {
|
|
|
2532
2709
|
var BATCH_SIZE2 = 12;
|
|
2533
2710
|
var DEFAULT_CONFIG_PATH = "~/.config/threadbase-scanner";
|
|
2534
2711
|
function defaultDbPath() {
|
|
2535
|
-
return process.env.TB_SCANNER_DB ?? (0,
|
|
2712
|
+
return process.env.TB_SCANNER_DB ?? (0, import_path9.join)((0, import_os2.homedir)(), ".config", "threadbase-scanner", "index.db");
|
|
2536
2713
|
}
|
|
2537
2714
|
var ConversationScanner = class {
|
|
2538
2715
|
metadataCache = /* @__PURE__ */ new Map();
|
|
2716
|
+
// Parsed conversations plus (persistent claude-code entries only) the resume
|
|
2717
|
+
// point that lets refreshFile extend them in place when the file grows.
|
|
2539
2718
|
conversationLRU;
|
|
2540
2719
|
// session_id is NOT unique, so this maps a sessionId to every active meta that
|
|
2541
2720
|
// carries it. Resolution picks deterministically (newest timestamp, then path
|
|
@@ -2567,7 +2746,9 @@ var ConversationScanner = class {
|
|
|
2567
2746
|
// can't hit a closed DB (the watch-mode half of Bug #4).
|
|
2568
2747
|
inFlightReconcile = null;
|
|
2569
2748
|
constructor(options) {
|
|
2570
|
-
this.conversationLRU = new LRUCache(
|
|
2749
|
+
this.conversationLRU = new LRUCache(
|
|
2750
|
+
options?.conversationCacheSize ?? 5
|
|
2751
|
+
);
|
|
2571
2752
|
if (options?.persistent === false) {
|
|
2572
2753
|
this.dbPath = null;
|
|
2573
2754
|
this.sidecarEnabled = false;
|
|
@@ -2824,7 +3005,7 @@ var ConversationScanner = class {
|
|
|
2824
3005
|
const cached = this.conversationLRU.get(id);
|
|
2825
3006
|
if (cached) {
|
|
2826
3007
|
log.debug({ id }, "getConversation: cache hit");
|
|
2827
|
-
return cached;
|
|
3008
|
+
return cached.conversation;
|
|
2828
3009
|
}
|
|
2829
3010
|
const meta = this.persistent ? this.engine().getByIdOrSession(id) : this.metadataCache.get(id) ?? this.resolveSessionId(id);
|
|
2830
3011
|
if (!meta) {
|
|
@@ -2833,9 +3014,14 @@ var ConversationScanner = class {
|
|
|
2833
3014
|
}
|
|
2834
3015
|
log.debug({ id, filePath: meta.filePath }, "getConversation: cache miss, parsing");
|
|
2835
3016
|
try {
|
|
3017
|
+
if (this.persistent && meta.provider !== CODEX_CLI_PROVIDER) {
|
|
3018
|
+
const parsed = await parseConversationResumable(meta.filePath, meta.account);
|
|
3019
|
+
if (parsed) this.conversationLRU.set(id, parsed);
|
|
3020
|
+
return parsed?.conversation ?? null;
|
|
3021
|
+
}
|
|
2836
3022
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
2837
3023
|
if (conversation) {
|
|
2838
|
-
this.conversationLRU.set(id, conversation);
|
|
3024
|
+
this.conversationLRU.set(id, { conversation });
|
|
2839
3025
|
}
|
|
2840
3026
|
return conversation;
|
|
2841
3027
|
} catch (err) {
|
|
@@ -2907,20 +3093,30 @@ var ConversationScanner = class {
|
|
|
2907
3093
|
// not seen before. Returns the fresh ConversationMeta, or null when the file
|
|
2908
3094
|
// no longer parses (missing/empty) — in which case any prior entry for it is
|
|
2909
3095
|
// dropped from all indexes.
|
|
2910
|
-
|
|
3096
|
+
//
|
|
3097
|
+
// Single-flighted per path: concurrent callers (stacked client retries, a
|
|
3098
|
+
// watcher tick racing a caller) await the one in-flight refresh instead of
|
|
3099
|
+
// each re-reading the file.
|
|
3100
|
+
refreshesInFlight = /* @__PURE__ */ new Map();
|
|
3101
|
+
refreshFile(filePath, account) {
|
|
3102
|
+
const inFlight = this.refreshesInFlight.get(filePath);
|
|
3103
|
+
if (inFlight) return inFlight;
|
|
3104
|
+
const refresh = this.doRefreshFile(filePath, account).finally(() => {
|
|
3105
|
+
if (this.refreshesInFlight.get(filePath) === refresh) {
|
|
3106
|
+
this.refreshesInFlight.delete(filePath);
|
|
3107
|
+
}
|
|
3108
|
+
});
|
|
3109
|
+
this.refreshesInFlight.set(filePath, refresh);
|
|
3110
|
+
return refresh;
|
|
3111
|
+
}
|
|
3112
|
+
async doRefreshFile(filePath, account) {
|
|
2911
3113
|
const log = getLogger();
|
|
2912
3114
|
if (this.persistent) {
|
|
2913
3115
|
const engine = this.engine();
|
|
2914
3116
|
const previous2 = engine.getByIdOrSession(filePath);
|
|
2915
3117
|
const resolvedAccount2 = account ?? previous2?.account ?? "default";
|
|
2916
|
-
const evict2 = (m) => {
|
|
2917
|
-
if (!m) return;
|
|
2918
|
-
this.conversationLRU.delete(m.id);
|
|
2919
|
-
this.conversationLRU.delete(m.sessionId);
|
|
2920
|
-
};
|
|
2921
|
-
evict2(previous2);
|
|
2922
3118
|
const provider = await this.resolveProviderForFile(filePath, previous2);
|
|
2923
|
-
const meta2 = await engine.indexFile(
|
|
3119
|
+
const { meta: meta2, change } = await engine.indexFile(
|
|
2924
3120
|
filePath,
|
|
2925
3121
|
resolvedAccount2,
|
|
2926
3122
|
this.lastTier.name,
|
|
@@ -2929,8 +3125,19 @@ var ConversationScanner = class {
|
|
|
2929
3125
|
false,
|
|
2930
3126
|
provider
|
|
2931
3127
|
);
|
|
2932
|
-
|
|
2933
|
-
|
|
3128
|
+
const cacheKeys = /* @__PURE__ */ new Set();
|
|
3129
|
+
for (const m of [previous2, meta2]) {
|
|
3130
|
+
if (m) {
|
|
3131
|
+
cacheKeys.add(m.id);
|
|
3132
|
+
cacheKeys.add(m.sessionId);
|
|
3133
|
+
}
|
|
3134
|
+
}
|
|
3135
|
+
if (!meta2 || change === "reindex" || change === "vanished") {
|
|
3136
|
+
for (const key of cacheKeys) this.conversationLRU.delete(key);
|
|
3137
|
+
} else if (change === "appended") {
|
|
3138
|
+
await this.extendCachedConversations(cacheKeys, filePath, meta2.account);
|
|
3139
|
+
}
|
|
3140
|
+
log.debug({ filePath, change, kept: !!meta2 }, "refreshFile: updated persistent index");
|
|
2934
3141
|
return meta2;
|
|
2935
3142
|
}
|
|
2936
3143
|
const previous = this.metadataCache.get(filePath);
|
|
@@ -2974,6 +3181,39 @@ var ConversationScanner = class {
|
|
|
2974
3181
|
);
|
|
2975
3182
|
return meta;
|
|
2976
3183
|
}
|
|
3184
|
+
// Advance every cached parse of an appended file by folding only the new
|
|
3185
|
+
// bytes through the conversation reducer — the in-memory analogue of the
|
|
3186
|
+
// persisted metadata fold. Entries without resume state (Codex) and entries
|
|
3187
|
+
// whose extension fails are evicted so the next read re-parses from scratch.
|
|
3188
|
+
async extendCachedConversations(cacheKeys, filePath, account) {
|
|
3189
|
+
const wrappers = /* @__PURE__ */ new Map();
|
|
3190
|
+
for (const key of cacheKeys) {
|
|
3191
|
+
const wrapper = this.conversationLRU.get(key);
|
|
3192
|
+
if (!wrapper) continue;
|
|
3193
|
+
const keys = wrappers.get(wrapper) ?? [];
|
|
3194
|
+
keys.push(key);
|
|
3195
|
+
wrappers.set(wrapper, keys);
|
|
3196
|
+
}
|
|
3197
|
+
for (const [wrapper, keys] of wrappers) {
|
|
3198
|
+
if (!wrapper.resume) {
|
|
3199
|
+
for (const key of keys) this.conversationLRU.delete(key);
|
|
3200
|
+
continue;
|
|
3201
|
+
}
|
|
3202
|
+
try {
|
|
3203
|
+
const extended = await extendConversation(
|
|
3204
|
+
wrapper.conversation,
|
|
3205
|
+
wrapper.resume,
|
|
3206
|
+
filePath,
|
|
3207
|
+
account
|
|
3208
|
+
);
|
|
3209
|
+
wrapper.conversation = extended.conversation;
|
|
3210
|
+
wrapper.resume = extended.resume;
|
|
3211
|
+
} catch (err) {
|
|
3212
|
+
getLogger().warn({ filePath, err }, "refreshFile: cache extension failed, evicting");
|
|
3213
|
+
for (const key of keys) this.conversationLRU.delete(key);
|
|
3214
|
+
}
|
|
3215
|
+
}
|
|
3216
|
+
}
|
|
2977
3217
|
getMetadataCache() {
|
|
2978
3218
|
if (this.persistent) {
|
|
2979
3219
|
const map = /* @__PURE__ */ new Map();
|
|
@@ -3271,12 +3511,14 @@ async function getConversation(id, options, scanner) {
|
|
|
3271
3511
|
applySinceFilter,
|
|
3272
3512
|
applySort,
|
|
3273
3513
|
cleanSystemTags,
|
|
3514
|
+
createJsonlParseState,
|
|
3274
3515
|
createLogger,
|
|
3275
3516
|
detectDefaultProfile,
|
|
3276
3517
|
getConversation,
|
|
3277
3518
|
getLogger,
|
|
3278
3519
|
getProjectsDir,
|
|
3279
3520
|
loadProfiles,
|
|
3521
|
+
parseJsonlLine,
|
|
3280
3522
|
readGitBranch,
|
|
3281
3523
|
readSidecar,
|
|
3282
3524
|
resetDefaultScanner,
|