@threadbase-sh/scanner 0.9.4 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +376 -138
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +963 -712
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +16 -1
- package/dist/index.d.ts +16 -1
- package/dist/index.js +1059 -810
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -289,94 +289,137 @@ var SearchIndexer = class {
|
|
|
289
289
|
}
|
|
290
290
|
};
|
|
291
291
|
|
|
292
|
-
// src/
|
|
293
|
-
import {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
292
|
+
// src/parser.ts
|
|
293
|
+
import { createReadStream } from "fs";
|
|
294
|
+
import { basename as basename2 } from "path";
|
|
295
|
+
import { createInterface } from "readline";
|
|
296
|
+
|
|
297
|
+
// src/persistent/metadata-reducer.ts
|
|
298
|
+
import { basename, dirname as dirname2, join as join2 } from "path";
|
|
299
|
+
|
|
300
|
+
// src/providers/provider.ts
|
|
301
|
+
var CLAUDE_CODE_PROVIDER = "claude-code";
|
|
302
|
+
var CODEX_CLI_PROVIDER = "codex-cli";
|
|
303
|
+
|
|
304
|
+
// src/persistent/metadata-reducer.ts
|
|
305
|
+
function initialReducerState() {
|
|
299
306
|
return {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
messageCount:
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
307
|
+
sessionId: "",
|
|
308
|
+
sessionName: "",
|
|
309
|
+
latestTimestamp: "",
|
|
310
|
+
cwd: "",
|
|
311
|
+
teamName: "",
|
|
312
|
+
model: null,
|
|
313
|
+
messageCount: 0,
|
|
314
|
+
lastMessageSender: "user",
|
|
315
|
+
isTeammate: false,
|
|
316
|
+
firstUserSeen: false,
|
|
317
|
+
firstMessage: null,
|
|
318
|
+
lastMessage: null,
|
|
319
|
+
lastPrompt: "",
|
|
320
|
+
pageMessageCount: 0,
|
|
321
|
+
toolNames: [],
|
|
322
|
+
previewParts: [],
|
|
323
|
+
snippetParts: [],
|
|
324
|
+
previewLength: 0,
|
|
325
|
+
snippetLength: 0,
|
|
326
|
+
badJsonLines: 0
|
|
315
327
|
};
|
|
316
328
|
}
|
|
317
|
-
function
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
329
|
+
function reduceLine(state, entry, tier) {
|
|
330
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
331
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
332
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
333
|
+
if (entry.teamName && !state.teamName) state.teamName = entry.teamName;
|
|
334
|
+
if (entry.timestamp) {
|
|
335
|
+
const ts = entry.timestamp;
|
|
336
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
322
337
|
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
return
|
|
327
|
-
}
|
|
328
|
-
|
|
338
|
+
const type = entry.type;
|
|
339
|
+
if (type === "last-prompt") {
|
|
340
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
if (type !== "user" && type !== "assistant") return;
|
|
344
|
+
if (entry.isMeta) return;
|
|
345
|
+
const msg = entry.message;
|
|
346
|
+
if (state.model === null && msg?.model) state.model = msg.model;
|
|
347
|
+
if (type === "user" && !state.firstUserSeen) {
|
|
348
|
+
state.firstUserSeen = true;
|
|
349
|
+
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
350
|
+
}
|
|
351
|
+
const content = extractTextContent(msg?.content);
|
|
352
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
353
|
+
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
354
|
+
const toolSet = new Set(state.toolNames);
|
|
355
|
+
collectToolNames(msg?.content, toolSet);
|
|
356
|
+
state.toolNames = Array.from(toolSet);
|
|
357
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
358
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
359
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
360
|
+
if (content || isOnlyToolResult || toolUseBlocks.length > 0 || hasThinking) {
|
|
361
|
+
state.pageMessageCount++;
|
|
362
|
+
}
|
|
363
|
+
if (content || isOnlyToolResult) {
|
|
364
|
+
state.messageCount++;
|
|
365
|
+
state.lastMessageSender = type;
|
|
366
|
+
if (content) {
|
|
367
|
+
const ts = entry.timestamp || "";
|
|
368
|
+
if (!state.firstMessage) state.firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
369
|
+
state.lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
370
|
+
if (state.previewLength < tier.previewMax) {
|
|
371
|
+
state.previewParts.push(content);
|
|
372
|
+
state.previewLength += content.length;
|
|
373
|
+
}
|
|
374
|
+
if (state.snippetLength < tier.snippetMax) {
|
|
375
|
+
const remaining = tier.snippetMax - state.snippetLength;
|
|
376
|
+
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
377
|
+
state.snippetParts.push(chunk);
|
|
378
|
+
state.snippetLength += chunk.length;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
329
381
|
}
|
|
330
382
|
}
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
function getProjectsDir(profile) {
|
|
341
|
-
return join2(resolveConfigDir(profile.configDir), "projects");
|
|
342
|
-
}
|
|
343
|
-
async function detectDefaultProfile() {
|
|
383
|
+
function finalizeMeta(state, filePath, account, tier) {
|
|
384
|
+
if (state.messageCount === 0) return null;
|
|
385
|
+
const isSubagent = filePath.includes("/subagents/");
|
|
386
|
+
let parentSessionId = null;
|
|
387
|
+
if (isSubagent) {
|
|
388
|
+
const uuidDir = dirname2(dirname2(filePath));
|
|
389
|
+
parentSessionId = join2(dirname2(uuidDir), `${basename(uuidDir)}.jsonl`);
|
|
390
|
+
}
|
|
391
|
+
const projectPath = state.cwd;
|
|
344
392
|
return {
|
|
345
|
-
id:
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
393
|
+
id: filePath,
|
|
394
|
+
filePath,
|
|
395
|
+
provider: CLAUDE_CODE_PROVIDER,
|
|
396
|
+
sessionId: state.sessionId || basename(filePath, ".jsonl"),
|
|
397
|
+
sessionName: state.sessionName,
|
|
398
|
+
projectPath,
|
|
399
|
+
projectName: getShortProjectName(projectPath),
|
|
400
|
+
account,
|
|
401
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
402
|
+
messageCount: state.messageCount,
|
|
403
|
+
lastMessageSender: state.lastMessageSender,
|
|
404
|
+
preview: state.previewParts.join(" ").slice(0, tier.previewMax),
|
|
405
|
+
contentSnippet: state.snippetParts.join(" "),
|
|
406
|
+
gitBranch: null,
|
|
407
|
+
model: state.model,
|
|
408
|
+
isSubagent,
|
|
409
|
+
parentSessionId,
|
|
410
|
+
isTeammate: state.isTeammate,
|
|
411
|
+
teamName: state.teamName || null,
|
|
412
|
+
toolNames: state.toolNames,
|
|
413
|
+
firstMessage: state.firstMessage,
|
|
414
|
+
lastMessage: state.lastMessage,
|
|
415
|
+
lastPrompt: state.lastPrompt || void 0
|
|
350
416
|
};
|
|
351
417
|
}
|
|
352
|
-
|
|
353
|
-
const
|
|
354
|
-
|
|
355
|
-
const resolved = resolveConfigDir(configPath);
|
|
356
|
-
const data = await readFile(join2(resolved, PROFILES_FILE), "utf-8");
|
|
357
|
-
const profiles = JSON.parse(data);
|
|
358
|
-
log.debug({ configPath, count: profiles.length }, "profiles: loaded");
|
|
359
|
-
return profiles;
|
|
360
|
-
} catch (err) {
|
|
361
|
-
log.debug({ configPath, err }, "profiles: load failed, using default");
|
|
362
|
-
const defaultProfile = await detectDefaultProfile();
|
|
363
|
-
return [defaultProfile];
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
async function saveProfiles(profiles, configPath) {
|
|
367
|
-
const resolved = resolveConfigDir(configPath);
|
|
368
|
-
await mkdir(resolved, { recursive: true });
|
|
369
|
-
await writeFile(join2(resolved, PROFILES_FILE), JSON.stringify(profiles, null, 2));
|
|
370
|
-
getLogger().debug({ configPath, count: profiles.length }, "profiles: saved");
|
|
418
|
+
function getShortProjectName(fullPath) {
|
|
419
|
+
const parts = fullPath.split("/").filter(Boolean);
|
|
420
|
+
return parts.slice(-3).join("/");
|
|
371
421
|
}
|
|
372
422
|
|
|
373
|
-
// src/providers/codex-cli.ts
|
|
374
|
-
import fg from "fast-glob";
|
|
375
|
-
import { createReadStream } from "fs";
|
|
376
|
-
import { stat } from "fs/promises";
|
|
377
|
-
import { basename } from "path";
|
|
378
|
-
import { createInterface } from "readline";
|
|
379
|
-
|
|
380
423
|
// src/tags.ts
|
|
381
424
|
var SYSTEM_TAGS = [
|
|
382
425
|
"system-reminder",
|
|
@@ -405,186 +448,49 @@ function cleanSystemTags(text) {
|
|
|
405
448
|
return text.replace(SYSTEM_TAG_RE, "").replace(/[^\S\n]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
|
|
406
449
|
}
|
|
407
450
|
|
|
408
|
-
// src/
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
const
|
|
417
|
-
const results = [];
|
|
418
|
-
for (const root of roots) {
|
|
419
|
-
let paths;
|
|
420
|
-
try {
|
|
421
|
-
paths = await fg(["**/rollout-*.jsonl", "**/*.jsonl"], {
|
|
422
|
-
cwd: root,
|
|
423
|
-
absolute: true,
|
|
424
|
-
dot: false,
|
|
425
|
-
unique: true
|
|
426
|
-
});
|
|
427
|
-
} catch (err) {
|
|
428
|
-
log.warn({ root, err }, "codex discovery: glob failed");
|
|
429
|
-
continue;
|
|
430
|
-
}
|
|
431
|
-
for (const filePath of paths) {
|
|
432
|
-
try {
|
|
433
|
-
const s = await stat(filePath);
|
|
434
|
-
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
435
|
-
} catch (err) {
|
|
436
|
-
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
return results;
|
|
441
|
-
}
|
|
442
|
-
// Codex rollout lines carry distinctive top-level types.
|
|
443
|
-
canParse(_filePath, sample) {
|
|
444
|
-
for (const line of sample.split("\n")) {
|
|
451
|
+
// src/parser.ts
|
|
452
|
+
async function parseMeta(filePath, account, tier) {
|
|
453
|
+
const log = getLogger();
|
|
454
|
+
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
455
|
+
const state = initialReducerState();
|
|
456
|
+
const fileStream = createReadStream(filePath);
|
|
457
|
+
const rl = createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
458
|
+
try {
|
|
459
|
+
for await (const line of rl) {
|
|
445
460
|
if (!line.trim()) continue;
|
|
461
|
+
let entry;
|
|
446
462
|
try {
|
|
447
|
-
|
|
448
|
-
if (e.type === "session_meta" || e.type === "response_item" || e.type === "event_msg") {
|
|
449
|
-
return true;
|
|
450
|
-
}
|
|
451
|
-
if (e.type === "user" || e.type === "assistant") return false;
|
|
463
|
+
entry = JSON.parse(line);
|
|
452
464
|
} catch {
|
|
465
|
+
state.badJsonLines++;
|
|
466
|
+
continue;
|
|
453
467
|
}
|
|
468
|
+
reduceLine(state, entry, tier);
|
|
454
469
|
}
|
|
455
|
-
|
|
470
|
+
} catch (err) {
|
|
471
|
+
log.warn({ filePath, err }, "parseMeta: read failed");
|
|
472
|
+
return null;
|
|
456
473
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
model: null,
|
|
463
|
-
latestTimestamp: "",
|
|
464
|
-
messageCount: 0,
|
|
465
|
-
lastMessageSender: "user",
|
|
466
|
-
firstUser: null,
|
|
467
|
-
lastUser: null,
|
|
468
|
-
lastAssistant: null,
|
|
469
|
-
toolNames: [],
|
|
470
|
-
previewParts: [],
|
|
471
|
-
previewLength: 0,
|
|
472
|
-
snippetParts: [],
|
|
473
|
-
snippetLength: 0
|
|
474
|
-
};
|
|
475
|
-
}
|
|
476
|
-
reduceEntry(acc, entry, tier) {
|
|
477
|
-
reduceCodexEntry(acc, entry, tier);
|
|
478
|
-
}
|
|
479
|
-
finalize(acc, filePath, account, tier) {
|
|
480
|
-
return finalizeCodexMeta(acc, filePath, account, tier);
|
|
481
|
-
}
|
|
482
|
-
};
|
|
483
|
-
var asString = (v) => typeof v === "string" ? v : "";
|
|
484
|
-
function extractCodexText(content) {
|
|
485
|
-
if (typeof content === "string") return cleanSystemTags(content);
|
|
486
|
-
if (!Array.isArray(content)) return "";
|
|
487
|
-
return content.map((item) => {
|
|
488
|
-
if (typeof item === "string") return item;
|
|
489
|
-
const t = item?.type;
|
|
490
|
-
if ((t === "input_text" || t === "output_text" || t === "text") && item?.text) {
|
|
491
|
-
return item.text;
|
|
492
|
-
}
|
|
493
|
-
return "";
|
|
494
|
-
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
495
|
-
}
|
|
496
|
-
function reduceCodexEntry(acc, entry, tier) {
|
|
497
|
-
const ts = asString(entry.timestamp);
|
|
498
|
-
if (ts && (!acc.latestTimestamp || ts > acc.latestTimestamp)) acc.latestTimestamp = ts;
|
|
499
|
-
const payload = entry.payload;
|
|
500
|
-
if (!payload || typeof payload !== "object") return;
|
|
501
|
-
const type = entry.type;
|
|
502
|
-
if (type === "session_meta") {
|
|
503
|
-
if (!acc.sessionId) acc.sessionId = asString(payload.id);
|
|
504
|
-
if (!acc.cwd) acc.cwd = asString(payload.cwd);
|
|
505
|
-
const git = payload.git;
|
|
506
|
-
if (acc.gitBranch === null && git?.branch) acc.gitBranch = asString(git.branch) || null;
|
|
507
|
-
return;
|
|
508
|
-
}
|
|
509
|
-
if (acc.model === null && payload.model) acc.model = asString(payload.model) || null;
|
|
510
|
-
if (type !== "response_item") return;
|
|
511
|
-
const ptype = payload.type;
|
|
512
|
-
if (ptype === "function_call" || ptype === "custom_tool_call") {
|
|
513
|
-
const name = asString(payload.name);
|
|
514
|
-
if (name && !acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
515
|
-
return;
|
|
516
|
-
}
|
|
517
|
-
if (ptype !== "message") return;
|
|
518
|
-
const role = payload.role;
|
|
519
|
-
if (role !== "user" && role !== "assistant") return;
|
|
520
|
-
const text = extractCodexText(payload.content);
|
|
521
|
-
if (!text) return;
|
|
522
|
-
const sender = role;
|
|
523
|
-
acc.messageCount++;
|
|
524
|
-
acc.lastMessageSender = sender;
|
|
525
|
-
const snapshot = { text: text.slice(0, 200), timestamp: ts };
|
|
526
|
-
if (sender === "user") {
|
|
527
|
-
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
528
|
-
acc.lastUser = snapshot;
|
|
529
|
-
} else {
|
|
530
|
-
acc.lastAssistant = snapshot;
|
|
531
|
-
}
|
|
532
|
-
if (acc.previewLength < tier.previewMax) {
|
|
533
|
-
acc.previewParts.push(text);
|
|
534
|
-
acc.previewLength += text.length;
|
|
535
|
-
}
|
|
536
|
-
if (acc.snippetLength < tier.snippetMax) {
|
|
537
|
-
const remaining = tier.snippetMax - acc.snippetLength;
|
|
538
|
-
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
539
|
-
acc.snippetParts.push(chunk);
|
|
540
|
-
acc.snippetLength += chunk.length;
|
|
474
|
+
if (state.badJsonLines > 0) {
|
|
475
|
+
log.warn(
|
|
476
|
+
{ filePath, badJsonLines: state.badJsonLines },
|
|
477
|
+
"parseMeta: skipped malformed JSON lines"
|
|
478
|
+
);
|
|
541
479
|
}
|
|
480
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
481
|
+
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
482
|
+
return meta;
|
|
542
483
|
}
|
|
543
|
-
function
|
|
544
|
-
if (acc.messageCount === 0) return null;
|
|
545
|
-
const sessionId = acc.sessionId || basename(filePath, ".jsonl");
|
|
546
|
-
const projectPath = acc.cwd;
|
|
547
|
-
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
548
|
-
return {
|
|
549
|
-
id: filePath,
|
|
550
|
-
filePath,
|
|
551
|
-
provider: CODEX_CLI_PROVIDER,
|
|
552
|
-
kind,
|
|
553
|
-
externalSessionId: acc.sessionId || void 0,
|
|
554
|
-
sessionId,
|
|
555
|
-
sessionName: "",
|
|
556
|
-
projectPath,
|
|
557
|
-
projectName: getShortProjectName(projectPath),
|
|
558
|
-
account,
|
|
559
|
-
timestamp: acc.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
560
|
-
messageCount: acc.messageCount,
|
|
561
|
-
lastMessageSender: acc.lastMessageSender,
|
|
562
|
-
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
563
|
-
contentSnippet: acc.snippetParts.join(" "),
|
|
564
|
-
gitBranch: acc.gitBranch,
|
|
565
|
-
model: acc.model,
|
|
566
|
-
isSubagent: false,
|
|
567
|
-
parentSessionId: null,
|
|
568
|
-
isTeammate: false,
|
|
569
|
-
teamName: null,
|
|
570
|
-
toolNames: acc.toolNames,
|
|
571
|
-
firstMessage: acc.firstUser,
|
|
572
|
-
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
573
|
-
lastPrompt: acc.lastUser?.text || void 0
|
|
574
|
-
};
|
|
575
|
-
}
|
|
576
|
-
function getShortProjectName(fullPath) {
|
|
577
|
-
return fullPath.split("/").filter(Boolean).slice(-3).join("/");
|
|
578
|
-
}
|
|
579
|
-
async function parseCodexConversation(filePath, account) {
|
|
484
|
+
async function parseConversation(filePath, account) {
|
|
580
485
|
const log = getLogger();
|
|
486
|
+
log.trace({ filePath, account }, "parseConversation: start");
|
|
581
487
|
const messages = [];
|
|
488
|
+
let badJsonLines = 0;
|
|
582
489
|
const textParts = [];
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
const rl = createInterface({ input: createReadStream(filePath), crlfDelay: Infinity });
|
|
490
|
+
const turnDurations = [];
|
|
491
|
+
const state = initialConvState();
|
|
492
|
+
const fileStream = createReadStream(filePath);
|
|
493
|
+
const rl = createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
588
494
|
try {
|
|
589
495
|
for await (const line of rl) {
|
|
590
496
|
if (!line.trim()) continue;
|
|
@@ -592,120 +498,146 @@ async function parseCodexConversation(filePath, account) {
|
|
|
592
498
|
try {
|
|
593
499
|
entry = JSON.parse(line);
|
|
594
500
|
} catch {
|
|
501
|
+
badJsonLines++;
|
|
595
502
|
continue;
|
|
596
503
|
}
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
if (!cwd) cwd = asString(payload.cwd);
|
|
504
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
505
|
+
turnDurations.push({
|
|
506
|
+
durationMs: entry.durationMs,
|
|
507
|
+
messageCount: entry.messageCount || 0,
|
|
508
|
+
uuid: entry.uuid
|
|
509
|
+
});
|
|
604
510
|
continue;
|
|
605
511
|
}
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
messages.push({ role, text, timestamp: ts });
|
|
612
|
-
textParts.push(text);
|
|
613
|
-
if (role === "user") lastUserText = text;
|
|
512
|
+
const message = reduceConvLine(state, entry);
|
|
513
|
+
if (message) {
|
|
514
|
+
messages.push(message);
|
|
515
|
+
if (message.text) textParts.push(message.text);
|
|
516
|
+
}
|
|
614
517
|
}
|
|
615
518
|
} catch (err) {
|
|
616
|
-
log.warn({ filePath, err }, "
|
|
519
|
+
log.warn({ filePath, err }, "parseConversation: read failed");
|
|
617
520
|
return null;
|
|
618
521
|
}
|
|
619
|
-
if (
|
|
522
|
+
if (badJsonLines > 0) {
|
|
523
|
+
log.warn({ filePath, badJsonLines }, "parseConversation: skipped malformed JSON lines");
|
|
524
|
+
}
|
|
525
|
+
if (messages.length === 0) {
|
|
526
|
+
log.trace({ filePath }, "parseConversation: no messages");
|
|
527
|
+
return null;
|
|
528
|
+
}
|
|
529
|
+
log.debug({ filePath, messageCount: messages.length }, "parseConversation: complete");
|
|
530
|
+
applyTeamInfo(messages, state);
|
|
620
531
|
return {
|
|
621
532
|
id: filePath,
|
|
622
533
|
filePath,
|
|
623
|
-
projectPath: cwd,
|
|
624
|
-
projectName:
|
|
625
|
-
sessionId: sessionId ||
|
|
626
|
-
sessionName:
|
|
534
|
+
projectPath: state.cwd,
|
|
535
|
+
projectName: getShortProjectName2(state.cwd),
|
|
536
|
+
sessionId: state.sessionId || basename2(filePath, ".jsonl"),
|
|
537
|
+
sessionName: state.sessionName,
|
|
627
538
|
messages,
|
|
628
539
|
fullText: textParts.join(" "),
|
|
629
|
-
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
540
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
630
541
|
messageCount: messages.length,
|
|
631
542
|
account,
|
|
632
|
-
|
|
543
|
+
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
544
|
+
lastPrompt: state.lastPrompt || void 0
|
|
633
545
|
};
|
|
634
546
|
}
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
let filePaths;
|
|
646
|
-
try {
|
|
647
|
-
filePaths = await fg2("**/*.jsonl", {
|
|
648
|
-
cwd: projectsDir,
|
|
649
|
-
absolute: true,
|
|
650
|
-
dot: false
|
|
651
|
-
});
|
|
652
|
-
} catch (err) {
|
|
653
|
-
log.warn({ projectsDir, account, err }, "discovery: glob failed");
|
|
654
|
-
continue;
|
|
655
|
-
}
|
|
656
|
-
const filtered = filePaths.filter((fp) => !EXCLUDED_SEGMENTS.some((seg) => fp.includes(seg)));
|
|
657
|
-
let kept = 0;
|
|
658
|
-
let skippedEmpty = 0;
|
|
659
|
-
let skippedInaccessible = 0;
|
|
660
|
-
for (let i = 0; i < filtered.length; i += STAT_CONCURRENCY) {
|
|
661
|
-
const chunk = filtered.slice(i, i + STAT_CONCURRENCY);
|
|
662
|
-
const statted = await Promise.all(
|
|
663
|
-
chunk.map(async (filePath) => {
|
|
664
|
-
try {
|
|
665
|
-
const s = await stat2(filePath);
|
|
666
|
-
return { filePath, size: s.size };
|
|
667
|
-
} catch (err) {
|
|
668
|
-
log.warn({ filePath, err }, "discovery: stat failed");
|
|
669
|
-
return { filePath, size: -1 };
|
|
670
|
-
}
|
|
671
|
-
})
|
|
672
|
-
);
|
|
673
|
-
for (const { filePath, size } of statted) {
|
|
674
|
-
if (size < 0) {
|
|
675
|
-
skippedInaccessible++;
|
|
676
|
-
} else if (size > 0) {
|
|
677
|
-
results.push({ filePath, account });
|
|
678
|
-
kept++;
|
|
679
|
-
} else {
|
|
680
|
-
skippedEmpty++;
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
|
-
}
|
|
684
|
-
log.debug(
|
|
685
|
-
{
|
|
686
|
-
projectsDir,
|
|
687
|
-
account,
|
|
688
|
-
globMatches: filePaths.length,
|
|
689
|
-
afterExclusions: filtered.length,
|
|
690
|
-
kept,
|
|
691
|
-
skippedEmpty,
|
|
692
|
-
skippedInaccessible
|
|
693
|
-
},
|
|
694
|
-
"discovery: directory scanned"
|
|
695
|
-
);
|
|
696
|
-
onProgress?.(results.length);
|
|
547
|
+
function extractTextContent(content) {
|
|
548
|
+
if (!content) return "";
|
|
549
|
+
if (typeof content === "string") return cleanSystemTags(content);
|
|
550
|
+
if (Array.isArray(content)) {
|
|
551
|
+
return content.map((item) => {
|
|
552
|
+
if (typeof item === "string") return item;
|
|
553
|
+
if (item?.type === "text" && item?.text) return item.text;
|
|
554
|
+
if (item?.type === "tool_result" && typeof item?.content === "string") return item.content;
|
|
555
|
+
return "";
|
|
556
|
+
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
697
557
|
}
|
|
698
|
-
|
|
699
|
-
|
|
558
|
+
return "";
|
|
559
|
+
}
|
|
560
|
+
function extractToolUseNames(content) {
|
|
561
|
+
if (!Array.isArray(content)) return [];
|
|
562
|
+
return content.filter((item) => item?.type === "tool_use" && item?.name).map((item) => item.name);
|
|
563
|
+
}
|
|
564
|
+
function extractToolUseBlocks(content) {
|
|
565
|
+
if (!Array.isArray(content)) return [];
|
|
566
|
+
return content.filter((item) => item?.type === "tool_use" && item?.name && item?.id).map((item) => ({
|
|
567
|
+
id: item.id,
|
|
568
|
+
name: item.name,
|
|
569
|
+
input: item.input || {}
|
|
570
|
+
}));
|
|
571
|
+
}
|
|
572
|
+
var TOOL_NAME_TO_TYPE = {
|
|
573
|
+
Edit: "edit",
|
|
574
|
+
Write: "write",
|
|
575
|
+
Read: "read",
|
|
576
|
+
Bash: "bash",
|
|
577
|
+
Grep: "grep",
|
|
578
|
+
Glob: "glob",
|
|
579
|
+
Agent: "taskAgent",
|
|
580
|
+
TaskCreate: "taskCreate",
|
|
581
|
+
TaskUpdate: "taskUpdate"
|
|
582
|
+
};
|
|
583
|
+
function extractToolResultBlocks(content, pendingToolUses) {
|
|
584
|
+
if (!Array.isArray(content)) return [];
|
|
585
|
+
return content.filter((item) => item?.type === "tool_result" && item?.tool_use_id).map((item) => {
|
|
586
|
+
const toolName = pendingToolUses.get(item.tool_use_id)?.name ?? "";
|
|
587
|
+
return {
|
|
588
|
+
toolUseId: item.tool_use_id,
|
|
589
|
+
type: TOOL_NAME_TO_TYPE[toolName] ?? "generic",
|
|
590
|
+
content: typeof item.content === "string" ? { text: item.content } : item.content ?? {},
|
|
591
|
+
isError: typeof item.is_error === "boolean" ? item.is_error : void 0
|
|
592
|
+
};
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
function collectToolNames(content, toolSet) {
|
|
596
|
+
if (!Array.isArray(content)) return;
|
|
597
|
+
for (const item of content) {
|
|
598
|
+
if (item?.type === "tool_use" && item?.name) {
|
|
599
|
+
toolSet.add(item.name);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
function isOnlyToolResultContent(content) {
|
|
604
|
+
if (!Array.isArray(content)) return false;
|
|
605
|
+
return content.length > 0 && content.every((item) => item?.type === "tool_result");
|
|
606
|
+
}
|
|
607
|
+
function isTeammateContent(content) {
|
|
608
|
+
const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
|
|
609
|
+
(item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
|
|
610
|
+
).join("") : "";
|
|
611
|
+
return raw.includes("<teammate-message");
|
|
612
|
+
}
|
|
613
|
+
function extractThinking(content) {
|
|
614
|
+
if (!Array.isArray(content)) return { content: "", signature: "" };
|
|
615
|
+
const blocks = content.filter((item) => item?.type === "thinking");
|
|
616
|
+
return {
|
|
617
|
+
content: blocks.map((b) => b.thinking).filter(Boolean).join("\n\n"),
|
|
618
|
+
signature: blocks.map((b) => b.signature).filter(Boolean).join("")
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
function hasImageBlocks(content) {
|
|
622
|
+
if (!Array.isArray(content)) return false;
|
|
623
|
+
return content.some(
|
|
624
|
+
(item) => item?.type === "image" && (item?.source?.type === "base64" || item?.file?.base64 !== void 0)
|
|
625
|
+
);
|
|
626
|
+
}
|
|
627
|
+
function parseTeammateMessageTag(content) {
|
|
628
|
+
const match = content.match(/<teammate-message\s+([^>]*)>/);
|
|
629
|
+
if (!match) return null;
|
|
630
|
+
const attrs = match[1];
|
|
631
|
+
const id = attrs.match(/teammate_id="([^"]*)"/)?.[1];
|
|
632
|
+
if (!id) return null;
|
|
633
|
+
const summary = attrs.match(/summary="([^"]*)"/)?.[1];
|
|
634
|
+
const color = attrs.match(/color="([^"]*)"/)?.[1];
|
|
635
|
+
return { teammateId: id, summary, color };
|
|
636
|
+
}
|
|
637
|
+
function getShortProjectName2(fullPath) {
|
|
638
|
+
const parts = fullPath.split("/").filter(Boolean);
|
|
639
|
+
return parts.slice(-3).join("/");
|
|
700
640
|
}
|
|
701
|
-
|
|
702
|
-
// src/persistent/metadata-reducer.ts
|
|
703
|
-
import { basename as basename3, dirname as dirname2, join as join3 } from "path";
|
|
704
|
-
|
|
705
|
-
// src/parser.ts
|
|
706
|
-
import { createReadStream as createReadStream2 } from "fs";
|
|
707
|
-
import { basename as basename2 } from "path";
|
|
708
|
-
import { createInterface as createInterface2 } from "readline";
|
|
709
641
|
|
|
710
642
|
// src/persistent/conversation-reducer.ts
|
|
711
643
|
function initialConvState() {
|
|
@@ -762,7 +694,10 @@ function reduceConvLine(state, entry) {
|
|
|
762
694
|
if (isToolResultOnly) {
|
|
763
695
|
const pending = new Map(Object.entries(state.pendingToolUses));
|
|
764
696
|
const toolResultBlocks = extractToolResultBlocks(msg?.content, pending);
|
|
765
|
-
if (toolResultBlocks.length > 0)
|
|
697
|
+
if (toolResultBlocks.length > 0) {
|
|
698
|
+
metadata.toolResults = toolResultBlocks;
|
|
699
|
+
for (const block of toolResultBlocks) delete state.pendingToolUses[block.toolUseId];
|
|
700
|
+
}
|
|
766
701
|
}
|
|
767
702
|
if (entry.teamName) {
|
|
768
703
|
metadata.teamName = entry.teamName;
|
|
@@ -793,6 +728,17 @@ function reduceConvLine(state, entry) {
|
|
|
793
728
|
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
794
729
|
};
|
|
795
730
|
}
|
|
731
|
+
function parseJsonlLine(line, state = initialConvState()) {
|
|
732
|
+
const text = line.trimEnd();
|
|
733
|
+
if (text.trim().length === 0) return null;
|
|
734
|
+
let entry;
|
|
735
|
+
try {
|
|
736
|
+
entry = JSON.parse(text);
|
|
737
|
+
} catch {
|
|
738
|
+
return null;
|
|
739
|
+
}
|
|
740
|
+
return reduceConvLine(state, entry);
|
|
741
|
+
}
|
|
796
742
|
function applyTeamInfo(messages, state) {
|
|
797
743
|
if (Object.keys(state.teamInfo).length === 0) return;
|
|
798
744
|
for (const m of messages) {
|
|
@@ -801,394 +747,662 @@ function applyTeamInfo(messages, state) {
|
|
|
801
747
|
}
|
|
802
748
|
}
|
|
803
749
|
|
|
804
|
-
// src/
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
750
|
+
// src/persistent/sidecar.ts
|
|
751
|
+
import { readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
752
|
+
var SIDECAR_VERSION = 1;
|
|
753
|
+
function sidecarPath(jsonlPath) {
|
|
754
|
+
return `${jsonlPath}.idx.json`;
|
|
755
|
+
}
|
|
756
|
+
function buildSidecar(meta, cursor, updatedAt) {
|
|
757
|
+
return {
|
|
758
|
+
version: SIDECAR_VERSION,
|
|
759
|
+
sourcePath: meta.filePath,
|
|
760
|
+
sizeBytes: cursor.sizeBytes,
|
|
761
|
+
mtimeMs: cursor.mtimeMs,
|
|
762
|
+
lastIndexedOffset: cursor.offset,
|
|
763
|
+
lastIndexedLine: cursor.line,
|
|
764
|
+
messageCount: meta.messageCount,
|
|
765
|
+
projectPath: meta.projectPath,
|
|
766
|
+
projectName: meta.projectName,
|
|
767
|
+
branch: meta.gitBranch,
|
|
768
|
+
firstSentAt: meta.firstMessage?.timestamp ?? null,
|
|
769
|
+
firstSentText: meta.firstMessage?.text ?? null,
|
|
770
|
+
lastSentAt: meta.lastMessage?.timestamp ?? null,
|
|
771
|
+
lastSentText: meta.lastMessage?.text ?? null,
|
|
772
|
+
updatedAt
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
function writeSidecar(jsonlPath, sidecar) {
|
|
811
776
|
try {
|
|
812
|
-
|
|
813
|
-
if (!line.trim()) continue;
|
|
814
|
-
let entry;
|
|
815
|
-
try {
|
|
816
|
-
entry = JSON.parse(line);
|
|
817
|
-
} catch {
|
|
818
|
-
state.badJsonLines++;
|
|
819
|
-
continue;
|
|
820
|
-
}
|
|
821
|
-
reduceLine(state, entry, tier);
|
|
822
|
-
}
|
|
777
|
+
writeFileSync(sidecarPath(jsonlPath), JSON.stringify(sidecar, null, 2));
|
|
823
778
|
} catch (err) {
|
|
824
|
-
|
|
825
|
-
return null;
|
|
779
|
+
getLogger().warn({ jsonlPath, err }, "sidecar: write failed");
|
|
826
780
|
}
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
781
|
+
}
|
|
782
|
+
function readSidecar(jsonlPath) {
|
|
783
|
+
try {
|
|
784
|
+
return JSON.parse(readFileSync2(sidecarPath(jsonlPath), "utf-8"));
|
|
785
|
+
} catch {
|
|
786
|
+
return null;
|
|
832
787
|
}
|
|
833
|
-
const meta = finalizeMeta(state, filePath, account, tier);
|
|
834
|
-
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
835
|
-
return meta;
|
|
836
788
|
}
|
|
837
|
-
|
|
789
|
+
|
|
790
|
+
// src/profiles.ts
|
|
791
|
+
import { mkdir, readFile, writeFile } from "fs/promises";
|
|
792
|
+
import { homedir } from "os";
|
|
793
|
+
import { join as join3 } from "path";
|
|
794
|
+
var PROFILES_FILE = "profiles.json";
|
|
795
|
+
function resolveConfigDir(configDir) {
|
|
796
|
+
return configDir.replace(/^~/, homedir());
|
|
797
|
+
}
|
|
798
|
+
function getProjectsDir(profile) {
|
|
799
|
+
return join3(resolveConfigDir(profile.configDir), "projects");
|
|
800
|
+
}
|
|
801
|
+
async function detectDefaultProfile() {
|
|
802
|
+
return {
|
|
803
|
+
id: "default",
|
|
804
|
+
label: "Default",
|
|
805
|
+
configDir: join3(homedir(), ".claude"),
|
|
806
|
+
enabled: true,
|
|
807
|
+
emoji: "\u{1F916}"
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
async function loadProfiles(configPath) {
|
|
838
811
|
const log = getLogger();
|
|
839
|
-
log.trace({ filePath, account }, "parseConversation: start");
|
|
840
|
-
const messages = [];
|
|
841
|
-
let badJsonLines = 0;
|
|
842
|
-
const textParts = [];
|
|
843
|
-
const turnDurations = [];
|
|
844
|
-
const state = initialConvState();
|
|
845
|
-
const fileStream = createReadStream2(filePath);
|
|
846
|
-
const rl = createInterface2({ input: fileStream, crlfDelay: Infinity });
|
|
847
812
|
try {
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
813
|
+
const resolved = resolveConfigDir(configPath);
|
|
814
|
+
const data = await readFile(join3(resolved, PROFILES_FILE), "utf-8");
|
|
815
|
+
const profiles = JSON.parse(data);
|
|
816
|
+
log.debug({ configPath, count: profiles.length }, "profiles: loaded");
|
|
817
|
+
return profiles;
|
|
818
|
+
} catch (err) {
|
|
819
|
+
log.debug({ configPath, err }, "profiles: load failed, using default");
|
|
820
|
+
const defaultProfile = await detectDefaultProfile();
|
|
821
|
+
return [defaultProfile];
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
async function saveProfiles(profiles, configPath) {
|
|
825
|
+
const resolved = resolveConfigDir(configPath);
|
|
826
|
+
await mkdir(resolved, { recursive: true });
|
|
827
|
+
await writeFile(join3(resolved, PROFILES_FILE), JSON.stringify(profiles, null, 2));
|
|
828
|
+
getLogger().debug({ configPath, count: profiles.length }, "profiles: saved");
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
// src/providers/codex-cli.ts
|
|
832
|
+
import fg from "fast-glob";
|
|
833
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
834
|
+
import { stat } from "fs/promises";
|
|
835
|
+
import { basename as basename3 } from "path";
|
|
836
|
+
import { createInterface as createInterface2 } from "readline";
|
|
837
|
+
var CodexCliProvider = class {
|
|
838
|
+
name = CODEX_CLI_PROVIDER;
|
|
839
|
+
async discover(roots) {
|
|
840
|
+
const log = getLogger();
|
|
841
|
+
const results = [];
|
|
842
|
+
for (const root of roots) {
|
|
843
|
+
let paths;
|
|
851
844
|
try {
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
858
|
-
turnDurations.push({
|
|
859
|
-
durationMs: entry.durationMs,
|
|
860
|
-
messageCount: entry.messageCount || 0,
|
|
861
|
-
uuid: entry.uuid
|
|
845
|
+
paths = await fg(["**/rollout-*.jsonl", "**/*.jsonl"], {
|
|
846
|
+
cwd: root,
|
|
847
|
+
absolute: true,
|
|
848
|
+
dot: false,
|
|
849
|
+
unique: true
|
|
862
850
|
});
|
|
851
|
+
} catch (err) {
|
|
852
|
+
log.warn({ root, err }, "codex discovery: glob failed");
|
|
863
853
|
continue;
|
|
864
854
|
}
|
|
865
|
-
const
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
855
|
+
for (const filePath of paths) {
|
|
856
|
+
try {
|
|
857
|
+
const s = await stat(filePath);
|
|
858
|
+
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
859
|
+
} catch (err) {
|
|
860
|
+
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
861
|
+
}
|
|
869
862
|
}
|
|
870
863
|
}
|
|
871
|
-
|
|
872
|
-
log.warn({ filePath, err }, "parseConversation: read failed");
|
|
873
|
-
return null;
|
|
864
|
+
return results;
|
|
874
865
|
}
|
|
875
|
-
|
|
876
|
-
|
|
866
|
+
// Codex rollout lines carry distinctive top-level types.
|
|
867
|
+
canParse(_filePath, sample) {
|
|
868
|
+
for (const line of sample.split("\n")) {
|
|
869
|
+
if (!line.trim()) continue;
|
|
870
|
+
try {
|
|
871
|
+
const e = JSON.parse(line);
|
|
872
|
+
if (e.type === "session_meta" || e.type === "response_item" || e.type === "event_msg") {
|
|
873
|
+
return true;
|
|
874
|
+
}
|
|
875
|
+
if (e.type === "user" || e.type === "assistant") return false;
|
|
876
|
+
} catch {
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
return false;
|
|
877
880
|
}
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
+
createEmptyAccumulator() {
|
|
882
|
+
return {
|
|
883
|
+
sessionId: "",
|
|
884
|
+
cwd: "",
|
|
885
|
+
gitBranch: null,
|
|
886
|
+
model: null,
|
|
887
|
+
latestTimestamp: "",
|
|
888
|
+
messageCount: 0,
|
|
889
|
+
lastMessageSender: "user",
|
|
890
|
+
firstUser: null,
|
|
891
|
+
lastUser: null,
|
|
892
|
+
lastAssistant: null,
|
|
893
|
+
toolNames: [],
|
|
894
|
+
previewParts: [],
|
|
895
|
+
previewLength: 0,
|
|
896
|
+
snippetParts: [],
|
|
897
|
+
snippetLength: 0
|
|
898
|
+
};
|
|
881
899
|
}
|
|
882
|
-
|
|
883
|
-
|
|
900
|
+
reduceEntry(acc, entry, tier) {
|
|
901
|
+
reduceCodexEntry(acc, entry, tier);
|
|
902
|
+
}
|
|
903
|
+
finalize(acc, filePath, account, tier) {
|
|
904
|
+
return finalizeCodexMeta(acc, filePath, account, tier);
|
|
905
|
+
}
|
|
906
|
+
};
|
|
907
|
+
var asString = (v) => typeof v === "string" ? v : "";
|
|
908
|
+
function extractCodexText(content) {
|
|
909
|
+
if (typeof content === "string") return cleanSystemTags(content);
|
|
910
|
+
if (!Array.isArray(content)) return "";
|
|
911
|
+
return content.map((item) => {
|
|
912
|
+
if (typeof item === "string") return item;
|
|
913
|
+
const t = item?.type;
|
|
914
|
+
if ((t === "input_text" || t === "output_text" || t === "text") && item?.text) {
|
|
915
|
+
return item.text;
|
|
916
|
+
}
|
|
917
|
+
return "";
|
|
918
|
+
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
919
|
+
}
|
|
920
|
+
function reduceCodexEntry(acc, entry, tier) {
|
|
921
|
+
const ts = asString(entry.timestamp);
|
|
922
|
+
if (ts && (!acc.latestTimestamp || ts > acc.latestTimestamp)) acc.latestTimestamp = ts;
|
|
923
|
+
const payload = entry.payload;
|
|
924
|
+
if (!payload || typeof payload !== "object") return;
|
|
925
|
+
const type = entry.type;
|
|
926
|
+
if (type === "session_meta") {
|
|
927
|
+
if (!acc.sessionId) acc.sessionId = asString(payload.id);
|
|
928
|
+
if (!acc.cwd) acc.cwd = asString(payload.cwd);
|
|
929
|
+
const git = payload.git;
|
|
930
|
+
if (acc.gitBranch === null && git?.branch) acc.gitBranch = asString(git.branch) || null;
|
|
931
|
+
return;
|
|
932
|
+
}
|
|
933
|
+
if (acc.model === null && payload.model) acc.model = asString(payload.model) || null;
|
|
934
|
+
if (type !== "response_item") return;
|
|
935
|
+
const ptype = payload.type;
|
|
936
|
+
if (ptype === "function_call" || ptype === "custom_tool_call") {
|
|
937
|
+
const name = asString(payload.name);
|
|
938
|
+
if (name && !acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
941
|
+
if (ptype !== "message") return;
|
|
942
|
+
const role = payload.role;
|
|
943
|
+
if (role !== "user" && role !== "assistant") return;
|
|
944
|
+
const text = extractCodexText(payload.content);
|
|
945
|
+
if (!text) return;
|
|
946
|
+
const sender = role;
|
|
947
|
+
acc.messageCount++;
|
|
948
|
+
acc.lastMessageSender = sender;
|
|
949
|
+
const snapshot = { text: text.slice(0, 200), timestamp: ts };
|
|
950
|
+
if (sender === "user") {
|
|
951
|
+
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
952
|
+
acc.lastUser = snapshot;
|
|
953
|
+
} else {
|
|
954
|
+
acc.lastAssistant = snapshot;
|
|
955
|
+
}
|
|
956
|
+
if (acc.previewLength < tier.previewMax) {
|
|
957
|
+
acc.previewParts.push(text);
|
|
958
|
+
acc.previewLength += text.length;
|
|
959
|
+
}
|
|
960
|
+
if (acc.snippetLength < tier.snippetMax) {
|
|
961
|
+
const remaining = tier.snippetMax - acc.snippetLength;
|
|
962
|
+
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
963
|
+
acc.snippetParts.push(chunk);
|
|
964
|
+
acc.snippetLength += chunk.length;
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
968
|
+
if (acc.messageCount === 0) return null;
|
|
969
|
+
const sessionId = acc.sessionId || basename3(filePath, ".jsonl");
|
|
970
|
+
const projectPath = acc.cwd;
|
|
971
|
+
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
884
972
|
return {
|
|
885
973
|
id: filePath,
|
|
886
974
|
filePath,
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
messageCount: messages.length,
|
|
975
|
+
provider: CODEX_CLI_PROVIDER,
|
|
976
|
+
kind,
|
|
977
|
+
externalSessionId: acc.sessionId || void 0,
|
|
978
|
+
sessionId,
|
|
979
|
+
sessionName: "",
|
|
980
|
+
projectPath,
|
|
981
|
+
projectName: getShortProjectName3(projectPath),
|
|
895
982
|
account,
|
|
896
|
-
|
|
897
|
-
|
|
983
|
+
timestamp: acc.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
984
|
+
messageCount: acc.messageCount,
|
|
985
|
+
lastMessageSender: acc.lastMessageSender,
|
|
986
|
+
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
987
|
+
contentSnippet: acc.snippetParts.join(" "),
|
|
988
|
+
gitBranch: acc.gitBranch,
|
|
989
|
+
model: acc.model,
|
|
990
|
+
isSubagent: false,
|
|
991
|
+
parentSessionId: null,
|
|
992
|
+
isTeammate: false,
|
|
993
|
+
teamName: null,
|
|
994
|
+
toolNames: acc.toolNames,
|
|
995
|
+
firstMessage: acc.firstUser,
|
|
996
|
+
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
997
|
+
lastPrompt: acc.lastUser?.text || void 0
|
|
898
998
|
};
|
|
899
999
|
}
|
|
900
|
-
function
|
|
901
|
-
|
|
902
|
-
if (typeof content === "string") return cleanSystemTags(content);
|
|
903
|
-
if (Array.isArray(content)) {
|
|
904
|
-
return content.map((item) => {
|
|
905
|
-
if (typeof item === "string") return item;
|
|
906
|
-
if (item?.type === "text" && item?.text) return item.text;
|
|
907
|
-
if (item?.type === "tool_result" && typeof item?.content === "string") return item.content;
|
|
908
|
-
return "";
|
|
909
|
-
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
910
|
-
}
|
|
911
|
-
return "";
|
|
912
|
-
}
|
|
913
|
-
function extractToolUseNames(content) {
|
|
914
|
-
if (!Array.isArray(content)) return [];
|
|
915
|
-
return content.filter((item) => item?.type === "tool_use" && item?.name).map((item) => item.name);
|
|
916
|
-
}
|
|
917
|
-
function extractToolUseBlocks(content) {
|
|
918
|
-
if (!Array.isArray(content)) return [];
|
|
919
|
-
return content.filter((item) => item?.type === "tool_use" && item?.name && item?.id).map((item) => ({
|
|
920
|
-
id: item.id,
|
|
921
|
-
name: item.name,
|
|
922
|
-
input: item.input || {}
|
|
923
|
-
}));
|
|
924
|
-
}
|
|
925
|
-
var TOOL_NAME_TO_TYPE = {
|
|
926
|
-
Edit: "edit",
|
|
927
|
-
Write: "write",
|
|
928
|
-
Read: "read",
|
|
929
|
-
Bash: "bash",
|
|
930
|
-
Grep: "grep",
|
|
931
|
-
Glob: "glob",
|
|
932
|
-
Agent: "taskAgent",
|
|
933
|
-
TaskCreate: "taskCreate",
|
|
934
|
-
TaskUpdate: "taskUpdate"
|
|
935
|
-
};
|
|
936
|
-
function extractToolResultBlocks(content, pendingToolUses) {
|
|
937
|
-
if (!Array.isArray(content)) return [];
|
|
938
|
-
return content.filter((item) => item?.type === "tool_result" && item?.tool_use_id).map((item) => {
|
|
939
|
-
const toolName = pendingToolUses.get(item.tool_use_id)?.name ?? "";
|
|
940
|
-
return {
|
|
941
|
-
toolUseId: item.tool_use_id,
|
|
942
|
-
type: TOOL_NAME_TO_TYPE[toolName] ?? "generic",
|
|
943
|
-
content: typeof item.content === "string" ? { text: item.content } : item.content ?? {},
|
|
944
|
-
isError: typeof item.is_error === "boolean" ? item.is_error : void 0
|
|
945
|
-
};
|
|
946
|
-
});
|
|
1000
|
+
function getShortProjectName3(fullPath) {
|
|
1001
|
+
return fullPath.split("/").filter(Boolean).slice(-3).join("/");
|
|
947
1002
|
}
|
|
948
|
-
function
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
1003
|
+
async function parseCodexConversation(filePath, account) {
|
|
1004
|
+
const log = getLogger();
|
|
1005
|
+
const messages = [];
|
|
1006
|
+
const textParts = [];
|
|
1007
|
+
let sessionId = "";
|
|
1008
|
+
let cwd = "";
|
|
1009
|
+
let latestTimestamp = "";
|
|
1010
|
+
let lastUserText = "";
|
|
1011
|
+
const rl = createInterface2({ input: createReadStream2(filePath), crlfDelay: Infinity });
|
|
1012
|
+
try {
|
|
1013
|
+
for await (const line of rl) {
|
|
1014
|
+
if (!line.trim()) continue;
|
|
1015
|
+
let entry;
|
|
1016
|
+
try {
|
|
1017
|
+
entry = JSON.parse(line);
|
|
1018
|
+
} catch {
|
|
1019
|
+
continue;
|
|
1020
|
+
}
|
|
1021
|
+
const ts = asString(entry.timestamp);
|
|
1022
|
+
if (ts && (!latestTimestamp || ts > latestTimestamp)) latestTimestamp = ts;
|
|
1023
|
+
const payload = entry.payload;
|
|
1024
|
+
if (!payload || typeof payload !== "object") continue;
|
|
1025
|
+
if (entry.type === "session_meta") {
|
|
1026
|
+
if (!sessionId) sessionId = asString(payload.id);
|
|
1027
|
+
if (!cwd) cwd = asString(payload.cwd);
|
|
1028
|
+
continue;
|
|
1029
|
+
}
|
|
1030
|
+
if (entry.type !== "response_item" || payload.type !== "message") continue;
|
|
1031
|
+
const role = payload.role;
|
|
1032
|
+
if (role !== "user" && role !== "assistant") continue;
|
|
1033
|
+
const text = extractCodexText(payload.content);
|
|
1034
|
+
if (!text) continue;
|
|
1035
|
+
messages.push({ role, text, timestamp: ts });
|
|
1036
|
+
textParts.push(text);
|
|
1037
|
+
if (role === "user") lastUserText = text;
|
|
953
1038
|
}
|
|
1039
|
+
} catch (err) {
|
|
1040
|
+
log.warn({ filePath, err }, "parseCodexConversation: read failed");
|
|
1041
|
+
return null;
|
|
954
1042
|
}
|
|
955
|
-
|
|
956
|
-
function isOnlyToolResultContent(content) {
|
|
957
|
-
if (!Array.isArray(content)) return false;
|
|
958
|
-
return content.length > 0 && content.every((item) => item?.type === "tool_result");
|
|
959
|
-
}
|
|
960
|
-
function isTeammateContent(content) {
|
|
961
|
-
const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
|
|
962
|
-
(item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
|
|
963
|
-
).join("") : "";
|
|
964
|
-
return raw.includes("<teammate-message");
|
|
965
|
-
}
|
|
966
|
-
function extractThinking(content) {
|
|
967
|
-
if (!Array.isArray(content)) return { content: "", signature: "" };
|
|
968
|
-
const blocks = content.filter((item) => item?.type === "thinking");
|
|
1043
|
+
if (messages.length === 0) return null;
|
|
969
1044
|
return {
|
|
970
|
-
|
|
971
|
-
|
|
1045
|
+
id: filePath,
|
|
1046
|
+
filePath,
|
|
1047
|
+
projectPath: cwd,
|
|
1048
|
+
projectName: getShortProjectName3(cwd),
|
|
1049
|
+
sessionId: sessionId || basename3(filePath, ".jsonl"),
|
|
1050
|
+
sessionName: "",
|
|
1051
|
+
messages,
|
|
1052
|
+
fullText: textParts.join(" "),
|
|
1053
|
+
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1054
|
+
messageCount: messages.length,
|
|
1055
|
+
account,
|
|
1056
|
+
lastPrompt: lastUserText || void 0
|
|
972
1057
|
};
|
|
973
1058
|
}
|
|
974
|
-
function hasImageBlocks(content) {
|
|
975
|
-
if (!Array.isArray(content)) return false;
|
|
976
|
-
return content.some(
|
|
977
|
-
(item) => item?.type === "image" && (item?.source?.type === "base64" || item?.file?.base64 !== void 0)
|
|
978
|
-
);
|
|
979
|
-
}
|
|
980
|
-
function parseTeammateMessageTag(content) {
|
|
981
|
-
const match = content.match(/<teammate-message\s+([^>]*)>/);
|
|
982
|
-
if (!match) return null;
|
|
983
|
-
const attrs = match[1];
|
|
984
|
-
const id = attrs.match(/teammate_id="([^"]*)"/)?.[1];
|
|
985
|
-
if (!id) return null;
|
|
986
|
-
const summary = attrs.match(/summary="([^"]*)"/)?.[1];
|
|
987
|
-
const color = attrs.match(/color="([^"]*)"/)?.[1];
|
|
988
|
-
return { teammateId: id, summary, color };
|
|
989
|
-
}
|
|
990
|
-
function getShortProjectName2(fullPath) {
|
|
991
|
-
const parts = fullPath.split("/").filter(Boolean);
|
|
992
|
-
return parts.slice(-3).join("/");
|
|
993
|
-
}
|
|
994
1059
|
|
|
995
|
-
// src/
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1060
|
+
// src/discovery.ts
|
|
1061
|
+
import fg2 from "fast-glob";
|
|
1062
|
+
import { stat as stat2 } from "fs/promises";
|
|
1063
|
+
var EXCLUDED_SEGMENTS = ["/memory/", "/tool-results/"];
|
|
1064
|
+
var STAT_CONCURRENCY = 32;
|
|
1065
|
+
async function discoverJsonlFiles(dirs, onProgress) {
|
|
1066
|
+
const log = getLogger();
|
|
1067
|
+
const results = [];
|
|
1068
|
+
for (const { projectsDir, account } of dirs) {
|
|
1069
|
+
let filePaths;
|
|
1070
|
+
try {
|
|
1071
|
+
filePaths = await fg2("**/*.jsonl", {
|
|
1072
|
+
cwd: projectsDir,
|
|
1073
|
+
absolute: true,
|
|
1074
|
+
dot: false
|
|
1075
|
+
});
|
|
1076
|
+
} catch (err) {
|
|
1077
|
+
log.warn({ projectsDir, account, err }, "discovery: glob failed");
|
|
1078
|
+
continue;
|
|
1079
|
+
}
|
|
1080
|
+
const filtered = filePaths.filter((fp) => !EXCLUDED_SEGMENTS.some((seg) => fp.includes(seg)));
|
|
1081
|
+
let kept = 0;
|
|
1082
|
+
let skippedEmpty = 0;
|
|
1083
|
+
let skippedInaccessible = 0;
|
|
1084
|
+
for (let i = 0; i < filtered.length; i += STAT_CONCURRENCY) {
|
|
1085
|
+
const chunk = filtered.slice(i, i + STAT_CONCURRENCY);
|
|
1086
|
+
const statted = await Promise.all(
|
|
1087
|
+
chunk.map(async (filePath) => {
|
|
1088
|
+
try {
|
|
1089
|
+
const s = await stat2(filePath);
|
|
1090
|
+
return { filePath, size: s.size };
|
|
1091
|
+
} catch (err) {
|
|
1092
|
+
log.warn({ filePath, err }, "discovery: stat failed");
|
|
1093
|
+
return { filePath, size: -1 };
|
|
1094
|
+
}
|
|
1095
|
+
})
|
|
1096
|
+
);
|
|
1097
|
+
for (const { filePath, size } of statted) {
|
|
1098
|
+
if (size < 0) {
|
|
1099
|
+
skippedInaccessible++;
|
|
1100
|
+
} else if (size > 0) {
|
|
1101
|
+
results.push({ filePath, account });
|
|
1102
|
+
kept++;
|
|
1103
|
+
} else {
|
|
1104
|
+
skippedEmpty++;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
log.debug(
|
|
1109
|
+
{
|
|
1110
|
+
projectsDir,
|
|
1111
|
+
account,
|
|
1112
|
+
globMatches: filePaths.length,
|
|
1113
|
+
afterExclusions: filtered.length,
|
|
1114
|
+
kept,
|
|
1115
|
+
skippedEmpty,
|
|
1116
|
+
skippedInaccessible
|
|
1117
|
+
},
|
|
1118
|
+
"discovery: directory scanned"
|
|
1119
|
+
);
|
|
1120
|
+
onProgress?.(results.length);
|
|
1121
|
+
}
|
|
1122
|
+
log.debug({ totalFiles: results.length, dirs: dirs.length }, "discovery: complete");
|
|
1123
|
+
return results;
|
|
1019
1124
|
}
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1125
|
+
|
|
1126
|
+
// src/providers/threadbase.ts
|
|
1127
|
+
var ThreadbaseProvider = class {
|
|
1128
|
+
name = CLAUDE_CODE_PROVIDER;
|
|
1129
|
+
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
1130
|
+
// per-root account through the shared interface. The scanner builds these.
|
|
1131
|
+
async discover(roots) {
|
|
1132
|
+
const dirs = roots.map((r) => {
|
|
1133
|
+
const [projectsDir, account = "default"] = r.split("\0");
|
|
1134
|
+
return { projectsDir, account };
|
|
1135
|
+
});
|
|
1136
|
+
return discoverJsonlFiles(dirs);
|
|
1028
1137
|
}
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1138
|
+
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
1139
|
+
canParse(_filePath, sample) {
|
|
1140
|
+
for (const line of sample.split("\n")) {
|
|
1141
|
+
if (!line.trim()) continue;
|
|
1142
|
+
try {
|
|
1143
|
+
const e = JSON.parse(line);
|
|
1144
|
+
if (e.type === "user" || e.type === "assistant") return true;
|
|
1145
|
+
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
1146
|
+
} catch {
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
return false;
|
|
1033
1150
|
}
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
const msg = entry.message;
|
|
1037
|
-
if (state.model === null && msg?.model) state.model = msg.model;
|
|
1038
|
-
if (type === "user" && !state.firstUserSeen) {
|
|
1039
|
-
state.firstUserSeen = true;
|
|
1040
|
-
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
1151
|
+
createEmptyAccumulator() {
|
|
1152
|
+
return initialReducerState();
|
|
1041
1153
|
}
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
1045
|
-
const toolSet = new Set(state.toolNames);
|
|
1046
|
-
collectToolNames(msg?.content, toolSet);
|
|
1047
|
-
state.toolNames = Array.from(toolSet);
|
|
1048
|
-
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
1049
|
-
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
1050
|
-
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
1051
|
-
if (content || isOnlyToolResult || toolUseBlocks.length > 0 || hasThinking) {
|
|
1052
|
-
state.pageMessageCount++;
|
|
1154
|
+
reduceEntry(acc, entry, tier) {
|
|
1155
|
+
reduceLine(acc, entry, tier);
|
|
1053
1156
|
}
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1157
|
+
finalize(acc, filePath, account, tier) {
|
|
1158
|
+
return finalizeMeta(acc, filePath, account, tier);
|
|
1159
|
+
}
|
|
1160
|
+
};
|
|
1161
|
+
|
|
1162
|
+
// src/scanner.ts
|
|
1163
|
+
import { EventEmitter } from "events";
|
|
1164
|
+
import { closeSync as closeSync2, openSync as openSync2, readSync as readSync2, statSync as statSync3 } from "fs";
|
|
1165
|
+
import { homedir as homedir2 } from "os";
|
|
1166
|
+
import { join as join4 } from "path";
|
|
1167
|
+
|
|
1168
|
+
// src/cache.ts
|
|
1169
|
+
var LRUCache = class {
|
|
1170
|
+
map = /* @__PURE__ */ new Map();
|
|
1171
|
+
capacity;
|
|
1172
|
+
constructor(capacity) {
|
|
1173
|
+
this.capacity = capacity;
|
|
1174
|
+
}
|
|
1175
|
+
get(key) {
|
|
1176
|
+
const value = this.map.get(key);
|
|
1177
|
+
if (value === void 0) return void 0;
|
|
1178
|
+
this.map.delete(key);
|
|
1179
|
+
this.map.set(key, value);
|
|
1180
|
+
return value;
|
|
1181
|
+
}
|
|
1182
|
+
set(key, value) {
|
|
1183
|
+
this.map.delete(key);
|
|
1184
|
+
this.map.set(key, value);
|
|
1185
|
+
if (this.map.size > this.capacity) {
|
|
1186
|
+
const oldest = this.map.keys().next();
|
|
1187
|
+
if (!oldest.done) this.map.delete(oldest.value);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
has(key) {
|
|
1191
|
+
return this.map.has(key);
|
|
1192
|
+
}
|
|
1193
|
+
delete(key) {
|
|
1194
|
+
return this.map.delete(key);
|
|
1195
|
+
}
|
|
1196
|
+
clear() {
|
|
1197
|
+
this.map.clear();
|
|
1198
|
+
}
|
|
1199
|
+
get size() {
|
|
1200
|
+
return this.map.size;
|
|
1201
|
+
}
|
|
1202
|
+
};
|
|
1203
|
+
|
|
1204
|
+
// src/persistent/conversation-stream.ts
|
|
1205
|
+
import { basename as basename4 } from "path";
|
|
1206
|
+
|
|
1207
|
+
// src/persistent/paged-reader.ts
|
|
1208
|
+
import { createReadStream as createReadStream4 } from "fs";
|
|
1209
|
+
import { setImmediate as yieldToEventLoop2 } from "timers/promises";
|
|
1210
|
+
|
|
1211
|
+
// src/persistent/jsonl-tail-reader.ts
|
|
1212
|
+
import { createReadStream as createReadStream3 } from "fs";
|
|
1213
|
+
import { setImmediate as yieldToEventLoop } from "timers/promises";
|
|
1214
|
+
var YIELD_EVERY_LINES = 500;
|
|
1215
|
+
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1216
|
+
const stream = createReadStream3(filePath, { start: startOffset, encoding: "utf8" });
|
|
1217
|
+
let buffer = "";
|
|
1218
|
+
let offset = startOffset;
|
|
1219
|
+
let line = startLine;
|
|
1220
|
+
let parsedLines = 0;
|
|
1221
|
+
let sinceYield = 0;
|
|
1222
|
+
for await (const chunk of stream) {
|
|
1223
|
+
buffer += chunk;
|
|
1224
|
+
let nl;
|
|
1225
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1226
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1227
|
+
const text = lineWithNewline.trimEnd();
|
|
1228
|
+
buffer = buffer.slice(nl + 1);
|
|
1229
|
+
if (text.length > 0) {
|
|
1230
|
+
try {
|
|
1231
|
+
reduceLine(state, JSON.parse(text), tier);
|
|
1232
|
+
} catch {
|
|
1233
|
+
state.badJsonLines++;
|
|
1234
|
+
}
|
|
1235
|
+
parsedLines++;
|
|
1064
1236
|
}
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1237
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1238
|
+
line++;
|
|
1239
|
+
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
1240
|
+
sinceYield = 0;
|
|
1241
|
+
await yieldToEventLoop();
|
|
1070
1242
|
}
|
|
1071
1243
|
}
|
|
1072
1244
|
}
|
|
1245
|
+
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
1073
1246
|
}
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1247
|
+
|
|
1248
|
+
// src/persistent/paged-reader.ts
|
|
1249
|
+
var CHECKPOINT_INTERVAL = 500;
|
|
1250
|
+
async function streamMessages(filePath, startOffset, startLine, state, onMessage, onEntry) {
|
|
1251
|
+
const stream = createReadStream4(filePath, { start: startOffset, encoding: "utf8" });
|
|
1252
|
+
let buffer = "";
|
|
1253
|
+
let offset = startOffset;
|
|
1254
|
+
let line = startLine;
|
|
1255
|
+
let sinceYield = 0;
|
|
1256
|
+
for await (const chunk of stream) {
|
|
1257
|
+
buffer += chunk;
|
|
1258
|
+
let nl;
|
|
1259
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1260
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1261
|
+
const text = lineWithNewline.trimEnd();
|
|
1262
|
+
buffer = buffer.slice(nl + 1);
|
|
1263
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1264
|
+
line += 1;
|
|
1265
|
+
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
1266
|
+
sinceYield = 0;
|
|
1267
|
+
await yieldToEventLoop2();
|
|
1268
|
+
}
|
|
1269
|
+
if (text.length === 0) continue;
|
|
1270
|
+
let entry;
|
|
1271
|
+
try {
|
|
1272
|
+
entry = JSON.parse(text);
|
|
1273
|
+
} catch {
|
|
1274
|
+
continue;
|
|
1275
|
+
}
|
|
1276
|
+
if (onEntry?.(entry)) continue;
|
|
1277
|
+
const message = reduceConvLine(state, entry);
|
|
1278
|
+
if (message && onMessage(message, offset, line)) {
|
|
1279
|
+
stream.destroy();
|
|
1280
|
+
return { offset, line };
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1081
1283
|
}
|
|
1082
|
-
|
|
1284
|
+
return { offset, line };
|
|
1285
|
+
}
|
|
1286
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
1287
|
+
const checkpoints = [];
|
|
1288
|
+
const state = from ? from.state : initialConvState();
|
|
1289
|
+
let index = from ? from.messageIndex : 0;
|
|
1290
|
+
await streamMessages(
|
|
1291
|
+
filePath,
|
|
1292
|
+
from?.byteOffset ?? 0,
|
|
1293
|
+
from?.lineNumber ?? 0,
|
|
1294
|
+
state,
|
|
1295
|
+
(_msg, nextOffset, nextLine) => {
|
|
1296
|
+
index += 1;
|
|
1297
|
+
if (index % interval === 0) {
|
|
1298
|
+
checkpoints.push({
|
|
1299
|
+
messageIndex: index,
|
|
1300
|
+
byteOffset: nextOffset,
|
|
1301
|
+
lineNumber: nextLine,
|
|
1302
|
+
state: structuredClone(state)
|
|
1303
|
+
});
|
|
1304
|
+
}
|
|
1305
|
+
return false;
|
|
1306
|
+
}
|
|
1307
|
+
);
|
|
1308
|
+
return checkpoints;
|
|
1309
|
+
}
|
|
1310
|
+
async function readPage(filePath, total, options, floor) {
|
|
1311
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
1312
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1313
|
+
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1314
|
+
const startOffset = floor ? floor.byteOffset : 0;
|
|
1315
|
+
const startLine = floor ? floor.lineNumber : 0;
|
|
1316
|
+
let index = floor ? floor.messageIndex : 0;
|
|
1317
|
+
const window = [];
|
|
1318
|
+
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1319
|
+
const current = index;
|
|
1320
|
+
index += 1;
|
|
1321
|
+
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1322
|
+
return index >= beforeIndex;
|
|
1323
|
+
});
|
|
1324
|
+
applyTeamInfo(window, state);
|
|
1325
|
+
return { messages: window, total, fromIndex };
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
// src/persistent/conversation-stream.ts
|
|
1329
|
+
async function foldTail(filePath, resume) {
|
|
1330
|
+
const messages = [];
|
|
1331
|
+
const textParts = [];
|
|
1332
|
+
const turnDurations = [];
|
|
1333
|
+
const end = await streamMessages(
|
|
1334
|
+
filePath,
|
|
1335
|
+
resume.offset,
|
|
1336
|
+
resume.line,
|
|
1337
|
+
resume.state,
|
|
1338
|
+
(message) => {
|
|
1339
|
+
messages.push(message);
|
|
1340
|
+
if (message.text) textParts.push(message.text);
|
|
1341
|
+
return false;
|
|
1342
|
+
},
|
|
1343
|
+
(entry) => {
|
|
1344
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
1345
|
+
turnDurations.push({
|
|
1346
|
+
durationMs: entry.durationMs,
|
|
1347
|
+
messageCount: entry.messageCount || 0,
|
|
1348
|
+
uuid: entry.uuid
|
|
1349
|
+
});
|
|
1350
|
+
return true;
|
|
1351
|
+
}
|
|
1352
|
+
return false;
|
|
1353
|
+
}
|
|
1354
|
+
);
|
|
1355
|
+
return { messages, textParts, turnDurations, end };
|
|
1356
|
+
}
|
|
1357
|
+
function assemble(filePath, account, messages, fullText, turnDurations, state) {
|
|
1083
1358
|
return {
|
|
1084
1359
|
id: filePath,
|
|
1085
1360
|
filePath,
|
|
1086
|
-
|
|
1087
|
-
|
|
1361
|
+
projectPath: state.cwd,
|
|
1362
|
+
projectName: getShortProjectName2(state.cwd),
|
|
1363
|
+
sessionId: state.sessionId || basename4(filePath, ".jsonl"),
|
|
1088
1364
|
sessionName: state.sessionName,
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
account,
|
|
1365
|
+
messages,
|
|
1366
|
+
fullText,
|
|
1092
1367
|
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1093
|
-
messageCount:
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
contentSnippet: state.snippetParts.join(" "),
|
|
1097
|
-
gitBranch: null,
|
|
1098
|
-
model: state.model,
|
|
1099
|
-
isSubagent,
|
|
1100
|
-
parentSessionId,
|
|
1101
|
-
isTeammate: state.isTeammate,
|
|
1102
|
-
teamName: state.teamName || null,
|
|
1103
|
-
toolNames: state.toolNames,
|
|
1104
|
-
firstMessage: state.firstMessage,
|
|
1105
|
-
lastMessage: state.lastMessage,
|
|
1368
|
+
messageCount: messages.length,
|
|
1369
|
+
account,
|
|
1370
|
+
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
1106
1371
|
lastPrompt: state.lastPrompt || void 0
|
|
1107
1372
|
};
|
|
1108
1373
|
}
|
|
1109
|
-
function
|
|
1110
|
-
const
|
|
1111
|
-
|
|
1374
|
+
async function parseConversationResumable(filePath, account) {
|
|
1375
|
+
const resume = { state: initialConvState(), offset: 0, line: 0 };
|
|
1376
|
+
const { messages, textParts, turnDurations, end } = await foldTail(filePath, resume);
|
|
1377
|
+
if (messages.length === 0) return null;
|
|
1378
|
+
applyTeamInfo(messages, resume.state);
|
|
1379
|
+
const conversation = assemble(
|
|
1380
|
+
filePath,
|
|
1381
|
+
account,
|
|
1382
|
+
messages,
|
|
1383
|
+
textParts.join(" "),
|
|
1384
|
+
turnDurations,
|
|
1385
|
+
resume.state
|
|
1386
|
+
);
|
|
1387
|
+
return { conversation, resume: { state: resume.state, offset: end.offset, line: end.line } };
|
|
1388
|
+
}
|
|
1389
|
+
async function extendConversation(previous, resume, filePath, account) {
|
|
1390
|
+
const { messages: fresh, textParts, turnDurations, end } = await foldTail(filePath, resume);
|
|
1391
|
+
const messages = fresh.length > 0 ? previous.messages.concat(fresh) : previous.messages;
|
|
1392
|
+
applyTeamInfo(messages, resume.state);
|
|
1393
|
+
const fullText = textParts.length === 0 ? previous.fullText : previous.fullText ? `${previous.fullText} ${textParts.join(" ")}` : textParts.join(" ");
|
|
1394
|
+
const allTurnDurations = (previous.turnDurations ?? []).concat(turnDurations);
|
|
1395
|
+
const conversation = assemble(
|
|
1396
|
+
filePath,
|
|
1397
|
+
account,
|
|
1398
|
+
messages,
|
|
1399
|
+
fullText,
|
|
1400
|
+
allTurnDurations,
|
|
1401
|
+
resume.state
|
|
1402
|
+
);
|
|
1403
|
+
return { conversation, resume: { state: resume.state, offset: end.offset, line: end.line } };
|
|
1112
1404
|
}
|
|
1113
1405
|
|
|
1114
|
-
// src/providers/threadbase.ts
|
|
1115
|
-
var ThreadbaseProvider = class {
|
|
1116
|
-
name = CLAUDE_CODE_PROVIDER;
|
|
1117
|
-
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
1118
|
-
// per-root account through the shared interface. The scanner builds these.
|
|
1119
|
-
async discover(roots) {
|
|
1120
|
-
const dirs = roots.map((r) => {
|
|
1121
|
-
const [projectsDir, account = "default"] = r.split("\0");
|
|
1122
|
-
return { projectsDir, account };
|
|
1123
|
-
});
|
|
1124
|
-
return discoverJsonlFiles(dirs);
|
|
1125
|
-
}
|
|
1126
|
-
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
1127
|
-
canParse(_filePath, sample) {
|
|
1128
|
-
for (const line of sample.split("\n")) {
|
|
1129
|
-
if (!line.trim()) continue;
|
|
1130
|
-
try {
|
|
1131
|
-
const e = JSON.parse(line);
|
|
1132
|
-
if (e.type === "user" || e.type === "assistant") return true;
|
|
1133
|
-
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
1134
|
-
} catch {
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
return false;
|
|
1138
|
-
}
|
|
1139
|
-
createEmptyAccumulator() {
|
|
1140
|
-
return initialReducerState();
|
|
1141
|
-
}
|
|
1142
|
-
reduceEntry(acc, entry, tier) {
|
|
1143
|
-
reduceLine(acc, entry, tier);
|
|
1144
|
-
}
|
|
1145
|
-
finalize(acc, filePath, account, tier) {
|
|
1146
|
-
return finalizeMeta(acc, filePath, account, tier);
|
|
1147
|
-
}
|
|
1148
|
-
};
|
|
1149
|
-
|
|
1150
|
-
// src/scanner.ts
|
|
1151
|
-
import { EventEmitter } from "events";
|
|
1152
|
-
import { closeSync as closeSync2, openSync as openSync2, readSync as readSync2, statSync as statSync2 } from "fs";
|
|
1153
|
-
import { homedir as homedir2 } from "os";
|
|
1154
|
-
import { join as join4 } from "path";
|
|
1155
|
-
|
|
1156
|
-
// src/cache.ts
|
|
1157
|
-
var LRUCache = class {
|
|
1158
|
-
map = /* @__PURE__ */ new Map();
|
|
1159
|
-
capacity;
|
|
1160
|
-
constructor(capacity) {
|
|
1161
|
-
this.capacity = capacity;
|
|
1162
|
-
}
|
|
1163
|
-
get(key) {
|
|
1164
|
-
const value = this.map.get(key);
|
|
1165
|
-
if (value === void 0) return void 0;
|
|
1166
|
-
this.map.delete(key);
|
|
1167
|
-
this.map.set(key, value);
|
|
1168
|
-
return value;
|
|
1169
|
-
}
|
|
1170
|
-
set(key, value) {
|
|
1171
|
-
this.map.delete(key);
|
|
1172
|
-
this.map.set(key, value);
|
|
1173
|
-
if (this.map.size > this.capacity) {
|
|
1174
|
-
const oldest = this.map.keys().next();
|
|
1175
|
-
if (!oldest.done) this.map.delete(oldest.value);
|
|
1176
|
-
}
|
|
1177
|
-
}
|
|
1178
|
-
has(key) {
|
|
1179
|
-
return this.map.has(key);
|
|
1180
|
-
}
|
|
1181
|
-
delete(key) {
|
|
1182
|
-
return this.map.delete(key);
|
|
1183
|
-
}
|
|
1184
|
-
clear() {
|
|
1185
|
-
this.map.clear();
|
|
1186
|
-
}
|
|
1187
|
-
get size() {
|
|
1188
|
-
return this.map.size;
|
|
1189
|
-
}
|
|
1190
|
-
};
|
|
1191
|
-
|
|
1192
1406
|
// src/persistent/cursor.ts
|
|
1193
1407
|
import { createHash } from "crypto";
|
|
1194
1408
|
import { closeSync, openSync, readSync, statSync } from "fs";
|
|
@@ -1244,14 +1458,17 @@ function classify(filePath, existing) {
|
|
|
1244
1458
|
return { change: "reindex", stat: stat4 };
|
|
1245
1459
|
}
|
|
1246
1460
|
|
|
1461
|
+
// src/persistent/index-engine.ts
|
|
1462
|
+
import { statSync as statSync2 } from "fs";
|
|
1463
|
+
|
|
1247
1464
|
// src/providers/parse.ts
|
|
1248
|
-
import { createReadStream as
|
|
1465
|
+
import { createReadStream as createReadStream5 } from "fs";
|
|
1249
1466
|
import { createInterface as createInterface3 } from "readline";
|
|
1250
1467
|
async function parseMetaWithProvider(provider, filePath, account, tier) {
|
|
1251
1468
|
const log = getLogger();
|
|
1252
1469
|
const acc = provider.createEmptyAccumulator();
|
|
1253
1470
|
const rl = createInterface3({
|
|
1254
|
-
input:
|
|
1471
|
+
input: createReadStream5(filePath),
|
|
1255
1472
|
crlfDelay: Infinity
|
|
1256
1473
|
});
|
|
1257
1474
|
try {
|
|
@@ -1578,104 +1795,6 @@ function joinPath(dir, name) {
|
|
|
1578
1795
|
return dir.endsWith("/") ? `${dir}${name}` : `${dir}/${name}`;
|
|
1579
1796
|
}
|
|
1580
1797
|
|
|
1581
|
-
// src/persistent/jsonl-tail-reader.ts
|
|
1582
|
-
import { createReadStream as createReadStream4 } from "fs";
|
|
1583
|
-
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1584
|
-
const stream = createReadStream4(filePath, { start: startOffset, encoding: "utf8" });
|
|
1585
|
-
let buffer = "";
|
|
1586
|
-
let offset = startOffset;
|
|
1587
|
-
let line = startLine;
|
|
1588
|
-
let parsedLines = 0;
|
|
1589
|
-
for await (const chunk of stream) {
|
|
1590
|
-
buffer += chunk;
|
|
1591
|
-
let nl;
|
|
1592
|
-
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1593
|
-
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1594
|
-
const text = lineWithNewline.trimEnd();
|
|
1595
|
-
buffer = buffer.slice(nl + 1);
|
|
1596
|
-
if (text.length > 0) {
|
|
1597
|
-
try {
|
|
1598
|
-
reduceLine(state, JSON.parse(text), tier);
|
|
1599
|
-
} catch {
|
|
1600
|
-
state.badJsonLines++;
|
|
1601
|
-
}
|
|
1602
|
-
parsedLines++;
|
|
1603
|
-
}
|
|
1604
|
-
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1605
|
-
line++;
|
|
1606
|
-
}
|
|
1607
|
-
}
|
|
1608
|
-
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
1609
|
-
}
|
|
1610
|
-
|
|
1611
|
-
// src/persistent/paged-reader.ts
|
|
1612
|
-
import { createReadStream as createReadStream5 } from "fs";
|
|
1613
|
-
var CHECKPOINT_INTERVAL = 500;
|
|
1614
|
-
async function streamMessages(filePath, startOffset, startLine, state, onMessage) {
|
|
1615
|
-
const stream = createReadStream5(filePath, { start: startOffset, encoding: "utf8" });
|
|
1616
|
-
let buffer = "";
|
|
1617
|
-
let offset = startOffset;
|
|
1618
|
-
let line = startLine;
|
|
1619
|
-
for await (const chunk of stream) {
|
|
1620
|
-
buffer += chunk;
|
|
1621
|
-
let nl;
|
|
1622
|
-
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1623
|
-
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1624
|
-
const text = lineWithNewline.trimEnd();
|
|
1625
|
-
buffer = buffer.slice(nl + 1);
|
|
1626
|
-
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1627
|
-
line += 1;
|
|
1628
|
-
if (text.length === 0) continue;
|
|
1629
|
-
let entry;
|
|
1630
|
-
try {
|
|
1631
|
-
entry = JSON.parse(text);
|
|
1632
|
-
} catch {
|
|
1633
|
-
continue;
|
|
1634
|
-
}
|
|
1635
|
-
const message = reduceConvLine(state, entry);
|
|
1636
|
-
if (message && onMessage(message, offset, line)) {
|
|
1637
|
-
stream.destroy();
|
|
1638
|
-
return;
|
|
1639
|
-
}
|
|
1640
|
-
}
|
|
1641
|
-
}
|
|
1642
|
-
}
|
|
1643
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL) {
|
|
1644
|
-
const checkpoints = [];
|
|
1645
|
-
const state = initialConvState();
|
|
1646
|
-
let index = 0;
|
|
1647
|
-
await streamMessages(filePath, 0, 0, state, (_msg, nextOffset, nextLine) => {
|
|
1648
|
-
index += 1;
|
|
1649
|
-
if (index % interval === 0) {
|
|
1650
|
-
checkpoints.push({
|
|
1651
|
-
messageIndex: index,
|
|
1652
|
-
byteOffset: nextOffset,
|
|
1653
|
-
lineNumber: nextLine,
|
|
1654
|
-
state: structuredClone(state)
|
|
1655
|
-
});
|
|
1656
|
-
}
|
|
1657
|
-
return false;
|
|
1658
|
-
});
|
|
1659
|
-
return checkpoints;
|
|
1660
|
-
}
|
|
1661
|
-
async function readPage(filePath, total, options, floor) {
|
|
1662
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
1663
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1664
|
-
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1665
|
-
const startOffset = floor ? floor.byteOffset : 0;
|
|
1666
|
-
const startLine = floor ? floor.lineNumber : 0;
|
|
1667
|
-
let index = floor ? floor.messageIndex : 0;
|
|
1668
|
-
const window = [];
|
|
1669
|
-
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1670
|
-
const current = index;
|
|
1671
|
-
index += 1;
|
|
1672
|
-
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1673
|
-
return index >= beforeIndex;
|
|
1674
|
-
});
|
|
1675
|
-
applyTeamInfo(window, state);
|
|
1676
|
-
return { messages: window, total, fromIndex };
|
|
1677
|
-
}
|
|
1678
|
-
|
|
1679
1798
|
// src/persistent/repositories/checkpoints.repo.ts
|
|
1680
1799
|
var CheckpointsRepo = class {
|
|
1681
1800
|
constructor(db) {
|
|
@@ -1696,6 +1815,22 @@ var CheckpointsRepo = class {
|
|
|
1696
1815
|
});
|
|
1697
1816
|
tx();
|
|
1698
1817
|
}
|
|
1818
|
+
// Insert checkpoints without touching existing rows. Appends never invalidate
|
|
1819
|
+
// the chain covering the immutable prefix (Kafka sparse-index style); rows are
|
|
1820
|
+
// only ever removed on truncation/replace or deletion.
|
|
1821
|
+
append(sourcePath, checkpoints) {
|
|
1822
|
+
const tx = this.db.transaction(() => {
|
|
1823
|
+
const insert = this.db.prepare(
|
|
1824
|
+
`INSERT INTO message_checkpoints
|
|
1825
|
+
(source_path, message_index, byte_offset, line_number, parser_state)
|
|
1826
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
1827
|
+
);
|
|
1828
|
+
for (const c of checkpoints) {
|
|
1829
|
+
insert.run(sourcePath, c.messageIndex, c.byteOffset, c.lineNumber, JSON.stringify(c.state));
|
|
1830
|
+
}
|
|
1831
|
+
});
|
|
1832
|
+
tx();
|
|
1833
|
+
}
|
|
1699
1834
|
// The latest checkpoint at or before `messageIndex`, or null if none (read
|
|
1700
1835
|
// from the file start). Lets a page seek to the nearest prior anchor.
|
|
1701
1836
|
floor(sourcePath, messageIndex) {
|
|
@@ -1707,6 +1842,17 @@ var CheckpointsRepo = class {
|
|
|
1707
1842
|
).get(sourcePath, messageIndex);
|
|
1708
1843
|
return row ? toCheckpoint(row) : null;
|
|
1709
1844
|
}
|
|
1845
|
+
// The highest-index checkpoint for a file, or null if none. The resume point
|
|
1846
|
+
// for extending the chain after an append.
|
|
1847
|
+
last(sourcePath) {
|
|
1848
|
+
const row = this.db.prepare(
|
|
1849
|
+
`SELECT message_index, byte_offset, line_number, parser_state
|
|
1850
|
+
FROM message_checkpoints
|
|
1851
|
+
WHERE source_path = ?
|
|
1852
|
+
ORDER BY message_index DESC LIMIT 1`
|
|
1853
|
+
).get(sourcePath);
|
|
1854
|
+
return row ? toCheckpoint(row) : null;
|
|
1855
|
+
}
|
|
1710
1856
|
count(sourcePath) {
|
|
1711
1857
|
return this.db.prepare("SELECT COUNT(*) AS n FROM message_checkpoints WHERE source_path = ?").get(sourcePath).n;
|
|
1712
1858
|
}
|
|
@@ -1724,7 +1870,7 @@ function toCheckpoint(row) {
|
|
|
1724
1870
|
}
|
|
1725
1871
|
|
|
1726
1872
|
// src/persistent/repositories/conversation-files.repo.ts
|
|
1727
|
-
import { basename as
|
|
1873
|
+
import { basename as basename5, dirname as dirname4 } from "path";
|
|
1728
1874
|
var ConversationFilesRepo = class {
|
|
1729
1875
|
constructor(db) {
|
|
1730
1876
|
this.db = db;
|
|
@@ -1741,7 +1887,7 @@ var ConversationFilesRepo = class {
|
|
|
1741
1887
|
const info = this.db.prepare(
|
|
1742
1888
|
`INSERT INTO conversation_files (absolute_path, parent_dir, file_name, account)
|
|
1743
1889
|
VALUES (?, ?, ?, ?)`
|
|
1744
|
-
).run(absolutePath, dirname4(absolutePath),
|
|
1890
|
+
).run(absolutePath, dirname4(absolutePath), basename5(absolutePath), account);
|
|
1745
1891
|
return Number(info.lastInsertRowid);
|
|
1746
1892
|
}
|
|
1747
1893
|
// Advance the cursor + persisted reducer state after a successful index pass.
|
|
@@ -2100,6 +2246,9 @@ var PersistentEngine = class {
|
|
|
2100
2246
|
// restart just means the first few post-restart scans don't force an early
|
|
2101
2247
|
// backstop pass, which is harmless (watermarks themselves persist in the DB).
|
|
2102
2248
|
scanCount = 0;
|
|
2249
|
+
// In-flight checkpoint build/extension per file, so concurrent getPage
|
|
2250
|
+
// callers share one stream instead of each walking the file.
|
|
2251
|
+
checkpointBuilds = /* @__PURE__ */ new Map();
|
|
2103
2252
|
constructor(dbPath, options = {}) {
|
|
2104
2253
|
this.db = openDatabase(dbPath);
|
|
2105
2254
|
this.files = new ConversationFilesRepo(this.db);
|
|
@@ -2152,7 +2301,7 @@ var PersistentEngine = class {
|
|
|
2152
2301
|
const batch = discovered.slice(i, i + BATCH_SIZE);
|
|
2153
2302
|
const results = await Promise.all(
|
|
2154
2303
|
batch.map(async ({ filePath, account, provider }) => {
|
|
2155
|
-
const meta = await this.indexFile(
|
|
2304
|
+
const { meta } = await this.indexFile(
|
|
2156
2305
|
filePath,
|
|
2157
2306
|
account,
|
|
2158
2307
|
tier.name,
|
|
@@ -2187,7 +2336,9 @@ var PersistentEngine = class {
|
|
|
2187
2336
|
// unchanged → return the stored summary; appended → resume the fold and read
|
|
2188
2337
|
// only new bytes; reindex/force → fold from offset 0. Writes the summary +
|
|
2189
2338
|
// cursor + reducer state in one transaction so a crash never leaves a
|
|
2190
|
-
// half-written row or an over-advanced cursor.
|
|
2339
|
+
// half-written row or an over-advanced cursor. Returns the classification
|
|
2340
|
+
// alongside the meta so callers (refreshFile) can keep, extend, or evict
|
|
2341
|
+
// their own per-file caches without re-stat'ing the file (racy) themselves.
|
|
2191
2342
|
async indexFile(filePath, account, tierName, customTiers, resolveGitBranch, force = false, provider) {
|
|
2192
2343
|
const log = getLogger();
|
|
2193
2344
|
const tier = resolveTier(tierName, customTiers);
|
|
@@ -2195,13 +2346,21 @@ var PersistentEngine = class {
|
|
|
2195
2346
|
const { change, stat: stat4 } = classify(filePath, existing);
|
|
2196
2347
|
if (change === "vanished" || !stat4) {
|
|
2197
2348
|
this.markDeleted(filePath);
|
|
2198
|
-
return null;
|
|
2349
|
+
return { meta: null, change: "vanished" };
|
|
2199
2350
|
}
|
|
2200
2351
|
if (change === "unchanged" && !force) {
|
|
2201
|
-
return this.conversations.getBySourcePath(filePath);
|
|
2352
|
+
return { meta: this.conversations.getBySourcePath(filePath), change };
|
|
2202
2353
|
}
|
|
2203
2354
|
if (provider && provider.name !== CLAUDE_CODE_PROVIDER) {
|
|
2204
|
-
|
|
2355
|
+
const meta2 = await this.indexFileWithProvider(
|
|
2356
|
+
provider,
|
|
2357
|
+
filePath,
|
|
2358
|
+
account,
|
|
2359
|
+
tier,
|
|
2360
|
+
stat4,
|
|
2361
|
+
resolveGitBranch
|
|
2362
|
+
);
|
|
2363
|
+
return { meta: meta2, change };
|
|
2205
2364
|
}
|
|
2206
2365
|
const resume = change === "appended" && !force && existing?.reducer_state;
|
|
2207
2366
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
@@ -2212,12 +2371,12 @@ var PersistentEngine = class {
|
|
|
2212
2371
|
result = await tailReduce(filePath, startOffset, startLine, state, tier);
|
|
2213
2372
|
} catch (err) {
|
|
2214
2373
|
log.warn({ filePath, err }, "persistent: tail read failed");
|
|
2215
|
-
return null;
|
|
2374
|
+
return { meta: null, change };
|
|
2216
2375
|
}
|
|
2217
2376
|
const meta = finalizeMeta(state, filePath, account, tier);
|
|
2218
2377
|
if (!meta) {
|
|
2219
2378
|
this.markDeleted(filePath);
|
|
2220
|
-
return null;
|
|
2379
|
+
return { meta: null, change };
|
|
2221
2380
|
}
|
|
2222
2381
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2223
2382
|
const fp = stat4.size > 0 ? fingerprint(filePath, stat4.size) : null;
|
|
@@ -2225,7 +2384,7 @@ var PersistentEngine = class {
|
|
|
2225
2384
|
const upsert = this.db.transaction(() => {
|
|
2226
2385
|
this.conversations.upsert(fileId, meta, state.pageMessageCount);
|
|
2227
2386
|
this.fts.upsert(meta);
|
|
2228
|
-
this.checkpoints.remove(filePath);
|
|
2387
|
+
if (!resume) this.checkpoints.remove(filePath);
|
|
2229
2388
|
this.files.updateCursor(fileId, {
|
|
2230
2389
|
sizeBytes: stat4.size,
|
|
2231
2390
|
mtimeMs: stat4.mtimeMs,
|
|
@@ -2258,7 +2417,7 @@ var PersistentEngine = class {
|
|
|
2258
2417
|
{ filePath, change, bytesRead: result.newOffset - startOffset, msgs: meta.messageCount },
|
|
2259
2418
|
"persistent: indexed file"
|
|
2260
2419
|
);
|
|
2261
|
-
return meta;
|
|
2420
|
+
return { meta, change };
|
|
2262
2421
|
}
|
|
2263
2422
|
// Index a non-Threadbase provider file: full reparse from offset 0 through the
|
|
2264
2423
|
// provider's reducer/finalize, then the same upsert + FTS write + cursor bump
|
|
@@ -2358,15 +2517,40 @@ var PersistentEngine = class {
|
|
|
2358
2517
|
return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
|
|
2359
2518
|
}
|
|
2360
2519
|
const total = this.conversations.pageMessageCount(filePath);
|
|
2361
|
-
|
|
2362
|
-
const built = await buildCheckpoints(filePath);
|
|
2363
|
-
if (built.length > 0) this.checkpoints.replaceAll(filePath, built);
|
|
2364
|
-
}
|
|
2520
|
+
await this.ensureCheckpoints(filePath, total);
|
|
2365
2521
|
const beforeIndex = options.beforeIndex ?? total;
|
|
2366
2522
|
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2367
|
-
|
|
2523
|
+
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
2524
|
+
if (floor) {
|
|
2525
|
+
try {
|
|
2526
|
+
if (floor.byteOffset > statSync2(filePath).size) floor = null;
|
|
2527
|
+
} catch {
|
|
2528
|
+
}
|
|
2529
|
+
}
|
|
2368
2530
|
return readPage(filePath, total, options, floor);
|
|
2369
2531
|
}
|
|
2532
|
+
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
2533
|
+
// file → full build; a file that grew → extend from the last persisted
|
|
2534
|
+
// checkpoint (reads only past its offset, never the prefix). Single-flighted
|
|
2535
|
+
// per path: concurrent getPage callers await the same build instead of
|
|
2536
|
+
// streaming the file in parallel.
|
|
2537
|
+
ensureCheckpoints(filePath, total) {
|
|
2538
|
+
if (total <= CHECKPOINT_INTERVAL) return Promise.resolve();
|
|
2539
|
+
const inFlight = this.checkpointBuilds.get(filePath);
|
|
2540
|
+
if (inFlight) return inFlight;
|
|
2541
|
+
const build = (async () => {
|
|
2542
|
+
const last = this.checkpoints.last(filePath);
|
|
2543
|
+
if (last && total < last.messageIndex + CHECKPOINT_INTERVAL) return;
|
|
2544
|
+
const fresh = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, last);
|
|
2545
|
+
if (fresh.length === 0) return;
|
|
2546
|
+
if (last) this.checkpoints.append(filePath, fresh);
|
|
2547
|
+
else this.checkpoints.replaceAll(filePath, fresh);
|
|
2548
|
+
})().finally(() => {
|
|
2549
|
+
if (this.checkpointBuilds.get(filePath) === build) this.checkpointBuilds.delete(filePath);
|
|
2550
|
+
});
|
|
2551
|
+
this.checkpointBuilds.set(filePath, build);
|
|
2552
|
+
return build;
|
|
2553
|
+
}
|
|
2370
2554
|
};
|
|
2371
2555
|
|
|
2372
2556
|
// src/watcher/file-watcher.ts
|
|
@@ -2472,6 +2656,8 @@ function defaultDbPath() {
|
|
|
2472
2656
|
}
|
|
2473
2657
|
var ConversationScanner = class {
|
|
2474
2658
|
metadataCache = /* @__PURE__ */ new Map();
|
|
2659
|
+
// Parsed conversations plus (persistent claude-code entries only) the resume
|
|
2660
|
+
// point that lets refreshFile extend them in place when the file grows.
|
|
2475
2661
|
conversationLRU;
|
|
2476
2662
|
// session_id is NOT unique, so this maps a sessionId to every active meta that
|
|
2477
2663
|
// carries it. Resolution picks deterministically (newest timestamp, then path
|
|
@@ -2503,7 +2689,9 @@ var ConversationScanner = class {
|
|
|
2503
2689
|
// can't hit a closed DB (the watch-mode half of Bug #4).
|
|
2504
2690
|
inFlightReconcile = null;
|
|
2505
2691
|
constructor(options) {
|
|
2506
|
-
this.conversationLRU = new LRUCache(
|
|
2692
|
+
this.conversationLRU = new LRUCache(
|
|
2693
|
+
options?.conversationCacheSize ?? 5
|
|
2694
|
+
);
|
|
2507
2695
|
if (options?.persistent === false) {
|
|
2508
2696
|
this.dbPath = null;
|
|
2509
2697
|
this.sidecarEnabled = false;
|
|
@@ -2635,7 +2823,7 @@ var ConversationScanner = class {
|
|
|
2635
2823
|
const cached = statCache.get(filePath);
|
|
2636
2824
|
if (cached) {
|
|
2637
2825
|
try {
|
|
2638
|
-
const s =
|
|
2826
|
+
const s = statSync3(filePath);
|
|
2639
2827
|
if (s.mtimeMs === cached.stat.mtimeMs && s.size === cached.stat.size) {
|
|
2640
2828
|
return cached.meta;
|
|
2641
2829
|
}
|
|
@@ -2760,7 +2948,7 @@ var ConversationScanner = class {
|
|
|
2760
2948
|
const cached = this.conversationLRU.get(id);
|
|
2761
2949
|
if (cached) {
|
|
2762
2950
|
log.debug({ id }, "getConversation: cache hit");
|
|
2763
|
-
return cached;
|
|
2951
|
+
return cached.conversation;
|
|
2764
2952
|
}
|
|
2765
2953
|
const meta = this.persistent ? this.engine().getByIdOrSession(id) : this.metadataCache.get(id) ?? this.resolveSessionId(id);
|
|
2766
2954
|
if (!meta) {
|
|
@@ -2769,9 +2957,14 @@ var ConversationScanner = class {
|
|
|
2769
2957
|
}
|
|
2770
2958
|
log.debug({ id, filePath: meta.filePath }, "getConversation: cache miss, parsing");
|
|
2771
2959
|
try {
|
|
2960
|
+
if (this.persistent && meta.provider !== CODEX_CLI_PROVIDER) {
|
|
2961
|
+
const parsed = await parseConversationResumable(meta.filePath, meta.account);
|
|
2962
|
+
if (parsed) this.conversationLRU.set(id, parsed);
|
|
2963
|
+
return parsed?.conversation ?? null;
|
|
2964
|
+
}
|
|
2772
2965
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
2773
2966
|
if (conversation) {
|
|
2774
|
-
this.conversationLRU.set(id, conversation);
|
|
2967
|
+
this.conversationLRU.set(id, { conversation });
|
|
2775
2968
|
}
|
|
2776
2969
|
return conversation;
|
|
2777
2970
|
} catch (err) {
|
|
@@ -2843,20 +3036,30 @@ var ConversationScanner = class {
|
|
|
2843
3036
|
// not seen before. Returns the fresh ConversationMeta, or null when the file
|
|
2844
3037
|
// no longer parses (missing/empty) — in which case any prior entry for it is
|
|
2845
3038
|
// dropped from all indexes.
|
|
2846
|
-
|
|
3039
|
+
//
|
|
3040
|
+
// Single-flighted per path: concurrent callers (stacked client retries, a
|
|
3041
|
+
// watcher tick racing a caller) await the one in-flight refresh instead of
|
|
3042
|
+
// each re-reading the file.
|
|
3043
|
+
refreshesInFlight = /* @__PURE__ */ new Map();
|
|
3044
|
+
refreshFile(filePath, account) {
|
|
3045
|
+
const inFlight = this.refreshesInFlight.get(filePath);
|
|
3046
|
+
if (inFlight) return inFlight;
|
|
3047
|
+
const refresh = this.doRefreshFile(filePath, account).finally(() => {
|
|
3048
|
+
if (this.refreshesInFlight.get(filePath) === refresh) {
|
|
3049
|
+
this.refreshesInFlight.delete(filePath);
|
|
3050
|
+
}
|
|
3051
|
+
});
|
|
3052
|
+
this.refreshesInFlight.set(filePath, refresh);
|
|
3053
|
+
return refresh;
|
|
3054
|
+
}
|
|
3055
|
+
async doRefreshFile(filePath, account) {
|
|
2847
3056
|
const log = getLogger();
|
|
2848
3057
|
if (this.persistent) {
|
|
2849
3058
|
const engine = this.engine();
|
|
2850
3059
|
const previous2 = engine.getByIdOrSession(filePath);
|
|
2851
3060
|
const resolvedAccount2 = account ?? previous2?.account ?? "default";
|
|
2852
|
-
const evict2 = (m) => {
|
|
2853
|
-
if (!m) return;
|
|
2854
|
-
this.conversationLRU.delete(m.id);
|
|
2855
|
-
this.conversationLRU.delete(m.sessionId);
|
|
2856
|
-
};
|
|
2857
|
-
evict2(previous2);
|
|
2858
3061
|
const provider = await this.resolveProviderForFile(filePath, previous2);
|
|
2859
|
-
const meta2 = await engine.indexFile(
|
|
3062
|
+
const { meta: meta2, change } = await engine.indexFile(
|
|
2860
3063
|
filePath,
|
|
2861
3064
|
resolvedAccount2,
|
|
2862
3065
|
this.lastTier.name,
|
|
@@ -2865,8 +3068,19 @@ var ConversationScanner = class {
|
|
|
2865
3068
|
false,
|
|
2866
3069
|
provider
|
|
2867
3070
|
);
|
|
2868
|
-
|
|
2869
|
-
|
|
3071
|
+
const cacheKeys = /* @__PURE__ */ new Set();
|
|
3072
|
+
for (const m of [previous2, meta2]) {
|
|
3073
|
+
if (m) {
|
|
3074
|
+
cacheKeys.add(m.id);
|
|
3075
|
+
cacheKeys.add(m.sessionId);
|
|
3076
|
+
}
|
|
3077
|
+
}
|
|
3078
|
+
if (!meta2 || change === "reindex" || change === "vanished") {
|
|
3079
|
+
for (const key of cacheKeys) this.conversationLRU.delete(key);
|
|
3080
|
+
} else if (change === "appended") {
|
|
3081
|
+
await this.extendCachedConversations(cacheKeys, filePath, meta2.account);
|
|
3082
|
+
}
|
|
3083
|
+
log.debug({ filePath, change, kept: !!meta2 }, "refreshFile: updated persistent index");
|
|
2870
3084
|
return meta2;
|
|
2871
3085
|
}
|
|
2872
3086
|
const previous = this.metadataCache.get(filePath);
|
|
@@ -2910,6 +3124,39 @@ var ConversationScanner = class {
|
|
|
2910
3124
|
);
|
|
2911
3125
|
return meta;
|
|
2912
3126
|
}
|
|
3127
|
+
// Advance every cached parse of an appended file by folding only the new
|
|
3128
|
+
// bytes through the conversation reducer — the in-memory analogue of the
|
|
3129
|
+
// persisted metadata fold. Entries without resume state (Codex) and entries
|
|
3130
|
+
// whose extension fails are evicted so the next read re-parses from scratch.
|
|
3131
|
+
async extendCachedConversations(cacheKeys, filePath, account) {
|
|
3132
|
+
const wrappers = /* @__PURE__ */ new Map();
|
|
3133
|
+
for (const key of cacheKeys) {
|
|
3134
|
+
const wrapper = this.conversationLRU.get(key);
|
|
3135
|
+
if (!wrapper) continue;
|
|
3136
|
+
const keys = wrappers.get(wrapper) ?? [];
|
|
3137
|
+
keys.push(key);
|
|
3138
|
+
wrappers.set(wrapper, keys);
|
|
3139
|
+
}
|
|
3140
|
+
for (const [wrapper, keys] of wrappers) {
|
|
3141
|
+
if (!wrapper.resume) {
|
|
3142
|
+
for (const key of keys) this.conversationLRU.delete(key);
|
|
3143
|
+
continue;
|
|
3144
|
+
}
|
|
3145
|
+
try {
|
|
3146
|
+
const extended = await extendConversation(
|
|
3147
|
+
wrapper.conversation,
|
|
3148
|
+
wrapper.resume,
|
|
3149
|
+
filePath,
|
|
3150
|
+
account
|
|
3151
|
+
);
|
|
3152
|
+
wrapper.conversation = extended.conversation;
|
|
3153
|
+
wrapper.resume = extended.resume;
|
|
3154
|
+
} catch (err) {
|
|
3155
|
+
getLogger().warn({ filePath, err }, "refreshFile: cache extension failed, evicting");
|
|
3156
|
+
for (const key of keys) this.conversationLRU.delete(key);
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
3159
|
+
}
|
|
2913
3160
|
getMetadataCache() {
|
|
2914
3161
|
if (this.persistent) {
|
|
2915
3162
|
const map = /* @__PURE__ */ new Map();
|
|
@@ -3206,12 +3453,14 @@ export {
|
|
|
3206
3453
|
applySinceFilter,
|
|
3207
3454
|
applySort,
|
|
3208
3455
|
cleanSystemTags,
|
|
3456
|
+
initialConvState as createJsonlParseState,
|
|
3209
3457
|
createLogger,
|
|
3210
3458
|
detectDefaultProfile,
|
|
3211
3459
|
getConversation,
|
|
3212
3460
|
getLogger,
|
|
3213
3461
|
getProjectsDir,
|
|
3214
3462
|
loadProfiles,
|
|
3463
|
+
parseJsonlLine,
|
|
3215
3464
|
readGitBranch,
|
|
3216
3465
|
readSidecar,
|
|
3217
3466
|
resetDefaultScanner,
|