@threadbase-sh/scanner 0.7.2 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -10
- package/dist/cli.js +2060 -339
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +2115 -377
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +154 -6
- package/dist/index.d.ts +154 -6
- package/dist/index.js +2112 -378
- package/dist/index.js.map +1 -1
- package/package.json +4 -1
package/dist/index.js
CHANGED
|
@@ -1,23 +1,25 @@
|
|
|
1
1
|
// src/filters.ts
|
|
2
2
|
function applySort(metas, order) {
|
|
3
3
|
const out = [...metas];
|
|
4
|
+
const tie = (a, b) => a.id.localeCompare(b.id);
|
|
4
5
|
switch (order) {
|
|
5
6
|
case "recent":
|
|
6
|
-
out.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
7
|
+
out.sort((a, b) => b.timestamp.localeCompare(a.timestamp) || tie(a, b));
|
|
7
8
|
break;
|
|
8
9
|
case "oldest":
|
|
9
|
-
out.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
10
|
+
out.sort((a, b) => a.timestamp.localeCompare(b.timestamp) || tie(a, b));
|
|
10
11
|
break;
|
|
11
12
|
case "messages-desc":
|
|
12
|
-
out.sort((a, b) => b.messageCount - a.messageCount);
|
|
13
|
+
out.sort((a, b) => b.messageCount - a.messageCount || tie(a, b));
|
|
13
14
|
break;
|
|
14
15
|
case "messages-asc":
|
|
15
|
-
out.sort((a, b) => a.messageCount - b.messageCount);
|
|
16
|
+
out.sort((a, b) => a.messageCount - b.messageCount || tie(a, b));
|
|
16
17
|
break;
|
|
17
18
|
case "alpha":
|
|
18
19
|
out.sort((a, b) => {
|
|
19
20
|
const cmp = a.projectName.localeCompare(b.projectName);
|
|
20
|
-
|
|
21
|
+
if (cmp !== 0) return cmp;
|
|
22
|
+
return a.preview.localeCompare(b.preview) || tie(a, b);
|
|
21
23
|
});
|
|
22
24
|
break;
|
|
23
25
|
}
|
|
@@ -142,6 +144,36 @@ function readGitBranch(projectPath) {
|
|
|
142
144
|
|
|
143
145
|
// src/indexer.ts
|
|
144
146
|
import FlexSearchModule from "flexsearch";
|
|
147
|
+
|
|
148
|
+
// src/search-matches.ts
|
|
149
|
+
function generateMatches(meta, query) {
|
|
150
|
+
const matches = [];
|
|
151
|
+
const lowerQuery = query.toLowerCase();
|
|
152
|
+
const fields = [
|
|
153
|
+
["contentSnippet", meta.contentSnippet],
|
|
154
|
+
["projectName", meta.projectName],
|
|
155
|
+
["sessionId", meta.sessionId],
|
|
156
|
+
["sessionName", meta.sessionName],
|
|
157
|
+
["account", meta.account],
|
|
158
|
+
["model", meta.model || ""],
|
|
159
|
+
["gitBranch", meta.gitBranch || ""],
|
|
160
|
+
["toolNames", meta.toolNames.join(" ")]
|
|
161
|
+
];
|
|
162
|
+
for (const [field, value] of fields) {
|
|
163
|
+
const idx = value.toLowerCase().indexOf(lowerQuery);
|
|
164
|
+
if (idx !== -1) {
|
|
165
|
+
const start = Math.max(0, idx - 80);
|
|
166
|
+
const end = Math.min(value.length, idx + query.length + 120);
|
|
167
|
+
let snippet = value.slice(start, end);
|
|
168
|
+
if (start > 0) snippet = `...${snippet}`;
|
|
169
|
+
if (end < value.length) snippet = `${snippet}...`;
|
|
170
|
+
matches.push({ field, snippet });
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return matches.length > 0 ? matches : [{ field: "preview", snippet: meta.preview }];
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// src/indexer.ts
|
|
145
177
|
var FlexSearch = FlexSearchModule.default ?? FlexSearchModule;
|
|
146
178
|
var SearchIndexer = class {
|
|
147
179
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -210,7 +242,7 @@ var SearchIndexer = class {
|
|
|
210
242
|
seen.add(id);
|
|
211
243
|
const meta = this.documents.get(id);
|
|
212
244
|
if (!meta) continue;
|
|
213
|
-
const matches =
|
|
245
|
+
const matches = generateMatches(meta, query);
|
|
214
246
|
searchResults.push({ meta, score: 1, matches });
|
|
215
247
|
if (searchResults.length >= limit) break;
|
|
216
248
|
}
|
|
@@ -225,32 +257,6 @@ var SearchIndexer = class {
|
|
|
225
257
|
matches: [{ field: "timestamp", snippet: meta.preview }]
|
|
226
258
|
}));
|
|
227
259
|
}
|
|
228
|
-
generateMatches(meta, query) {
|
|
229
|
-
const matches = [];
|
|
230
|
-
const lowerQuery = query.toLowerCase();
|
|
231
|
-
const fields = [
|
|
232
|
-
["contentSnippet", meta.contentSnippet],
|
|
233
|
-
["projectName", meta.projectName],
|
|
234
|
-
["sessionId", meta.sessionId],
|
|
235
|
-
["sessionName", meta.sessionName],
|
|
236
|
-
["account", meta.account],
|
|
237
|
-
["model", meta.model || ""],
|
|
238
|
-
["gitBranch", meta.gitBranch || ""],
|
|
239
|
-
["toolNames", meta.toolNames.join(" ")]
|
|
240
|
-
];
|
|
241
|
-
for (const [field, value] of fields) {
|
|
242
|
-
const idx = value.toLowerCase().indexOf(lowerQuery);
|
|
243
|
-
if (idx !== -1) {
|
|
244
|
-
const start = Math.max(0, idx - 80);
|
|
245
|
-
const end = Math.min(value.length, idx + query.length + 120);
|
|
246
|
-
let snippet = value.slice(start, end);
|
|
247
|
-
if (start > 0) snippet = `...${snippet}`;
|
|
248
|
-
if (end < value.length) snippet = `${snippet}...`;
|
|
249
|
-
matches.push({ field, snippet });
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
return matches.length > 0 ? matches : [{ field: "preview", snippet: meta.preview }];
|
|
253
|
-
}
|
|
254
260
|
getDocumentCount() {
|
|
255
261
|
return this.documents.size;
|
|
256
262
|
}
|
|
@@ -283,6 +289,46 @@ var SearchIndexer = class {
|
|
|
283
289
|
}
|
|
284
290
|
};
|
|
285
291
|
|
|
292
|
+
// src/persistent/sidecar.ts
|
|
293
|
+
import { readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
294
|
+
var SIDECAR_VERSION = 1;
|
|
295
|
+
function sidecarPath(jsonlPath) {
|
|
296
|
+
return `${jsonlPath}.idx.json`;
|
|
297
|
+
}
|
|
298
|
+
function buildSidecar(meta, cursor, updatedAt) {
|
|
299
|
+
return {
|
|
300
|
+
version: SIDECAR_VERSION,
|
|
301
|
+
sourcePath: meta.filePath,
|
|
302
|
+
sizeBytes: cursor.sizeBytes,
|
|
303
|
+
mtimeMs: cursor.mtimeMs,
|
|
304
|
+
lastIndexedOffset: cursor.offset,
|
|
305
|
+
lastIndexedLine: cursor.line,
|
|
306
|
+
messageCount: meta.messageCount,
|
|
307
|
+
projectPath: meta.projectPath,
|
|
308
|
+
projectName: meta.projectName,
|
|
309
|
+
branch: meta.gitBranch,
|
|
310
|
+
firstSentAt: meta.firstMessage?.timestamp ?? null,
|
|
311
|
+
firstSentText: meta.firstMessage?.text ?? null,
|
|
312
|
+
lastSentAt: meta.lastMessage?.timestamp ?? null,
|
|
313
|
+
lastSentText: meta.lastMessage?.text ?? null,
|
|
314
|
+
updatedAt
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
function writeSidecar(jsonlPath, sidecar) {
|
|
318
|
+
try {
|
|
319
|
+
writeFileSync(sidecarPath(jsonlPath), JSON.stringify(sidecar, null, 2));
|
|
320
|
+
} catch (err) {
|
|
321
|
+
getLogger().warn({ jsonlPath, err }, "sidecar: write failed");
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
function readSidecar(jsonlPath) {
|
|
325
|
+
try {
|
|
326
|
+
return JSON.parse(readFileSync2(sidecarPath(jsonlPath), "utf-8"));
|
|
327
|
+
} catch {
|
|
328
|
+
return null;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
286
332
|
// src/profiles.ts
|
|
287
333
|
import { mkdir, readFile, writeFile } from "fs/promises";
|
|
288
334
|
import { homedir } from "os";
|
|
@@ -324,48 +370,272 @@ async function saveProfiles(profiles, configPath) {
|
|
|
324
370
|
getLogger().debug({ configPath, count: profiles.length }, "profiles: saved");
|
|
325
371
|
}
|
|
326
372
|
|
|
327
|
-
// src/
|
|
328
|
-
import
|
|
373
|
+
// src/providers/codex-cli.ts
|
|
374
|
+
import fg from "fast-glob";
|
|
375
|
+
import { createReadStream } from "fs";
|
|
376
|
+
import { stat } from "fs/promises";
|
|
377
|
+
import { basename } from "path";
|
|
378
|
+
import { createInterface } from "readline";
|
|
329
379
|
|
|
330
|
-
// src/
|
|
331
|
-
var
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
380
|
+
// src/tags.ts
|
|
381
|
+
var SYSTEM_TAGS = [
|
|
382
|
+
"system-reminder",
|
|
383
|
+
"command-name",
|
|
384
|
+
"command-message",
|
|
385
|
+
"command-args",
|
|
386
|
+
"ide_selection",
|
|
387
|
+
"ide_opened_file",
|
|
388
|
+
"local-command-stdout",
|
|
389
|
+
"local-command-caveat",
|
|
390
|
+
"retrieval_status",
|
|
391
|
+
"task_id",
|
|
392
|
+
"task_type",
|
|
393
|
+
"task-id",
|
|
394
|
+
"task-notification",
|
|
395
|
+
"fast_mode_info",
|
|
396
|
+
"persisted-output",
|
|
397
|
+
"tool_use_error",
|
|
398
|
+
"user-prompt-submit-hook",
|
|
399
|
+
"thinking",
|
|
400
|
+
"ask_user",
|
|
401
|
+
"teammate-message"
|
|
402
|
+
];
|
|
403
|
+
var SYSTEM_TAG_RE = new RegExp(`<(${SYSTEM_TAGS.join("|")})[^>]*>[\\s\\S]*?<\\/\\1>`, "g");
|
|
404
|
+
function cleanSystemTags(text) {
|
|
405
|
+
return text.replace(SYSTEM_TAG_RE, "").replace(/[^\S\n]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// src/providers/provider.ts
|
|
409
|
+
var CLAUDE_CODE_PROVIDER = "claude-code";
|
|
410
|
+
var CODEX_CLI_PROVIDER = "codex-cli";
|
|
411
|
+
|
|
412
|
+
// src/providers/codex-cli.ts
|
|
413
|
+
var CodexCliProvider = class {
|
|
414
|
+
name = CODEX_CLI_PROVIDER;
|
|
415
|
+
async discover(roots) {
|
|
416
|
+
const log = getLogger();
|
|
417
|
+
const results = [];
|
|
418
|
+
for (const root of roots) {
|
|
419
|
+
let paths;
|
|
420
|
+
try {
|
|
421
|
+
paths = await fg(["**/rollout-*.jsonl", "**/*.jsonl"], {
|
|
422
|
+
cwd: root,
|
|
423
|
+
absolute: true,
|
|
424
|
+
dot: false,
|
|
425
|
+
unique: true
|
|
426
|
+
});
|
|
427
|
+
} catch (err) {
|
|
428
|
+
log.warn({ root, err }, "codex discovery: glob failed");
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
for (const filePath of paths) {
|
|
432
|
+
try {
|
|
433
|
+
const s = await stat(filePath);
|
|
434
|
+
if (s.size > 0) results.push({ filePath, account: "codex" });
|
|
435
|
+
} catch (err) {
|
|
436
|
+
log.warn({ filePath, err }, "codex discovery: stat failed");
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
return results;
|
|
336
441
|
}
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
442
|
+
// Codex rollout lines carry distinctive top-level types.
|
|
443
|
+
canParse(_filePath, sample) {
|
|
444
|
+
for (const line of sample.split("\n")) {
|
|
445
|
+
if (!line.trim()) continue;
|
|
446
|
+
try {
|
|
447
|
+
const e = JSON.parse(line);
|
|
448
|
+
if (e.type === "session_meta" || e.type === "response_item" || e.type === "event_msg") {
|
|
449
|
+
return true;
|
|
450
|
+
}
|
|
451
|
+
if (e.type === "user" || e.type === "assistant") return false;
|
|
452
|
+
} catch {
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
return false;
|
|
343
456
|
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
457
|
+
createEmptyAccumulator() {
|
|
458
|
+
return {
|
|
459
|
+
sessionId: "",
|
|
460
|
+
cwd: "",
|
|
461
|
+
gitBranch: null,
|
|
462
|
+
model: null,
|
|
463
|
+
latestTimestamp: "",
|
|
464
|
+
messageCount: 0,
|
|
465
|
+
lastMessageSender: "user",
|
|
466
|
+
firstUser: null,
|
|
467
|
+
lastUser: null,
|
|
468
|
+
lastAssistant: null,
|
|
469
|
+
toolNames: [],
|
|
470
|
+
previewParts: [],
|
|
471
|
+
previewLength: 0,
|
|
472
|
+
snippetParts: [],
|
|
473
|
+
snippetLength: 0
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
reduceEntry(acc, entry, tier) {
|
|
477
|
+
reduceCodexEntry(acc, entry, tier);
|
|
478
|
+
}
|
|
479
|
+
finalize(acc, filePath, account, tier) {
|
|
480
|
+
return finalizeCodexMeta(acc, filePath, account, tier);
|
|
481
|
+
}
|
|
482
|
+
};
|
|
483
|
+
var asString = (v) => typeof v === "string" ? v : "";
|
|
484
|
+
function extractCodexText(content) {
|
|
485
|
+
if (typeof content === "string") return cleanSystemTags(content);
|
|
486
|
+
if (!Array.isArray(content)) return "";
|
|
487
|
+
return content.map((item) => {
|
|
488
|
+
if (typeof item === "string") return item;
|
|
489
|
+
const t = item?.type;
|
|
490
|
+
if ((t === "input_text" || t === "output_text" || t === "text") && item?.text) {
|
|
491
|
+
return item.text;
|
|
350
492
|
}
|
|
493
|
+
return "";
|
|
494
|
+
}).filter(Boolean).map(cleanSystemTags).join(" ");
|
|
495
|
+
}
|
|
496
|
+
function reduceCodexEntry(acc, entry, tier) {
|
|
497
|
+
const ts = asString(entry.timestamp);
|
|
498
|
+
if (ts && (!acc.latestTimestamp || ts > acc.latestTimestamp)) acc.latestTimestamp = ts;
|
|
499
|
+
const payload = entry.payload;
|
|
500
|
+
if (!payload || typeof payload !== "object") return;
|
|
501
|
+
const type = entry.type;
|
|
502
|
+
if (type === "session_meta") {
|
|
503
|
+
if (!acc.sessionId) acc.sessionId = asString(payload.id);
|
|
504
|
+
if (!acc.cwd) acc.cwd = asString(payload.cwd);
|
|
505
|
+
const git = payload.git;
|
|
506
|
+
if (acc.gitBranch === null && git?.branch) acc.gitBranch = asString(git.branch) || null;
|
|
507
|
+
return;
|
|
351
508
|
}
|
|
352
|
-
|
|
353
|
-
|
|
509
|
+
if (acc.model === null && payload.model) acc.model = asString(payload.model) || null;
|
|
510
|
+
if (type !== "response_item") return;
|
|
511
|
+
const ptype = payload.type;
|
|
512
|
+
if (ptype === "function_call" || ptype === "custom_tool_call") {
|
|
513
|
+
const name = asString(payload.name);
|
|
514
|
+
if (name && !acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
515
|
+
return;
|
|
354
516
|
}
|
|
355
|
-
|
|
356
|
-
|
|
517
|
+
if (ptype !== "message") return;
|
|
518
|
+
const role = payload.role;
|
|
519
|
+
if (role !== "user" && role !== "assistant") return;
|
|
520
|
+
const text = extractCodexText(payload.content);
|
|
521
|
+
if (!text) return;
|
|
522
|
+
const sender = role;
|
|
523
|
+
acc.messageCount++;
|
|
524
|
+
acc.lastMessageSender = sender;
|
|
525
|
+
const snapshot = { text: text.slice(0, 200), timestamp: ts };
|
|
526
|
+
if (sender === "user") {
|
|
527
|
+
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
528
|
+
acc.lastUser = snapshot;
|
|
529
|
+
} else {
|
|
530
|
+
acc.lastAssistant = snapshot;
|
|
357
531
|
}
|
|
358
|
-
|
|
359
|
-
|
|
532
|
+
if (acc.previewLength < tier.previewMax) {
|
|
533
|
+
acc.previewParts.push(text);
|
|
534
|
+
acc.previewLength += text.length;
|
|
360
535
|
}
|
|
361
|
-
|
|
362
|
-
|
|
536
|
+
if (acc.snippetLength < tier.snippetMax) {
|
|
537
|
+
const remaining = tier.snippetMax - acc.snippetLength;
|
|
538
|
+
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
539
|
+
acc.snippetParts.push(chunk);
|
|
540
|
+
acc.snippetLength += chunk.length;
|
|
363
541
|
}
|
|
364
|
-
}
|
|
542
|
+
}
|
|
543
|
+
function finalizeCodexMeta(acc, filePath, account, tier) {
|
|
544
|
+
if (acc.messageCount === 0) return null;
|
|
545
|
+
const sessionId = acc.sessionId || basename(filePath, ".jsonl");
|
|
546
|
+
const projectPath = acc.cwd;
|
|
547
|
+
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
548
|
+
return {
|
|
549
|
+
id: filePath,
|
|
550
|
+
filePath,
|
|
551
|
+
provider: CODEX_CLI_PROVIDER,
|
|
552
|
+
kind,
|
|
553
|
+
externalSessionId: acc.sessionId || void 0,
|
|
554
|
+
sessionId,
|
|
555
|
+
sessionName: "",
|
|
556
|
+
projectPath,
|
|
557
|
+
projectName: getShortProjectName(projectPath),
|
|
558
|
+
account,
|
|
559
|
+
timestamp: acc.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
560
|
+
messageCount: acc.messageCount,
|
|
561
|
+
lastMessageSender: acc.lastMessageSender,
|
|
562
|
+
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
563
|
+
contentSnippet: acc.snippetParts.join(" "),
|
|
564
|
+
gitBranch: acc.gitBranch,
|
|
565
|
+
model: acc.model,
|
|
566
|
+
isSubagent: false,
|
|
567
|
+
parentSessionId: null,
|
|
568
|
+
isTeammate: false,
|
|
569
|
+
teamName: null,
|
|
570
|
+
toolNames: acc.toolNames,
|
|
571
|
+
firstMessage: acc.firstUser,
|
|
572
|
+
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
573
|
+
lastPrompt: acc.lastUser?.text || void 0
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
function getShortProjectName(fullPath) {
|
|
577
|
+
return fullPath.split("/").filter(Boolean).slice(-3).join("/");
|
|
578
|
+
}
|
|
579
|
+
async function parseCodexConversation(filePath, account) {
|
|
580
|
+
const log = getLogger();
|
|
581
|
+
const messages = [];
|
|
582
|
+
const textParts = [];
|
|
583
|
+
let sessionId = "";
|
|
584
|
+
let cwd = "";
|
|
585
|
+
let latestTimestamp = "";
|
|
586
|
+
let lastUserText = "";
|
|
587
|
+
const rl = createInterface({ input: createReadStream(filePath), crlfDelay: Infinity });
|
|
588
|
+
try {
|
|
589
|
+
for await (const line of rl) {
|
|
590
|
+
if (!line.trim()) continue;
|
|
591
|
+
let entry;
|
|
592
|
+
try {
|
|
593
|
+
entry = JSON.parse(line);
|
|
594
|
+
} catch {
|
|
595
|
+
continue;
|
|
596
|
+
}
|
|
597
|
+
const ts = asString(entry.timestamp);
|
|
598
|
+
if (ts && (!latestTimestamp || ts > latestTimestamp)) latestTimestamp = ts;
|
|
599
|
+
const payload = entry.payload;
|
|
600
|
+
if (!payload || typeof payload !== "object") continue;
|
|
601
|
+
if (entry.type === "session_meta") {
|
|
602
|
+
if (!sessionId) sessionId = asString(payload.id);
|
|
603
|
+
if (!cwd) cwd = asString(payload.cwd);
|
|
604
|
+
continue;
|
|
605
|
+
}
|
|
606
|
+
if (entry.type !== "response_item" || payload.type !== "message") continue;
|
|
607
|
+
const role = payload.role;
|
|
608
|
+
if (role !== "user" && role !== "assistant") continue;
|
|
609
|
+
const text = extractCodexText(payload.content);
|
|
610
|
+
if (!text) continue;
|
|
611
|
+
messages.push({ role, text, timestamp: ts });
|
|
612
|
+
textParts.push(text);
|
|
613
|
+
if (role === "user") lastUserText = text;
|
|
614
|
+
}
|
|
615
|
+
} catch (err) {
|
|
616
|
+
log.warn({ filePath, err }, "parseCodexConversation: read failed");
|
|
617
|
+
return null;
|
|
618
|
+
}
|
|
619
|
+
if (messages.length === 0) return null;
|
|
620
|
+
return {
|
|
621
|
+
id: filePath,
|
|
622
|
+
filePath,
|
|
623
|
+
projectPath: cwd,
|
|
624
|
+
projectName: getShortProjectName(cwd),
|
|
625
|
+
sessionId: sessionId || basename(filePath, ".jsonl"),
|
|
626
|
+
sessionName: "",
|
|
627
|
+
messages,
|
|
628
|
+
fullText: textParts.join(" "),
|
|
629
|
+
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
630
|
+
messageCount: messages.length,
|
|
631
|
+
account,
|
|
632
|
+
lastPrompt: lastUserText || void 0
|
|
633
|
+
};
|
|
634
|
+
}
|
|
365
635
|
|
|
366
636
|
// src/discovery.ts
|
|
367
|
-
import
|
|
368
|
-
import { stat } from "fs/promises";
|
|
637
|
+
import fg2 from "fast-glob";
|
|
638
|
+
import { stat as stat2 } from "fs/promises";
|
|
369
639
|
var EXCLUDED_SEGMENTS = ["/memory/", "/tool-results/"];
|
|
370
640
|
var STAT_CONCURRENCY = 32;
|
|
371
641
|
async function discoverJsonlFiles(dirs, onProgress) {
|
|
@@ -374,7 +644,7 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
374
644
|
for (const { projectsDir, account } of dirs) {
|
|
375
645
|
let filePaths;
|
|
376
646
|
try {
|
|
377
|
-
filePaths = await
|
|
647
|
+
filePaths = await fg2("**/*.jsonl", {
|
|
378
648
|
cwd: projectsDir,
|
|
379
649
|
absolute: true,
|
|
380
650
|
dot: false
|
|
@@ -392,7 +662,7 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
392
662
|
const statted = await Promise.all(
|
|
393
663
|
chunk.map(async (filePath) => {
|
|
394
664
|
try {
|
|
395
|
-
const s = await
|
|
665
|
+
const s = await stat2(filePath);
|
|
396
666
|
return { filePath, size: s.size };
|
|
397
667
|
} catch (err) {
|
|
398
668
|
log.warn({ filePath, err }, "discovery: stat failed");
|
|
@@ -429,64 +699,115 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
429
699
|
return results;
|
|
430
700
|
}
|
|
431
701
|
|
|
702
|
+
// src/persistent/metadata-reducer.ts
|
|
703
|
+
import { basename as basename3, dirname as dirname2, join as join3 } from "path";
|
|
704
|
+
|
|
432
705
|
// src/parser.ts
|
|
433
|
-
import { createReadStream } from "fs";
|
|
434
|
-
import { basename
|
|
435
|
-
import { createInterface } from "readline";
|
|
706
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
707
|
+
import { basename as basename2 } from "path";
|
|
708
|
+
import { createInterface as createInterface2 } from "readline";
|
|
436
709
|
|
|
437
|
-
// src/
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
"
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
710
|
+
// src/persistent/conversation-reducer.ts
|
|
711
|
+
function initialConvState() {
|
|
712
|
+
return {
|
|
713
|
+
cwd: "",
|
|
714
|
+
sessionId: "",
|
|
715
|
+
sessionName: "",
|
|
716
|
+
latestTimestamp: "",
|
|
717
|
+
lastPrompt: "",
|
|
718
|
+
pendingToolUses: {},
|
|
719
|
+
teamInfo: {}
|
|
720
|
+
};
|
|
721
|
+
}
|
|
722
|
+
function reduceConvLine(state, entry) {
|
|
723
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
724
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
725
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
726
|
+
if (entry.timestamp) {
|
|
727
|
+
const ts = entry.timestamp;
|
|
728
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
729
|
+
}
|
|
730
|
+
const type = entry.type;
|
|
731
|
+
if (type === "last-prompt") {
|
|
732
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
733
|
+
return null;
|
|
734
|
+
}
|
|
735
|
+
if (type !== "user" && type !== "assistant") return null;
|
|
736
|
+
if (entry.isMeta) return null;
|
|
737
|
+
const msg = entry.message;
|
|
738
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
739
|
+
for (const block of toolUseBlocks) state.pendingToolUses[block.id] = block;
|
|
740
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
741
|
+
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
742
|
+
const content = extractTextContent(msg?.content);
|
|
743
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
744
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
745
|
+
if (!(content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking)) return null;
|
|
746
|
+
const metadata = {};
|
|
747
|
+
if (msg?.model) metadata.model = msg.model;
|
|
748
|
+
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
749
|
+
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
750
|
+
if (entry.version) metadata.version = entry.version;
|
|
751
|
+
const usage = msg?.usage;
|
|
752
|
+
if (usage) {
|
|
753
|
+
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
754
|
+
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
755
|
+
if (usage.cache_read_input_tokens) metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
756
|
+
if (usage.cache_creation_input_tokens)
|
|
757
|
+
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
758
|
+
}
|
|
759
|
+
const toolUseNames = extractToolUseNames(msg?.content);
|
|
760
|
+
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
761
|
+
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
762
|
+
if (isToolResultOnly) {
|
|
763
|
+
const pending = new Map(Object.entries(state.pendingToolUses));
|
|
764
|
+
const toolResultBlocks = extractToolResultBlocks(msg?.content, pending);
|
|
765
|
+
if (toolResultBlocks.length > 0) metadata.toolResults = toolResultBlocks;
|
|
766
|
+
}
|
|
767
|
+
if (entry.teamName) {
|
|
768
|
+
metadata.teamName = entry.teamName;
|
|
769
|
+
if (!state.teamInfo[metadata.teamName] && content) {
|
|
770
|
+
const info = parseTeammateMessageTag(content);
|
|
771
|
+
if (info) state.teamInfo[metadata.teamName] = info;
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
const thinkingContent = thinking?.content || void 0;
|
|
775
|
+
const thinkingSignature = thinking?.signature || void 0;
|
|
776
|
+
const hasMetadata = Object.keys(metadata).length > 0;
|
|
777
|
+
return {
|
|
778
|
+
role: type,
|
|
779
|
+
text: content || "",
|
|
780
|
+
timestamp: entry.timestamp || "",
|
|
781
|
+
uuid: entry.uuid || void 0,
|
|
782
|
+
metadata: hasMetadata ? metadata : void 0,
|
|
783
|
+
isToolResult: isToolResultOnly || void 0,
|
|
784
|
+
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
785
|
+
thinkingContent,
|
|
786
|
+
thinkingSignature,
|
|
787
|
+
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
788
|
+
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
789
|
+
promptId: type === "user" ? entry.promptId : void 0,
|
|
790
|
+
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
791
|
+
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
792
|
+
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
793
|
+
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
function applyTeamInfo(messages, state) {
|
|
797
|
+
if (Object.keys(state.teamInfo).length === 0) return;
|
|
798
|
+
for (const m of messages) {
|
|
799
|
+
const name = m.metadata?.teamName;
|
|
800
|
+
if (name && state.teamInfo[name] && m.metadata) m.metadata.teamInfo = state.teamInfo[name];
|
|
801
|
+
}
|
|
463
802
|
}
|
|
464
803
|
|
|
465
804
|
// src/parser.ts
|
|
466
805
|
async function parseMeta(filePath, account, tier) {
|
|
467
806
|
const log = getLogger();
|
|
468
807
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
let latestTimestamp = "";
|
|
473
|
-
let cwd = "";
|
|
474
|
-
let teamName = "";
|
|
475
|
-
let model = null;
|
|
476
|
-
let messageCount = 0;
|
|
477
|
-
let lastMessageSender = "user";
|
|
478
|
-
let isTeammate = false;
|
|
479
|
-
let firstUserSeen = false;
|
|
480
|
-
let firstMessage = null;
|
|
481
|
-
let lastMessage = null;
|
|
482
|
-
let lastPrompt = "";
|
|
483
|
-
const toolNameSet = /* @__PURE__ */ new Set();
|
|
484
|
-
const previewParts = [];
|
|
485
|
-
const snippetParts = [];
|
|
486
|
-
let snippetLength = 0;
|
|
487
|
-
let previewLength = 0;
|
|
488
|
-
const fileStream = createReadStream(filePath);
|
|
489
|
-
const rl = createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
808
|
+
const state = initialReducerState();
|
|
809
|
+
const fileStream = createReadStream2(filePath);
|
|
810
|
+
const rl = createInterface2({ input: fileStream, crlfDelay: Infinity });
|
|
490
811
|
try {
|
|
491
812
|
for await (const line of rl) {
|
|
492
813
|
if (!line.trim()) continue;
|
|
@@ -494,121 +815,35 @@ async function parseMeta(filePath, account, tier) {
|
|
|
494
815
|
try {
|
|
495
816
|
entry = JSON.parse(line);
|
|
496
817
|
} catch {
|
|
497
|
-
badJsonLines++;
|
|
818
|
+
state.badJsonLines++;
|
|
498
819
|
continue;
|
|
499
820
|
}
|
|
500
|
-
|
|
501
|
-
if (entry.sessionId && !sessionId) sessionId = entry.sessionId;
|
|
502
|
-
if (entry.slug && !sessionName) sessionName = entry.slug;
|
|
503
|
-
if (entry.teamName && !teamName) teamName = entry.teamName;
|
|
504
|
-
if (entry.timestamp) {
|
|
505
|
-
const ts = entry.timestamp;
|
|
506
|
-
if (!latestTimestamp || ts > latestTimestamp) latestTimestamp = ts;
|
|
507
|
-
}
|
|
508
|
-
const type = entry.type;
|
|
509
|
-
if (type === "last-prompt") {
|
|
510
|
-
if (entry.lastPrompt && !lastPrompt) lastPrompt = entry.lastPrompt;
|
|
511
|
-
continue;
|
|
512
|
-
}
|
|
513
|
-
if (type !== "user" && type !== "assistant") continue;
|
|
514
|
-
if (entry.isMeta) continue;
|
|
515
|
-
if (model === null) {
|
|
516
|
-
const msg2 = entry.message;
|
|
517
|
-
if (msg2?.model) model = msg2.model;
|
|
518
|
-
}
|
|
519
|
-
if (type === "user" && !firstUserSeen) {
|
|
520
|
-
firstUserSeen = true;
|
|
521
|
-
if (isTeammateContent(entry.message?.content)) {
|
|
522
|
-
isTeammate = true;
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
const msg = entry.message;
|
|
526
|
-
const content = extractTextContent(msg?.content);
|
|
527
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
528
|
-
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
529
|
-
collectToolNames(msg?.content, toolNameSet);
|
|
530
|
-
if (content || isOnlyToolResult) {
|
|
531
|
-
messageCount++;
|
|
532
|
-
lastMessageSender = type;
|
|
533
|
-
if (content) {
|
|
534
|
-
const ts = entry.timestamp || "";
|
|
535
|
-
if (!firstMessage) {
|
|
536
|
-
firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
537
|
-
}
|
|
538
|
-
lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
539
|
-
if (previewLength < tier.previewMax) {
|
|
540
|
-
previewParts.push(content);
|
|
541
|
-
previewLength += content.length;
|
|
542
|
-
}
|
|
543
|
-
if (snippetLength < tier.snippetMax) {
|
|
544
|
-
const remaining = tier.snippetMax - snippetLength;
|
|
545
|
-
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
546
|
-
snippetParts.push(chunk);
|
|
547
|
-
snippetLength += chunk.length;
|
|
548
|
-
}
|
|
549
|
-
}
|
|
550
|
-
}
|
|
821
|
+
reduceLine(state, entry, tier);
|
|
551
822
|
}
|
|
552
823
|
} catch (err) {
|
|
553
824
|
log.warn({ filePath, err }, "parseMeta: read failed");
|
|
554
825
|
return null;
|
|
555
826
|
}
|
|
556
|
-
if (badJsonLines > 0) {
|
|
557
|
-
log.warn(
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
return null;
|
|
562
|
-
}
|
|
563
|
-
const isSubagent = filePath.includes("/subagents/");
|
|
564
|
-
let parentSessionId = null;
|
|
565
|
-
if (isSubagent) {
|
|
566
|
-
const uuidDir = dirname2(dirname2(filePath));
|
|
567
|
-
parentSessionId = join3(dirname2(uuidDir), `${basename(uuidDir)}.jsonl`);
|
|
827
|
+
if (state.badJsonLines > 0) {
|
|
828
|
+
log.warn(
|
|
829
|
+
{ filePath, badJsonLines: state.badJsonLines },
|
|
830
|
+
"parseMeta: skipped malformed JSON lines"
|
|
831
|
+
);
|
|
568
832
|
}
|
|
569
|
-
const
|
|
570
|
-
|
|
571
|
-
return
|
|
572
|
-
id: filePath,
|
|
573
|
-
filePath,
|
|
574
|
-
sessionId: sessionId || basename(filePath, ".jsonl"),
|
|
575
|
-
sessionName,
|
|
576
|
-
projectPath,
|
|
577
|
-
projectName: getShortProjectName(projectPath),
|
|
578
|
-
account,
|
|
579
|
-
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
580
|
-
messageCount,
|
|
581
|
-
lastMessageSender,
|
|
582
|
-
preview,
|
|
583
|
-
contentSnippet: snippetParts.join(" "),
|
|
584
|
-
gitBranch: null,
|
|
585
|
-
model,
|
|
586
|
-
isSubagent,
|
|
587
|
-
parentSessionId,
|
|
588
|
-
isTeammate,
|
|
589
|
-
teamName: teamName || null,
|
|
590
|
-
toolNames: Array.from(toolNameSet),
|
|
591
|
-
firstMessage,
|
|
592
|
-
lastMessage,
|
|
593
|
-
lastPrompt: lastPrompt || void 0
|
|
594
|
-
};
|
|
833
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
834
|
+
if (!meta) log.trace({ filePath }, "parseMeta: no messages");
|
|
835
|
+
return meta;
|
|
595
836
|
}
|
|
596
837
|
async function parseConversation(filePath, account) {
|
|
597
838
|
const log = getLogger();
|
|
598
839
|
log.trace({ filePath, account }, "parseConversation: start");
|
|
599
840
|
const messages = [];
|
|
600
841
|
let badJsonLines = 0;
|
|
601
|
-
let sessionId = "";
|
|
602
|
-
let sessionName = "";
|
|
603
|
-
let latestTimestamp = "";
|
|
604
|
-
let cwd = "";
|
|
605
842
|
const textParts = [];
|
|
606
|
-
const pendingToolUses = /* @__PURE__ */ new Map();
|
|
607
|
-
const teamInfoMap = /* @__PURE__ */ new Map();
|
|
608
843
|
const turnDurations = [];
|
|
609
|
-
|
|
610
|
-
const fileStream =
|
|
611
|
-
const rl =
|
|
844
|
+
const state = initialConvState();
|
|
845
|
+
const fileStream = createReadStream2(filePath);
|
|
846
|
+
const rl = createInterface2({ input: fileStream, crlfDelay: Infinity });
|
|
612
847
|
try {
|
|
613
848
|
for await (const line of rl) {
|
|
614
849
|
if (!line.trim()) continue;
|
|
@@ -619,91 +854,18 @@ async function parseConversation(filePath, account) {
|
|
|
619
854
|
badJsonLines++;
|
|
620
855
|
continue;
|
|
621
856
|
}
|
|
622
|
-
if (entry.
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
}
|
|
629
|
-
const type = entry.type;
|
|
630
|
-
if (type === "last-prompt") {
|
|
631
|
-
if (entry.lastPrompt && !lastPrompt) lastPrompt = entry.lastPrompt;
|
|
632
|
-
continue;
|
|
633
|
-
}
|
|
634
|
-
if (type === "system") {
|
|
635
|
-
if (entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
636
|
-
turnDurations.push({
|
|
637
|
-
durationMs: entry.durationMs,
|
|
638
|
-
messageCount: entry.messageCount || 0,
|
|
639
|
-
uuid: entry.uuid
|
|
640
|
-
});
|
|
641
|
-
}
|
|
857
|
+
if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
|
|
858
|
+
turnDurations.push({
|
|
859
|
+
durationMs: entry.durationMs,
|
|
860
|
+
messageCount: entry.messageCount || 0,
|
|
861
|
+
uuid: entry.uuid
|
|
862
|
+
});
|
|
642
863
|
continue;
|
|
643
864
|
}
|
|
644
|
-
|
|
645
|
-
if (
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
for (const block of toolUseBlocks) {
|
|
649
|
-
pendingToolUses.set(block.id, block);
|
|
650
|
-
}
|
|
651
|
-
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
652
|
-
const isToolResultOnly = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
653
|
-
const content = extractTextContent(msg?.content);
|
|
654
|
-
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
655
|
-
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
656
|
-
if (content || isToolResultOnly || toolUseBlocks.length > 0 || hasThinking) {
|
|
657
|
-
const metadata = {};
|
|
658
|
-
if (msg?.model) metadata.model = msg.model;
|
|
659
|
-
if (msg?.stop_reason !== void 0) metadata.stopReason = msg.stop_reason;
|
|
660
|
-
if (entry.gitBranch) metadata.gitBranch = entry.gitBranch;
|
|
661
|
-
if (entry.version) metadata.version = entry.version;
|
|
662
|
-
const usage = msg?.usage;
|
|
663
|
-
if (usage) {
|
|
664
|
-
if (usage.input_tokens) metadata.inputTokens = usage.input_tokens;
|
|
665
|
-
if (usage.output_tokens) metadata.outputTokens = usage.output_tokens;
|
|
666
|
-
if (usage.cache_read_input_tokens)
|
|
667
|
-
metadata.cacheReadTokens = usage.cache_read_input_tokens;
|
|
668
|
-
if (usage.cache_creation_input_tokens)
|
|
669
|
-
metadata.cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
670
|
-
}
|
|
671
|
-
const toolUseNames = extractToolUseNames(msg?.content);
|
|
672
|
-
if (toolUseNames.length > 0) metadata.toolUses = toolUseNames;
|
|
673
|
-
if (toolUseBlocks.length > 0) metadata.toolUseBlocks = toolUseBlocks;
|
|
674
|
-
if (isToolResultOnly) {
|
|
675
|
-
const toolResultBlocks = extractToolResultBlocks(msg?.content, pendingToolUses);
|
|
676
|
-
if (toolResultBlocks.length > 0) metadata.toolResults = toolResultBlocks;
|
|
677
|
-
}
|
|
678
|
-
if (entry.teamName) {
|
|
679
|
-
metadata.teamName = entry.teamName;
|
|
680
|
-
if (!teamInfoMap.has(metadata.teamName) && content) {
|
|
681
|
-
const info = parseTeammateMessageTag(content);
|
|
682
|
-
if (info) teamInfoMap.set(metadata.teamName, info);
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
const thinkingContent = thinking?.content || void 0;
|
|
686
|
-
const thinkingSignature = thinking?.signature || void 0;
|
|
687
|
-
const hasMetadata = Object.keys(metadata).length > 0;
|
|
688
|
-
messages.push({
|
|
689
|
-
role: type,
|
|
690
|
-
text: content || "",
|
|
691
|
-
timestamp: entry.timestamp || "",
|
|
692
|
-
uuid: entry.uuid || void 0,
|
|
693
|
-
metadata: hasMetadata ? metadata : void 0,
|
|
694
|
-
isToolResult: isToolResultOnly || void 0,
|
|
695
|
-
isThinking: thinkingContent || thinkingSignature ? true : void 0,
|
|
696
|
-
thinkingContent,
|
|
697
|
-
thinkingSignature,
|
|
698
|
-
parentUuid: entry.parentUuid !== void 0 ? entry.parentUuid : void 0,
|
|
699
|
-
requestId: type === "assistant" ? entry.requestId : void 0,
|
|
700
|
-
promptId: type === "user" ? entry.promptId : void 0,
|
|
701
|
-
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
702
|
-
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
703
|
-
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
704
|
-
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
705
|
-
});
|
|
706
|
-
if (content) textParts.push(content);
|
|
865
|
+
const message = reduceConvLine(state, entry);
|
|
866
|
+
if (message) {
|
|
867
|
+
messages.push(message);
|
|
868
|
+
if (message.text) textParts.push(message.text);
|
|
707
869
|
}
|
|
708
870
|
}
|
|
709
871
|
} catch (err) {
|
|
@@ -718,28 +880,21 @@ async function parseConversation(filePath, account) {
|
|
|
718
880
|
return null;
|
|
719
881
|
}
|
|
720
882
|
log.debug({ filePath, messageCount: messages.length }, "parseConversation: complete");
|
|
721
|
-
|
|
722
|
-
for (const msg of messages) {
|
|
723
|
-
if (msg.metadata?.teamName) {
|
|
724
|
-
const info = teamInfoMap.get(msg.metadata.teamName);
|
|
725
|
-
if (info) msg.metadata.teamInfo = info;
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
}
|
|
883
|
+
applyTeamInfo(messages, state);
|
|
729
884
|
return {
|
|
730
885
|
id: filePath,
|
|
731
886
|
filePath,
|
|
732
|
-
projectPath: cwd,
|
|
733
|
-
projectName:
|
|
734
|
-
sessionId: sessionId ||
|
|
735
|
-
sessionName,
|
|
887
|
+
projectPath: state.cwd,
|
|
888
|
+
projectName: getShortProjectName2(state.cwd),
|
|
889
|
+
sessionId: state.sessionId || basename2(filePath, ".jsonl"),
|
|
890
|
+
sessionName: state.sessionName,
|
|
736
891
|
messages,
|
|
737
892
|
fullText: textParts.join(" "),
|
|
738
|
-
timestamp: latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
893
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
739
894
|
messageCount: messages.length,
|
|
740
895
|
account,
|
|
741
896
|
turnDurations: turnDurations.length > 0 ? turnDurations : void 0,
|
|
742
|
-
lastPrompt: lastPrompt || void 0
|
|
897
|
+
lastPrompt: state.lastPrompt || void 0
|
|
743
898
|
};
|
|
744
899
|
}
|
|
745
900
|
function extractTextContent(content) {
|
|
@@ -832,13 +987,294 @@ function parseTeammateMessageTag(content) {
|
|
|
832
987
|
const color = attrs.match(/color="([^"]*)"/)?.[1];
|
|
833
988
|
return { teammateId: id, summary, color };
|
|
834
989
|
}
|
|
835
|
-
function
|
|
990
|
+
function getShortProjectName2(fullPath) {
|
|
836
991
|
const parts = fullPath.split("/").filter(Boolean);
|
|
837
992
|
return parts.slice(-3).join("/");
|
|
838
993
|
}
|
|
839
994
|
|
|
840
|
-
// src/
|
|
841
|
-
|
|
995
|
+
// src/persistent/metadata-reducer.ts
|
|
996
|
+
function initialReducerState() {
|
|
997
|
+
return {
|
|
998
|
+
sessionId: "",
|
|
999
|
+
sessionName: "",
|
|
1000
|
+
latestTimestamp: "",
|
|
1001
|
+
cwd: "",
|
|
1002
|
+
teamName: "",
|
|
1003
|
+
model: null,
|
|
1004
|
+
messageCount: 0,
|
|
1005
|
+
lastMessageSender: "user",
|
|
1006
|
+
isTeammate: false,
|
|
1007
|
+
firstUserSeen: false,
|
|
1008
|
+
firstMessage: null,
|
|
1009
|
+
lastMessage: null,
|
|
1010
|
+
lastPrompt: "",
|
|
1011
|
+
pageMessageCount: 0,
|
|
1012
|
+
toolNames: [],
|
|
1013
|
+
previewParts: [],
|
|
1014
|
+
snippetParts: [],
|
|
1015
|
+
previewLength: 0,
|
|
1016
|
+
snippetLength: 0,
|
|
1017
|
+
badJsonLines: 0
|
|
1018
|
+
};
|
|
1019
|
+
}
|
|
1020
|
+
function reduceLine(state, entry, tier) {
|
|
1021
|
+
if (entry.cwd && !state.cwd) state.cwd = entry.cwd;
|
|
1022
|
+
if (entry.sessionId && !state.sessionId) state.sessionId = entry.sessionId;
|
|
1023
|
+
if (entry.slug && !state.sessionName) state.sessionName = entry.slug;
|
|
1024
|
+
if (entry.teamName && !state.teamName) state.teamName = entry.teamName;
|
|
1025
|
+
if (entry.timestamp) {
|
|
1026
|
+
const ts = entry.timestamp;
|
|
1027
|
+
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
1028
|
+
}
|
|
1029
|
+
const type = entry.type;
|
|
1030
|
+
if (type === "last-prompt") {
|
|
1031
|
+
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
1032
|
+
return;
|
|
1033
|
+
}
|
|
1034
|
+
if (type !== "user" && type !== "assistant") return;
|
|
1035
|
+
if (entry.isMeta) return;
|
|
1036
|
+
const msg = entry.message;
|
|
1037
|
+
if (state.model === null && msg?.model) state.model = msg.model;
|
|
1038
|
+
if (type === "user" && !state.firstUserSeen) {
|
|
1039
|
+
state.firstUserSeen = true;
|
|
1040
|
+
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
1041
|
+
}
|
|
1042
|
+
const content = extractTextContent(msg?.content);
|
|
1043
|
+
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
1044
|
+
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
1045
|
+
const toolSet = new Set(state.toolNames);
|
|
1046
|
+
collectToolNames(msg?.content, toolSet);
|
|
1047
|
+
state.toolNames = Array.from(toolSet);
|
|
1048
|
+
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
1049
|
+
const thinking = type === "assistant" ? extractThinking(msg?.content) : null;
|
|
1050
|
+
const hasThinking = !!(thinking?.content || thinking?.signature);
|
|
1051
|
+
if (content || isOnlyToolResult || toolUseBlocks.length > 0 || hasThinking) {
|
|
1052
|
+
state.pageMessageCount++;
|
|
1053
|
+
}
|
|
1054
|
+
if (content || isOnlyToolResult) {
|
|
1055
|
+
state.messageCount++;
|
|
1056
|
+
state.lastMessageSender = type;
|
|
1057
|
+
if (content) {
|
|
1058
|
+
const ts = entry.timestamp || "";
|
|
1059
|
+
if (!state.firstMessage) state.firstMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
1060
|
+
state.lastMessage = { text: content.slice(0, 200), timestamp: ts };
|
|
1061
|
+
if (state.previewLength < tier.previewMax) {
|
|
1062
|
+
state.previewParts.push(content);
|
|
1063
|
+
state.previewLength += content.length;
|
|
1064
|
+
}
|
|
1065
|
+
if (state.snippetLength < tier.snippetMax) {
|
|
1066
|
+
const remaining = tier.snippetMax - state.snippetLength;
|
|
1067
|
+
const chunk = content.length > remaining ? content.slice(0, remaining) : content;
|
|
1068
|
+
state.snippetParts.push(chunk);
|
|
1069
|
+
state.snippetLength += chunk.length;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
function finalizeMeta(state, filePath, account, tier) {
|
|
1075
|
+
if (state.messageCount === 0) return null;
|
|
1076
|
+
const isSubagent = filePath.includes("/subagents/");
|
|
1077
|
+
let parentSessionId = null;
|
|
1078
|
+
if (isSubagent) {
|
|
1079
|
+
const uuidDir = dirname2(dirname2(filePath));
|
|
1080
|
+
parentSessionId = join3(dirname2(uuidDir), `${basename3(uuidDir)}.jsonl`);
|
|
1081
|
+
}
|
|
1082
|
+
const projectPath = state.cwd;
|
|
1083
|
+
return {
|
|
1084
|
+
id: filePath,
|
|
1085
|
+
filePath,
|
|
1086
|
+
provider: CLAUDE_CODE_PROVIDER,
|
|
1087
|
+
sessionId: state.sessionId || basename3(filePath, ".jsonl"),
|
|
1088
|
+
sessionName: state.sessionName,
|
|
1089
|
+
projectPath,
|
|
1090
|
+
projectName: getShortProjectName3(projectPath),
|
|
1091
|
+
account,
|
|
1092
|
+
timestamp: state.latestTimestamp || (/* @__PURE__ */ new Date()).toISOString(),
|
|
1093
|
+
messageCount: state.messageCount,
|
|
1094
|
+
lastMessageSender: state.lastMessageSender,
|
|
1095
|
+
preview: state.previewParts.join(" ").slice(0, tier.previewMax),
|
|
1096
|
+
contentSnippet: state.snippetParts.join(" "),
|
|
1097
|
+
gitBranch: null,
|
|
1098
|
+
model: state.model,
|
|
1099
|
+
isSubagent,
|
|
1100
|
+
parentSessionId,
|
|
1101
|
+
isTeammate: state.isTeammate,
|
|
1102
|
+
teamName: state.teamName || null,
|
|
1103
|
+
toolNames: state.toolNames,
|
|
1104
|
+
firstMessage: state.firstMessage,
|
|
1105
|
+
lastMessage: state.lastMessage,
|
|
1106
|
+
lastPrompt: state.lastPrompt || void 0
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
function getShortProjectName3(fullPath) {
|
|
1110
|
+
const parts = fullPath.split("/").filter(Boolean);
|
|
1111
|
+
return parts.slice(-3).join("/");
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
// src/providers/threadbase.ts
|
|
1115
|
+
var ThreadbaseProvider = class {
|
|
1116
|
+
name = CLAUDE_CODE_PROVIDER;
|
|
1117
|
+
// Roots are passed as "<projectsDir>\0<account>" so the scanner can carry the
|
|
1118
|
+
// per-root account through the shared interface. The scanner builds these.
|
|
1119
|
+
async discover(roots) {
|
|
1120
|
+
const dirs = roots.map((r) => {
|
|
1121
|
+
const [projectsDir, account = "default"] = r.split("\0");
|
|
1122
|
+
return { projectsDir, account };
|
|
1123
|
+
});
|
|
1124
|
+
return discoverJsonlFiles(dirs);
|
|
1125
|
+
}
|
|
1126
|
+
// Threadbase JSONL has top-level type "user"/"assistant" with a cwd/sessionId.
|
|
1127
|
+
canParse(_filePath, sample) {
|
|
1128
|
+
for (const line of sample.split("\n")) {
|
|
1129
|
+
if (!line.trim()) continue;
|
|
1130
|
+
try {
|
|
1131
|
+
const e = JSON.parse(line);
|
|
1132
|
+
if (e.type === "user" || e.type === "assistant") return true;
|
|
1133
|
+
if (e.type === "session_meta" || e.type === "response_item") return false;
|
|
1134
|
+
} catch {
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
return false;
|
|
1138
|
+
}
|
|
1139
|
+
createEmptyAccumulator() {
|
|
1140
|
+
return initialReducerState();
|
|
1141
|
+
}
|
|
1142
|
+
reduceEntry(acc, entry, tier) {
|
|
1143
|
+
reduceLine(acc, entry, tier);
|
|
1144
|
+
}
|
|
1145
|
+
finalize(acc, filePath, account, tier) {
|
|
1146
|
+
return finalizeMeta(acc, filePath, account, tier);
|
|
1147
|
+
}
|
|
1148
|
+
};
|
|
1149
|
+
|
|
1150
|
+
// src/scanner.ts
|
|
1151
|
+
import { EventEmitter } from "events";
|
|
1152
|
+
import { closeSync as closeSync2, openSync as openSync2, readSync as readSync2, statSync as statSync2 } from "fs";
|
|
1153
|
+
import { homedir as homedir2 } from "os";
|
|
1154
|
+
import { join as join4 } from "path";
|
|
1155
|
+
|
|
1156
|
+
// src/cache.ts
|
|
1157
|
+
var LRUCache = class {
|
|
1158
|
+
map = /* @__PURE__ */ new Map();
|
|
1159
|
+
capacity;
|
|
1160
|
+
constructor(capacity) {
|
|
1161
|
+
this.capacity = capacity;
|
|
1162
|
+
}
|
|
1163
|
+
get(key) {
|
|
1164
|
+
const value = this.map.get(key);
|
|
1165
|
+
if (value === void 0) return void 0;
|
|
1166
|
+
this.map.delete(key);
|
|
1167
|
+
this.map.set(key, value);
|
|
1168
|
+
return value;
|
|
1169
|
+
}
|
|
1170
|
+
set(key, value) {
|
|
1171
|
+
this.map.delete(key);
|
|
1172
|
+
this.map.set(key, value);
|
|
1173
|
+
if (this.map.size > this.capacity) {
|
|
1174
|
+
const oldest = this.map.keys().next();
|
|
1175
|
+
if (!oldest.done) this.map.delete(oldest.value);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
has(key) {
|
|
1179
|
+
return this.map.has(key);
|
|
1180
|
+
}
|
|
1181
|
+
delete(key) {
|
|
1182
|
+
return this.map.delete(key);
|
|
1183
|
+
}
|
|
1184
|
+
clear() {
|
|
1185
|
+
this.map.clear();
|
|
1186
|
+
}
|
|
1187
|
+
get size() {
|
|
1188
|
+
return this.map.size;
|
|
1189
|
+
}
|
|
1190
|
+
};
|
|
1191
|
+
|
|
1192
|
+
// src/persistent/cursor.ts
|
|
1193
|
+
import { createHash } from "crypto";
|
|
1194
|
+
import { closeSync, openSync, readSync, statSync } from "fs";
|
|
1195
|
+
var FP_BYTES = 4096;
|
|
1196
|
+
function fingerprint(filePath, size) {
|
|
1197
|
+
const hash = createHash("sha1");
|
|
1198
|
+
hash.update(String(size));
|
|
1199
|
+
const fd = openSync(filePath, "r");
|
|
1200
|
+
try {
|
|
1201
|
+
const head = Buffer.alloc(Math.min(FP_BYTES, size));
|
|
1202
|
+
if (head.length > 0) {
|
|
1203
|
+
readSync(fd, head, 0, head.length, 0);
|
|
1204
|
+
hash.update(head);
|
|
1205
|
+
}
|
|
1206
|
+
if (size > FP_BYTES) {
|
|
1207
|
+
const tailLen = Math.min(FP_BYTES, size);
|
|
1208
|
+
const tail = Buffer.alloc(tailLen);
|
|
1209
|
+
readSync(fd, tail, 0, tailLen, size - tailLen);
|
|
1210
|
+
hash.update(tail);
|
|
1211
|
+
}
|
|
1212
|
+
} finally {
|
|
1213
|
+
closeSync(fd);
|
|
1214
|
+
}
|
|
1215
|
+
return hash.digest("hex");
|
|
1216
|
+
}
|
|
1217
|
+
function classify(filePath, existing) {
|
|
1218
|
+
let stat3;
|
|
1219
|
+
try {
|
|
1220
|
+
const s = statSync(filePath);
|
|
1221
|
+
stat3 = { size: s.size, mtimeMs: s.mtimeMs };
|
|
1222
|
+
} catch {
|
|
1223
|
+
return { change: "vanished" };
|
|
1224
|
+
}
|
|
1225
|
+
if (!existing || existing.status !== "active" || existing.last_indexed_offset === 0) {
|
|
1226
|
+
return { change: "reindex", stat: stat3 };
|
|
1227
|
+
}
|
|
1228
|
+
if (stat3.size < existing.last_indexed_offset) {
|
|
1229
|
+
return { change: "reindex", stat: stat3 };
|
|
1230
|
+
}
|
|
1231
|
+
if (stat3.size === existing.size_bytes && stat3.mtimeMs === existing.mtime_ms) {
|
|
1232
|
+
return { change: "unchanged", stat: stat3 };
|
|
1233
|
+
}
|
|
1234
|
+
if (stat3.size === existing.last_indexed_offset) {
|
|
1235
|
+
const fp = fingerprint(filePath, stat3.size);
|
|
1236
|
+
if (existing.content_fingerprint && fp !== existing.content_fingerprint) {
|
|
1237
|
+
return { change: "reindex", stat: stat3 };
|
|
1238
|
+
}
|
|
1239
|
+
return { change: "unchanged", stat: stat3 };
|
|
1240
|
+
}
|
|
1241
|
+
return { change: "appended", stat: stat3 };
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
// src/providers/parse.ts
|
|
1245
|
+
import { createReadStream as createReadStream3 } from "fs";
|
|
1246
|
+
import { createInterface as createInterface3 } from "readline";
|
|
1247
|
+
async function parseMetaWithProvider(provider, filePath, account, tier) {
|
|
1248
|
+
const log = getLogger();
|
|
1249
|
+
const acc = provider.createEmptyAccumulator();
|
|
1250
|
+
const rl = createInterface3({
|
|
1251
|
+
input: createReadStream3(filePath),
|
|
1252
|
+
crlfDelay: Infinity
|
|
1253
|
+
});
|
|
1254
|
+
try {
|
|
1255
|
+
for await (const line of rl) {
|
|
1256
|
+
if (!line.trim()) continue;
|
|
1257
|
+
let entry;
|
|
1258
|
+
try {
|
|
1259
|
+
entry = JSON.parse(line);
|
|
1260
|
+
} catch {
|
|
1261
|
+
continue;
|
|
1262
|
+
}
|
|
1263
|
+
try {
|
|
1264
|
+
provider.reduceEntry(acc, entry, tier);
|
|
1265
|
+
} catch (err) {
|
|
1266
|
+
log.warn({ filePath, provider: provider.name, err }, "provider reduce threw; line skipped");
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
} catch (err) {
|
|
1270
|
+
log.warn({ filePath, provider: provider.name, err }, "provider parse: read failed");
|
|
1271
|
+
return null;
|
|
1272
|
+
}
|
|
1273
|
+
return provider.finalize(acc, filePath, account, tier);
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
// src/tiers.ts
|
|
1277
|
+
var DEFAULT_TIERS = {
|
|
842
1278
|
standard: { name: "standard", previewMax: 200, snippetMax: 5e3 },
|
|
843
1279
|
full: { name: "full", previewMax: 1200, snippetMax: 5e4 }
|
|
844
1280
|
};
|
|
@@ -852,28 +1288,1108 @@ function resolveTier(tierName, customTiers) {
|
|
|
852
1288
|
return tier;
|
|
853
1289
|
}
|
|
854
1290
|
|
|
855
|
-
// src/
|
|
1291
|
+
// src/persistent/db.ts
|
|
1292
|
+
import Database from "better-sqlite3";
|
|
1293
|
+
import { mkdirSync } from "fs";
|
|
1294
|
+
import { dirname as dirname3 } from "path";
|
|
1295
|
+
|
|
1296
|
+
// src/persistent/schema.ts
|
|
1297
|
+
var SCHEMA_VERSION = 3;
|
|
1298
|
+
var SCHEMA_SQL = `
|
|
1299
|
+
CREATE TABLE IF NOT EXISTS conversation_files (
|
|
1300
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1301
|
+
|
|
1302
|
+
absolute_path TEXT NOT NULL UNIQUE,
|
|
1303
|
+
parent_dir TEXT NOT NULL,
|
|
1304
|
+
file_name TEXT NOT NULL,
|
|
1305
|
+
|
|
1306
|
+
account TEXT NOT NULL DEFAULT 'default',
|
|
1307
|
+
|
|
1308
|
+
size_bytes INTEGER NOT NULL DEFAULT 0,
|
|
1309
|
+
mtime_ms INTEGER NOT NULL DEFAULT 0,
|
|
1310
|
+
|
|
1311
|
+
last_indexed_offset INTEGER NOT NULL DEFAULT 0,
|
|
1312
|
+
last_indexed_line INTEGER NOT NULL DEFAULT 0,
|
|
1313
|
+
|
|
1314
|
+
reducer_state TEXT,
|
|
1315
|
+
|
|
1316
|
+
content_fingerprint TEXT,
|
|
1317
|
+
|
|
1318
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
1319
|
+
|
|
1320
|
+
last_indexed_at TEXT,
|
|
1321
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1322
|
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1323
|
+
deleted_at TEXT
|
|
1324
|
+
);
|
|
1325
|
+
|
|
1326
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_status ON conversation_files(status);
|
|
1327
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_parent_dir ON conversation_files(parent_dir);
|
|
1328
|
+
CREATE INDEX IF NOT EXISTS idx_conversation_files_account ON conversation_files(account);
|
|
1329
|
+
|
|
1330
|
+
CREATE TABLE IF NOT EXISTS conversations (
|
|
1331
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1332
|
+
|
|
1333
|
+
file_id INTEGER NOT NULL UNIQUE,
|
|
1334
|
+
|
|
1335
|
+
source_path TEXT NOT NULL UNIQUE,
|
|
1336
|
+
-- Which provider produced this row. Canonical identity is (provider,
|
|
1337
|
+
-- source_path); session_id stays non-unique across providers too.
|
|
1338
|
+
provider TEXT NOT NULL DEFAULT 'claude-code',
|
|
1339
|
+
kind TEXT,
|
|
1340
|
+
external_session_id TEXT,
|
|
1341
|
+
session_id TEXT NOT NULL,
|
|
1342
|
+
session_name TEXT,
|
|
1343
|
+
|
|
1344
|
+
project_path TEXT,
|
|
1345
|
+
project_name TEXT,
|
|
1346
|
+
account TEXT NOT NULL DEFAULT 'default',
|
|
1347
|
+
branch TEXT,
|
|
1348
|
+
|
|
1349
|
+
preview TEXT,
|
|
1350
|
+
content_snippet TEXT,
|
|
1351
|
+
|
|
1352
|
+
message_count INTEGER NOT NULL DEFAULT 0,
|
|
1353
|
+
-- Count of messages parseConversation() produces (broader than message_count;
|
|
1354
|
+
-- includes tool_use-only and thinking-only lines). The total for bounded paging.
|
|
1355
|
+
page_message_count INTEGER NOT NULL DEFAULT 0,
|
|
1356
|
+
last_message_sender TEXT NOT NULL DEFAULT 'user',
|
|
1357
|
+
timestamp TEXT,
|
|
1358
|
+
|
|
1359
|
+
-- Monotonic write counter; the highest value is the most recently indexed
|
|
1360
|
+
-- row. Drives last-writer-wins resolution for shared session_ids (matching
|
|
1361
|
+
-- the in-memory sessionId map), at the sub-second precision updated_at lacks.
|
|
1362
|
+
index_seq INTEGER NOT NULL DEFAULT 0,
|
|
1363
|
+
|
|
1364
|
+
first_sent_at TEXT,
|
|
1365
|
+
first_sent_text TEXT,
|
|
1366
|
+
|
|
1367
|
+
last_sent_at TEXT,
|
|
1368
|
+
last_sent_text TEXT,
|
|
1369
|
+
|
|
1370
|
+
model TEXT,
|
|
1371
|
+
is_subagent INTEGER NOT NULL DEFAULT 0,
|
|
1372
|
+
parent_session_id TEXT,
|
|
1373
|
+
is_teammate INTEGER NOT NULL DEFAULT 0,
|
|
1374
|
+
team_name TEXT,
|
|
1375
|
+
tool_names_json TEXT,
|
|
1376
|
+
last_prompt TEXT,
|
|
1377
|
+
|
|
1378
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
1379
|
+
|
|
1380
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1381
|
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
1382
|
+
|
|
1383
|
+
FOREIGN KEY (file_id) REFERENCES conversation_files(id)
|
|
1384
|
+
);
|
|
1385
|
+
|
|
1386
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_session_id ON conversations(session_id);
|
|
1387
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_session ON conversations(provider, session_id);
|
|
1388
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_recent ON conversations(provider, timestamp DESC);
|
|
1389
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_provider_project_branch ON conversations(provider, project_path, branch);
|
|
1390
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_recent ON conversations(timestamp DESC);
|
|
1391
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_project_recent ON conversations(project_path, timestamp DESC);
|
|
1392
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_project_branch_recent ON conversations(project_path, branch, timestamp DESC);
|
|
1393
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_account_recent ON conversations(account, timestamp DESC);
|
|
1394
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_subagent_recent ON conversations(is_subagent, timestamp DESC);
|
|
1395
|
+
CREATE INDEX IF NOT EXISTS idx_conversations_team_recent ON conversations(team_name, timestamp DESC);
|
|
1396
|
+
|
|
1397
|
+
-- Full-text search index over conversation content + metadata. Kept separate
|
|
1398
|
+
-- from the metadata tables so list-screen queries stay small and fast.
|
|
1399
|
+
-- source_path is UNINDEXED (stored, not tokenized) and links back to a
|
|
1400
|
+
-- conversations row. One FTS row per conversation, replaced on each upsert.
|
|
1401
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS conversation_messages_fts USING fts5(
|
|
1402
|
+
source_path UNINDEXED,
|
|
1403
|
+
content,
|
|
1404
|
+
project_name,
|
|
1405
|
+
session_id,
|
|
1406
|
+
session_name,
|
|
1407
|
+
account,
|
|
1408
|
+
model,
|
|
1409
|
+
branch,
|
|
1410
|
+
tool_names,
|
|
1411
|
+
tokenize = 'unicode61'
|
|
1412
|
+
);
|
|
1413
|
+
|
|
1414
|
+
-- Seek index for large conversations: every N messages, record the byte offset
|
|
1415
|
+
-- where the next message's line begins plus the parser's cross-line state
|
|
1416
|
+
-- (pending tool_use blocks, team info) needed to resume an equivalent parse
|
|
1417
|
+
-- from that point. Lets getConversationPage() read a window near the end of a
|
|
1418
|
+
-- huge file without parsing from byte 0.
|
|
1419
|
+
CREATE TABLE IF NOT EXISTS message_checkpoints (
|
|
1420
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1421
|
+
source_path TEXT NOT NULL,
|
|
1422
|
+
message_index INTEGER NOT NULL,
|
|
1423
|
+
byte_offset INTEGER NOT NULL,
|
|
1424
|
+
line_number INTEGER NOT NULL,
|
|
1425
|
+
parser_state TEXT NOT NULL,
|
|
1426
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
1427
|
+
);
|
|
1428
|
+
|
|
1429
|
+
CREATE INDEX IF NOT EXISTS idx_message_checkpoints_lookup
|
|
1430
|
+
ON message_checkpoints(source_path, message_index);
|
|
1431
|
+
`;
|
|
1432
|
+
|
|
1433
|
+
// src/persistent/migrations.ts
|
|
1434
|
+
function runMigrations(db) {
|
|
1435
|
+
const current = db.pragma("user_version", { simple: true });
|
|
1436
|
+
if (current >= SCHEMA_VERSION) return;
|
|
1437
|
+
const log = getLogger();
|
|
1438
|
+
log.info({ from: current, to: SCHEMA_VERSION }, "migrations: applying");
|
|
1439
|
+
if (current >= 1 && current < 2 && tableExists(db, "conversations")) {
|
|
1440
|
+
for (const [col, ddl] of [
|
|
1441
|
+
[
|
|
1442
|
+
"provider",
|
|
1443
|
+
// Keep the historical DEFAULT 'threadbase' — v2→v3 below updates these rows.
|
|
1444
|
+
"ALTER TABLE conversations ADD COLUMN provider TEXT NOT NULL DEFAULT 'threadbase'"
|
|
1445
|
+
],
|
|
1446
|
+
["kind", "ALTER TABLE conversations ADD COLUMN kind TEXT"],
|
|
1447
|
+
["external_session_id", "ALTER TABLE conversations ADD COLUMN external_session_id TEXT"]
|
|
1448
|
+
]) {
|
|
1449
|
+
if (!hasColumn(db, "conversations", col)) db.exec(ddl);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
if (current >= 1 && current < 3 && tableExists(db, "conversations")) {
|
|
1453
|
+
db.exec("UPDATE conversations SET provider = 'claude-code' WHERE provider = 'threadbase'");
|
|
1454
|
+
}
|
|
1455
|
+
db.exec(SCHEMA_SQL);
|
|
1456
|
+
db.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
1457
|
+
}
|
|
1458
|
+
function tableExists(db, table) {
|
|
1459
|
+
return db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== void 0;
|
|
1460
|
+
}
|
|
1461
|
+
function hasColumn(db, table, column) {
|
|
1462
|
+
const cols = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
1463
|
+
return cols.some((c) => c.name === column);
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
// src/persistent/db.ts
|
|
1467
|
+
function openDatabase(dbPath) {
|
|
1468
|
+
if (dbPath !== ":memory:") {
|
|
1469
|
+
mkdirSync(dirname3(dbPath), { recursive: true });
|
|
1470
|
+
}
|
|
1471
|
+
const db = new Database(dbPath);
|
|
1472
|
+
db.pragma("journal_mode = WAL");
|
|
1473
|
+
db.pragma("synchronous = NORMAL");
|
|
1474
|
+
db.pragma("temp_store = MEMORY");
|
|
1475
|
+
db.pragma("foreign_keys = ON");
|
|
1476
|
+
runMigrations(db);
|
|
1477
|
+
getLogger().debug({ dbPath }, "db: opened");
|
|
1478
|
+
return db;
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
// src/persistent/jsonl-tail-reader.ts
|
|
1482
|
+
import { createReadStream as createReadStream4 } from "fs";
|
|
1483
|
+
async function tailReduce(filePath, startOffset, startLine, state, tier) {
|
|
1484
|
+
const stream = createReadStream4(filePath, { start: startOffset, encoding: "utf8" });
|
|
1485
|
+
let buffer = "";
|
|
1486
|
+
let offset = startOffset;
|
|
1487
|
+
let line = startLine;
|
|
1488
|
+
let parsedLines = 0;
|
|
1489
|
+
for await (const chunk of stream) {
|
|
1490
|
+
buffer += chunk;
|
|
1491
|
+
let nl;
|
|
1492
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1493
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1494
|
+
const text = lineWithNewline.trimEnd();
|
|
1495
|
+
buffer = buffer.slice(nl + 1);
|
|
1496
|
+
if (text.length > 0) {
|
|
1497
|
+
try {
|
|
1498
|
+
reduceLine(state, JSON.parse(text), tier);
|
|
1499
|
+
} catch {
|
|
1500
|
+
state.badJsonLines++;
|
|
1501
|
+
}
|
|
1502
|
+
parsedLines++;
|
|
1503
|
+
}
|
|
1504
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1505
|
+
line++;
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
return { newOffset: offset, newLine: line, parsedLines, badJsonLines: state.badJsonLines };
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
// src/persistent/paged-reader.ts
|
|
1512
|
+
import { createReadStream as createReadStream5 } from "fs";
|
|
1513
|
+
var CHECKPOINT_INTERVAL = 500;
|
|
1514
|
+
async function streamMessages(filePath, startOffset, startLine, state, onMessage) {
|
|
1515
|
+
const stream = createReadStream5(filePath, { start: startOffset, encoding: "utf8" });
|
|
1516
|
+
let buffer = "";
|
|
1517
|
+
let offset = startOffset;
|
|
1518
|
+
let line = startLine;
|
|
1519
|
+
for await (const chunk of stream) {
|
|
1520
|
+
buffer += chunk;
|
|
1521
|
+
let nl;
|
|
1522
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1523
|
+
const lineWithNewline = buffer.slice(0, nl + 1);
|
|
1524
|
+
const text = lineWithNewline.trimEnd();
|
|
1525
|
+
buffer = buffer.slice(nl + 1);
|
|
1526
|
+
offset += Buffer.byteLength(lineWithNewline, "utf8");
|
|
1527
|
+
line += 1;
|
|
1528
|
+
if (text.length === 0) continue;
|
|
1529
|
+
let entry;
|
|
1530
|
+
try {
|
|
1531
|
+
entry = JSON.parse(text);
|
|
1532
|
+
} catch {
|
|
1533
|
+
continue;
|
|
1534
|
+
}
|
|
1535
|
+
const message = reduceConvLine(state, entry);
|
|
1536
|
+
if (message && onMessage(message, offset, line)) {
|
|
1537
|
+
stream.destroy();
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL) {
|
|
1544
|
+
const checkpoints = [];
|
|
1545
|
+
const state = initialConvState();
|
|
1546
|
+
let index = 0;
|
|
1547
|
+
await streamMessages(filePath, 0, 0, state, (_msg, nextOffset, nextLine) => {
|
|
1548
|
+
index += 1;
|
|
1549
|
+
if (index % interval === 0) {
|
|
1550
|
+
checkpoints.push({
|
|
1551
|
+
messageIndex: index,
|
|
1552
|
+
byteOffset: nextOffset,
|
|
1553
|
+
lineNumber: nextLine,
|
|
1554
|
+
state: structuredClone(state)
|
|
1555
|
+
});
|
|
1556
|
+
}
|
|
1557
|
+
return false;
|
|
1558
|
+
});
|
|
1559
|
+
return checkpoints;
|
|
1560
|
+
}
|
|
1561
|
+
async function readPage(filePath, total, options, floor) {
|
|
1562
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
1563
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1564
|
+
const state = floor ? structuredClone(floor.state) : initialConvState();
|
|
1565
|
+
const startOffset = floor ? floor.byteOffset : 0;
|
|
1566
|
+
const startLine = floor ? floor.lineNumber : 0;
|
|
1567
|
+
let index = floor ? floor.messageIndex : 0;
|
|
1568
|
+
const window = [];
|
|
1569
|
+
await streamMessages(filePath, startOffset, startLine, state, (message) => {
|
|
1570
|
+
const current = index;
|
|
1571
|
+
index += 1;
|
|
1572
|
+
if (current >= fromIndex && current < beforeIndex) window.push(message);
|
|
1573
|
+
return index >= beforeIndex;
|
|
1574
|
+
});
|
|
1575
|
+
applyTeamInfo(window, state);
|
|
1576
|
+
return { messages: window, total, fromIndex };
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
// src/persistent/repositories/checkpoints.repo.ts
|
|
1580
|
+
var CheckpointsRepo = class {
|
|
1581
|
+
constructor(db) {
|
|
1582
|
+
this.db = db;
|
|
1583
|
+
}
|
|
1584
|
+
db;
|
|
1585
|
+
replaceAll(sourcePath, checkpoints) {
|
|
1586
|
+
const tx = this.db.transaction(() => {
|
|
1587
|
+
this.db.prepare("DELETE FROM message_checkpoints WHERE source_path = ?").run(sourcePath);
|
|
1588
|
+
const insert = this.db.prepare(
|
|
1589
|
+
`INSERT INTO message_checkpoints
|
|
1590
|
+
(source_path, message_index, byte_offset, line_number, parser_state)
|
|
1591
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
1592
|
+
);
|
|
1593
|
+
for (const c of checkpoints) {
|
|
1594
|
+
insert.run(sourcePath, c.messageIndex, c.byteOffset, c.lineNumber, JSON.stringify(c.state));
|
|
1595
|
+
}
|
|
1596
|
+
});
|
|
1597
|
+
tx();
|
|
1598
|
+
}
|
|
1599
|
+
// The latest checkpoint at or before `messageIndex`, or null if none (read
|
|
1600
|
+
// from the file start). Lets a page seek to the nearest prior anchor.
|
|
1601
|
+
floor(sourcePath, messageIndex) {
|
|
1602
|
+
const row = this.db.prepare(
|
|
1603
|
+
`SELECT message_index, byte_offset, line_number, parser_state
|
|
1604
|
+
FROM message_checkpoints
|
|
1605
|
+
WHERE source_path = ? AND message_index <= ?
|
|
1606
|
+
ORDER BY message_index DESC LIMIT 1`
|
|
1607
|
+
).get(sourcePath, messageIndex);
|
|
1608
|
+
return row ? toCheckpoint(row) : null;
|
|
1609
|
+
}
|
|
1610
|
+
count(sourcePath) {
|
|
1611
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM message_checkpoints WHERE source_path = ?").get(sourcePath).n;
|
|
1612
|
+
}
|
|
1613
|
+
remove(sourcePath) {
|
|
1614
|
+
this.db.prepare("DELETE FROM message_checkpoints WHERE source_path = ?").run(sourcePath);
|
|
1615
|
+
}
|
|
1616
|
+
};
|
|
1617
|
+
function toCheckpoint(row) {
|
|
1618
|
+
return {
|
|
1619
|
+
messageIndex: row.message_index,
|
|
1620
|
+
byteOffset: row.byte_offset,
|
|
1621
|
+
lineNumber: row.line_number,
|
|
1622
|
+
state: JSON.parse(row.parser_state)
|
|
1623
|
+
};
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
// src/persistent/repositories/conversation-files.repo.ts
|
|
1627
|
+
import { basename as basename4, dirname as dirname4 } from "path";
|
|
1628
|
+
var ConversationFilesRepo = class {
|
|
1629
|
+
constructor(db) {
|
|
1630
|
+
this.db = db;
|
|
1631
|
+
}
|
|
1632
|
+
db;
|
|
1633
|
+
getByPath(absolutePath) {
|
|
1634
|
+
return this.db.prepare("SELECT * FROM conversation_files WHERE absolute_path = ?").get(absolutePath);
|
|
1635
|
+
}
|
|
1636
|
+
// Insert a freshly-discovered file at offset 0; returns its row id. Existing
|
|
1637
|
+
// path is left untouched (returns the existing id) so a re-discovery is safe.
|
|
1638
|
+
ensure(absolutePath, account) {
|
|
1639
|
+
const existing = this.getByPath(absolutePath);
|
|
1640
|
+
if (existing) return existing.id;
|
|
1641
|
+
const info = this.db.prepare(
|
|
1642
|
+
`INSERT INTO conversation_files (absolute_path, parent_dir, file_name, account)
|
|
1643
|
+
VALUES (?, ?, ?, ?)`
|
|
1644
|
+
).run(absolutePath, dirname4(absolutePath), basename4(absolutePath), account);
|
|
1645
|
+
return Number(info.lastInsertRowid);
|
|
1646
|
+
}
|
|
1647
|
+
// Advance the cursor + persisted reducer state after a successful index pass.
|
|
1648
|
+
updateCursor(id, fields) {
|
|
1649
|
+
this.db.prepare(
|
|
1650
|
+
`UPDATE conversation_files
|
|
1651
|
+
SET size_bytes = ?, mtime_ms = ?, last_indexed_offset = ?, last_indexed_line = ?,
|
|
1652
|
+
reducer_state = ?, content_fingerprint = ?, status = ?,
|
|
1653
|
+
last_indexed_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
|
|
1654
|
+
WHERE id = ?`
|
|
1655
|
+
).run(
|
|
1656
|
+
fields.sizeBytes,
|
|
1657
|
+
fields.mtimeMs,
|
|
1658
|
+
fields.offset,
|
|
1659
|
+
fields.line,
|
|
1660
|
+
fields.reducerState,
|
|
1661
|
+
fields.fingerprint,
|
|
1662
|
+
fields.status ?? "active",
|
|
1663
|
+
id
|
|
1664
|
+
);
|
|
1665
|
+
}
|
|
1666
|
+
// Reset the cursor to 0 for a truncated/replaced file before a full reindex.
|
|
1667
|
+
resetCursor(id) {
|
|
1668
|
+
this.db.prepare(
|
|
1669
|
+
`UPDATE conversation_files
|
|
1670
|
+
SET last_indexed_offset = 0, last_indexed_line = 0, reducer_state = NULL,
|
|
1671
|
+
status = 'needs_reindex', updated_at = CURRENT_TIMESTAMP
|
|
1672
|
+
WHERE id = ?`
|
|
1673
|
+
).run(id);
|
|
1674
|
+
}
|
|
1675
|
+
setStatus(id, status) {
|
|
1676
|
+
const deletedAt = status === "deleted" ? "CURRENT_TIMESTAMP" : "deleted_at";
|
|
1677
|
+
this.db.prepare(
|
|
1678
|
+
`UPDATE conversation_files
|
|
1679
|
+
SET status = ?, deleted_at = ${deletedAt}, updated_at = CURRENT_TIMESTAMP
|
|
1680
|
+
WHERE id = ?`
|
|
1681
|
+
).run(status, id);
|
|
1682
|
+
}
|
|
1683
|
+
allActivePaths() {
|
|
1684
|
+
const rows = this.db.prepare("SELECT absolute_path FROM conversation_files WHERE status != 'deleted'").all();
|
|
1685
|
+
return rows.map((r) => r.absolute_path);
|
|
1686
|
+
}
|
|
1687
|
+
};
|
|
1688
|
+
|
|
1689
|
+
// src/persistent/repositories/conversations.repo.ts
|
|
1690
|
+
function rowToMeta(row) {
|
|
1691
|
+
return {
|
|
1692
|
+
id: row.source_path,
|
|
1693
|
+
filePath: row.source_path,
|
|
1694
|
+
provider: row.provider ?? CLAUDE_CODE_PROVIDER,
|
|
1695
|
+
kind: row.kind ?? void 0,
|
|
1696
|
+
externalSessionId: row.external_session_id ?? void 0,
|
|
1697
|
+
sessionId: row.session_id,
|
|
1698
|
+
sessionName: row.session_name ?? "",
|
|
1699
|
+
projectPath: row.project_path ?? "",
|
|
1700
|
+
projectName: row.project_name ?? "",
|
|
1701
|
+
account: row.account,
|
|
1702
|
+
timestamp: row.timestamp ?? "",
|
|
1703
|
+
messageCount: row.message_count,
|
|
1704
|
+
lastMessageSender: row.last_message_sender,
|
|
1705
|
+
preview: row.preview ?? "",
|
|
1706
|
+
contentSnippet: row.content_snippet ?? "",
|
|
1707
|
+
gitBranch: row.branch,
|
|
1708
|
+
model: row.model,
|
|
1709
|
+
isSubagent: row.is_subagent === 1,
|
|
1710
|
+
parentSessionId: row.parent_session_id,
|
|
1711
|
+
isTeammate: row.is_teammate === 1,
|
|
1712
|
+
teamName: row.team_name,
|
|
1713
|
+
toolNames: row.tool_names_json ? JSON.parse(row.tool_names_json) : [],
|
|
1714
|
+
firstMessage: row.first_sent_text ? { text: row.first_sent_text, timestamp: row.first_sent_at ?? "" } : null,
|
|
1715
|
+
lastMessage: row.last_sent_text ? { text: row.last_sent_text, timestamp: row.last_sent_at ?? "" } : null,
|
|
1716
|
+
lastPrompt: row.last_prompt ?? void 0
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
var ConversationsRepo = class {
|
|
1720
|
+
constructor(db) {
|
|
1721
|
+
this.db = db;
|
|
1722
|
+
}
|
|
1723
|
+
db;
|
|
1724
|
+
// Upsert by file_id (1 file = 1 conversation). Keyed on the unique file_id so
|
|
1725
|
+
// a re-index overwrites the prior summary in place.
|
|
1726
|
+
upsert(fileId, meta, pageMessageCount = meta.messageCount) {
|
|
1727
|
+
this.db.prepare(
|
|
1728
|
+
`INSERT INTO conversations (
|
|
1729
|
+
file_id, source_path, provider, kind, external_session_id,
|
|
1730
|
+
session_id, session_name, project_path, project_name,
|
|
1731
|
+
account, branch, preview, content_snippet, message_count, page_message_count,
|
|
1732
|
+
last_message_sender,
|
|
1733
|
+
timestamp, index_seq, first_sent_at, first_sent_text, last_sent_at, last_sent_text,
|
|
1734
|
+
model, is_subagent, parent_session_id, is_teammate, team_name, tool_names_json,
|
|
1735
|
+
last_prompt, status, updated_at
|
|
1736
|
+
) VALUES (
|
|
1737
|
+
@file_id, @source_path, @provider, @kind, @external_session_id,
|
|
1738
|
+
@session_id, @session_name, @project_path, @project_name,
|
|
1739
|
+
@account, @branch, @preview, @content_snippet, @message_count, @page_message_count,
|
|
1740
|
+
@last_message_sender,
|
|
1741
|
+
@timestamp,
|
|
1742
|
+
(SELECT COALESCE(MAX(index_seq), 0) + 1 FROM conversations),
|
|
1743
|
+
@first_sent_at, @first_sent_text, @last_sent_at, @last_sent_text,
|
|
1744
|
+
@model, @is_subagent, @parent_session_id, @is_teammate, @team_name, @tool_names_json,
|
|
1745
|
+
@last_prompt, 'active', CURRENT_TIMESTAMP
|
|
1746
|
+
)
|
|
1747
|
+
ON CONFLICT(file_id) DO UPDATE SET
|
|
1748
|
+
source_path = excluded.source_path,
|
|
1749
|
+
provider = excluded.provider,
|
|
1750
|
+
kind = excluded.kind,
|
|
1751
|
+
external_session_id = excluded.external_session_id,
|
|
1752
|
+
session_id = excluded.session_id,
|
|
1753
|
+
session_name = excluded.session_name,
|
|
1754
|
+
project_path = excluded.project_path,
|
|
1755
|
+
project_name = excluded.project_name,
|
|
1756
|
+
account = excluded.account,
|
|
1757
|
+
branch = excluded.branch,
|
|
1758
|
+
preview = excluded.preview,
|
|
1759
|
+
content_snippet = excluded.content_snippet,
|
|
1760
|
+
message_count = excluded.message_count,
|
|
1761
|
+
page_message_count = excluded.page_message_count,
|
|
1762
|
+
last_message_sender = excluded.last_message_sender,
|
|
1763
|
+
timestamp = excluded.timestamp,
|
|
1764
|
+
first_sent_at = excluded.first_sent_at,
|
|
1765
|
+
first_sent_text = excluded.first_sent_text,
|
|
1766
|
+
last_sent_at = excluded.last_sent_at,
|
|
1767
|
+
last_sent_text = excluded.last_sent_text,
|
|
1768
|
+
model = excluded.model,
|
|
1769
|
+
is_subagent = excluded.is_subagent,
|
|
1770
|
+
parent_session_id = excluded.parent_session_id,
|
|
1771
|
+
is_teammate = excluded.is_teammate,
|
|
1772
|
+
team_name = excluded.team_name,
|
|
1773
|
+
tool_names_json = excluded.tool_names_json,
|
|
1774
|
+
last_prompt = excluded.last_prompt,
|
|
1775
|
+
status = 'active',
|
|
1776
|
+
index_seq = (SELECT COALESCE(MAX(index_seq), 0) + 1 FROM conversations),
|
|
1777
|
+
updated_at = CURRENT_TIMESTAMP`
|
|
1778
|
+
).run({
|
|
1779
|
+
file_id: fileId,
|
|
1780
|
+
source_path: meta.id,
|
|
1781
|
+
provider: meta.provider ?? CLAUDE_CODE_PROVIDER,
|
|
1782
|
+
kind: meta.kind ?? null,
|
|
1783
|
+
external_session_id: meta.externalSessionId ?? null,
|
|
1784
|
+
session_id: meta.sessionId,
|
|
1785
|
+
session_name: meta.sessionName || null,
|
|
1786
|
+
project_path: meta.projectPath || null,
|
|
1787
|
+
project_name: meta.projectName || null,
|
|
1788
|
+
account: meta.account,
|
|
1789
|
+
branch: meta.gitBranch,
|
|
1790
|
+
preview: meta.preview || null,
|
|
1791
|
+
content_snippet: meta.contentSnippet || null,
|
|
1792
|
+
message_count: meta.messageCount,
|
|
1793
|
+
page_message_count: pageMessageCount,
|
|
1794
|
+
last_message_sender: meta.lastMessageSender,
|
|
1795
|
+
timestamp: meta.timestamp || null,
|
|
1796
|
+
first_sent_at: meta.firstMessage?.timestamp ?? null,
|
|
1797
|
+
first_sent_text: meta.firstMessage?.text ?? null,
|
|
1798
|
+
last_sent_at: meta.lastMessage?.timestamp ?? null,
|
|
1799
|
+
last_sent_text: meta.lastMessage?.text ?? null,
|
|
1800
|
+
model: meta.model,
|
|
1801
|
+
is_subagent: meta.isSubagent ? 1 : 0,
|
|
1802
|
+
parent_session_id: meta.parentSessionId,
|
|
1803
|
+
is_teammate: meta.isTeammate ? 1 : 0,
|
|
1804
|
+
team_name: meta.teamName,
|
|
1805
|
+
tool_names_json: JSON.stringify(meta.toolNames),
|
|
1806
|
+
last_prompt: meta.lastPrompt ?? null
|
|
1807
|
+
});
|
|
1808
|
+
}
|
|
1809
|
+
getBySourcePath(sourcePath) {
|
|
1810
|
+
const row = this.db.prepare("SELECT * FROM conversations WHERE source_path = ? AND status = 'active'").get(sourcePath);
|
|
1811
|
+
return row ? rowToMeta(row) : null;
|
|
1812
|
+
}
|
|
1813
|
+
// Dual lookup matching scanner.getConversation: resolve by source_path (the
|
|
1814
|
+
// canonical id) first, then by session_id.
|
|
1815
|
+
//
|
|
1816
|
+
// session_id is NOT unique (see schema header) — the sessionId form is a
|
|
1817
|
+
// compatibility convenience. Resolution is deterministic and matches the
|
|
1818
|
+
// in-memory scanner: among active rows sharing the session_id, newest
|
|
1819
|
+
// timestamp wins, index_seq breaks sub-second ties (precision updated_at
|
|
1820
|
+
// lacks), then source_path ascending. Collision-safe callers should use
|
|
1821
|
+
// getAllBySessionId() instead.
|
|
1822
|
+
getByIdOrSession(id) {
|
|
1823
|
+
const direct = this.getBySourcePath(id);
|
|
1824
|
+
if (direct) return direct;
|
|
1825
|
+
const row = this.db.prepare(
|
|
1826
|
+
`SELECT * FROM conversations WHERE session_id = ? AND status = 'active'
|
|
1827
|
+
ORDER BY COALESCE(timestamp, '') DESC, index_seq DESC, source_path ASC LIMIT 1`
|
|
1828
|
+
).get(id);
|
|
1829
|
+
return row ? rowToMeta(row) : null;
|
|
1830
|
+
}
|
|
1831
|
+
// All active conversations sharing a session_id, newest first (same ordering
|
|
1832
|
+
// as getByIdOrSession). Collision-safe counterpart to the convenience
|
|
1833
|
+
// getByIdOrSession() sessionId lookup.
|
|
1834
|
+
getAllBySessionId(sessionId) {
|
|
1835
|
+
const rows = this.db.prepare(
|
|
1836
|
+
`SELECT * FROM conversations WHERE session_id = ? AND status = 'active'
|
|
1837
|
+
ORDER BY COALESCE(timestamp, '') DESC, index_seq DESC, source_path ASC`
|
|
1838
|
+
).all(sessionId);
|
|
1839
|
+
return rows.map(rowToMeta);
|
|
1840
|
+
}
|
|
1841
|
+
// All active metas (unsorted/unfiltered) — callers apply the existing
|
|
1842
|
+
// filters/view transforms. Used by scan() before SQL filtering is wired in.
|
|
1843
|
+
allActive() {
|
|
1844
|
+
const rows = this.db.prepare("SELECT * FROM conversations WHERE status = 'active'").all();
|
|
1845
|
+
return rows.map(rowToMeta);
|
|
1846
|
+
}
|
|
1847
|
+
// Most recent active conversations, newest first. Backs the empty-query
|
|
1848
|
+
// search path (mirrors the in-memory indexer's getRecent).
|
|
1849
|
+
recent(limit) {
|
|
1850
|
+
const rows = this.db.prepare(
|
|
1851
|
+
`SELECT * FROM conversations WHERE status = 'active'
|
|
1852
|
+
ORDER BY COALESCE(timestamp, '') DESC, source_path ASC LIMIT ?`
|
|
1853
|
+
).all(limit);
|
|
1854
|
+
return rows.map(rowToMeta);
|
|
1855
|
+
}
|
|
1856
|
+
distinctProjects() {
|
|
1857
|
+
const rows = this.db.prepare(
|
|
1858
|
+
`SELECT DISTINCT project_path FROM conversations
|
|
1859
|
+
WHERE status = 'active' AND project_path IS NOT NULL AND project_path != ''
|
|
1860
|
+
ORDER BY project_path ASC`
|
|
1861
|
+
).all();
|
|
1862
|
+
return rows.map((r) => r.project_path);
|
|
1863
|
+
}
|
|
1864
|
+
deleteByFileId(fileId) {
|
|
1865
|
+
this.db.prepare(
|
|
1866
|
+
"UPDATE conversations SET status = 'deleted', updated_at = CURRENT_TIMESTAMP WHERE file_id = ?"
|
|
1867
|
+
).run(fileId);
|
|
1868
|
+
}
|
|
1869
|
+
// The parseConversation message total for a file (for bounded paging), or 0
|
|
1870
|
+
// if not indexed.
|
|
1871
|
+
pageMessageCount(sourcePath) {
|
|
1872
|
+
const row = this.db.prepare(
|
|
1873
|
+
"SELECT page_message_count AS n FROM conversations WHERE source_path = ? AND status = 'active'"
|
|
1874
|
+
).get(sourcePath);
|
|
1875
|
+
return row?.n ?? 0;
|
|
1876
|
+
}
|
|
1877
|
+
count() {
|
|
1878
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM conversations WHERE status = 'active'").get().n;
|
|
1879
|
+
}
|
|
1880
|
+
};
|
|
1881
|
+
|
|
1882
|
+
// src/persistent/repositories/fts.repo.ts
|
|
1883
|
+
var FtsRepo = class {
|
|
1884
|
+
constructor(db) {
|
|
1885
|
+
this.db = db;
|
|
1886
|
+
}
|
|
1887
|
+
db;
|
|
1888
|
+
upsert(meta) {
|
|
1889
|
+
const tx = this.db.transaction(() => {
|
|
1890
|
+
this.db.prepare("DELETE FROM conversation_messages_fts WHERE source_path = ?").run(meta.id);
|
|
1891
|
+
this.db.prepare(
|
|
1892
|
+
`INSERT INTO conversation_messages_fts
|
|
1893
|
+
(source_path, content, project_name, session_id, session_name, account, model, branch, tool_names)
|
|
1894
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1895
|
+
).run(
|
|
1896
|
+
meta.id,
|
|
1897
|
+
meta.contentSnippet ?? "",
|
|
1898
|
+
meta.projectName ?? "",
|
|
1899
|
+
meta.sessionId ?? "",
|
|
1900
|
+
meta.sessionName ?? "",
|
|
1901
|
+
meta.account ?? "",
|
|
1902
|
+
meta.model ?? "",
|
|
1903
|
+
meta.gitBranch ?? "",
|
|
1904
|
+
meta.toolNames.join(" ")
|
|
1905
|
+
);
|
|
1906
|
+
});
|
|
1907
|
+
tx();
|
|
1908
|
+
}
|
|
1909
|
+
remove(sourcePath) {
|
|
1910
|
+
this.db.prepare("DELETE FROM conversation_messages_fts WHERE source_path = ?").run(sourcePath);
|
|
1911
|
+
}
|
|
1912
|
+
// Ranked source_paths matching the query, best first. Returns [] on an empty
|
|
1913
|
+
// query (callers fall back to a recency listing).
|
|
1914
|
+
search(query, limit) {
|
|
1915
|
+
const match = toMatchQuery(query);
|
|
1916
|
+
if (!match) return [];
|
|
1917
|
+
const rows = this.db.prepare(
|
|
1918
|
+
`SELECT source_path FROM conversation_messages_fts
|
|
1919
|
+
WHERE conversation_messages_fts MATCH ?
|
|
1920
|
+
ORDER BY rank
|
|
1921
|
+
LIMIT ?`
|
|
1922
|
+
).all(match, limit);
|
|
1923
|
+
return rows.map((r) => r.source_path);
|
|
1924
|
+
}
|
|
1925
|
+
count() {
|
|
1926
|
+
return this.db.prepare("SELECT COUNT(*) AS n FROM conversation_messages_fts").get().n;
|
|
1927
|
+
}
|
|
1928
|
+
};
|
|
1929
|
+
function toMatchQuery(query) {
|
|
1930
|
+
const terms = query.trim().split(/\s+/).map((t) => t.replace(/"/g, "").trim()).filter(Boolean);
|
|
1931
|
+
if (terms.length === 0) return "";
|
|
1932
|
+
return terms.map((t) => `"${t}"*`).join(" AND ");
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
// src/persistent/index-engine.ts
|
|
856
1936
|
var BATCH_SIZE = 12;
|
|
1937
|
+
var PersistentEngine = class {
|
|
1938
|
+
db;
|
|
1939
|
+
files;
|
|
1940
|
+
conversations;
|
|
1941
|
+
fts;
|
|
1942
|
+
checkpoints;
|
|
1943
|
+
// When true, write a portable <file>.idx.json sidecar next to each indexed
|
|
1944
|
+
// JSONL. Off by default.
|
|
1945
|
+
sidecar;
|
|
1946
|
+
constructor(dbPath, options = {}) {
|
|
1947
|
+
this.db = openDatabase(dbPath);
|
|
1948
|
+
this.files = new ConversationFilesRepo(this.db);
|
|
1949
|
+
this.conversations = new ConversationsRepo(this.db);
|
|
1950
|
+
this.fts = new FtsRepo(this.db);
|
|
1951
|
+
this.checkpoints = new CheckpointsRepo(this.db);
|
|
1952
|
+
this.sidecar = options.sidecar ?? false;
|
|
1953
|
+
}
|
|
1954
|
+
close() {
|
|
1955
|
+
this.db.close();
|
|
1956
|
+
}
|
|
1957
|
+
// Discover all JSONL files under the active profiles, (re)parse files that
|
|
1958
|
+
// changed since the last index, and upsert their metadata. Returns the number
|
|
1959
|
+
// of files seen on disk this pass (the scan "scanned" count).
|
|
1960
|
+
async indexAll(activeProfiles, options) {
|
|
1961
|
+
const log = getLogger();
|
|
1962
|
+
const tier = resolveTier(options.tier ?? "standard", options.tiers);
|
|
1963
|
+
const enabled = options.providers ?? [CLAUDE_CODE_PROVIDER];
|
|
1964
|
+
const discovered = [];
|
|
1965
|
+
if (enabled.includes(CLAUDE_CODE_PROVIDER)) {
|
|
1966
|
+
const configDirs = activeProfiles.map((p) => ({
|
|
1967
|
+
projectsDir: getProjectsDir(p),
|
|
1968
|
+
account: p.id
|
|
1969
|
+
}));
|
|
1970
|
+
for (const f of await discoverJsonlFiles(configDirs)) discovered.push(f);
|
|
1971
|
+
}
|
|
1972
|
+
const codex = new CodexCliProvider();
|
|
1973
|
+
if (enabled.includes(CODEX_CLI_PROVIDER) && (options.codexRoots?.length ?? 0) > 0) {
|
|
1974
|
+
for (const f of await codex.discover(options.codexRoots)) {
|
|
1975
|
+
discovered.push({ ...f, provider: codex });
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
let scanned = 0;
|
|
1979
|
+
const gitBranchMemo = /* @__PURE__ */ new Map();
|
|
1980
|
+
const resolveGitBranch = (projectPath) => {
|
|
1981
|
+
if (gitBranchMemo.has(projectPath)) {
|
|
1982
|
+
return gitBranchMemo.get(projectPath) ?? null;
|
|
1983
|
+
}
|
|
1984
|
+
const branch = readGitBranch(projectPath);
|
|
1985
|
+
gitBranchMemo.set(projectPath, branch);
|
|
1986
|
+
return branch;
|
|
1987
|
+
};
|
|
1988
|
+
for (let i = 0; i < discovered.length; i += BATCH_SIZE) {
|
|
1989
|
+
const batch = discovered.slice(i, i + BATCH_SIZE);
|
|
1990
|
+
const results = await Promise.all(
|
|
1991
|
+
batch.map(async ({ filePath, account, provider }) => {
|
|
1992
|
+
const meta = await this.indexFile(
|
|
1993
|
+
filePath,
|
|
1994
|
+
account,
|
|
1995
|
+
tier.name,
|
|
1996
|
+
options.tiers,
|
|
1997
|
+
resolveGitBranch,
|
|
1998
|
+
false,
|
|
1999
|
+
provider
|
|
2000
|
+
);
|
|
2001
|
+
return meta;
|
|
2002
|
+
})
|
|
2003
|
+
);
|
|
2004
|
+
const kept = results.filter((m) => m != null);
|
|
2005
|
+
if (kept.length > 0) options.onBatch?.(kept);
|
|
2006
|
+
scanned += batch.length;
|
|
2007
|
+
options.onProgress?.(scanned, discovered.length);
|
|
2008
|
+
}
|
|
2009
|
+
const seen = new Set(discovered.map((d) => d.filePath));
|
|
2010
|
+
for (const path of this.files.allActivePaths()) {
|
|
2011
|
+
if (!seen.has(path)) this.markDeleted(path);
|
|
2012
|
+
}
|
|
2013
|
+
log.info({ scanned, indexed: this.conversations.count() }, "persistent: indexAll complete");
|
|
2014
|
+
return { scanned };
|
|
2015
|
+
}
|
|
2016
|
+
// (Re)index a single file. Classifies the change vs. the persisted cursor:
|
|
2017
|
+
// unchanged → return the stored summary; appended → resume the fold and read
|
|
2018
|
+
// only new bytes; reindex/force → fold from offset 0. Writes the summary +
|
|
2019
|
+
// cursor + reducer state in one transaction so a crash never leaves a
|
|
2020
|
+
// half-written row or an over-advanced cursor.
|
|
2021
|
+
async indexFile(filePath, account, tierName, customTiers, resolveGitBranch, force = false, provider) {
|
|
2022
|
+
const log = getLogger();
|
|
2023
|
+
const tier = resolveTier(tierName, customTiers);
|
|
2024
|
+
const existing = this.files.getByPath(filePath);
|
|
2025
|
+
const { change, stat: stat3 } = classify(filePath, existing);
|
|
2026
|
+
if (change === "vanished" || !stat3) {
|
|
2027
|
+
this.markDeleted(filePath);
|
|
2028
|
+
return null;
|
|
2029
|
+
}
|
|
2030
|
+
if (change === "unchanged" && !force) {
|
|
2031
|
+
return this.conversations.getBySourcePath(filePath);
|
|
2032
|
+
}
|
|
2033
|
+
if (provider && provider.name !== CLAUDE_CODE_PROVIDER) {
|
|
2034
|
+
return this.indexFileWithProvider(provider, filePath, account, tier, stat3, resolveGitBranch);
|
|
2035
|
+
}
|
|
2036
|
+
const resume = change === "appended" && !force && existing?.reducer_state;
|
|
2037
|
+
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
2038
|
+
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
2039
|
+
const startLine = resume ? existing.last_indexed_line : 0;
|
|
2040
|
+
let result;
|
|
2041
|
+
try {
|
|
2042
|
+
result = await tailReduce(filePath, startOffset, startLine, state, tier);
|
|
2043
|
+
} catch (err) {
|
|
2044
|
+
log.warn({ filePath, err }, "persistent: tail read failed");
|
|
2045
|
+
return null;
|
|
2046
|
+
}
|
|
2047
|
+
const meta = finalizeMeta(state, filePath, account, tier);
|
|
2048
|
+
if (!meta) {
|
|
2049
|
+
this.markDeleted(filePath);
|
|
2050
|
+
return null;
|
|
2051
|
+
}
|
|
2052
|
+
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2053
|
+
const fp = stat3.size > 0 ? fingerprint(filePath, stat3.size) : null;
|
|
2054
|
+
const fileId = this.files.ensure(filePath, account);
|
|
2055
|
+
const upsert = this.db.transaction(() => {
|
|
2056
|
+
this.conversations.upsert(fileId, meta, state.pageMessageCount);
|
|
2057
|
+
this.fts.upsert(meta);
|
|
2058
|
+
this.checkpoints.remove(filePath);
|
|
2059
|
+
this.files.updateCursor(fileId, {
|
|
2060
|
+
sizeBytes: stat3.size,
|
|
2061
|
+
mtimeMs: stat3.mtimeMs,
|
|
2062
|
+
// Advance only to the last fully-parsed line; a trailing partial line
|
|
2063
|
+
// is left for the next pass.
|
|
2064
|
+
offset: result.newOffset,
|
|
2065
|
+
line: result.newLine,
|
|
2066
|
+
reducerState: JSON.stringify(state),
|
|
2067
|
+
fingerprint: fp,
|
|
2068
|
+
status: "active"
|
|
2069
|
+
});
|
|
2070
|
+
});
|
|
2071
|
+
upsert();
|
|
2072
|
+
if (this.sidecar) {
|
|
2073
|
+
writeSidecar(
|
|
2074
|
+
filePath,
|
|
2075
|
+
buildSidecar(
|
|
2076
|
+
meta,
|
|
2077
|
+
{
|
|
2078
|
+
sizeBytes: stat3.size,
|
|
2079
|
+
mtimeMs: stat3.mtimeMs,
|
|
2080
|
+
offset: result.newOffset,
|
|
2081
|
+
line: result.newLine
|
|
2082
|
+
},
|
|
2083
|
+
(/* @__PURE__ */ new Date()).toISOString()
|
|
2084
|
+
)
|
|
2085
|
+
);
|
|
2086
|
+
}
|
|
2087
|
+
log.debug(
|
|
2088
|
+
{ filePath, change, bytesRead: result.newOffset - startOffset, msgs: meta.messageCount },
|
|
2089
|
+
"persistent: indexed file"
|
|
2090
|
+
);
|
|
2091
|
+
return meta;
|
|
2092
|
+
}
|
|
2093
|
+
// Index a non-Threadbase provider file: full reparse from offset 0 through the
|
|
2094
|
+
// provider's reducer/finalize, then the same upsert + FTS write + cursor bump
|
|
2095
|
+
// the Threadbase path uses. The cursor records size/mtime (and offset = size)
|
|
2096
|
+
// so the next pass classifies an unchanged file as "unchanged" and skips it;
|
|
2097
|
+
// any change reparses from 0 again. No reducer_state is persisted.
|
|
2098
|
+
async indexFileWithProvider(provider, filePath, account, tier, stat3, resolveGitBranch) {
|
|
2099
|
+
const log = getLogger();
|
|
2100
|
+
const meta = await parseMetaWithProvider(provider, filePath, account, tier);
|
|
2101
|
+
if (!meta) {
|
|
2102
|
+
this.markDeleted(filePath);
|
|
2103
|
+
return null;
|
|
2104
|
+
}
|
|
2105
|
+
if (meta.gitBranch === null && meta.projectPath) {
|
|
2106
|
+
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
2107
|
+
}
|
|
2108
|
+
const fp = stat3.size > 0 ? fingerprint(filePath, stat3.size) : null;
|
|
2109
|
+
const fileId = this.files.ensure(filePath, account);
|
|
2110
|
+
const upsert = this.db.transaction(() => {
|
|
2111
|
+
this.conversations.upsert(fileId, meta, meta.messageCount);
|
|
2112
|
+
this.fts.upsert(meta);
|
|
2113
|
+
this.checkpoints.remove(filePath);
|
|
2114
|
+
this.files.updateCursor(fileId, {
|
|
2115
|
+
sizeBytes: stat3.size,
|
|
2116
|
+
mtimeMs: stat3.mtimeMs,
|
|
2117
|
+
// offset = size marks the file fully consumed (non-zero so the next pass
|
|
2118
|
+
// can classify it "unchanged"); no resumable reducer state is kept.
|
|
2119
|
+
offset: stat3.size,
|
|
2120
|
+
line: 0,
|
|
2121
|
+
reducerState: null,
|
|
2122
|
+
fingerprint: fp,
|
|
2123
|
+
status: "active"
|
|
2124
|
+
});
|
|
2125
|
+
});
|
|
2126
|
+
upsert();
|
|
2127
|
+
log.debug(
|
|
2128
|
+
{ filePath, provider: provider.name, msgs: meta.messageCount },
|
|
2129
|
+
"persistent: indexed provider file"
|
|
2130
|
+
);
|
|
2131
|
+
return meta;
|
|
2132
|
+
}
|
|
2133
|
+
markDeleted(filePath) {
|
|
2134
|
+
const existing = this.files.getByPath(filePath);
|
|
2135
|
+
if (!existing) return;
|
|
2136
|
+
const tx = this.db.transaction(() => {
|
|
2137
|
+
this.conversations.deleteByFileId(existing.id);
|
|
2138
|
+
this.fts.remove(filePath);
|
|
2139
|
+
this.checkpoints.remove(filePath);
|
|
2140
|
+
this.files.setStatus(existing.id, "deleted");
|
|
2141
|
+
});
|
|
2142
|
+
tx();
|
|
2143
|
+
}
|
|
2144
|
+
// ── Query helpers (read straight from SQLite) ───────────────────────────
|
|
2145
|
+
allActive() {
|
|
2146
|
+
return this.conversations.allActive();
|
|
2147
|
+
}
|
|
2148
|
+
getByIdOrSession(id) {
|
|
2149
|
+
return this.conversations.getByIdOrSession(id);
|
|
2150
|
+
}
|
|
2151
|
+
getAllBySessionId(sessionId) {
|
|
2152
|
+
return this.conversations.getAllBySessionId(sessionId);
|
|
2153
|
+
}
|
|
2154
|
+
// Ranked metas matching the FTS query, best first. Empty query returns the
|
|
2155
|
+
// most recent conversations (mirroring the in-memory indexer's empty-query
|
|
2156
|
+
// behavior). Resolves each FTS hit to its active conversation row.
|
|
2157
|
+
searchMetas(query, limit) {
|
|
2158
|
+
if (!query.trim()) {
|
|
2159
|
+
return this.conversations.recent(limit);
|
|
2160
|
+
}
|
|
2161
|
+
const paths = this.fts.search(query, limit);
|
|
2162
|
+
const metas = [];
|
|
2163
|
+
for (const path of paths) {
|
|
2164
|
+
const meta = this.conversations.getBySourcePath(path);
|
|
2165
|
+
if (meta) metas.push(meta);
|
|
2166
|
+
}
|
|
2167
|
+
return metas;
|
|
2168
|
+
}
|
|
2169
|
+
getProjects() {
|
|
2170
|
+
return this.conversations.distinctProjects();
|
|
2171
|
+
}
|
|
2172
|
+
// Bounded conversation page: read only the requested window from the file,
|
|
2173
|
+
// seeking from the nearest checkpoint. Returns null if the id can't be
|
|
2174
|
+
// resolved to an indexed conversation. For conversations large enough to span
|
|
2175
|
+
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
2176
|
+
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
2177
|
+
async getPage(id, options) {
|
|
2178
|
+
const meta = this.conversations.getByIdOrSession(id);
|
|
2179
|
+
if (!meta) return null;
|
|
2180
|
+
const filePath = meta.filePath;
|
|
2181
|
+
const total = this.conversations.pageMessageCount(filePath);
|
|
2182
|
+
if (total > CHECKPOINT_INTERVAL && this.checkpoints.count(filePath) === 0) {
|
|
2183
|
+
const built = await buildCheckpoints(filePath);
|
|
2184
|
+
if (built.length > 0) this.checkpoints.replaceAll(filePath, built);
|
|
2185
|
+
}
|
|
2186
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
2187
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2188
|
+
const floor = this.checkpoints.floor(filePath, fromIndex);
|
|
2189
|
+
return readPage(filePath, total, options, floor);
|
|
2190
|
+
}
|
|
2191
|
+
};
|
|
2192
|
+
|
|
2193
|
+
// src/watcher/file-watcher.ts
|
|
2194
|
+
import chokidar from "chokidar";
|
|
2195
|
+
var EXCLUDED_SEGMENTS2 = ["/memory/", "/tool-results/"];
|
|
2196
|
+
var FileWatcher = class {
|
|
2197
|
+
constructor(profiles, onEvent, options = {}) {
|
|
2198
|
+
this.profiles = profiles;
|
|
2199
|
+
this.onEvent = onEvent;
|
|
2200
|
+
this.debounceMs = options.debounceMs ?? 400;
|
|
2201
|
+
}
|
|
2202
|
+
profiles;
|
|
2203
|
+
onEvent;
|
|
2204
|
+
watchers = [];
|
|
2205
|
+
timers = /* @__PURE__ */ new Map();
|
|
2206
|
+
debounceMs;
|
|
2207
|
+
// Resolves once every underlying chokidar watcher has finished its initial
|
|
2208
|
+
// scan, so the caller knows subsequent FS changes will be observed.
|
|
2209
|
+
async start() {
|
|
2210
|
+
const log = getLogger();
|
|
2211
|
+
const ready = [];
|
|
2212
|
+
for (const profile of this.profiles) {
|
|
2213
|
+
const dir = getProjectsDir(profile);
|
|
2214
|
+
const watcher = chokidar.watch(dir, {
|
|
2215
|
+
ignoreInitial: true,
|
|
2216
|
+
ignored: (p) => EXCLUDED_SEGMENTS2.some((seg) => p.includes(seg)),
|
|
2217
|
+
awaitWriteFinish: { stabilityThreshold: this.debounceMs, pollInterval: 100 }
|
|
2218
|
+
});
|
|
2219
|
+
watcher.on("add", (p) => this.dispatch(p, profile.id, "add")).on("change", (p) => this.dispatch(p, profile.id, "change")).on("unlink", (p) => this.dispatch(p, profile.id, "unlink"));
|
|
2220
|
+
ready.push(new Promise((resolve) => watcher.once("ready", () => resolve())));
|
|
2221
|
+
this.watchers.push(watcher);
|
|
2222
|
+
log.debug({ dir, account: profile.id }, "watcher: watching");
|
|
2223
|
+
}
|
|
2224
|
+
await Promise.all(ready);
|
|
2225
|
+
}
|
|
2226
|
+
async stop() {
|
|
2227
|
+
for (const t of this.timers.values()) clearTimeout(t);
|
|
2228
|
+
this.timers.clear();
|
|
2229
|
+
await Promise.all(this.watchers.map((w) => w.close()));
|
|
2230
|
+
this.watchers = [];
|
|
2231
|
+
}
|
|
2232
|
+
dispatch(filePath, account, type) {
|
|
2233
|
+
if (!filePath.endsWith(".jsonl")) return;
|
|
2234
|
+
const existing = this.timers.get(filePath);
|
|
2235
|
+
if (existing) clearTimeout(existing);
|
|
2236
|
+
this.timers.set(
|
|
2237
|
+
filePath,
|
|
2238
|
+
setTimeout(() => {
|
|
2239
|
+
this.timers.delete(filePath);
|
|
2240
|
+
this.onEvent({ filePath, account, type });
|
|
2241
|
+
}, this.debounceMs)
|
|
2242
|
+
);
|
|
2243
|
+
}
|
|
2244
|
+
};
|
|
2245
|
+
|
|
2246
|
+
// src/watcher/index-queue.ts
|
|
2247
|
+
var IndexQueue = class {
|
|
2248
|
+
constructor(process2) {
|
|
2249
|
+
this.process = process2;
|
|
2250
|
+
}
|
|
2251
|
+
process;
|
|
2252
|
+
pending = /* @__PURE__ */ new Map();
|
|
2253
|
+
running = false;
|
|
2254
|
+
idleResolvers = [];
|
|
2255
|
+
enqueue(job) {
|
|
2256
|
+
this.pending.set(job.filePath, job);
|
|
2257
|
+
void this.drain();
|
|
2258
|
+
}
|
|
2259
|
+
// Resolves when the queue has fully drained — useful for tests and for a
|
|
2260
|
+
// clean shutdown.
|
|
2261
|
+
onIdle() {
|
|
2262
|
+
if (!this.running && this.pending.size === 0) return Promise.resolve();
|
|
2263
|
+
return new Promise((resolve) => this.idleResolvers.push(resolve));
|
|
2264
|
+
}
|
|
2265
|
+
get size() {
|
|
2266
|
+
return this.pending.size;
|
|
2267
|
+
}
|
|
2268
|
+
async drain() {
|
|
2269
|
+
if (this.running) return;
|
|
2270
|
+
this.running = true;
|
|
2271
|
+
const log = getLogger();
|
|
2272
|
+
while (this.pending.size > 0) {
|
|
2273
|
+
const [path, job] = this.pending.entries().next().value;
|
|
2274
|
+
this.pending.delete(path);
|
|
2275
|
+
try {
|
|
2276
|
+
await this.process(job);
|
|
2277
|
+
} catch (err) {
|
|
2278
|
+
log.warn({ filePath: job.filePath, err }, "index-queue: job failed");
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
this.running = false;
|
|
2282
|
+
const resolvers = this.idleResolvers;
|
|
2283
|
+
this.idleResolvers = [];
|
|
2284
|
+
for (const r of resolvers) r();
|
|
2285
|
+
}
|
|
2286
|
+
};
|
|
2287
|
+
|
|
2288
|
+
// src/scanner.ts
|
|
2289
|
+
var BATCH_SIZE2 = 12;
|
|
857
2290
|
var DEFAULT_CONFIG_PATH = "~/.config/threadbase-scanner";
|
|
2291
|
+
function defaultDbPath() {
|
|
2292
|
+
return process.env.TB_SCANNER_DB ?? join4(homedir2(), ".config", "threadbase-scanner", "index.db");
|
|
2293
|
+
}
|
|
858
2294
|
var ConversationScanner = class {
|
|
859
2295
|
metadataCache = /* @__PURE__ */ new Map();
|
|
860
2296
|
conversationLRU;
|
|
2297
|
+
// session_id is NOT unique, so this maps a sessionId to every active meta that
|
|
2298
|
+
// carries it. Resolution picks deterministically (newest timestamp, then path
|
|
2299
|
+
// ascending) so dropping one file never hides another with the same id.
|
|
861
2300
|
sessionIdIndex = /* @__PURE__ */ new Map();
|
|
862
2301
|
projects = /* @__PURE__ */ new Set();
|
|
863
2302
|
indexer = new SearchIndexer();
|
|
864
2303
|
// Tier the most recent scan() ran with, so refreshFile() re-parses a single
|
|
865
2304
|
// file at the same content depth. Defaults to the standard tier.
|
|
866
2305
|
lastTier = resolveTier("standard");
|
|
2306
|
+
// null when persistent mode is disabled (legacy in-memory path). Lazily
|
|
2307
|
+
// opened on first use so merely constructing a scanner never touches disk.
|
|
2308
|
+
dbPath;
|
|
2309
|
+
sidecarEnabled;
|
|
2310
|
+
engineInstance = null;
|
|
2311
|
+
emitter = new EventEmitter();
|
|
2312
|
+
watcher = null;
|
|
2313
|
+
queue = null;
|
|
2314
|
+
periodicTimer = null;
|
|
867
2315
|
constructor(options) {
|
|
868
2316
|
this.conversationLRU = new LRUCache(options?.conversationCacheSize ?? 5);
|
|
2317
|
+
if (options?.persistent === false) {
|
|
2318
|
+
this.dbPath = null;
|
|
2319
|
+
this.sidecarEnabled = false;
|
|
2320
|
+
} else {
|
|
2321
|
+
this.dbPath = options?.persistent?.dbPath ?? defaultDbPath();
|
|
2322
|
+
this.sidecarEnabled = options?.persistent?.sidecar ?? false;
|
|
2323
|
+
}
|
|
2324
|
+
}
|
|
2325
|
+
get persistent() {
|
|
2326
|
+
return this.dbPath !== null;
|
|
2327
|
+
}
|
|
2328
|
+
engine() {
|
|
2329
|
+
if (!this.engineInstance) {
|
|
2330
|
+
this.engineInstance = new PersistentEngine(this.dbPath, {
|
|
2331
|
+
sidecar: this.sidecarEnabled
|
|
2332
|
+
});
|
|
2333
|
+
}
|
|
2334
|
+
return this.engineInstance;
|
|
2335
|
+
}
|
|
2336
|
+
// Release the SQLite connection. No-op in legacy mode. Safe to call
|
|
2337
|
+
// repeatedly. Stops the watcher first if one is running; call unwatch()
|
|
2338
|
+
// explicitly beforehand if you need to await watcher teardown.
|
|
2339
|
+
close() {
|
|
2340
|
+
if (this.watcher || this.queue || this.periodicTimer) void this.unwatch();
|
|
2341
|
+
this.engineInstance?.close();
|
|
2342
|
+
this.engineInstance = null;
|
|
869
2343
|
}
|
|
870
2344
|
async scan(options = {}) {
|
|
871
|
-
const log = getLogger();
|
|
872
|
-
const startedAt = Date.now();
|
|
873
2345
|
const profiles = await this.resolveProfiles(options.profiles);
|
|
874
2346
|
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
875
|
-
|
|
876
|
-
this.
|
|
2347
|
+
this.lastTier = resolveTier(options.tier ?? "standard", options.tiers);
|
|
2348
|
+
if (this.persistent) {
|
|
2349
|
+
return this.scanPersistent(activeProfiles, options);
|
|
2350
|
+
}
|
|
2351
|
+
return this.scanInMemory(activeProfiles, options);
|
|
2352
|
+
}
|
|
2353
|
+
// SQLite-backed scan: (re)index changed files into the DB, then query all
|
|
2354
|
+
// active metas and run the identical filter/sort/view/paginate pipeline as
|
|
2355
|
+
// the in-memory path — guaranteeing an identical ScanResult shape.
|
|
2356
|
+
async scanPersistent(activeProfiles, options) {
|
|
2357
|
+
const log = getLogger();
|
|
2358
|
+
const startedAt = Date.now();
|
|
2359
|
+
const engine = this.engine();
|
|
2360
|
+
const { scanned } = await engine.indexAll(activeProfiles, options);
|
|
2361
|
+
const allMetas = engine.allActive();
|
|
2362
|
+
const { conversations, total } = this.finalize(allMetas, options);
|
|
2363
|
+
log.info(
|
|
2364
|
+
{ scanned, kept: allMetas.length, filteredTotal: total, elapsedMs: Date.now() - startedAt },
|
|
2365
|
+
"scan: complete (persistent)"
|
|
2366
|
+
);
|
|
2367
|
+
return { conversations, total, scanned };
|
|
2368
|
+
}
|
|
2369
|
+
// Apply include/project/account/since filters, sort, view transform, and
|
|
2370
|
+
// pagination. Shared by both backends so results never diverge.
|
|
2371
|
+
finalize(allMetas, options) {
|
|
2372
|
+
let filtered = allMetas;
|
|
2373
|
+
if (options.include && options.include !== "all") {
|
|
2374
|
+
filtered = applyIncludeFilter(filtered, options.include);
|
|
2375
|
+
}
|
|
2376
|
+
if (options.project) filtered = applyProjectFilter(filtered, options.project);
|
|
2377
|
+
if (options.account) filtered = applyAccountFilter(filtered, options.account);
|
|
2378
|
+
if (options.since) filtered = applySinceFilter(filtered, options.since);
|
|
2379
|
+
filtered = applySort(filtered, options.sort ?? "recent");
|
|
2380
|
+
const total = filtered.length;
|
|
2381
|
+
const conversations = this.transformView(filtered, options);
|
|
2382
|
+
if (Array.isArray(conversations)) {
|
|
2383
|
+
const limit = options.limit ?? 50;
|
|
2384
|
+
const offset = options.offset ?? 0;
|
|
2385
|
+
return { conversations: applyPagination(conversations, limit, offset).items, total };
|
|
2386
|
+
}
|
|
2387
|
+
return { conversations, total };
|
|
2388
|
+
}
|
|
2389
|
+
async scanInMemory(activeProfiles, options) {
|
|
2390
|
+
const log = getLogger();
|
|
2391
|
+
const startedAt = Date.now();
|
|
2392
|
+
const tier = this.lastTier;
|
|
877
2393
|
log.info(
|
|
878
2394
|
{
|
|
879
2395
|
activeProfiles: activeProfiles.length,
|
|
@@ -889,11 +2405,7 @@ var ConversationScanner = class {
|
|
|
889
2405
|
this.sessionIdIndex.clear();
|
|
890
2406
|
this.projects.clear();
|
|
891
2407
|
this.indexer.clear();
|
|
892
|
-
const
|
|
893
|
-
projectsDir: getProjectsDir(p),
|
|
894
|
-
account: p.id
|
|
895
|
-
}));
|
|
896
|
-
const files = await discoverJsonlFiles(configDirs);
|
|
2408
|
+
const files = await this.discoverWithProviders(activeProfiles, options);
|
|
897
2409
|
const totalFiles = files.length;
|
|
898
2410
|
let scanned = 0;
|
|
899
2411
|
let parseFailures = 0;
|
|
@@ -908,15 +2420,15 @@ var ConversationScanner = class {
|
|
|
908
2420
|
return branch;
|
|
909
2421
|
};
|
|
910
2422
|
const { statCache } = options;
|
|
911
|
-
for (let i = 0; i < files.length; i +=
|
|
912
|
-
const batch = files.slice(i, i +
|
|
2423
|
+
for (let i = 0; i < files.length; i += BATCH_SIZE2) {
|
|
2424
|
+
const batch = files.slice(i, i + BATCH_SIZE2);
|
|
913
2425
|
const results = await Promise.all(
|
|
914
|
-
batch.map(async ({ filePath, account }) => {
|
|
2426
|
+
batch.map(async ({ filePath, account, provider }) => {
|
|
915
2427
|
if (statCache) {
|
|
916
2428
|
const cached = statCache.get(filePath);
|
|
917
2429
|
if (cached) {
|
|
918
2430
|
try {
|
|
919
|
-
const s =
|
|
2431
|
+
const s = statSync2(filePath);
|
|
920
2432
|
if (s.mtimeMs === cached.stat.mtimeMs && s.size === cached.stat.size) {
|
|
921
2433
|
return cached.meta;
|
|
922
2434
|
}
|
|
@@ -925,14 +2437,14 @@ var ConversationScanner = class {
|
|
|
925
2437
|
}
|
|
926
2438
|
}
|
|
927
2439
|
try {
|
|
928
|
-
const meta = await
|
|
929
|
-
if (meta) {
|
|
2440
|
+
const meta = await parseMetaWithProvider(provider, filePath, account, tier);
|
|
2441
|
+
if (meta && meta.gitBranch === null && meta.projectPath) {
|
|
930
2442
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
931
2443
|
}
|
|
932
2444
|
return meta;
|
|
933
2445
|
} catch (err) {
|
|
934
2446
|
parseFailures++;
|
|
935
|
-
log.warn({ filePath, account, err }, "scan:
|
|
2447
|
+
log.warn({ filePath, account, provider: provider.name, err }, "scan: parse threw");
|
|
936
2448
|
return null;
|
|
937
2449
|
}
|
|
938
2450
|
})
|
|
@@ -941,7 +2453,7 @@ var ConversationScanner = class {
|
|
|
941
2453
|
for (const meta of results) {
|
|
942
2454
|
if (meta && meta.messageCount > 0) {
|
|
943
2455
|
this.metadataCache.set(meta.id, meta);
|
|
944
|
-
this.
|
|
2456
|
+
this.addToSessionIndex(meta);
|
|
945
2457
|
this.projects.add(meta.projectPath);
|
|
946
2458
|
allMetas.push(meta);
|
|
947
2459
|
batchMetas.push(meta);
|
|
@@ -955,22 +2467,7 @@ var ConversationScanner = class {
|
|
|
955
2467
|
log.debug({ scanned, totalFiles, batchKept: batchMetas.length }, "scan: batch complete");
|
|
956
2468
|
options.onProgress?.(scanned, totalFiles);
|
|
957
2469
|
}
|
|
958
|
-
|
|
959
|
-
if (options.include && options.include !== "all") {
|
|
960
|
-
filtered = applyIncludeFilter(filtered, options.include);
|
|
961
|
-
}
|
|
962
|
-
if (options.project) {
|
|
963
|
-
filtered = applyProjectFilter(filtered, options.project);
|
|
964
|
-
}
|
|
965
|
-
if (options.account) {
|
|
966
|
-
filtered = applyAccountFilter(filtered, options.account);
|
|
967
|
-
}
|
|
968
|
-
if (options.since) {
|
|
969
|
-
filtered = applySinceFilter(filtered, options.since);
|
|
970
|
-
}
|
|
971
|
-
filtered = applySort(filtered, options.sort ?? "recent");
|
|
972
|
-
const total = filtered.length;
|
|
973
|
-
const conversations = this.transformView(filtered, options);
|
|
2470
|
+
const { conversations, total } = this.finalize(allMetas, options);
|
|
974
2471
|
const elapsedMs = Date.now() - startedAt;
|
|
975
2472
|
log.info(
|
|
976
2473
|
{
|
|
@@ -983,25 +2480,36 @@ var ConversationScanner = class {
|
|
|
983
2480
|
},
|
|
984
2481
|
"scan: complete"
|
|
985
2482
|
);
|
|
986
|
-
if (Array.isArray(conversations)) {
|
|
987
|
-
const limit = options.limit ?? 50;
|
|
988
|
-
const offset = options.offset ?? 0;
|
|
989
|
-
const paginated = applyPagination(conversations, limit, offset);
|
|
990
|
-
return { conversations: paginated.items, total, scanned };
|
|
991
|
-
}
|
|
992
2483
|
return { conversations, total, scanned };
|
|
993
2484
|
}
|
|
994
2485
|
async search(query, options = {}) {
|
|
995
2486
|
const log = getLogger();
|
|
996
2487
|
log.debug({ query, indexSize: this.indexer.getDocumentCount() }, "search: start");
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
2488
|
+
let results;
|
|
2489
|
+
if (this.persistent) {
|
|
2490
|
+
const engine = this.engine();
|
|
2491
|
+
if (engine.conversations.count() === 0) {
|
|
2492
|
+
log.debug("search: persistent index empty, triggering scan");
|
|
2493
|
+
const profiles = await this.resolveProfiles(options.profiles);
|
|
2494
|
+
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
2495
|
+
await engine.indexAll(activeProfiles, { ...options, limit: void 0, offset: void 0 });
|
|
2496
|
+
}
|
|
2497
|
+
const metas = engine.searchMetas(query, (options.limit ?? 50) * 2);
|
|
2498
|
+
results = query.trim() ? metas.map((meta) => ({ meta, score: 1, matches: generateMatches(meta, query) })) : metas.map((meta) => ({
|
|
2499
|
+
meta,
|
|
2500
|
+
score: 1,
|
|
2501
|
+
matches: [{ field: "timestamp", snippet: meta.preview }]
|
|
2502
|
+
}));
|
|
2503
|
+
} else {
|
|
2504
|
+
if (this.indexer.getDocumentCount() === 0) {
|
|
2505
|
+
log.debug("search: index empty, triggering scan");
|
|
2506
|
+
await this.scan({ ...options, limit: void 0, offset: void 0 });
|
|
2507
|
+
}
|
|
2508
|
+
results = this.indexer.search(query, {
|
|
2509
|
+
fields: options.fields,
|
|
2510
|
+
limit: (options.limit ?? 50) * 2
|
|
2511
|
+
});
|
|
1000
2512
|
}
|
|
1001
|
-
let results = this.indexer.search(query, {
|
|
1002
|
-
fields: options.fields,
|
|
1003
|
-
limit: (options.limit ?? 50) * 2
|
|
1004
|
-
});
|
|
1005
2513
|
if (options.include && options.include !== "all") {
|
|
1006
2514
|
results = results.filter((r) => {
|
|
1007
2515
|
switch (options.include) {
|
|
@@ -1025,6 +2533,11 @@ var ConversationScanner = class {
|
|
|
1025
2533
|
if (options.account) {
|
|
1026
2534
|
results = results.filter((r) => r.meta.account === options.account);
|
|
1027
2535
|
}
|
|
2536
|
+
if (options.provider) {
|
|
2537
|
+
results = results.filter(
|
|
2538
|
+
(r) => (r.meta.provider ?? CLAUDE_CODE_PROVIDER) === options.provider
|
|
2539
|
+
);
|
|
2540
|
+
}
|
|
1028
2541
|
if (options.since) {
|
|
1029
2542
|
const cutoff = parseSinceCutoff(options.since);
|
|
1030
2543
|
results = results.filter((r) => new Date(r.meta.timestamp).getTime() >= cutoff.getTime());
|
|
@@ -1042,14 +2555,14 @@ var ConversationScanner = class {
|
|
|
1042
2555
|
log.debug({ id }, "getConversation: cache hit");
|
|
1043
2556
|
return cached;
|
|
1044
2557
|
}
|
|
1045
|
-
const meta = this.metadataCache.get(id) ?? this.
|
|
2558
|
+
const meta = this.persistent ? this.engine().getByIdOrSession(id) : this.metadataCache.get(id) ?? this.resolveSessionId(id);
|
|
1046
2559
|
if (!meta) {
|
|
1047
2560
|
log.debug({ id }, "getConversation: not found in metadata");
|
|
1048
2561
|
return null;
|
|
1049
2562
|
}
|
|
1050
2563
|
log.debug({ id, filePath: meta.filePath }, "getConversation: cache miss, parsing");
|
|
1051
2564
|
try {
|
|
1052
|
-
const conversation = await parseConversation(meta.filePath, meta.account);
|
|
2565
|
+
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
1053
2566
|
if (conversation) {
|
|
1054
2567
|
this.conversationLRU.set(id, conversation);
|
|
1055
2568
|
}
|
|
@@ -1069,14 +2582,18 @@ var ConversationScanner = class {
|
|
|
1069
2582
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
1070
2583
|
// same contract as getConversation.
|
|
1071
2584
|
//
|
|
1072
|
-
//
|
|
1073
|
-
//
|
|
1074
|
-
//
|
|
1075
|
-
// parseConversation() by
|
|
1076
|
-
//
|
|
1077
|
-
//
|
|
1078
|
-
//
|
|
2585
|
+
// Persistent mode: a true bounded read — the engine seeks from the nearest
|
|
2586
|
+
// checkpoint and parses only the requested window (checkpoints are built
|
|
2587
|
+
// lazily for large conversations). Windowed message indices are proven
|
|
2588
|
+
// identical to parseConversation().messages.slice(...) by the paged-reader
|
|
2589
|
+
// equivalence test.
|
|
2590
|
+
//
|
|
2591
|
+
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
2592
|
+
// which is identical by construction.
|
|
1079
2593
|
async getConversationPage(id, options) {
|
|
2594
|
+
if (this.persistent) {
|
|
2595
|
+
return this.engine().getPage(id, options);
|
|
2596
|
+
}
|
|
1080
2597
|
const conversation = await this.getConversation(id);
|
|
1081
2598
|
if (!conversation) return null;
|
|
1082
2599
|
const { messages } = conversation;
|
|
@@ -1121,6 +2638,30 @@ var ConversationScanner = class {
|
|
|
1121
2638
|
// dropped from all indexes.
|
|
1122
2639
|
async refreshFile(filePath, account) {
|
|
1123
2640
|
const log = getLogger();
|
|
2641
|
+
if (this.persistent) {
|
|
2642
|
+
const engine = this.engine();
|
|
2643
|
+
const previous2 = engine.getByIdOrSession(filePath);
|
|
2644
|
+
const resolvedAccount2 = account ?? previous2?.account ?? "default";
|
|
2645
|
+
const evict2 = (m) => {
|
|
2646
|
+
if (!m) return;
|
|
2647
|
+
this.conversationLRU.delete(m.id);
|
|
2648
|
+
this.conversationLRU.delete(m.sessionId);
|
|
2649
|
+
};
|
|
2650
|
+
evict2(previous2);
|
|
2651
|
+
const provider = await this.resolveProviderForFile(filePath, previous2);
|
|
2652
|
+
const meta2 = await engine.indexFile(
|
|
2653
|
+
filePath,
|
|
2654
|
+
resolvedAccount2,
|
|
2655
|
+
this.lastTier.name,
|
|
2656
|
+
void 0,
|
|
2657
|
+
readGitBranch,
|
|
2658
|
+
true,
|
|
2659
|
+
provider
|
|
2660
|
+
);
|
|
2661
|
+
evict2(meta2);
|
|
2662
|
+
log.debug({ filePath, kept: !!meta2 }, "refreshFile: updated persistent index");
|
|
2663
|
+
return meta2;
|
|
2664
|
+
}
|
|
1124
2665
|
const previous = this.metadataCache.get(filePath);
|
|
1125
2666
|
const resolvedAccount = account ?? previous?.account ?? "default";
|
|
1126
2667
|
let meta = null;
|
|
@@ -1140,18 +2681,16 @@ var ConversationScanner = class {
|
|
|
1140
2681
|
if (!meta || meta.messageCount === 0) {
|
|
1141
2682
|
if (previous) {
|
|
1142
2683
|
this.metadataCache.delete(previous.id);
|
|
1143
|
-
this.
|
|
2684
|
+
this.removeFromSessionIndex(previous);
|
|
1144
2685
|
this.indexer.removeDocument(previous.id);
|
|
1145
2686
|
}
|
|
1146
2687
|
log.debug({ filePath }, "refreshFile: dropped (no parseable messages)");
|
|
1147
2688
|
return null;
|
|
1148
2689
|
}
|
|
1149
2690
|
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
1150
|
-
if (previous
|
|
1151
|
-
this.sessionIdIndex.delete(previous.sessionId);
|
|
1152
|
-
}
|
|
2691
|
+
if (previous) this.removeFromSessionIndex(previous);
|
|
1153
2692
|
this.metadataCache.set(meta.id, meta);
|
|
1154
|
-
this.
|
|
2693
|
+
this.addToSessionIndex(meta);
|
|
1155
2694
|
this.projects.add(meta.projectPath);
|
|
1156
2695
|
if (previous) {
|
|
1157
2696
|
this.indexer.updateDocument(meta);
|
|
@@ -1165,17 +2704,208 @@ var ConversationScanner = class {
|
|
|
1165
2704
|
return meta;
|
|
1166
2705
|
}
|
|
1167
2706
|
getMetadataCache() {
|
|
2707
|
+
if (this.persistent) {
|
|
2708
|
+
const map = /* @__PURE__ */ new Map();
|
|
2709
|
+
for (const meta of this.engine().allActive()) map.set(meta.id, meta);
|
|
2710
|
+
return map;
|
|
2711
|
+
}
|
|
1168
2712
|
return this.metadataCache;
|
|
1169
2713
|
}
|
|
2714
|
+
// Collision-safe sessionId lookup. session_id is NOT unique (the parser falls
|
|
2715
|
+
// back to the file basename, and resumed/subagent sessions repeat ids), so
|
|
2716
|
+
// getConversation(sessionId) is a convenience that resolves to one match;
|
|
2717
|
+
// this returns every active conversation sharing the id, newest first.
|
|
2718
|
+
getConversationsBySessionId(sessionId) {
|
|
2719
|
+
if (this.persistent) {
|
|
2720
|
+
return this.engine().getAllBySessionId(sessionId);
|
|
2721
|
+
}
|
|
2722
|
+
return this.sortBySessionPriority(this.sessionIdIndex.get(sessionId) ?? []);
|
|
2723
|
+
}
|
|
1170
2724
|
getProjects() {
|
|
2725
|
+
const source = this.persistent ? this.engine().getProjects() : this.projects;
|
|
1171
2726
|
const normalized = /* @__PURE__ */ new Set();
|
|
1172
|
-
for (const p of
|
|
2727
|
+
for (const p of source) {
|
|
1173
2728
|
normalized.add(p.replace(/\/+$/, ""));
|
|
1174
2729
|
}
|
|
1175
2730
|
return Array.from(normalized).sort();
|
|
1176
2731
|
}
|
|
2732
|
+
on(event, listener) {
|
|
2733
|
+
this.emitter.on(event, listener);
|
|
2734
|
+
return this;
|
|
2735
|
+
}
|
|
2736
|
+
off(event, listener) {
|
|
2737
|
+
this.emitter.off(event, listener);
|
|
2738
|
+
return this;
|
|
2739
|
+
}
|
|
2740
|
+
// Start watching the active profiles' project dirs. A filesystem watcher
|
|
2741
|
+
// feeds debounced, path-deduplicated index jobs through a single-writer
|
|
2742
|
+
// queue; a periodic full rescan backstops any events the watcher misses
|
|
2743
|
+
// (sleep/wake, network FS, restarts). Emits "change" per indexed file.
|
|
2744
|
+
// Persistent mode only — throws in legacy mode (no durable index to update).
|
|
2745
|
+
async watch(options = {}) {
|
|
2746
|
+
if (!this.persistent) {
|
|
2747
|
+
throw new Error("watch() requires persistent mode; construct with persistent enabled");
|
|
2748
|
+
}
|
|
2749
|
+
if (this.watcher) return;
|
|
2750
|
+
const profiles = await this.resolveProfiles(options.profiles);
|
|
2751
|
+
const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
|
|
2752
|
+
this.queue = new IndexQueue(async (job) => {
|
|
2753
|
+
try {
|
|
2754
|
+
const meta = await this.refreshFile(job.filePath, job.account);
|
|
2755
|
+
this.emitter.emit("change", {
|
|
2756
|
+
filePath: job.filePath,
|
|
2757
|
+
account: job.account,
|
|
2758
|
+
meta,
|
|
2759
|
+
reason: job.reason
|
|
2760
|
+
});
|
|
2761
|
+
} catch (err) {
|
|
2762
|
+
this.emitter.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
2763
|
+
}
|
|
2764
|
+
});
|
|
2765
|
+
this.watcher = new FileWatcher(
|
|
2766
|
+
activeProfiles,
|
|
2767
|
+
(e) => {
|
|
2768
|
+
this.queue?.enqueue({
|
|
2769
|
+
filePath: e.filePath,
|
|
2770
|
+
account: e.account,
|
|
2771
|
+
reason: "watcher"
|
|
2772
|
+
});
|
|
2773
|
+
},
|
|
2774
|
+
{ debounceMs: options.debounceMs }
|
|
2775
|
+
);
|
|
2776
|
+
await this.watcher.start();
|
|
2777
|
+
const periodicMs = options.periodicMs ?? 6e4;
|
|
2778
|
+
if (periodicMs > 0) {
|
|
2779
|
+
this.periodicTimer = setInterval(() => {
|
|
2780
|
+
void this.periodicReconcile(activeProfiles);
|
|
2781
|
+
}, periodicMs);
|
|
2782
|
+
this.periodicTimer.unref?.();
|
|
2783
|
+
}
|
|
2784
|
+
getLogger().info({ profiles: activeProfiles.length, periodicMs }, "watch: started");
|
|
2785
|
+
}
|
|
2786
|
+
// Periodic correctness backstop: re-discover all files and enqueue them
|
|
2787
|
+
// through the same queue the watcher uses, so any add/change the watcher
|
|
2788
|
+
// missed still gets indexed and emits a "change" event. Vanished files
|
|
2789
|
+
// (active in the DB but no longer on disk) are enqueued too — refreshFile
|
|
2790
|
+
// drops them. Routing through the queue (not a direct scan) keeps event
|
|
2791
|
+
// emission and single-writer serialization unified.
|
|
2792
|
+
async periodicReconcile(activeProfiles) {
|
|
2793
|
+
if (!this.queue) return;
|
|
2794
|
+
try {
|
|
2795
|
+
const engine = this.engine();
|
|
2796
|
+
const configDirs = activeProfiles.map((p) => ({
|
|
2797
|
+
projectsDir: getProjectsDir(p),
|
|
2798
|
+
account: p.id
|
|
2799
|
+
}));
|
|
2800
|
+
const discovered = await discoverJsonlFiles(configDirs);
|
|
2801
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2802
|
+
for (const { filePath, account } of discovered) {
|
|
2803
|
+
seen.add(filePath);
|
|
2804
|
+
const { change } = classify(filePath, engine.files.getByPath(filePath));
|
|
2805
|
+
if (change !== "unchanged") {
|
|
2806
|
+
this.queue.enqueue({ filePath, account, reason: "periodic" });
|
|
2807
|
+
}
|
|
2808
|
+
}
|
|
2809
|
+
for (const path of engine.files.allActivePaths()) {
|
|
2810
|
+
if (!seen.has(path)) {
|
|
2811
|
+
this.queue.enqueue({ filePath: path, account: "default", reason: "periodic" });
|
|
2812
|
+
}
|
|
2813
|
+
}
|
|
2814
|
+
} catch (err) {
|
|
2815
|
+
this.emitter.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2818
|
+
// Stop watching and drain any in-flight index jobs.
|
|
2819
|
+
async unwatch() {
|
|
2820
|
+
if (this.periodicTimer) {
|
|
2821
|
+
clearInterval(this.periodicTimer);
|
|
2822
|
+
this.periodicTimer = null;
|
|
2823
|
+
}
|
|
2824
|
+
if (this.watcher) {
|
|
2825
|
+
await this.watcher.stop();
|
|
2826
|
+
this.watcher = null;
|
|
2827
|
+
}
|
|
2828
|
+
if (this.queue) {
|
|
2829
|
+
await this.queue.onIdle();
|
|
2830
|
+
this.queue = null;
|
|
2831
|
+
}
|
|
2832
|
+
}
|
|
2833
|
+
// ── Provider plumbing (in-memory path) ──────────────────────────────────
|
|
2834
|
+
// Build the flat parse worklist for the enabled providers. Threadbase
|
|
2835
|
+
// discovers under the active profiles' project dirs; Codex discovers under
|
|
2836
|
+
// the explicit codexRoots (opt-in — no default home scan).
|
|
2837
|
+
async discoverWithProviders(activeProfiles, options) {
|
|
2838
|
+
const enabled = options.providers ?? [CLAUDE_CODE_PROVIDER];
|
|
2839
|
+
const work = [];
|
|
2840
|
+
if (enabled.includes(CLAUDE_CODE_PROVIDER)) {
|
|
2841
|
+
const provider = new ThreadbaseProvider();
|
|
2842
|
+
const roots = activeProfiles.map((p) => `${getProjectsDir(p)}\0${p.id}`);
|
|
2843
|
+
for (const f of await provider.discover(roots)) {
|
|
2844
|
+
work.push({ ...f, provider });
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
if (enabled.includes(CODEX_CLI_PROVIDER) && (options.codexRoots?.length ?? 0) > 0) {
|
|
2848
|
+
const provider = new CodexCliProvider();
|
|
2849
|
+
for (const f of await provider.discover(options.codexRoots)) {
|
|
2850
|
+
work.push({ ...f, provider });
|
|
2851
|
+
}
|
|
2852
|
+
}
|
|
2853
|
+
return work;
|
|
2854
|
+
}
|
|
2855
|
+
// Resolve which provider should (re)parse a file. Stored metadata wins; for a
|
|
2856
|
+
// file the index has not seen, sniff the first lines with each non-Threadbase
|
|
2857
|
+
// provider's canParse. Returns undefined for Threadbase (the engine's default
|
|
2858
|
+
// tail-read path).
|
|
2859
|
+
async resolveProviderForFile(filePath, previous) {
|
|
2860
|
+
if (previous?.provider === CODEX_CLI_PROVIDER) return new CodexCliProvider();
|
|
2861
|
+
if (previous?.provider) return void 0;
|
|
2862
|
+
let sample = "";
|
|
2863
|
+
try {
|
|
2864
|
+
const fd = openSync2(filePath, "r");
|
|
2865
|
+
try {
|
|
2866
|
+
const buf = Buffer.alloc(8192);
|
|
2867
|
+
const n = readSync2(fd, buf, 0, buf.length, 0);
|
|
2868
|
+
sample = buf.subarray(0, n).toString("utf8");
|
|
2869
|
+
} finally {
|
|
2870
|
+
closeSync2(fd);
|
|
2871
|
+
}
|
|
2872
|
+
} catch {
|
|
2873
|
+
return void 0;
|
|
2874
|
+
}
|
|
2875
|
+
const codex = new CodexCliProvider();
|
|
2876
|
+
if (codex.canParse(filePath, sample)) return codex;
|
|
2877
|
+
return void 0;
|
|
2878
|
+
}
|
|
2879
|
+
addToSessionIndex(meta) {
|
|
2880
|
+
const list = this.sessionIdIndex.get(meta.sessionId);
|
|
2881
|
+
if (list) {
|
|
2882
|
+
const i = list.findIndex((m) => m.id === meta.id);
|
|
2883
|
+
if (i >= 0) list[i] = meta;
|
|
2884
|
+
else list.push(meta);
|
|
2885
|
+
} else {
|
|
2886
|
+
this.sessionIdIndex.set(meta.sessionId, [meta]);
|
|
2887
|
+
}
|
|
2888
|
+
}
|
|
2889
|
+
removeFromSessionIndex(meta) {
|
|
2890
|
+
const list = this.sessionIdIndex.get(meta.sessionId);
|
|
2891
|
+
if (!list) return;
|
|
2892
|
+
const next = list.filter((m) => m.id !== meta.id);
|
|
2893
|
+
if (next.length > 0) this.sessionIdIndex.set(meta.sessionId, next);
|
|
2894
|
+
else this.sessionIdIndex.delete(meta.sessionId);
|
|
2895
|
+
}
|
|
2896
|
+
// Deterministic single-result sessionId resolution: newest timestamp first,
|
|
2897
|
+
// tie-broken by absolute path ascending.
|
|
2898
|
+
resolveSessionId(sessionId) {
|
|
2899
|
+
return this.sortBySessionPriority(this.sessionIdIndex.get(sessionId) ?? [])[0];
|
|
2900
|
+
}
|
|
2901
|
+
sortBySessionPriority(metas) {
|
|
2902
|
+
return [...metas].sort((a, b) => {
|
|
2903
|
+
if (a.timestamp !== b.timestamp) return a.timestamp < b.timestamp ? 1 : -1;
|
|
2904
|
+
return a.filePath < b.filePath ? -1 : a.filePath > b.filePath ? 1 : 0;
|
|
2905
|
+
});
|
|
2906
|
+
}
|
|
1177
2907
|
async resolveProfiles(profiles) {
|
|
1178
|
-
if (profiles
|
|
2908
|
+
if (profiles) return profiles;
|
|
1179
2909
|
return loadProfiles(DEFAULT_CONFIG_PATH);
|
|
1180
2910
|
}
|
|
1181
2911
|
transformView(metas, options) {
|
|
@@ -1250,9 +2980,11 @@ async function getConversation(id, options, scanner) {
|
|
|
1250
2980
|
return (scanner ?? getDefaultScanner()).getConversation(id, options);
|
|
1251
2981
|
}
|
|
1252
2982
|
export {
|
|
2983
|
+
CodexCliProvider,
|
|
1253
2984
|
ConversationScanner,
|
|
1254
2985
|
DEFAULT_TIERS,
|
|
1255
2986
|
SearchIndexer,
|
|
2987
|
+
ThreadbaseProvider,
|
|
1256
2988
|
VALID_SORT_ORDERS,
|
|
1257
2989
|
applyAccountFilter,
|
|
1258
2990
|
applyIncludeFilter,
|
|
@@ -1268,12 +3000,14 @@ export {
|
|
|
1268
3000
|
getProjectsDir,
|
|
1269
3001
|
loadProfiles,
|
|
1270
3002
|
readGitBranch,
|
|
3003
|
+
readSidecar,
|
|
1271
3004
|
resetDefaultScanner,
|
|
1272
3005
|
resolveConfigDir,
|
|
1273
3006
|
resolveTier,
|
|
1274
3007
|
saveProfiles,
|
|
1275
3008
|
scan,
|
|
1276
3009
|
search,
|
|
1277
|
-
setLogger
|
|
3010
|
+
setLogger,
|
|
3011
|
+
sidecarPath
|
|
1278
3012
|
};
|
|
1279
3013
|
//# sourceMappingURL=index.js.map
|