@threadbase-sh/scanner 0.19.5 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -4
- package/dist/cli.js +401 -75
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +437 -105
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +415 -86
- package/dist/index.js.map +1 -1
- package/dist/persistent/conversation-reducer.d.ts +2 -1
- package/dist/persistent/conversation-reducer.d.ts.map +1 -1
- package/dist/persistent/index-engine.d.ts.map +1 -1
- package/dist/persistent/metadata-reducer.d.ts.map +1 -1
- package/dist/providers/copilot.d.ts +33 -0
- package/dist/providers/copilot.d.ts.map +1 -0
- package/dist/providers/provider.d.ts +2 -1
- package/dist/providers/provider.d.ts.map +1 -1
- package/dist/scanner.d.ts.map +1 -1
- package/dist/search-document.d.ts.map +1 -1
- package/dist/types.d.ts +9 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -32,9 +32,11 @@ var index_exports = {};
|
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
CLAUDE_CODE_PROVIDER: () => CLAUDE_CODE_PROVIDER,
|
|
34
34
|
CODEX_CLI_PROVIDER: () => CODEX_CLI_PROVIDER,
|
|
35
|
+
COPILOT_PROVIDER: () => COPILOT_PROVIDER,
|
|
35
36
|
CURSOR_PROVIDER: () => CURSOR_PROVIDER,
|
|
36
37
|
CodexCliProvider: () => CodexCliProvider,
|
|
37
38
|
ConversationScanner: () => ConversationScanner,
|
|
39
|
+
CopilotProvider: () => CopilotProvider,
|
|
38
40
|
CursorProvider: () => CursorProvider,
|
|
39
41
|
DEFAULT_TIERS: () => DEFAULT_TIERS,
|
|
40
42
|
LEGACY_CURSOR_PROVIDER: () => LEGACY_CURSOR_PROVIDER,
|
|
@@ -59,6 +61,7 @@ __export(index_exports, {
|
|
|
59
61
|
isCodexInjectedContext: () => isCodexInjectedContext,
|
|
60
62
|
loadProfiles: () => loadProfiles,
|
|
61
63
|
parseCodexJsonlLine: () => parseCodexJsonlLine,
|
|
64
|
+
parseCopilotJsonlLine: () => parseCopilotJsonlLine,
|
|
62
65
|
parseCursorJsonlLine: () => parseCursorJsonlLine,
|
|
63
66
|
parseJsonlLine: () => parseJsonlLine,
|
|
64
67
|
providerMatches: () => providerMatches,
|
|
@@ -440,10 +443,11 @@ var import_path2 = require("path");
|
|
|
440
443
|
var CLAUDE_CODE_PROVIDER = "claude-code";
|
|
441
444
|
var CODEX_CLI_PROVIDER = "codex-cli";
|
|
442
445
|
var CURSOR_PROVIDER = "cursor";
|
|
446
|
+
var COPILOT_PROVIDER = "copilot";
|
|
443
447
|
var LEGACY_CURSOR_PROVIDER = "cursor-cli";
|
|
444
448
|
function canonicalizeProviderName(value) {
|
|
445
449
|
if (value === LEGACY_CURSOR_PROVIDER) return CURSOR_PROVIDER;
|
|
446
|
-
if (value === CLAUDE_CODE_PROVIDER || value === CODEX_CLI_PROVIDER || value === CURSOR_PROVIDER) {
|
|
450
|
+
if (value === CLAUDE_CODE_PROVIDER || value === CODEX_CLI_PROVIDER || value === CURSOR_PROVIDER || value === COPILOT_PROVIDER) {
|
|
447
451
|
return value;
|
|
448
452
|
}
|
|
449
453
|
return void 0;
|
|
@@ -509,6 +513,10 @@ function reduceLine(state, entry, tier) {
|
|
|
509
513
|
if (type !== "user" && type !== "assistant") return;
|
|
510
514
|
if (entry.isMeta) return;
|
|
511
515
|
const msg = entry.message;
|
|
516
|
+
if (entry.isCompactSummary === true) {
|
|
517
|
+
if (extractTextContent(msg?.content)) state.pageMessageCount++;
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
512
520
|
if (state.model === null && msg?.model) state.model = msg.model;
|
|
513
521
|
if (type === "user" && !state.firstUserSeen) {
|
|
514
522
|
state.firstUserSeen = true;
|
|
@@ -843,8 +851,20 @@ function reduceConvLine(state, entry) {
|
|
|
843
851
|
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
844
852
|
return null;
|
|
845
853
|
}
|
|
854
|
+
if (type === "system" && entry.subtype === "compact_boundary") {
|
|
855
|
+
const meta = entry.compactMetadata ?? {};
|
|
856
|
+
state.pendingCompaction = {
|
|
857
|
+
trigger: typeof meta.trigger === "string" ? meta.trigger : void 0,
|
|
858
|
+
preTokens: typeof meta.preTokens === "number" ? meta.preTokens : void 0,
|
|
859
|
+
postTokens: typeof meta.postTokens === "number" ? meta.postTokens : void 0
|
|
860
|
+
};
|
|
861
|
+
return null;
|
|
862
|
+
}
|
|
846
863
|
if (type !== "user" && type !== "assistant") return null;
|
|
847
864
|
if (entry.isMeta) return null;
|
|
865
|
+
const isCompactSummary = type === "user" && entry.isCompactSummary === true;
|
|
866
|
+
const compaction = isCompactSummary ? state.pendingCompaction : void 0;
|
|
867
|
+
if (isCompactSummary) delete state.pendingCompaction;
|
|
848
868
|
const msg = entry.message;
|
|
849
869
|
const toolUseBlocks = extractToolUseBlocks(msg?.content);
|
|
850
870
|
for (const block of toolUseBlocks) state.pendingToolUses[block.id] = block;
|
|
@@ -904,7 +924,9 @@ function reduceConvLine(state, entry) {
|
|
|
904
924
|
isSidechain: typeof entry.isSidechain === "boolean" ? entry.isSidechain : void 0,
|
|
905
925
|
permissionMode: type === "user" ? entry.permissionMode : void 0,
|
|
906
926
|
hasImages: hasImageBlocks(msg?.content) || void 0,
|
|
907
|
-
attachment: entry.attachment !== void 0 ? entry.attachment : void 0
|
|
927
|
+
attachment: entry.attachment !== void 0 ? entry.attachment : void 0,
|
|
928
|
+
isCompactSummary: isCompactSummary || void 0,
|
|
929
|
+
compaction
|
|
908
930
|
};
|
|
909
931
|
}
|
|
910
932
|
function parseJsonlLine(line, state = initialConvState()) {
|
|
@@ -1365,13 +1387,293 @@ async function parseCodexConversation(filePath, account) {
|
|
|
1365
1387
|
};
|
|
1366
1388
|
}
|
|
1367
1389
|
|
|
1368
|
-
// src/providers/
|
|
1369
|
-
var import_crypto = require("crypto");
|
|
1390
|
+
// src/providers/copilot.ts
|
|
1370
1391
|
var import_fast_glob2 = __toESM(require("fast-glob"), 1);
|
|
1371
1392
|
var import_fs5 = require("fs");
|
|
1372
1393
|
var import_promises3 = require("fs/promises");
|
|
1373
1394
|
var import_path7 = require("path");
|
|
1374
1395
|
var import_readline3 = require("readline");
|
|
1396
|
+
var SESSION_START = "session.start";
|
|
1397
|
+
var USER_MESSAGE = "user.message";
|
|
1398
|
+
var ASSISTANT_MESSAGE = "assistant.message";
|
|
1399
|
+
var CopilotProvider = class {
|
|
1400
|
+
name = COPILOT_PROVIDER;
|
|
1401
|
+
async discover(roots) {
|
|
1402
|
+
const log = getLogger();
|
|
1403
|
+
const results = [];
|
|
1404
|
+
for (const root of roots) {
|
|
1405
|
+
let paths;
|
|
1406
|
+
try {
|
|
1407
|
+
paths = await (0, import_fast_glob2.default)(["*/events.jsonl", "session-state/*/events.jsonl"], {
|
|
1408
|
+
cwd: root,
|
|
1409
|
+
absolute: true,
|
|
1410
|
+
dot: false,
|
|
1411
|
+
unique: true
|
|
1412
|
+
});
|
|
1413
|
+
} catch (err) {
|
|
1414
|
+
log.warn({ root, err }, "copilot discovery: glob failed");
|
|
1415
|
+
continue;
|
|
1416
|
+
}
|
|
1417
|
+
for (const filePath of paths) {
|
|
1418
|
+
try {
|
|
1419
|
+
const s = await (0, import_promises3.stat)(filePath);
|
|
1420
|
+
if (s.size > 0) results.push({ filePath: canonicalPath(filePath), account: "copilot" });
|
|
1421
|
+
} catch (err) {
|
|
1422
|
+
log.warn({ filePath, err }, "copilot discovery: stat failed");
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
}
|
|
1426
|
+
return results;
|
|
1427
|
+
}
|
|
1428
|
+
canParse(_filePath, sample) {
|
|
1429
|
+
for (const line of sample.split("\n")) {
|
|
1430
|
+
if (!line.trim()) continue;
|
|
1431
|
+
try {
|
|
1432
|
+
if (looksLikeCopilotEvent(JSON.parse(line))) return true;
|
|
1433
|
+
} catch {
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
return false;
|
|
1437
|
+
}
|
|
1438
|
+
createEmptyAccumulator() {
|
|
1439
|
+
return {
|
|
1440
|
+
sessionId: "",
|
|
1441
|
+
cwd: "",
|
|
1442
|
+
gitBranch: null,
|
|
1443
|
+
model: null,
|
|
1444
|
+
latestTimestamp: "",
|
|
1445
|
+
messageCount: 0,
|
|
1446
|
+
lastMessageSender: "user",
|
|
1447
|
+
firstUser: null,
|
|
1448
|
+
lastUser: null,
|
|
1449
|
+
lastAssistant: null,
|
|
1450
|
+
toolNames: [],
|
|
1451
|
+
previewParts: [],
|
|
1452
|
+
previewLength: 0,
|
|
1453
|
+
snippetParts: [],
|
|
1454
|
+
snippetLength: 0
|
|
1455
|
+
};
|
|
1456
|
+
}
|
|
1457
|
+
reduceEntry(acc, entry, tier) {
|
|
1458
|
+
reduceCopilotEntry(acc, entry, tier);
|
|
1459
|
+
}
|
|
1460
|
+
finalize(acc, filePath, account, tier) {
|
|
1461
|
+
return finalizeCopilotMeta(acc, filePath, account, tier);
|
|
1462
|
+
}
|
|
1463
|
+
};
|
|
1464
|
+
function looksLikeCopilotEvent(entry) {
|
|
1465
|
+
return typeof entry.type === "string" && typeof entry.id === "string" && "parentId" in entry && asRecord(entry.data) !== null;
|
|
1466
|
+
}
|
|
1467
|
+
var asString2 = (v) => typeof v === "string" ? v : "";
|
|
1468
|
+
function asRecord(v) {
|
|
1469
|
+
return v && typeof v === "object" && !Array.isArray(v) ? v : null;
|
|
1470
|
+
}
|
|
1471
|
+
function copilotToolUseBlocks(data) {
|
|
1472
|
+
if (!Array.isArray(data.toolRequests)) return [];
|
|
1473
|
+
const blocks = [];
|
|
1474
|
+
for (const item of data.toolRequests) {
|
|
1475
|
+
const request = asRecord(item);
|
|
1476
|
+
const name = asString2(request?.name);
|
|
1477
|
+
if (!request || !name) continue;
|
|
1478
|
+
const input = asRecord(request.arguments) ?? { arguments: asString2(request.arguments) };
|
|
1479
|
+
blocks.push({ id: asString2(request.toolCallId), name, input });
|
|
1480
|
+
}
|
|
1481
|
+
return blocks;
|
|
1482
|
+
}
|
|
1483
|
+
function copilotEntryToMessage(entry) {
|
|
1484
|
+
if (!looksLikeCopilotEvent(entry)) return null;
|
|
1485
|
+
const data = entry.data;
|
|
1486
|
+
const timestamp = asString2(entry.timestamp);
|
|
1487
|
+
const uuid = asString2(data.messageId) || asString2(entry.id);
|
|
1488
|
+
if (entry.type === USER_MESSAGE) {
|
|
1489
|
+
const text = cleanSystemTags(asString2(data.content));
|
|
1490
|
+
return text ? { role: "user", text, timestamp, uuid } : null;
|
|
1491
|
+
}
|
|
1492
|
+
if (entry.type === ASSISTANT_MESSAGE) {
|
|
1493
|
+
const text = asString2(data.content).trim();
|
|
1494
|
+
const blocks = copilotToolUseBlocks(data);
|
|
1495
|
+
if (!text && blocks.length === 0) return null;
|
|
1496
|
+
const model = asString2(data.model);
|
|
1497
|
+
const metadata = {
|
|
1498
|
+
...model ? { model } : {},
|
|
1499
|
+
...blocks.length > 0 ? { toolUses: blocks.map((b) => b.name), toolUseBlocks: blocks } : {}
|
|
1500
|
+
};
|
|
1501
|
+
return {
|
|
1502
|
+
role: "assistant",
|
|
1503
|
+
text,
|
|
1504
|
+
timestamp,
|
|
1505
|
+
uuid,
|
|
1506
|
+
...Object.keys(metadata).length > 0 ? { metadata } : {}
|
|
1507
|
+
};
|
|
1508
|
+
}
|
|
1509
|
+
return null;
|
|
1510
|
+
}
|
|
1511
|
+
function parseCopilotJsonlLine(line) {
|
|
1512
|
+
const trimmed = line.trim();
|
|
1513
|
+
if (!trimmed) return null;
|
|
1514
|
+
try {
|
|
1515
|
+
return copilotEntryToMessage(JSON.parse(trimmed));
|
|
1516
|
+
} catch {
|
|
1517
|
+
return null;
|
|
1518
|
+
}
|
|
1519
|
+
}
|
|
1520
|
+
function readSessionStart(entry) {
|
|
1521
|
+
if (entry.type !== SESSION_START) return null;
|
|
1522
|
+
const data = asRecord(entry.data);
|
|
1523
|
+
if (!data) return null;
|
|
1524
|
+
const context = asRecord(data.context);
|
|
1525
|
+
return {
|
|
1526
|
+
sessionId: asString2(data.sessionId),
|
|
1527
|
+
cwd: asString2(context?.cwd),
|
|
1528
|
+
branch: asString2(context?.branch)
|
|
1529
|
+
};
|
|
1530
|
+
}
|
|
1531
|
+
function reduceCopilotEntry(acc, entry, tier) {
|
|
1532
|
+
const start = readSessionStart(entry);
|
|
1533
|
+
if (start) {
|
|
1534
|
+
if (!acc.sessionId) acc.sessionId = start.sessionId;
|
|
1535
|
+
if (!acc.cwd) acc.cwd = start.cwd;
|
|
1536
|
+
if (!acc.gitBranch && start.branch) acc.gitBranch = start.branch;
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
const message = copilotEntryToMessage(entry);
|
|
1540
|
+
if (!message) return;
|
|
1541
|
+
for (const name of message.metadata?.toolUses ?? []) {
|
|
1542
|
+
if (!acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
1543
|
+
}
|
|
1544
|
+
if (message.metadata?.model) acc.model = message.metadata.model;
|
|
1545
|
+
const { role, text, timestamp } = message;
|
|
1546
|
+
if (!text) return;
|
|
1547
|
+
if (timestamp && (!acc.latestTimestamp || timestamp > acc.latestTimestamp)) {
|
|
1548
|
+
acc.latestTimestamp = timestamp;
|
|
1549
|
+
}
|
|
1550
|
+
acc.messageCount++;
|
|
1551
|
+
acc.lastMessageSender = role;
|
|
1552
|
+
const snapshot = { text: text.slice(0, 200), timestamp };
|
|
1553
|
+
if (role === "user") {
|
|
1554
|
+
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
1555
|
+
acc.lastUser = snapshot;
|
|
1556
|
+
} else {
|
|
1557
|
+
acc.lastAssistant = snapshot;
|
|
1558
|
+
}
|
|
1559
|
+
if (acc.previewLength < tier.previewMax) {
|
|
1560
|
+
acc.previewParts.push(text);
|
|
1561
|
+
acc.previewLength += text.length;
|
|
1562
|
+
}
|
|
1563
|
+
if (acc.snippetLength < tier.snippetMax) {
|
|
1564
|
+
const remaining = tier.snippetMax - acc.snippetLength;
|
|
1565
|
+
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
1566
|
+
acc.snippetParts.push(chunk);
|
|
1567
|
+
acc.snippetLength += chunk.length;
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
function finalizeCopilotMeta(acc, filePath, account, tier) {
|
|
1571
|
+
if (acc.messageCount === 0) return null;
|
|
1572
|
+
const sessionId = acc.sessionId || sessionIdFromPath(filePath);
|
|
1573
|
+
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
1574
|
+
return {
|
|
1575
|
+
id: filePath,
|
|
1576
|
+
filePath,
|
|
1577
|
+
provider: COPILOT_PROVIDER,
|
|
1578
|
+
kind,
|
|
1579
|
+
externalSessionId: sessionId,
|
|
1580
|
+
sessionId,
|
|
1581
|
+
sessionName: deriveSessionNameFromFirstMessage(acc.firstUser),
|
|
1582
|
+
projectPath: acc.cwd,
|
|
1583
|
+
projectName: getShortProjectName4(acc.cwd),
|
|
1584
|
+
account,
|
|
1585
|
+
timestamp: acc.latestTimestamp || fileMtimeIso(filePath),
|
|
1586
|
+
messageCount: acc.messageCount,
|
|
1587
|
+
lastMessageSender: acc.lastMessageSender,
|
|
1588
|
+
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
1589
|
+
contentSnippet: acc.snippetParts.join(" "),
|
|
1590
|
+
gitBranch: acc.gitBranch,
|
|
1591
|
+
model: acc.model,
|
|
1592
|
+
isSubagent: false,
|
|
1593
|
+
parentSessionId: null,
|
|
1594
|
+
isTeammate: false,
|
|
1595
|
+
teamName: null,
|
|
1596
|
+
toolNames: acc.toolNames,
|
|
1597
|
+
firstMessage: acc.firstUser,
|
|
1598
|
+
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
1599
|
+
lastPrompt: acc.lastUser?.text || void 0
|
|
1600
|
+
};
|
|
1601
|
+
}
|
|
1602
|
+
function sessionIdFromPath(filePath) {
|
|
1603
|
+
return (0, import_path7.basename)((0, import_path7.dirname)(filePath));
|
|
1604
|
+
}
|
|
1605
|
+
function getShortProjectName4(fullPath) {
|
|
1606
|
+
return fullPath.split(/[/\\]/).filter(Boolean).slice(-3).join("/");
|
|
1607
|
+
}
|
|
1608
|
+
function fileMtimeIso(filePath) {
|
|
1609
|
+
try {
|
|
1610
|
+
return new Date((0, import_fs5.statSync)(filePath).mtimeMs).toISOString();
|
|
1611
|
+
} catch {
|
|
1612
|
+
return (/* @__PURE__ */ new Date()).toISOString();
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
async function parseCopilotConversation(filePath, account) {
|
|
1616
|
+
const log = getLogger();
|
|
1617
|
+
const messages = [];
|
|
1618
|
+
const textParts = [];
|
|
1619
|
+
let sessionId = "";
|
|
1620
|
+
let cwd = "";
|
|
1621
|
+
let latestTimestamp = "";
|
|
1622
|
+
let lastUserText = "";
|
|
1623
|
+
const rl = (0, import_readline3.createInterface)({ input: (0, import_fs5.createReadStream)(filePath), crlfDelay: Infinity });
|
|
1624
|
+
try {
|
|
1625
|
+
for await (const line of rl) {
|
|
1626
|
+
if (!line.trim()) continue;
|
|
1627
|
+
let entry;
|
|
1628
|
+
try {
|
|
1629
|
+
entry = JSON.parse(line);
|
|
1630
|
+
} catch {
|
|
1631
|
+
continue;
|
|
1632
|
+
}
|
|
1633
|
+
const start = readSessionStart(entry);
|
|
1634
|
+
if (start) {
|
|
1635
|
+
if (!sessionId) sessionId = start.sessionId;
|
|
1636
|
+
if (!cwd) cwd = start.cwd;
|
|
1637
|
+
continue;
|
|
1638
|
+
}
|
|
1639
|
+
const message = copilotEntryToMessage(entry);
|
|
1640
|
+
if (!message) continue;
|
|
1641
|
+
if (message.timestamp && (!latestTimestamp || message.timestamp > latestTimestamp)) {
|
|
1642
|
+
latestTimestamp = message.timestamp;
|
|
1643
|
+
}
|
|
1644
|
+
messages.push(message);
|
|
1645
|
+
if (!message.text) continue;
|
|
1646
|
+
textParts.push(message.text);
|
|
1647
|
+
if (message.role === "user") lastUserText = message.text;
|
|
1648
|
+
}
|
|
1649
|
+
} catch (err) {
|
|
1650
|
+
log.warn({ filePath, err }, "parseCopilotConversation: read failed");
|
|
1651
|
+
return null;
|
|
1652
|
+
}
|
|
1653
|
+
if (messages.length === 0) return null;
|
|
1654
|
+
return {
|
|
1655
|
+
id: filePath,
|
|
1656
|
+
filePath,
|
|
1657
|
+
projectPath: cwd,
|
|
1658
|
+
projectName: getShortProjectName4(cwd),
|
|
1659
|
+
sessionId: sessionId || sessionIdFromPath(filePath),
|
|
1660
|
+
sessionName: "",
|
|
1661
|
+
messages,
|
|
1662
|
+
fullText: textParts.join(" "),
|
|
1663
|
+
timestamp: latestTimestamp || fileMtimeIso(filePath),
|
|
1664
|
+
messageCount: messages.length,
|
|
1665
|
+
account,
|
|
1666
|
+
lastPrompt: lastUserText || void 0
|
|
1667
|
+
};
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1670
|
+
// src/providers/cursor.ts
|
|
1671
|
+
var import_crypto = require("crypto");
|
|
1672
|
+
var import_fast_glob3 = __toESM(require("fast-glob"), 1);
|
|
1673
|
+
var import_fs6 = require("fs");
|
|
1674
|
+
var import_promises4 = require("fs/promises");
|
|
1675
|
+
var import_path8 = require("path");
|
|
1676
|
+
var import_readline4 = require("readline");
|
|
1375
1677
|
var TIMESTAMP_RE = /<timestamp>\s*([\s\S]*?)\s*<\/timestamp>/;
|
|
1376
1678
|
var USER_QUERY_RE = /<user_query>\s*([\s\S]*?)\s*<\/user_query>/;
|
|
1377
1679
|
var SUBAGENT_PATH_RE2 = /[\\/]subagents[\\/]/;
|
|
@@ -1383,7 +1685,7 @@ var CursorProvider = class {
|
|
|
1383
1685
|
for (const root of roots) {
|
|
1384
1686
|
let paths;
|
|
1385
1687
|
try {
|
|
1386
|
-
paths = await (0,
|
|
1688
|
+
paths = await (0, import_fast_glob3.default)(["**/agent-transcripts/**/*.jsonl"], {
|
|
1387
1689
|
cwd: root,
|
|
1388
1690
|
absolute: true,
|
|
1389
1691
|
dot: false,
|
|
@@ -1395,7 +1697,7 @@ var CursorProvider = class {
|
|
|
1395
1697
|
}
|
|
1396
1698
|
for (const filePath of paths) {
|
|
1397
1699
|
try {
|
|
1398
|
-
const s = await (0,
|
|
1700
|
+
const s = await (0, import_promises4.stat)(filePath);
|
|
1399
1701
|
if (s.size > 0) results.push({ filePath: canonicalPath(filePath), account: "cursor" });
|
|
1400
1702
|
} catch (err) {
|
|
1401
1703
|
log.warn({ filePath, err }, "cursor discovery: stat failed");
|
|
@@ -1459,7 +1761,7 @@ function looksLikeImportedClaudeEntry(entry) {
|
|
|
1459
1761
|
function looksLikeImportedCodexEntry(entry) {
|
|
1460
1762
|
return entry.type === "session_meta" || entry.type === "response_item" || entry.type === "event_msg";
|
|
1461
1763
|
}
|
|
1462
|
-
function
|
|
1764
|
+
function asString3(value) {
|
|
1463
1765
|
return typeof value === "string" ? value : "";
|
|
1464
1766
|
}
|
|
1465
1767
|
function provenanceToken(value) {
|
|
@@ -1573,7 +1875,7 @@ function cursorEntryToMessage(entry) {
|
|
|
1573
1875
|
const role = message?.role === "assistant" || entry.type === "assistant" ? "assistant" : "user";
|
|
1574
1876
|
const content = message?.content ?? entry.content;
|
|
1575
1877
|
return withToolUses(
|
|
1576
|
-
{ role, text: extractCursorText(content), timestamp:
|
|
1878
|
+
{ role, text: extractCursorText(content), timestamp: asString3(entry.timestamp) },
|
|
1577
1879
|
content
|
|
1578
1880
|
);
|
|
1579
1881
|
}
|
|
@@ -1650,11 +1952,11 @@ function reduceCursorEntry(acc, entry, tier) {
|
|
|
1650
1952
|
}
|
|
1651
1953
|
function finalizeCursorMeta(acc, filePath, account, tier) {
|
|
1652
1954
|
if (acc.messageCount === 0) return null;
|
|
1653
|
-
const sessionId = (0,
|
|
1955
|
+
const sessionId = (0, import_path8.basename)(filePath, ".jsonl");
|
|
1654
1956
|
const isSubagent = SUBAGENT_PATH_RE2.test(filePath);
|
|
1655
1957
|
const parent = isSubagent ? parentFromSubagentPath(filePath) : null;
|
|
1656
1958
|
const { projectPath, projectName } = projectFromCursorPath(filePath);
|
|
1657
|
-
const timestamp = acc.latestTimestamp ||
|
|
1959
|
+
const timestamp = acc.latestTimestamp || fileMtimeIso2(filePath);
|
|
1658
1960
|
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
1659
1961
|
return {
|
|
1660
1962
|
id: filePath,
|
|
@@ -1689,34 +1991,34 @@ function finalizeCursorMeta(acc, filePath, account, tier) {
|
|
|
1689
1991
|
};
|
|
1690
1992
|
}
|
|
1691
1993
|
function parentFromSubagentPath(filePath) {
|
|
1692
|
-
const subagentsDir = (0,
|
|
1693
|
-
const parentDir = (0,
|
|
1694
|
-
const parentSessionUuid = (0,
|
|
1994
|
+
const subagentsDir = (0, import_path8.dirname)(filePath);
|
|
1995
|
+
const parentDir = (0, import_path8.dirname)(subagentsDir);
|
|
1996
|
+
const parentSessionUuid = (0, import_path8.basename)(parentDir);
|
|
1695
1997
|
if (!parentSessionUuid) return null;
|
|
1696
1998
|
return {
|
|
1697
1999
|
parentSessionUuid,
|
|
1698
|
-
parentFilePath: canonicalPath((0,
|
|
2000
|
+
parentFilePath: canonicalPath((0, import_path8.join)(parentDir, `${parentSessionUuid}.jsonl`))
|
|
1699
2001
|
};
|
|
1700
2002
|
}
|
|
1701
2003
|
function projectFromCursorPath(filePath) {
|
|
1702
2004
|
const parts = filePath.split(/[/\\]/);
|
|
1703
2005
|
const idx = parts.lastIndexOf("agent-transcripts");
|
|
1704
|
-
const cursorProjectDir = idx > 0 ? parts.slice(0, idx).join(
|
|
1705
|
-
const slug = (0,
|
|
2006
|
+
const cursorProjectDir = idx > 0 ? parts.slice(0, idx).join(import_path8.sep) : (0, import_path8.dirname)((0, import_path8.dirname)(filePath));
|
|
2007
|
+
const slug = (0, import_path8.basename)(cursorProjectDir);
|
|
1706
2008
|
const decoded = decodeCursorProjectSlug(slug);
|
|
1707
2009
|
const projectPath = decoded ?? cursorProjectDir;
|
|
1708
|
-
return { projectPath, projectName:
|
|
2010
|
+
return { projectPath, projectName: getShortProjectName5(projectPath) || slug };
|
|
1709
2011
|
}
|
|
1710
2012
|
function decodeCursorProjectSlug(slug) {
|
|
1711
2013
|
if (!slug) return null;
|
|
1712
2014
|
const segments = slug.split("-");
|
|
1713
2015
|
if (segments.length === 0) return null;
|
|
1714
2016
|
const rooted = slug.startsWith("Users-") || slug.startsWith("home-") || /^[A-Za-z]-/.test(slug);
|
|
1715
|
-
let acc = rooted ?
|
|
2017
|
+
let acc = rooted ? import_path8.sep : "";
|
|
1716
2018
|
let i = 0;
|
|
1717
2019
|
if (rooted && segments[0] === "Users") {
|
|
1718
|
-
const users = `${
|
|
1719
|
-
if (!(0,
|
|
2020
|
+
const users = `${import_path8.sep}Users`;
|
|
2021
|
+
if (!(0, import_fs6.existsSync)(users)) return null;
|
|
1720
2022
|
acc = users;
|
|
1721
2023
|
i = 1;
|
|
1722
2024
|
}
|
|
@@ -1724,9 +2026,9 @@ function decodeCursorProjectSlug(slug) {
|
|
|
1724
2026
|
let found = null;
|
|
1725
2027
|
let next = i + 1;
|
|
1726
2028
|
for (let j = segments.length; j > i; j--) {
|
|
1727
|
-
const candidate = (0,
|
|
2029
|
+
const candidate = (0, import_path8.join)(acc, segments.slice(i, j).join("-"));
|
|
1728
2030
|
try {
|
|
1729
|
-
if ((0,
|
|
2031
|
+
if ((0, import_fs6.existsSync)(candidate) && (0, import_fs6.statSync)(candidate).isDirectory()) {
|
|
1730
2032
|
found = candidate;
|
|
1731
2033
|
next = j;
|
|
1732
2034
|
break;
|
|
@@ -1740,12 +2042,12 @@ function decodeCursorProjectSlug(slug) {
|
|
|
1740
2042
|
}
|
|
1741
2043
|
return acc;
|
|
1742
2044
|
}
|
|
1743
|
-
function
|
|
2045
|
+
function getShortProjectName5(fullPath) {
|
|
1744
2046
|
return fullPath.split(/[/\\]/).filter(Boolean).slice(-3).join("/");
|
|
1745
2047
|
}
|
|
1746
|
-
function
|
|
2048
|
+
function fileMtimeIso2(filePath) {
|
|
1747
2049
|
try {
|
|
1748
|
-
return new Date((0,
|
|
2050
|
+
return new Date((0, import_fs6.statSync)(filePath).mtimeMs).toISOString();
|
|
1749
2051
|
} catch {
|
|
1750
2052
|
return (/* @__PURE__ */ new Date()).toISOString();
|
|
1751
2053
|
}
|
|
@@ -1756,7 +2058,7 @@ async function parseCursorConversation(filePath, account) {
|
|
|
1756
2058
|
const textParts = [];
|
|
1757
2059
|
let latestTimestamp = "";
|
|
1758
2060
|
let lastUserText = "";
|
|
1759
|
-
const rl = (0,
|
|
2061
|
+
const rl = (0, import_readline4.createInterface)({ input: (0, import_fs6.createReadStream)(filePath), crlfDelay: Infinity });
|
|
1760
2062
|
try {
|
|
1761
2063
|
for await (const line of rl) {
|
|
1762
2064
|
const message = parseCursorJsonlLine(line, messages.length);
|
|
@@ -1775,7 +2077,7 @@ async function parseCursorConversation(filePath, account) {
|
|
|
1775
2077
|
}
|
|
1776
2078
|
if (messages.length === 0) return null;
|
|
1777
2079
|
const { projectPath, projectName } = projectFromCursorPath(filePath);
|
|
1778
|
-
const sessionId = (0,
|
|
2080
|
+
const sessionId = (0, import_path8.basename)(filePath, ".jsonl");
|
|
1779
2081
|
return {
|
|
1780
2082
|
id: filePath,
|
|
1781
2083
|
filePath,
|
|
@@ -1785,7 +2087,7 @@ async function parseCursorConversation(filePath, account) {
|
|
|
1785
2087
|
sessionName: "",
|
|
1786
2088
|
messages,
|
|
1787
2089
|
fullText: textParts.join(" "),
|
|
1788
|
-
timestamp: latestTimestamp ||
|
|
2090
|
+
timestamp: latestTimestamp || fileMtimeIso2(filePath),
|
|
1789
2091
|
messageCount: messages.length,
|
|
1790
2092
|
account,
|
|
1791
2093
|
lastPrompt: lastUserText || void 0
|
|
@@ -1793,8 +2095,8 @@ async function parseCursorConversation(filePath, account) {
|
|
|
1793
2095
|
}
|
|
1794
2096
|
|
|
1795
2097
|
// src/discovery.ts
|
|
1796
|
-
var
|
|
1797
|
-
var
|
|
2098
|
+
var import_fast_glob4 = __toESM(require("fast-glob"), 1);
|
|
2099
|
+
var import_promises5 = require("fs/promises");
|
|
1798
2100
|
var EXCLUDED_SEGMENTS = ["/memory/", "/tool-results/"];
|
|
1799
2101
|
var STAT_CONCURRENCY = 32;
|
|
1800
2102
|
async function discoverJsonlFiles(dirs, onProgress) {
|
|
@@ -1803,7 +2105,7 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
1803
2105
|
for (const { projectsDir, account } of dirs) {
|
|
1804
2106
|
let filePaths;
|
|
1805
2107
|
try {
|
|
1806
|
-
filePaths = await (0,
|
|
2108
|
+
filePaths = await (0, import_fast_glob4.default)("**/*.jsonl", {
|
|
1807
2109
|
cwd: projectsDir,
|
|
1808
2110
|
absolute: true,
|
|
1809
2111
|
dot: false
|
|
@@ -1821,7 +2123,7 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
1821
2123
|
const statted = await Promise.all(
|
|
1822
2124
|
chunk.map(async (filePath) => {
|
|
1823
2125
|
try {
|
|
1824
|
-
const s = await (0,
|
|
2126
|
+
const s = await (0, import_promises5.stat)(filePath);
|
|
1825
2127
|
return { filePath, size: s.size };
|
|
1826
2128
|
} catch (err) {
|
|
1827
2129
|
log.warn({ filePath, err }, "discovery: stat failed");
|
|
@@ -1896,9 +2198,9 @@ var ThreadbaseProvider = class {
|
|
|
1896
2198
|
|
|
1897
2199
|
// src/scanner.ts
|
|
1898
2200
|
var import_events = require("events");
|
|
1899
|
-
var
|
|
2201
|
+
var import_fs13 = require("fs");
|
|
1900
2202
|
var import_os2 = require("os");
|
|
1901
|
-
var
|
|
2203
|
+
var import_path13 = require("path");
|
|
1902
2204
|
|
|
1903
2205
|
// src/cache.ts
|
|
1904
2206
|
var LRUCache = class {
|
|
@@ -1937,18 +2239,18 @@ var LRUCache = class {
|
|
|
1937
2239
|
};
|
|
1938
2240
|
|
|
1939
2241
|
// src/persistent/conversation-stream.ts
|
|
1940
|
-
var
|
|
2242
|
+
var import_path9 = require("path");
|
|
1941
2243
|
|
|
1942
2244
|
// src/persistent/paged-reader.ts
|
|
1943
|
-
var
|
|
1944
|
-
var
|
|
2245
|
+
var import_fs8 = require("fs");
|
|
2246
|
+
var import_promises7 = require("timers/promises");
|
|
1945
2247
|
|
|
1946
2248
|
// src/persistent/jsonl-tail-reader.ts
|
|
1947
|
-
var
|
|
1948
|
-
var
|
|
2249
|
+
var import_fs7 = require("fs");
|
|
2250
|
+
var import_promises6 = require("timers/promises");
|
|
1949
2251
|
var YIELD_EVERY_LINES = 500;
|
|
1950
2252
|
async function tailReduce(filePath, startOffset, startLine, state, tier, onEntry) {
|
|
1951
|
-
const stream = (0,
|
|
2253
|
+
const stream = (0, import_fs7.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1952
2254
|
let buffer = "";
|
|
1953
2255
|
let offset = startOffset;
|
|
1954
2256
|
let line = startLine;
|
|
@@ -1975,7 +2277,7 @@ async function tailReduce(filePath, startOffset, startLine, state, tier, onEntry
|
|
|
1975
2277
|
line++;
|
|
1976
2278
|
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
1977
2279
|
sinceYield = 0;
|
|
1978
|
-
await (0,
|
|
2280
|
+
await (0, import_promises6.setImmediate)();
|
|
1979
2281
|
}
|
|
1980
2282
|
}
|
|
1981
2283
|
}
|
|
@@ -1985,7 +2287,7 @@ async function tailReduce(filePath, startOffset, startLine, state, tier, onEntry
|
|
|
1985
2287
|
// src/persistent/paged-reader.ts
|
|
1986
2288
|
var CHECKPOINT_INTERVAL = 500;
|
|
1987
2289
|
async function streamMessages(filePath, startOffset, startLine, state, onMessage, onEntry) {
|
|
1988
|
-
const stream = (0,
|
|
2290
|
+
const stream = (0, import_fs8.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1989
2291
|
let buffer = "";
|
|
1990
2292
|
let offset = startOffset;
|
|
1991
2293
|
let line = startLine;
|
|
@@ -2001,7 +2303,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
|
|
|
2001
2303
|
line += 1;
|
|
2002
2304
|
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
2003
2305
|
sinceYield = 0;
|
|
2004
|
-
await (0,
|
|
2306
|
+
await (0, import_promises7.setImmediate)();
|
|
2005
2307
|
}
|
|
2006
2308
|
if (text.length === 0) continue;
|
|
2007
2309
|
let entry;
|
|
@@ -2097,7 +2399,7 @@ function assemble(filePath, account, messages, fullText, turnDurations, state) {
|
|
|
2097
2399
|
filePath,
|
|
2098
2400
|
projectPath: state.cwd,
|
|
2099
2401
|
projectName: getShortProjectName2(state.cwd),
|
|
2100
|
-
sessionId: state.sessionId || (0,
|
|
2402
|
+
sessionId: state.sessionId || (0, import_path9.basename)(filePath, ".jsonl"),
|
|
2101
2403
|
sessionName: state.sessionName || deriveSessionNameFromFirstMessage(messages.find((m) => m.role === "user" && m.text) ?? null),
|
|
2102
2404
|
messages,
|
|
2103
2405
|
fullText,
|
|
@@ -2142,70 +2444,70 @@ async function extendConversation(previous, resume, filePath, account) {
|
|
|
2142
2444
|
|
|
2143
2445
|
// src/persistent/cursor.ts
|
|
2144
2446
|
var import_crypto2 = require("crypto");
|
|
2145
|
-
var
|
|
2447
|
+
var import_fs9 = require("fs");
|
|
2146
2448
|
var FP_BYTES = 4096;
|
|
2147
2449
|
function fingerprint(filePath, size) {
|
|
2148
2450
|
const hash = (0, import_crypto2.createHash)("sha1");
|
|
2149
2451
|
hash.update(String(size));
|
|
2150
|
-
const fd = (0,
|
|
2452
|
+
const fd = (0, import_fs9.openSync)(filePath, "r");
|
|
2151
2453
|
try {
|
|
2152
2454
|
const head = Buffer.alloc(Math.min(FP_BYTES, size));
|
|
2153
2455
|
if (head.length > 0) {
|
|
2154
|
-
(0,
|
|
2456
|
+
(0, import_fs9.readSync)(fd, head, 0, head.length, 0);
|
|
2155
2457
|
hash.update(head);
|
|
2156
2458
|
}
|
|
2157
2459
|
if (size > FP_BYTES) {
|
|
2158
2460
|
const tailLen = Math.min(FP_BYTES, size);
|
|
2159
2461
|
const tail = Buffer.alloc(tailLen);
|
|
2160
|
-
(0,
|
|
2462
|
+
(0, import_fs9.readSync)(fd, tail, 0, tailLen, size - tailLen);
|
|
2161
2463
|
hash.update(tail);
|
|
2162
2464
|
}
|
|
2163
2465
|
} finally {
|
|
2164
|
-
(0,
|
|
2466
|
+
(0, import_fs9.closeSync)(fd);
|
|
2165
2467
|
}
|
|
2166
2468
|
return hash.digest("hex");
|
|
2167
2469
|
}
|
|
2168
2470
|
function classify(filePath, existing) {
|
|
2169
|
-
let
|
|
2471
|
+
let stat6;
|
|
2170
2472
|
try {
|
|
2171
|
-
const s = (0,
|
|
2172
|
-
|
|
2473
|
+
const s = (0, import_fs9.statSync)(filePath);
|
|
2474
|
+
stat6 = { size: s.size, mtimeMs: s.mtimeMs };
|
|
2173
2475
|
} catch {
|
|
2174
2476
|
return { change: "vanished" };
|
|
2175
2477
|
}
|
|
2176
2478
|
if (!existing || existing.status !== "active" || existing.last_indexed_offset === 0) {
|
|
2177
|
-
return { change: "reindex", stat:
|
|
2479
|
+
return { change: "reindex", stat: stat6 };
|
|
2178
2480
|
}
|
|
2179
|
-
if (
|
|
2180
|
-
return { change: "reindex", stat:
|
|
2481
|
+
if (stat6.size < existing.last_indexed_offset) {
|
|
2482
|
+
return { change: "reindex", stat: stat6 };
|
|
2181
2483
|
}
|
|
2182
|
-
if (
|
|
2183
|
-
return { change: "unchanged", stat:
|
|
2484
|
+
if (stat6.size === existing.size_bytes && stat6.mtimeMs === existing.mtime_ms) {
|
|
2485
|
+
return { change: "unchanged", stat: stat6 };
|
|
2184
2486
|
}
|
|
2185
|
-
if (
|
|
2186
|
-
const fp = fingerprint(filePath,
|
|
2487
|
+
if (stat6.size === existing.last_indexed_offset) {
|
|
2488
|
+
const fp = fingerprint(filePath, stat6.size);
|
|
2187
2489
|
if (existing.content_fingerprint && fp !== existing.content_fingerprint) {
|
|
2188
|
-
return { change: "reindex", stat:
|
|
2490
|
+
return { change: "reindex", stat: stat6 };
|
|
2189
2491
|
}
|
|
2190
|
-
return { change: "unchanged", stat:
|
|
2492
|
+
return { change: "unchanged", stat: stat6 };
|
|
2191
2493
|
}
|
|
2192
2494
|
if (existing.content_fingerprint && existing.size_bytes === existing.last_indexed_offset && fingerprint(filePath, existing.last_indexed_offset) === existing.content_fingerprint) {
|
|
2193
|
-
return { change: "appended", stat:
|
|
2495
|
+
return { change: "appended", stat: stat6 };
|
|
2194
2496
|
}
|
|
2195
|
-
return { change: "reindex", stat:
|
|
2497
|
+
return { change: "reindex", stat: stat6 };
|
|
2196
2498
|
}
|
|
2197
2499
|
|
|
2198
2500
|
// src/persistent/index-engine.ts
|
|
2199
|
-
var
|
|
2501
|
+
var import_fs12 = require("fs");
|
|
2200
2502
|
|
|
2201
2503
|
// src/providers/parse.ts
|
|
2202
|
-
var
|
|
2203
|
-
var
|
|
2504
|
+
var import_fs10 = require("fs");
|
|
2505
|
+
var import_readline5 = require("readline");
|
|
2204
2506
|
async function parseMetaWithProvider(provider, filePath, account, tier, onEntry) {
|
|
2205
2507
|
const log = getLogger();
|
|
2206
2508
|
const acc = provider.createEmptyAccumulator();
|
|
2207
|
-
const rl = (0,
|
|
2208
|
-
input: (0,
|
|
2509
|
+
const rl = (0, import_readline5.createInterface)({
|
|
2510
|
+
input: (0, import_fs10.createReadStream)(filePath),
|
|
2209
2511
|
crlfDelay: Infinity
|
|
2210
2512
|
});
|
|
2211
2513
|
try {
|
|
@@ -2278,8 +2580,17 @@ function extractSearchDelta(entry) {
|
|
|
2278
2580
|
if (looksLikeCursorChatEntry(entry)) {
|
|
2279
2581
|
return extractCursorDelta(entry);
|
|
2280
2582
|
}
|
|
2583
|
+
if (looksLikeCopilotEvent(entry)) {
|
|
2584
|
+
return extractCopilotDelta(entry);
|
|
2585
|
+
}
|
|
2281
2586
|
return extractClaudeDelta(entry);
|
|
2282
2587
|
}
|
|
2588
|
+
function extractCopilotDelta(entry) {
|
|
2589
|
+
const message = parseCopilotJsonlLine(JSON.stringify(entry));
|
|
2590
|
+
if (!message) return emptySearchDocument();
|
|
2591
|
+
const tools = (message.metadata?.toolUseBlocks ?? []).map((block) => capToolPayload(block.input)).filter(Boolean).join(SEP);
|
|
2592
|
+
return { text: message.text, thinking: "", tools };
|
|
2593
|
+
}
|
|
2283
2594
|
function extractCursorDelta(entry) {
|
|
2284
2595
|
const msg = entry.message;
|
|
2285
2596
|
const content = msg?.content ?? entry.content;
|
|
@@ -2307,7 +2618,7 @@ function extractCursorSearchText(content) {
|
|
|
2307
2618
|
function extractClaudeDelta(entry) {
|
|
2308
2619
|
const type = entry.type;
|
|
2309
2620
|
if (type !== "user" && type !== "assistant") return emptySearchDocument();
|
|
2310
|
-
if (entry.isMeta) return emptySearchDocument();
|
|
2621
|
+
if (entry.isMeta || entry.isCompactSummary === true) return emptySearchDocument();
|
|
2311
2622
|
const msg = entry.message;
|
|
2312
2623
|
const content = msg?.content;
|
|
2313
2624
|
const tools = [
|
|
@@ -2420,8 +2731,8 @@ function resolveTier(tierName, customTiers) {
|
|
|
2420
2731
|
|
|
2421
2732
|
// src/persistent/db.ts
|
|
2422
2733
|
var import_better_sqlite3 = __toESM(require("better-sqlite3"), 1);
|
|
2423
|
-
var
|
|
2424
|
-
var
|
|
2734
|
+
var import_fs11 = require("fs");
|
|
2735
|
+
var import_path10 = require("path");
|
|
2425
2736
|
|
|
2426
2737
|
// src/persistent/schema.ts
|
|
2427
2738
|
var SCHEMA_VERSION = 10;
|
|
@@ -2746,7 +3057,7 @@ function hasColumn(db, table, column) {
|
|
|
2746
3057
|
// src/persistent/db.ts
|
|
2747
3058
|
function openDatabase(dbPath) {
|
|
2748
3059
|
if (dbPath !== ":memory:") {
|
|
2749
|
-
(0,
|
|
3060
|
+
(0, import_fs11.mkdirSync)((0, import_path10.dirname)(dbPath), { recursive: true });
|
|
2750
3061
|
}
|
|
2751
3062
|
let db;
|
|
2752
3063
|
try {
|
|
@@ -2790,8 +3101,8 @@ function nativeBindingError(err, dbPath) {
|
|
|
2790
3101
|
}
|
|
2791
3102
|
|
|
2792
3103
|
// src/persistent/dir-watermark.ts
|
|
2793
|
-
var
|
|
2794
|
-
var
|
|
3104
|
+
var import_promises8 = require("fs/promises");
|
|
3105
|
+
var import_path11 = require("path");
|
|
2795
3106
|
var FULL_RECONCILE_EVERY_N_SCANS = 20;
|
|
2796
3107
|
async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
2797
3108
|
const log = getLogger();
|
|
@@ -2807,7 +3118,7 @@ async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
|
2807
3118
|
for (const projectDir of resolved.entries) {
|
|
2808
3119
|
let dirStat;
|
|
2809
3120
|
try {
|
|
2810
|
-
dirStat = await (0,
|
|
3121
|
+
dirStat = await (0, import_promises8.stat)(projectDir);
|
|
2811
3122
|
} catch {
|
|
2812
3123
|
scannedDirs.remove(projectDir);
|
|
2813
3124
|
continue;
|
|
@@ -2845,7 +3156,7 @@ async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
|
2845
3156
|
async function resolveProjectDirs(projectsDir, scannedDirs) {
|
|
2846
3157
|
let rootStat;
|
|
2847
3158
|
try {
|
|
2848
|
-
rootStat = await (0,
|
|
3159
|
+
rootStat = await (0, import_promises8.stat)(projectsDir);
|
|
2849
3160
|
} catch {
|
|
2850
3161
|
return null;
|
|
2851
3162
|
}
|
|
@@ -2855,17 +3166,17 @@ async function resolveProjectDirs(projectsDir, scannedDirs) {
|
|
|
2855
3166
|
}
|
|
2856
3167
|
let entries;
|
|
2857
3168
|
try {
|
|
2858
|
-
entries = (await (0,
|
|
3169
|
+
entries = (await (0, import_promises8.readdir)(projectsDir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => joinPath(projectsDir, e.name));
|
|
2859
3170
|
} catch {
|
|
2860
3171
|
return null;
|
|
2861
3172
|
}
|
|
2862
3173
|
return { entries, commitRoot: { mtimeMs: rootStat.mtimeMs } };
|
|
2863
3174
|
}
|
|
2864
3175
|
function dirnameOf(filePath) {
|
|
2865
|
-
return (0,
|
|
3176
|
+
return (0, import_path11.dirname)(filePath);
|
|
2866
3177
|
}
|
|
2867
3178
|
function joinPath(dir, name) {
|
|
2868
|
-
return (0,
|
|
3179
|
+
return (0, import_path11.join)(dir, name);
|
|
2869
3180
|
}
|
|
2870
3181
|
|
|
2871
3182
|
// src/persistent/repositories/checkpoints.repo.ts
|
|
@@ -2945,7 +3256,7 @@ function toCheckpoint(row) {
|
|
|
2945
3256
|
}
|
|
2946
3257
|
|
|
2947
3258
|
// src/persistent/repositories/conversation-files.repo.ts
|
|
2948
|
-
var
|
|
3259
|
+
var import_path12 = require("path");
|
|
2949
3260
|
var ConversationFilesRepo = class {
|
|
2950
3261
|
constructor(db) {
|
|
2951
3262
|
this.db = db;
|
|
@@ -2968,7 +3279,7 @@ var ConversationFilesRepo = class {
|
|
|
2968
3279
|
const info = this.db.prepare(
|
|
2969
3280
|
`INSERT INTO conversation_files (absolute_path, parent_dir, file_name, account)
|
|
2970
3281
|
VALUES (?, ?, ?, ?)`
|
|
2971
|
-
).run(canonical, (0,
|
|
3282
|
+
).run(canonical, (0, import_path12.dirname)(canonical), (0, import_path12.basename)(canonical), account);
|
|
2972
3283
|
return Number(info.lastInsertRowid);
|
|
2973
3284
|
}
|
|
2974
3285
|
// Advance the cursor + persisted reducer state after a successful index pass.
|
|
@@ -3485,6 +3796,12 @@ var PersistentEngine = class {
|
|
|
3485
3796
|
discovered.push({ ...f, provider: cursor });
|
|
3486
3797
|
}
|
|
3487
3798
|
}
|
|
3799
|
+
const copilot = new CopilotProvider();
|
|
3800
|
+
if (enabled.includes(COPILOT_PROVIDER) && (options.copilotRoots?.length ?? 0) > 0) {
|
|
3801
|
+
for (const f of await copilot.discover(options.copilotRoots)) {
|
|
3802
|
+
discovered.push({ ...f, provider: copilot });
|
|
3803
|
+
}
|
|
3804
|
+
}
|
|
3488
3805
|
let scanned = 0;
|
|
3489
3806
|
const gitBranchMemo = /* @__PURE__ */ new Map();
|
|
3490
3807
|
const resolveGitBranch = (projectPath) => {
|
|
@@ -3526,6 +3843,9 @@ var PersistentEngine = class {
|
|
|
3526
3843
|
if (enabled.includes(CURSOR_PROVIDER) && (options.cursorRoots?.length ?? 0) > 0) {
|
|
3527
3844
|
coveredAccounts.add("cursor");
|
|
3528
3845
|
}
|
|
3846
|
+
if (enabled.includes(COPILOT_PROVIDER) && (options.copilotRoots?.length ?? 0) > 0) {
|
|
3847
|
+
coveredAccounts.add("copilot");
|
|
3848
|
+
}
|
|
3529
3849
|
const seen = new Set(discovered.map((d) => canonicalPath(d.filePath)));
|
|
3530
3850
|
for (const path of this.files.activePathsByAccounts([...coveredAccounts])) {
|
|
3531
3851
|
if (!seen.has(path)) this.markDeleted(path);
|
|
@@ -3545,8 +3865,8 @@ var PersistentEngine = class {
|
|
|
3545
3865
|
const tier = resolveTier(tierName, customTiers);
|
|
3546
3866
|
const filePath = canonicalPath(rawFilePath);
|
|
3547
3867
|
const existing = this.files.getByPath(filePath);
|
|
3548
|
-
const { change, stat:
|
|
3549
|
-
if (change === "vanished" || !
|
|
3868
|
+
const { change, stat: stat6 } = classify(filePath, existing);
|
|
3869
|
+
if (change === "vanished" || !stat6) {
|
|
3550
3870
|
this.markDeleted(filePath);
|
|
3551
3871
|
return { meta: null, change: "vanished" };
|
|
3552
3872
|
}
|
|
@@ -3559,7 +3879,7 @@ var PersistentEngine = class {
|
|
|
3559
3879
|
filePath,
|
|
3560
3880
|
account,
|
|
3561
3881
|
tier,
|
|
3562
|
-
|
|
3882
|
+
stat6,
|
|
3563
3883
|
resolveGitBranch
|
|
3564
3884
|
);
|
|
3565
3885
|
return { meta: meta2, change };
|
|
@@ -3585,7 +3905,7 @@ var PersistentEngine = class {
|
|
|
3585
3905
|
return { meta: null, change };
|
|
3586
3906
|
}
|
|
3587
3907
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
3588
|
-
const fp =
|
|
3908
|
+
const fp = stat6.size > 0 ? fingerprint(filePath, stat6.size) : null;
|
|
3589
3909
|
const fileId = this.files.ensure(filePath, account);
|
|
3590
3910
|
const base = resume ? this.fts.readDocument(fileId) ?? emptySearchDocument() : emptySearchDocument();
|
|
3591
3911
|
const searchDoc = appendSearchDelta(base, searchDelta);
|
|
@@ -3596,8 +3916,8 @@ var PersistentEngine = class {
|
|
|
3596
3916
|
}
|
|
3597
3917
|
if (!resume) this.checkpoints.remove(filePath);
|
|
3598
3918
|
this.files.updateCursor(fileId, {
|
|
3599
|
-
sizeBytes:
|
|
3600
|
-
mtimeMs:
|
|
3919
|
+
sizeBytes: stat6.size,
|
|
3920
|
+
mtimeMs: stat6.mtimeMs,
|
|
3601
3921
|
// Advance only to the last fully-parsed line; a trailing partial line
|
|
3602
3922
|
// is left for the next pass.
|
|
3603
3923
|
offset: result.newOffset,
|
|
@@ -3614,8 +3934,8 @@ var PersistentEngine = class {
|
|
|
3614
3934
|
buildSidecar(
|
|
3615
3935
|
meta,
|
|
3616
3936
|
{
|
|
3617
|
-
sizeBytes:
|
|
3618
|
-
mtimeMs:
|
|
3937
|
+
sizeBytes: stat6.size,
|
|
3938
|
+
mtimeMs: stat6.mtimeMs,
|
|
3619
3939
|
offset: result.newOffset,
|
|
3620
3940
|
line: result.newLine
|
|
3621
3941
|
},
|
|
@@ -3634,7 +3954,7 @@ var PersistentEngine = class {
|
|
|
3634
3954
|
// the Threadbase path uses. The cursor records size/mtime (and offset = size)
|
|
3635
3955
|
// so the next pass classifies an unchanged file as "unchanged" and skips it;
|
|
3636
3956
|
// any change reparses from 0 again. No reducer_state is persisted.
|
|
3637
|
-
async indexFileWithProvider(provider, filePath, account, tier,
|
|
3957
|
+
async indexFileWithProvider(provider, filePath, account, tier, stat6, resolveGitBranch) {
|
|
3638
3958
|
const log = getLogger();
|
|
3639
3959
|
let searchDoc = emptySearchDocument();
|
|
3640
3960
|
const meta = await parseMetaWithProvider(provider, filePath, account, tier, (entry) => {
|
|
@@ -3647,7 +3967,7 @@ var PersistentEngine = class {
|
|
|
3647
3967
|
if (meta.gitBranch === null && meta.projectPath) {
|
|
3648
3968
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
3649
3969
|
}
|
|
3650
|
-
const fp =
|
|
3970
|
+
const fp = stat6.size > 0 ? fingerprint(filePath, stat6.size) : null;
|
|
3651
3971
|
const fileId = this.files.ensure(filePath, account);
|
|
3652
3972
|
const upsert = this.db.transaction(() => {
|
|
3653
3973
|
this.conversations.upsert(fileId, meta, meta.messageCount);
|
|
@@ -3656,11 +3976,11 @@ var PersistentEngine = class {
|
|
|
3656
3976
|
}
|
|
3657
3977
|
this.checkpoints.remove(filePath);
|
|
3658
3978
|
this.files.updateCursor(fileId, {
|
|
3659
|
-
sizeBytes:
|
|
3660
|
-
mtimeMs:
|
|
3979
|
+
sizeBytes: stat6.size,
|
|
3980
|
+
mtimeMs: stat6.mtimeMs,
|
|
3661
3981
|
// offset = size marks the file fully consumed (non-zero so the next pass
|
|
3662
3982
|
// can classify it "unchanged"); no resumable reducer state is kept.
|
|
3663
|
-
offset:
|
|
3983
|
+
offset: stat6.size,
|
|
3664
3984
|
line: 0,
|
|
3665
3985
|
reducerState: null,
|
|
3666
3986
|
fingerprint: fp,
|
|
@@ -3726,8 +4046,8 @@ var PersistentEngine = class {
|
|
|
3726
4046
|
const meta = this.conversations.getByIdOrSession(id);
|
|
3727
4047
|
if (!meta) return null;
|
|
3728
4048
|
const filePath = meta.filePath;
|
|
3729
|
-
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER) {
|
|
3730
|
-
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : await parseCursorConversation(filePath, meta.account);
|
|
4049
|
+
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
|
|
4050
|
+
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
|
|
3731
4051
|
if (!conversation) return null;
|
|
3732
4052
|
const { messages } = conversation;
|
|
3733
4053
|
const total2 = messages.length;
|
|
@@ -3742,7 +4062,7 @@ var PersistentEngine = class {
|
|
|
3742
4062
|
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
3743
4063
|
if (floor) {
|
|
3744
4064
|
try {
|
|
3745
|
-
if (floor.byteOffset > (0,
|
|
4065
|
+
if (floor.byteOffset > (0, import_fs12.statSync)(filePath).size) floor = null;
|
|
3746
4066
|
} catch {
|
|
3747
4067
|
}
|
|
3748
4068
|
}
|
|
@@ -3881,7 +4201,7 @@ function defaultDbPath() {
|
|
|
3881
4201
|
"TB_SCANNER_DB must be set under a test runner. Refusing to open the default index at ~/.config/threadbase-scanner/index.db, because test fixtures written there are indistinguishable from real conversations and corrupt every measurement of the index. Point it at a temp directory in your test setup, or pass persistent.dbPath explicitly."
|
|
3882
4202
|
);
|
|
3883
4203
|
}
|
|
3884
|
-
return (0,
|
|
4204
|
+
return (0, import_path13.join)((0, import_os2.homedir)(), ".config", "threadbase-scanner", "index.db");
|
|
3885
4205
|
}
|
|
3886
4206
|
function capForMemory(doc, max) {
|
|
3887
4207
|
const combined = combineSearchContent(doc);
|
|
@@ -4060,7 +4380,7 @@ var ConversationScanner = class {
|
|
|
4060
4380
|
const cached = statCache.get(filePath);
|
|
4061
4381
|
if (cached) {
|
|
4062
4382
|
try {
|
|
4063
|
-
const s = (0,
|
|
4383
|
+
const s = (0, import_fs13.statSync)(filePath);
|
|
4064
4384
|
if (s.mtimeMs === cached.stat.mtimeMs && s.size === cached.stat.size) {
|
|
4065
4385
|
return {
|
|
4066
4386
|
meta: cached.meta,
|
|
@@ -4218,12 +4538,12 @@ var ConversationScanner = class {
|
|
|
4218
4538
|
}
|
|
4219
4539
|
log.debug({ id, filePath: meta.filePath }, "getConversation: cache miss, parsing");
|
|
4220
4540
|
try {
|
|
4221
|
-
if (this.persistent && meta.provider !== CODEX_CLI_PROVIDER && meta.provider !== CURSOR_PROVIDER) {
|
|
4541
|
+
if (this.persistent && meta.provider !== CODEX_CLI_PROVIDER && meta.provider !== CURSOR_PROVIDER && meta.provider !== COPILOT_PROVIDER) {
|
|
4222
4542
|
const parsed = await parseConversationResumable(meta.filePath, meta.account);
|
|
4223
4543
|
if (parsed) this.conversationLRU.set(cid, parsed);
|
|
4224
4544
|
return parsed?.conversation ?? null;
|
|
4225
4545
|
}
|
|
4226
|
-
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
4546
|
+
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(meta.filePath, meta.account) : meta.provider === COPILOT_PROVIDER ? await parseCopilotConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
4227
4547
|
if (conversation) {
|
|
4228
4548
|
this.conversationLRU.set(cid, { conversation });
|
|
4229
4549
|
}
|
|
@@ -4277,7 +4597,7 @@ var ConversationScanner = class {
|
|
|
4277
4597
|
// fallback, matching refreshFile.
|
|
4278
4598
|
async parseSingleFilePage(filePath, account, options) {
|
|
4279
4599
|
const provider = await this.resolveProviderForFile(filePath, null);
|
|
4280
|
-
const conversation = provider?.name === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, account ?? "default") : provider?.name === CURSOR_PROVIDER ? await parseCursorConversation(filePath, account ?? "default") : await parseConversation(filePath, account ?? "default");
|
|
4600
|
+
const conversation = provider?.name === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, account ?? "default") : provider?.name === CURSOR_PROVIDER ? await parseCursorConversation(filePath, account ?? "default") : provider?.name === COPILOT_PROVIDER ? await parseCopilotConversation(filePath, account ?? "default") : await parseConversation(filePath, account ?? "default");
|
|
4281
4601
|
if (!conversation) return null;
|
|
4282
4602
|
const { messages } = conversation;
|
|
4283
4603
|
const total = messages.length;
|
|
@@ -4587,6 +4907,12 @@ var ConversationScanner = class {
|
|
|
4587
4907
|
work.push({ ...f, provider });
|
|
4588
4908
|
}
|
|
4589
4909
|
}
|
|
4910
|
+
if (enabled.includes(COPILOT_PROVIDER) && (options.copilotRoots?.length ?? 0) > 0) {
|
|
4911
|
+
const provider = new CopilotProvider();
|
|
4912
|
+
for (const f of await provider.discover(options.copilotRoots)) {
|
|
4913
|
+
work.push({ ...f, provider });
|
|
4914
|
+
}
|
|
4915
|
+
}
|
|
4590
4916
|
return work;
|
|
4591
4917
|
}
|
|
4592
4918
|
// Resolve which provider should (re)parse a file. Stored metadata wins; for a
|
|
@@ -4598,16 +4924,17 @@ var ConversationScanner = class {
|
|
|
4598
4924
|
if (providerMatches(previous?.provider, CURSOR_PROVIDER)) {
|
|
4599
4925
|
return new CursorProvider();
|
|
4600
4926
|
}
|
|
4927
|
+
if (previous?.provider === COPILOT_PROVIDER) return new CopilotProvider();
|
|
4601
4928
|
if (previous?.provider) return void 0;
|
|
4602
4929
|
let sample = "";
|
|
4603
4930
|
try {
|
|
4604
|
-
const fd = (0,
|
|
4931
|
+
const fd = (0, import_fs13.openSync)(filePath, "r");
|
|
4605
4932
|
try {
|
|
4606
4933
|
const buf = Buffer.alloc(8192);
|
|
4607
|
-
const n = (0,
|
|
4934
|
+
const n = (0, import_fs13.readSync)(fd, buf, 0, buf.length, 0);
|
|
4608
4935
|
sample = buf.subarray(0, n).toString("utf8");
|
|
4609
4936
|
} finally {
|
|
4610
|
-
(0,
|
|
4937
|
+
(0, import_fs13.closeSync)(fd);
|
|
4611
4938
|
}
|
|
4612
4939
|
} catch {
|
|
4613
4940
|
return void 0;
|
|
@@ -4616,6 +4943,8 @@ var ConversationScanner = class {
|
|
|
4616
4943
|
if (cursor.canParse(filePath, sample)) return cursor;
|
|
4617
4944
|
const codex = new CodexCliProvider();
|
|
4618
4945
|
if (codex.canParse(filePath, sample)) return codex;
|
|
4946
|
+
const copilot = new CopilotProvider();
|
|
4947
|
+
if (copilot.canParse(filePath, sample)) return copilot;
|
|
4619
4948
|
return void 0;
|
|
4620
4949
|
}
|
|
4621
4950
|
addToSessionIndex(meta) {
|
|
@@ -4725,9 +5054,11 @@ async function getConversation(id, options, scanner) {
|
|
|
4725
5054
|
0 && (module.exports = {
|
|
4726
5055
|
CLAUDE_CODE_PROVIDER,
|
|
4727
5056
|
CODEX_CLI_PROVIDER,
|
|
5057
|
+
COPILOT_PROVIDER,
|
|
4728
5058
|
CURSOR_PROVIDER,
|
|
4729
5059
|
CodexCliProvider,
|
|
4730
5060
|
ConversationScanner,
|
|
5061
|
+
CopilotProvider,
|
|
4731
5062
|
CursorProvider,
|
|
4732
5063
|
DEFAULT_TIERS,
|
|
4733
5064
|
LEGACY_CURSOR_PROVIDER,
|
|
@@ -4752,6 +5083,7 @@ async function getConversation(id, options, scanner) {
|
|
|
4752
5083
|
isCodexInjectedContext,
|
|
4753
5084
|
loadProfiles,
|
|
4754
5085
|
parseCodexJsonlLine,
|
|
5086
|
+
parseCopilotJsonlLine,
|
|
4755
5087
|
parseCursorJsonlLine,
|
|
4756
5088
|
parseJsonlLine,
|
|
4757
5089
|
providerMatches,
|