@threadbase-sh/scanner 0.19.4 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -4
- package/dist/cli.js +381 -73
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +417 -103
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +395 -84
- package/dist/index.js.map +1 -1
- package/dist/persistent/index-engine.d.ts.map +1 -1
- package/dist/providers/copilot.d.ts +33 -0
- package/dist/providers/copilot.d.ts.map +1 -0
- package/dist/providers/provider.d.ts +2 -1
- package/dist/providers/provider.d.ts.map +1 -1
- package/dist/scanner.d.ts.map +1 -1
- package/dist/search-document.d.ts.map +1 -1
- package/dist/types.d.ts +2 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -32,9 +32,11 @@ var index_exports = {};
|
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
CLAUDE_CODE_PROVIDER: () => CLAUDE_CODE_PROVIDER,
|
|
34
34
|
CODEX_CLI_PROVIDER: () => CODEX_CLI_PROVIDER,
|
|
35
|
+
COPILOT_PROVIDER: () => COPILOT_PROVIDER,
|
|
35
36
|
CURSOR_PROVIDER: () => CURSOR_PROVIDER,
|
|
36
37
|
CodexCliProvider: () => CodexCliProvider,
|
|
37
38
|
ConversationScanner: () => ConversationScanner,
|
|
39
|
+
CopilotProvider: () => CopilotProvider,
|
|
38
40
|
CursorProvider: () => CursorProvider,
|
|
39
41
|
DEFAULT_TIERS: () => DEFAULT_TIERS,
|
|
40
42
|
LEGACY_CURSOR_PROVIDER: () => LEGACY_CURSOR_PROVIDER,
|
|
@@ -59,6 +61,7 @@ __export(index_exports, {
|
|
|
59
61
|
isCodexInjectedContext: () => isCodexInjectedContext,
|
|
60
62
|
loadProfiles: () => loadProfiles,
|
|
61
63
|
parseCodexJsonlLine: () => parseCodexJsonlLine,
|
|
64
|
+
parseCopilotJsonlLine: () => parseCopilotJsonlLine,
|
|
62
65
|
parseCursorJsonlLine: () => parseCursorJsonlLine,
|
|
63
66
|
parseJsonlLine: () => parseJsonlLine,
|
|
64
67
|
providerMatches: () => providerMatches,
|
|
@@ -440,10 +443,11 @@ var import_path2 = require("path");
|
|
|
440
443
|
var CLAUDE_CODE_PROVIDER = "claude-code";
|
|
441
444
|
var CODEX_CLI_PROVIDER = "codex-cli";
|
|
442
445
|
var CURSOR_PROVIDER = "cursor";
|
|
446
|
+
var COPILOT_PROVIDER = "copilot";
|
|
443
447
|
var LEGACY_CURSOR_PROVIDER = "cursor-cli";
|
|
444
448
|
function canonicalizeProviderName(value) {
|
|
445
449
|
if (value === LEGACY_CURSOR_PROVIDER) return CURSOR_PROVIDER;
|
|
446
|
-
if (value === CLAUDE_CODE_PROVIDER || value === CODEX_CLI_PROVIDER || value === CURSOR_PROVIDER) {
|
|
450
|
+
if (value === CLAUDE_CODE_PROVIDER || value === CODEX_CLI_PROVIDER || value === CURSOR_PROVIDER || value === COPILOT_PROVIDER) {
|
|
447
451
|
return value;
|
|
448
452
|
}
|
|
449
453
|
return void 0;
|
|
@@ -1365,13 +1369,293 @@ async function parseCodexConversation(filePath, account) {
|
|
|
1365
1369
|
};
|
|
1366
1370
|
}
|
|
1367
1371
|
|
|
1368
|
-
// src/providers/
|
|
1369
|
-
var import_crypto = require("crypto");
|
|
1372
|
+
// src/providers/copilot.ts
|
|
1370
1373
|
var import_fast_glob2 = __toESM(require("fast-glob"), 1);
|
|
1371
1374
|
var import_fs5 = require("fs");
|
|
1372
1375
|
var import_promises3 = require("fs/promises");
|
|
1373
1376
|
var import_path7 = require("path");
|
|
1374
1377
|
var import_readline3 = require("readline");
|
|
1378
|
+
var SESSION_START = "session.start";
|
|
1379
|
+
var USER_MESSAGE = "user.message";
|
|
1380
|
+
var ASSISTANT_MESSAGE = "assistant.message";
|
|
1381
|
+
var CopilotProvider = class {
|
|
1382
|
+
name = COPILOT_PROVIDER;
|
|
1383
|
+
async discover(roots) {
|
|
1384
|
+
const log = getLogger();
|
|
1385
|
+
const results = [];
|
|
1386
|
+
for (const root of roots) {
|
|
1387
|
+
let paths;
|
|
1388
|
+
try {
|
|
1389
|
+
paths = await (0, import_fast_glob2.default)(["*/events.jsonl", "session-state/*/events.jsonl"], {
|
|
1390
|
+
cwd: root,
|
|
1391
|
+
absolute: true,
|
|
1392
|
+
dot: false,
|
|
1393
|
+
unique: true
|
|
1394
|
+
});
|
|
1395
|
+
} catch (err) {
|
|
1396
|
+
log.warn({ root, err }, "copilot discovery: glob failed");
|
|
1397
|
+
continue;
|
|
1398
|
+
}
|
|
1399
|
+
for (const filePath of paths) {
|
|
1400
|
+
try {
|
|
1401
|
+
const s = await (0, import_promises3.stat)(filePath);
|
|
1402
|
+
if (s.size > 0) results.push({ filePath: canonicalPath(filePath), account: "copilot" });
|
|
1403
|
+
} catch (err) {
|
|
1404
|
+
log.warn({ filePath, err }, "copilot discovery: stat failed");
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
return results;
|
|
1409
|
+
}
|
|
1410
|
+
canParse(_filePath, sample) {
|
|
1411
|
+
for (const line of sample.split("\n")) {
|
|
1412
|
+
if (!line.trim()) continue;
|
|
1413
|
+
try {
|
|
1414
|
+
if (looksLikeCopilotEvent(JSON.parse(line))) return true;
|
|
1415
|
+
} catch {
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
return false;
|
|
1419
|
+
}
|
|
1420
|
+
createEmptyAccumulator() {
|
|
1421
|
+
return {
|
|
1422
|
+
sessionId: "",
|
|
1423
|
+
cwd: "",
|
|
1424
|
+
gitBranch: null,
|
|
1425
|
+
model: null,
|
|
1426
|
+
latestTimestamp: "",
|
|
1427
|
+
messageCount: 0,
|
|
1428
|
+
lastMessageSender: "user",
|
|
1429
|
+
firstUser: null,
|
|
1430
|
+
lastUser: null,
|
|
1431
|
+
lastAssistant: null,
|
|
1432
|
+
toolNames: [],
|
|
1433
|
+
previewParts: [],
|
|
1434
|
+
previewLength: 0,
|
|
1435
|
+
snippetParts: [],
|
|
1436
|
+
snippetLength: 0
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
reduceEntry(acc, entry, tier) {
|
|
1440
|
+
reduceCopilotEntry(acc, entry, tier);
|
|
1441
|
+
}
|
|
1442
|
+
finalize(acc, filePath, account, tier) {
|
|
1443
|
+
return finalizeCopilotMeta(acc, filePath, account, tier);
|
|
1444
|
+
}
|
|
1445
|
+
};
|
|
1446
|
+
function looksLikeCopilotEvent(entry) {
|
|
1447
|
+
return typeof entry.type === "string" && typeof entry.id === "string" && "parentId" in entry && asRecord(entry.data) !== null;
|
|
1448
|
+
}
|
|
1449
|
+
var asString2 = (v) => typeof v === "string" ? v : "";
|
|
1450
|
+
function asRecord(v) {
|
|
1451
|
+
return v && typeof v === "object" && !Array.isArray(v) ? v : null;
|
|
1452
|
+
}
|
|
1453
|
+
function copilotToolUseBlocks(data) {
|
|
1454
|
+
if (!Array.isArray(data.toolRequests)) return [];
|
|
1455
|
+
const blocks = [];
|
|
1456
|
+
for (const item of data.toolRequests) {
|
|
1457
|
+
const request = asRecord(item);
|
|
1458
|
+
const name = asString2(request?.name);
|
|
1459
|
+
if (!request || !name) continue;
|
|
1460
|
+
const input = asRecord(request.arguments) ?? { arguments: asString2(request.arguments) };
|
|
1461
|
+
blocks.push({ id: asString2(request.toolCallId), name, input });
|
|
1462
|
+
}
|
|
1463
|
+
return blocks;
|
|
1464
|
+
}
|
|
1465
|
+
function copilotEntryToMessage(entry) {
|
|
1466
|
+
if (!looksLikeCopilotEvent(entry)) return null;
|
|
1467
|
+
const data = entry.data;
|
|
1468
|
+
const timestamp = asString2(entry.timestamp);
|
|
1469
|
+
const uuid = asString2(data.messageId) || asString2(entry.id);
|
|
1470
|
+
if (entry.type === USER_MESSAGE) {
|
|
1471
|
+
const text = cleanSystemTags(asString2(data.content));
|
|
1472
|
+
return text ? { role: "user", text, timestamp, uuid } : null;
|
|
1473
|
+
}
|
|
1474
|
+
if (entry.type === ASSISTANT_MESSAGE) {
|
|
1475
|
+
const text = asString2(data.content).trim();
|
|
1476
|
+
const blocks = copilotToolUseBlocks(data);
|
|
1477
|
+
if (!text && blocks.length === 0) return null;
|
|
1478
|
+
const model = asString2(data.model);
|
|
1479
|
+
const metadata = {
|
|
1480
|
+
...model ? { model } : {},
|
|
1481
|
+
...blocks.length > 0 ? { toolUses: blocks.map((b) => b.name), toolUseBlocks: blocks } : {}
|
|
1482
|
+
};
|
|
1483
|
+
return {
|
|
1484
|
+
role: "assistant",
|
|
1485
|
+
text,
|
|
1486
|
+
timestamp,
|
|
1487
|
+
uuid,
|
|
1488
|
+
...Object.keys(metadata).length > 0 ? { metadata } : {}
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
return null;
|
|
1492
|
+
}
|
|
1493
|
+
function parseCopilotJsonlLine(line) {
|
|
1494
|
+
const trimmed = line.trim();
|
|
1495
|
+
if (!trimmed) return null;
|
|
1496
|
+
try {
|
|
1497
|
+
return copilotEntryToMessage(JSON.parse(trimmed));
|
|
1498
|
+
} catch {
|
|
1499
|
+
return null;
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
function readSessionStart(entry) {
|
|
1503
|
+
if (entry.type !== SESSION_START) return null;
|
|
1504
|
+
const data = asRecord(entry.data);
|
|
1505
|
+
if (!data) return null;
|
|
1506
|
+
const context = asRecord(data.context);
|
|
1507
|
+
return {
|
|
1508
|
+
sessionId: asString2(data.sessionId),
|
|
1509
|
+
cwd: asString2(context?.cwd),
|
|
1510
|
+
branch: asString2(context?.branch)
|
|
1511
|
+
};
|
|
1512
|
+
}
|
|
1513
|
+
function reduceCopilotEntry(acc, entry, tier) {
|
|
1514
|
+
const start = readSessionStart(entry);
|
|
1515
|
+
if (start) {
|
|
1516
|
+
if (!acc.sessionId) acc.sessionId = start.sessionId;
|
|
1517
|
+
if (!acc.cwd) acc.cwd = start.cwd;
|
|
1518
|
+
if (!acc.gitBranch && start.branch) acc.gitBranch = start.branch;
|
|
1519
|
+
return;
|
|
1520
|
+
}
|
|
1521
|
+
const message = copilotEntryToMessage(entry);
|
|
1522
|
+
if (!message) return;
|
|
1523
|
+
for (const name of message.metadata?.toolUses ?? []) {
|
|
1524
|
+
if (!acc.toolNames.includes(name)) acc.toolNames.push(name);
|
|
1525
|
+
}
|
|
1526
|
+
if (message.metadata?.model) acc.model = message.metadata.model;
|
|
1527
|
+
const { role, text, timestamp } = message;
|
|
1528
|
+
if (!text) return;
|
|
1529
|
+
if (timestamp && (!acc.latestTimestamp || timestamp > acc.latestTimestamp)) {
|
|
1530
|
+
acc.latestTimestamp = timestamp;
|
|
1531
|
+
}
|
|
1532
|
+
acc.messageCount++;
|
|
1533
|
+
acc.lastMessageSender = role;
|
|
1534
|
+
const snapshot = { text: text.slice(0, 200), timestamp };
|
|
1535
|
+
if (role === "user") {
|
|
1536
|
+
if (!acc.firstUser) acc.firstUser = snapshot;
|
|
1537
|
+
acc.lastUser = snapshot;
|
|
1538
|
+
} else {
|
|
1539
|
+
acc.lastAssistant = snapshot;
|
|
1540
|
+
}
|
|
1541
|
+
if (acc.previewLength < tier.previewMax) {
|
|
1542
|
+
acc.previewParts.push(text);
|
|
1543
|
+
acc.previewLength += text.length;
|
|
1544
|
+
}
|
|
1545
|
+
if (acc.snippetLength < tier.snippetMax) {
|
|
1546
|
+
const remaining = tier.snippetMax - acc.snippetLength;
|
|
1547
|
+
const chunk = text.length > remaining ? text.slice(0, remaining) : text;
|
|
1548
|
+
acc.snippetParts.push(chunk);
|
|
1549
|
+
acc.snippetLength += chunk.length;
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
function finalizeCopilotMeta(acc, filePath, account, tier) {
|
|
1553
|
+
if (acc.messageCount === 0) return null;
|
|
1554
|
+
const sessionId = acc.sessionId || sessionIdFromPath(filePath);
|
|
1555
|
+
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
1556
|
+
return {
|
|
1557
|
+
id: filePath,
|
|
1558
|
+
filePath,
|
|
1559
|
+
provider: COPILOT_PROVIDER,
|
|
1560
|
+
kind,
|
|
1561
|
+
externalSessionId: sessionId,
|
|
1562
|
+
sessionId,
|
|
1563
|
+
sessionName: deriveSessionNameFromFirstMessage(acc.firstUser),
|
|
1564
|
+
projectPath: acc.cwd,
|
|
1565
|
+
projectName: getShortProjectName4(acc.cwd),
|
|
1566
|
+
account,
|
|
1567
|
+
timestamp: acc.latestTimestamp || fileMtimeIso(filePath),
|
|
1568
|
+
messageCount: acc.messageCount,
|
|
1569
|
+
lastMessageSender: acc.lastMessageSender,
|
|
1570
|
+
preview: acc.previewParts.join(" ").slice(0, tier.previewMax),
|
|
1571
|
+
contentSnippet: acc.snippetParts.join(" "),
|
|
1572
|
+
gitBranch: acc.gitBranch,
|
|
1573
|
+
model: acc.model,
|
|
1574
|
+
isSubagent: false,
|
|
1575
|
+
parentSessionId: null,
|
|
1576
|
+
isTeammate: false,
|
|
1577
|
+
teamName: null,
|
|
1578
|
+
toolNames: acc.toolNames,
|
|
1579
|
+
firstMessage: acc.firstUser,
|
|
1580
|
+
lastMessage: acc.lastAssistant ?? acc.lastUser,
|
|
1581
|
+
lastPrompt: acc.lastUser?.text || void 0
|
|
1582
|
+
};
|
|
1583
|
+
}
|
|
1584
|
+
function sessionIdFromPath(filePath) {
|
|
1585
|
+
return (0, import_path7.basename)((0, import_path7.dirname)(filePath));
|
|
1586
|
+
}
|
|
1587
|
+
function getShortProjectName4(fullPath) {
|
|
1588
|
+
return fullPath.split(/[/\\]/).filter(Boolean).slice(-3).join("/");
|
|
1589
|
+
}
|
|
1590
|
+
function fileMtimeIso(filePath) {
|
|
1591
|
+
try {
|
|
1592
|
+
return new Date((0, import_fs5.statSync)(filePath).mtimeMs).toISOString();
|
|
1593
|
+
} catch {
|
|
1594
|
+
return (/* @__PURE__ */ new Date()).toISOString();
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
async function parseCopilotConversation(filePath, account) {
|
|
1598
|
+
const log = getLogger();
|
|
1599
|
+
const messages = [];
|
|
1600
|
+
const textParts = [];
|
|
1601
|
+
let sessionId = "";
|
|
1602
|
+
let cwd = "";
|
|
1603
|
+
let latestTimestamp = "";
|
|
1604
|
+
let lastUserText = "";
|
|
1605
|
+
const rl = (0, import_readline3.createInterface)({ input: (0, import_fs5.createReadStream)(filePath), crlfDelay: Infinity });
|
|
1606
|
+
try {
|
|
1607
|
+
for await (const line of rl) {
|
|
1608
|
+
if (!line.trim()) continue;
|
|
1609
|
+
let entry;
|
|
1610
|
+
try {
|
|
1611
|
+
entry = JSON.parse(line);
|
|
1612
|
+
} catch {
|
|
1613
|
+
continue;
|
|
1614
|
+
}
|
|
1615
|
+
const start = readSessionStart(entry);
|
|
1616
|
+
if (start) {
|
|
1617
|
+
if (!sessionId) sessionId = start.sessionId;
|
|
1618
|
+
if (!cwd) cwd = start.cwd;
|
|
1619
|
+
continue;
|
|
1620
|
+
}
|
|
1621
|
+
const message = copilotEntryToMessage(entry);
|
|
1622
|
+
if (!message) continue;
|
|
1623
|
+
if (message.timestamp && (!latestTimestamp || message.timestamp > latestTimestamp)) {
|
|
1624
|
+
latestTimestamp = message.timestamp;
|
|
1625
|
+
}
|
|
1626
|
+
messages.push(message);
|
|
1627
|
+
if (!message.text) continue;
|
|
1628
|
+
textParts.push(message.text);
|
|
1629
|
+
if (message.role === "user") lastUserText = message.text;
|
|
1630
|
+
}
|
|
1631
|
+
} catch (err) {
|
|
1632
|
+
log.warn({ filePath, err }, "parseCopilotConversation: read failed");
|
|
1633
|
+
return null;
|
|
1634
|
+
}
|
|
1635
|
+
if (messages.length === 0) return null;
|
|
1636
|
+
return {
|
|
1637
|
+
id: filePath,
|
|
1638
|
+
filePath,
|
|
1639
|
+
projectPath: cwd,
|
|
1640
|
+
projectName: getShortProjectName4(cwd),
|
|
1641
|
+
sessionId: sessionId || sessionIdFromPath(filePath),
|
|
1642
|
+
sessionName: "",
|
|
1643
|
+
messages,
|
|
1644
|
+
fullText: textParts.join(" "),
|
|
1645
|
+
timestamp: latestTimestamp || fileMtimeIso(filePath),
|
|
1646
|
+
messageCount: messages.length,
|
|
1647
|
+
account,
|
|
1648
|
+
lastPrompt: lastUserText || void 0
|
|
1649
|
+
};
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
// src/providers/cursor.ts
|
|
1653
|
+
var import_crypto = require("crypto");
|
|
1654
|
+
var import_fast_glob3 = __toESM(require("fast-glob"), 1);
|
|
1655
|
+
var import_fs6 = require("fs");
|
|
1656
|
+
var import_promises4 = require("fs/promises");
|
|
1657
|
+
var import_path8 = require("path");
|
|
1658
|
+
var import_readline4 = require("readline");
|
|
1375
1659
|
var TIMESTAMP_RE = /<timestamp>\s*([\s\S]*?)\s*<\/timestamp>/;
|
|
1376
1660
|
var USER_QUERY_RE = /<user_query>\s*([\s\S]*?)\s*<\/user_query>/;
|
|
1377
1661
|
var SUBAGENT_PATH_RE2 = /[\\/]subagents[\\/]/;
|
|
@@ -1383,7 +1667,7 @@ var CursorProvider = class {
|
|
|
1383
1667
|
for (const root of roots) {
|
|
1384
1668
|
let paths;
|
|
1385
1669
|
try {
|
|
1386
|
-
paths = await (0,
|
|
1670
|
+
paths = await (0, import_fast_glob3.default)(["**/agent-transcripts/**/*.jsonl"], {
|
|
1387
1671
|
cwd: root,
|
|
1388
1672
|
absolute: true,
|
|
1389
1673
|
dot: false,
|
|
@@ -1395,7 +1679,7 @@ var CursorProvider = class {
|
|
|
1395
1679
|
}
|
|
1396
1680
|
for (const filePath of paths) {
|
|
1397
1681
|
try {
|
|
1398
|
-
const s = await (0,
|
|
1682
|
+
const s = await (0, import_promises4.stat)(filePath);
|
|
1399
1683
|
if (s.size > 0) results.push({ filePath: canonicalPath(filePath), account: "cursor" });
|
|
1400
1684
|
} catch (err) {
|
|
1401
1685
|
log.warn({ filePath, err }, "cursor discovery: stat failed");
|
|
@@ -1459,7 +1743,7 @@ function looksLikeImportedClaudeEntry(entry) {
|
|
|
1459
1743
|
function looksLikeImportedCodexEntry(entry) {
|
|
1460
1744
|
return entry.type === "session_meta" || entry.type === "response_item" || entry.type === "event_msg";
|
|
1461
1745
|
}
|
|
1462
|
-
function
|
|
1746
|
+
function asString3(value) {
|
|
1463
1747
|
return typeof value === "string" ? value : "";
|
|
1464
1748
|
}
|
|
1465
1749
|
function provenanceToken(value) {
|
|
@@ -1573,7 +1857,7 @@ function cursorEntryToMessage(entry) {
|
|
|
1573
1857
|
const role = message?.role === "assistant" || entry.type === "assistant" ? "assistant" : "user";
|
|
1574
1858
|
const content = message?.content ?? entry.content;
|
|
1575
1859
|
return withToolUses(
|
|
1576
|
-
{ role, text: extractCursorText(content), timestamp:
|
|
1860
|
+
{ role, text: extractCursorText(content), timestamp: asString3(entry.timestamp) },
|
|
1577
1861
|
content
|
|
1578
1862
|
);
|
|
1579
1863
|
}
|
|
@@ -1650,11 +1934,11 @@ function reduceCursorEntry(acc, entry, tier) {
|
|
|
1650
1934
|
}
|
|
1651
1935
|
function finalizeCursorMeta(acc, filePath, account, tier) {
|
|
1652
1936
|
if (acc.messageCount === 0) return null;
|
|
1653
|
-
const sessionId = (0,
|
|
1937
|
+
const sessionId = (0, import_path8.basename)(filePath, ".jsonl");
|
|
1654
1938
|
const isSubagent = SUBAGENT_PATH_RE2.test(filePath);
|
|
1655
1939
|
const parent = isSubagent ? parentFromSubagentPath(filePath) : null;
|
|
1656
1940
|
const { projectPath, projectName } = projectFromCursorPath(filePath);
|
|
1657
|
-
const timestamp = acc.latestTimestamp ||
|
|
1941
|
+
const timestamp = acc.latestTimestamp || fileMtimeIso2(filePath);
|
|
1658
1942
|
const kind = acc.lastAssistant === null && acc.toolNames.length > 0 ? "task" : "conversation";
|
|
1659
1943
|
return {
|
|
1660
1944
|
id: filePath,
|
|
@@ -1689,34 +1973,34 @@ function finalizeCursorMeta(acc, filePath, account, tier) {
|
|
|
1689
1973
|
};
|
|
1690
1974
|
}
|
|
1691
1975
|
function parentFromSubagentPath(filePath) {
|
|
1692
|
-
const subagentsDir = (0,
|
|
1693
|
-
const parentDir = (0,
|
|
1694
|
-
const parentSessionUuid = (0,
|
|
1976
|
+
const subagentsDir = (0, import_path8.dirname)(filePath);
|
|
1977
|
+
const parentDir = (0, import_path8.dirname)(subagentsDir);
|
|
1978
|
+
const parentSessionUuid = (0, import_path8.basename)(parentDir);
|
|
1695
1979
|
if (!parentSessionUuid) return null;
|
|
1696
1980
|
return {
|
|
1697
1981
|
parentSessionUuid,
|
|
1698
|
-
parentFilePath: canonicalPath((0,
|
|
1982
|
+
parentFilePath: canonicalPath((0, import_path8.join)(parentDir, `${parentSessionUuid}.jsonl`))
|
|
1699
1983
|
};
|
|
1700
1984
|
}
|
|
1701
1985
|
function projectFromCursorPath(filePath) {
|
|
1702
1986
|
const parts = filePath.split(/[/\\]/);
|
|
1703
1987
|
const idx = parts.lastIndexOf("agent-transcripts");
|
|
1704
|
-
const cursorProjectDir = idx > 0 ? parts.slice(0, idx).join(
|
|
1705
|
-
const slug = (0,
|
|
1988
|
+
const cursorProjectDir = idx > 0 ? parts.slice(0, idx).join(import_path8.sep) : (0, import_path8.dirname)((0, import_path8.dirname)(filePath));
|
|
1989
|
+
const slug = (0, import_path8.basename)(cursorProjectDir);
|
|
1706
1990
|
const decoded = decodeCursorProjectSlug(slug);
|
|
1707
1991
|
const projectPath = decoded ?? cursorProjectDir;
|
|
1708
|
-
return { projectPath, projectName:
|
|
1992
|
+
return { projectPath, projectName: getShortProjectName5(projectPath) || slug };
|
|
1709
1993
|
}
|
|
1710
1994
|
function decodeCursorProjectSlug(slug) {
|
|
1711
1995
|
if (!slug) return null;
|
|
1712
1996
|
const segments = slug.split("-");
|
|
1713
1997
|
if (segments.length === 0) return null;
|
|
1714
1998
|
const rooted = slug.startsWith("Users-") || slug.startsWith("home-") || /^[A-Za-z]-/.test(slug);
|
|
1715
|
-
let acc = rooted ?
|
|
1999
|
+
let acc = rooted ? import_path8.sep : "";
|
|
1716
2000
|
let i = 0;
|
|
1717
2001
|
if (rooted && segments[0] === "Users") {
|
|
1718
|
-
const users = `${
|
|
1719
|
-
if (!(0,
|
|
2002
|
+
const users = `${import_path8.sep}Users`;
|
|
2003
|
+
if (!(0, import_fs6.existsSync)(users)) return null;
|
|
1720
2004
|
acc = users;
|
|
1721
2005
|
i = 1;
|
|
1722
2006
|
}
|
|
@@ -1724,9 +2008,9 @@ function decodeCursorProjectSlug(slug) {
|
|
|
1724
2008
|
let found = null;
|
|
1725
2009
|
let next = i + 1;
|
|
1726
2010
|
for (let j = segments.length; j > i; j--) {
|
|
1727
|
-
const candidate = (0,
|
|
2011
|
+
const candidate = (0, import_path8.join)(acc, segments.slice(i, j).join("-"));
|
|
1728
2012
|
try {
|
|
1729
|
-
if ((0,
|
|
2013
|
+
if ((0, import_fs6.existsSync)(candidate) && (0, import_fs6.statSync)(candidate).isDirectory()) {
|
|
1730
2014
|
found = candidate;
|
|
1731
2015
|
next = j;
|
|
1732
2016
|
break;
|
|
@@ -1740,12 +2024,12 @@ function decodeCursorProjectSlug(slug) {
|
|
|
1740
2024
|
}
|
|
1741
2025
|
return acc;
|
|
1742
2026
|
}
|
|
1743
|
-
function
|
|
2027
|
+
function getShortProjectName5(fullPath) {
|
|
1744
2028
|
return fullPath.split(/[/\\]/).filter(Boolean).slice(-3).join("/");
|
|
1745
2029
|
}
|
|
1746
|
-
function
|
|
2030
|
+
function fileMtimeIso2(filePath) {
|
|
1747
2031
|
try {
|
|
1748
|
-
return new Date((0,
|
|
2032
|
+
return new Date((0, import_fs6.statSync)(filePath).mtimeMs).toISOString();
|
|
1749
2033
|
} catch {
|
|
1750
2034
|
return (/* @__PURE__ */ new Date()).toISOString();
|
|
1751
2035
|
}
|
|
@@ -1756,7 +2040,7 @@ async function parseCursorConversation(filePath, account) {
|
|
|
1756
2040
|
const textParts = [];
|
|
1757
2041
|
let latestTimestamp = "";
|
|
1758
2042
|
let lastUserText = "";
|
|
1759
|
-
const rl = (0,
|
|
2043
|
+
const rl = (0, import_readline4.createInterface)({ input: (0, import_fs6.createReadStream)(filePath), crlfDelay: Infinity });
|
|
1760
2044
|
try {
|
|
1761
2045
|
for await (const line of rl) {
|
|
1762
2046
|
const message = parseCursorJsonlLine(line, messages.length);
|
|
@@ -1775,7 +2059,7 @@ async function parseCursorConversation(filePath, account) {
|
|
|
1775
2059
|
}
|
|
1776
2060
|
if (messages.length === 0) return null;
|
|
1777
2061
|
const { projectPath, projectName } = projectFromCursorPath(filePath);
|
|
1778
|
-
const sessionId = (0,
|
|
2062
|
+
const sessionId = (0, import_path8.basename)(filePath, ".jsonl");
|
|
1779
2063
|
return {
|
|
1780
2064
|
id: filePath,
|
|
1781
2065
|
filePath,
|
|
@@ -1785,7 +2069,7 @@ async function parseCursorConversation(filePath, account) {
|
|
|
1785
2069
|
sessionName: "",
|
|
1786
2070
|
messages,
|
|
1787
2071
|
fullText: textParts.join(" "),
|
|
1788
|
-
timestamp: latestTimestamp ||
|
|
2072
|
+
timestamp: latestTimestamp || fileMtimeIso2(filePath),
|
|
1789
2073
|
messageCount: messages.length,
|
|
1790
2074
|
account,
|
|
1791
2075
|
lastPrompt: lastUserText || void 0
|
|
@@ -1793,8 +2077,8 @@ async function parseCursorConversation(filePath, account) {
|
|
|
1793
2077
|
}
|
|
1794
2078
|
|
|
1795
2079
|
// src/discovery.ts
|
|
1796
|
-
var
|
|
1797
|
-
var
|
|
2080
|
+
var import_fast_glob4 = __toESM(require("fast-glob"), 1);
|
|
2081
|
+
var import_promises5 = require("fs/promises");
|
|
1798
2082
|
var EXCLUDED_SEGMENTS = ["/memory/", "/tool-results/"];
|
|
1799
2083
|
var STAT_CONCURRENCY = 32;
|
|
1800
2084
|
async function discoverJsonlFiles(dirs, onProgress) {
|
|
@@ -1803,7 +2087,7 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
1803
2087
|
for (const { projectsDir, account } of dirs) {
|
|
1804
2088
|
let filePaths;
|
|
1805
2089
|
try {
|
|
1806
|
-
filePaths = await (0,
|
|
2090
|
+
filePaths = await (0, import_fast_glob4.default)("**/*.jsonl", {
|
|
1807
2091
|
cwd: projectsDir,
|
|
1808
2092
|
absolute: true,
|
|
1809
2093
|
dot: false
|
|
@@ -1821,7 +2105,7 @@ async function discoverJsonlFiles(dirs, onProgress) {
|
|
|
1821
2105
|
const statted = await Promise.all(
|
|
1822
2106
|
chunk.map(async (filePath) => {
|
|
1823
2107
|
try {
|
|
1824
|
-
const s = await (0,
|
|
2108
|
+
const s = await (0, import_promises5.stat)(filePath);
|
|
1825
2109
|
return { filePath, size: s.size };
|
|
1826
2110
|
} catch (err) {
|
|
1827
2111
|
log.warn({ filePath, err }, "discovery: stat failed");
|
|
@@ -1896,9 +2180,9 @@ var ThreadbaseProvider = class {
|
|
|
1896
2180
|
|
|
1897
2181
|
// src/scanner.ts
|
|
1898
2182
|
var import_events = require("events");
|
|
1899
|
-
var
|
|
2183
|
+
var import_fs13 = require("fs");
|
|
1900
2184
|
var import_os2 = require("os");
|
|
1901
|
-
var
|
|
2185
|
+
var import_path13 = require("path");
|
|
1902
2186
|
|
|
1903
2187
|
// src/cache.ts
|
|
1904
2188
|
var LRUCache = class {
|
|
@@ -1937,18 +2221,18 @@ var LRUCache = class {
|
|
|
1937
2221
|
};
|
|
1938
2222
|
|
|
1939
2223
|
// src/persistent/conversation-stream.ts
|
|
1940
|
-
var
|
|
2224
|
+
var import_path9 = require("path");
|
|
1941
2225
|
|
|
1942
2226
|
// src/persistent/paged-reader.ts
|
|
1943
|
-
var
|
|
1944
|
-
var
|
|
2227
|
+
var import_fs8 = require("fs");
|
|
2228
|
+
var import_promises7 = require("timers/promises");
|
|
1945
2229
|
|
|
1946
2230
|
// src/persistent/jsonl-tail-reader.ts
|
|
1947
|
-
var
|
|
1948
|
-
var
|
|
2231
|
+
var import_fs7 = require("fs");
|
|
2232
|
+
var import_promises6 = require("timers/promises");
|
|
1949
2233
|
var YIELD_EVERY_LINES = 500;
|
|
1950
2234
|
async function tailReduce(filePath, startOffset, startLine, state, tier, onEntry) {
|
|
1951
|
-
const stream = (0,
|
|
2235
|
+
const stream = (0, import_fs7.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1952
2236
|
let buffer = "";
|
|
1953
2237
|
let offset = startOffset;
|
|
1954
2238
|
let line = startLine;
|
|
@@ -1975,7 +2259,7 @@ async function tailReduce(filePath, startOffset, startLine, state, tier, onEntry
|
|
|
1975
2259
|
line++;
|
|
1976
2260
|
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
1977
2261
|
sinceYield = 0;
|
|
1978
|
-
await (0,
|
|
2262
|
+
await (0, import_promises6.setImmediate)();
|
|
1979
2263
|
}
|
|
1980
2264
|
}
|
|
1981
2265
|
}
|
|
@@ -1985,7 +2269,7 @@ async function tailReduce(filePath, startOffset, startLine, state, tier, onEntry
|
|
|
1985
2269
|
// src/persistent/paged-reader.ts
|
|
1986
2270
|
var CHECKPOINT_INTERVAL = 500;
|
|
1987
2271
|
async function streamMessages(filePath, startOffset, startLine, state, onMessage, onEntry) {
|
|
1988
|
-
const stream = (0,
|
|
2272
|
+
const stream = (0, import_fs8.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
|
|
1989
2273
|
let buffer = "";
|
|
1990
2274
|
let offset = startOffset;
|
|
1991
2275
|
let line = startLine;
|
|
@@ -2001,7 +2285,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
|
|
|
2001
2285
|
line += 1;
|
|
2002
2286
|
if (++sinceYield >= YIELD_EVERY_LINES) {
|
|
2003
2287
|
sinceYield = 0;
|
|
2004
|
-
await (0,
|
|
2288
|
+
await (0, import_promises7.setImmediate)();
|
|
2005
2289
|
}
|
|
2006
2290
|
if (text.length === 0) continue;
|
|
2007
2291
|
let entry;
|
|
@@ -2097,7 +2381,7 @@ function assemble(filePath, account, messages, fullText, turnDurations, state) {
|
|
|
2097
2381
|
filePath,
|
|
2098
2382
|
projectPath: state.cwd,
|
|
2099
2383
|
projectName: getShortProjectName2(state.cwd),
|
|
2100
|
-
sessionId: state.sessionId || (0,
|
|
2384
|
+
sessionId: state.sessionId || (0, import_path9.basename)(filePath, ".jsonl"),
|
|
2101
2385
|
sessionName: state.sessionName || deriveSessionNameFromFirstMessage(messages.find((m) => m.role === "user" && m.text) ?? null),
|
|
2102
2386
|
messages,
|
|
2103
2387
|
fullText,
|
|
@@ -2142,70 +2426,70 @@ async function extendConversation(previous, resume, filePath, account) {
|
|
|
2142
2426
|
|
|
2143
2427
|
// src/persistent/cursor.ts
|
|
2144
2428
|
var import_crypto2 = require("crypto");
|
|
2145
|
-
var
|
|
2429
|
+
var import_fs9 = require("fs");
|
|
2146
2430
|
var FP_BYTES = 4096;
|
|
2147
2431
|
function fingerprint(filePath, size) {
|
|
2148
2432
|
const hash = (0, import_crypto2.createHash)("sha1");
|
|
2149
2433
|
hash.update(String(size));
|
|
2150
|
-
const fd = (0,
|
|
2434
|
+
const fd = (0, import_fs9.openSync)(filePath, "r");
|
|
2151
2435
|
try {
|
|
2152
2436
|
const head = Buffer.alloc(Math.min(FP_BYTES, size));
|
|
2153
2437
|
if (head.length > 0) {
|
|
2154
|
-
(0,
|
|
2438
|
+
(0, import_fs9.readSync)(fd, head, 0, head.length, 0);
|
|
2155
2439
|
hash.update(head);
|
|
2156
2440
|
}
|
|
2157
2441
|
if (size > FP_BYTES) {
|
|
2158
2442
|
const tailLen = Math.min(FP_BYTES, size);
|
|
2159
2443
|
const tail = Buffer.alloc(tailLen);
|
|
2160
|
-
(0,
|
|
2444
|
+
(0, import_fs9.readSync)(fd, tail, 0, tailLen, size - tailLen);
|
|
2161
2445
|
hash.update(tail);
|
|
2162
2446
|
}
|
|
2163
2447
|
} finally {
|
|
2164
|
-
(0,
|
|
2448
|
+
(0, import_fs9.closeSync)(fd);
|
|
2165
2449
|
}
|
|
2166
2450
|
return hash.digest("hex");
|
|
2167
2451
|
}
|
|
2168
2452
|
function classify(filePath, existing) {
|
|
2169
|
-
let
|
|
2453
|
+
let stat6;
|
|
2170
2454
|
try {
|
|
2171
|
-
const s = (0,
|
|
2172
|
-
|
|
2455
|
+
const s = (0, import_fs9.statSync)(filePath);
|
|
2456
|
+
stat6 = { size: s.size, mtimeMs: s.mtimeMs };
|
|
2173
2457
|
} catch {
|
|
2174
2458
|
return { change: "vanished" };
|
|
2175
2459
|
}
|
|
2176
2460
|
if (!existing || existing.status !== "active" || existing.last_indexed_offset === 0) {
|
|
2177
|
-
return { change: "reindex", stat:
|
|
2461
|
+
return { change: "reindex", stat: stat6 };
|
|
2178
2462
|
}
|
|
2179
|
-
if (
|
|
2180
|
-
return { change: "reindex", stat:
|
|
2463
|
+
if (stat6.size < existing.last_indexed_offset) {
|
|
2464
|
+
return { change: "reindex", stat: stat6 };
|
|
2181
2465
|
}
|
|
2182
|
-
if (
|
|
2183
|
-
return { change: "unchanged", stat:
|
|
2466
|
+
if (stat6.size === existing.size_bytes && stat6.mtimeMs === existing.mtime_ms) {
|
|
2467
|
+
return { change: "unchanged", stat: stat6 };
|
|
2184
2468
|
}
|
|
2185
|
-
if (
|
|
2186
|
-
const fp = fingerprint(filePath,
|
|
2469
|
+
if (stat6.size === existing.last_indexed_offset) {
|
|
2470
|
+
const fp = fingerprint(filePath, stat6.size);
|
|
2187
2471
|
if (existing.content_fingerprint && fp !== existing.content_fingerprint) {
|
|
2188
|
-
return { change: "reindex", stat:
|
|
2472
|
+
return { change: "reindex", stat: stat6 };
|
|
2189
2473
|
}
|
|
2190
|
-
return { change: "unchanged", stat:
|
|
2474
|
+
return { change: "unchanged", stat: stat6 };
|
|
2191
2475
|
}
|
|
2192
2476
|
if (existing.content_fingerprint && existing.size_bytes === existing.last_indexed_offset && fingerprint(filePath, existing.last_indexed_offset) === existing.content_fingerprint) {
|
|
2193
|
-
return { change: "appended", stat:
|
|
2477
|
+
return { change: "appended", stat: stat6 };
|
|
2194
2478
|
}
|
|
2195
|
-
return { change: "reindex", stat:
|
|
2479
|
+
return { change: "reindex", stat: stat6 };
|
|
2196
2480
|
}
|
|
2197
2481
|
|
|
2198
2482
|
// src/persistent/index-engine.ts
|
|
2199
|
-
var
|
|
2483
|
+
var import_fs12 = require("fs");
|
|
2200
2484
|
|
|
2201
2485
|
// src/providers/parse.ts
|
|
2202
|
-
var
|
|
2203
|
-
var
|
|
2486
|
+
var import_fs10 = require("fs");
|
|
2487
|
+
var import_readline5 = require("readline");
|
|
2204
2488
|
async function parseMetaWithProvider(provider, filePath, account, tier, onEntry) {
|
|
2205
2489
|
const log = getLogger();
|
|
2206
2490
|
const acc = provider.createEmptyAccumulator();
|
|
2207
|
-
const rl = (0,
|
|
2208
|
-
input: (0,
|
|
2491
|
+
const rl = (0, import_readline5.createInterface)({
|
|
2492
|
+
input: (0, import_fs10.createReadStream)(filePath),
|
|
2209
2493
|
crlfDelay: Infinity
|
|
2210
2494
|
});
|
|
2211
2495
|
try {
|
|
@@ -2278,8 +2562,17 @@ function extractSearchDelta(entry) {
|
|
|
2278
2562
|
if (looksLikeCursorChatEntry(entry)) {
|
|
2279
2563
|
return extractCursorDelta(entry);
|
|
2280
2564
|
}
|
|
2565
|
+
if (looksLikeCopilotEvent(entry)) {
|
|
2566
|
+
return extractCopilotDelta(entry);
|
|
2567
|
+
}
|
|
2281
2568
|
return extractClaudeDelta(entry);
|
|
2282
2569
|
}
|
|
2570
|
+
function extractCopilotDelta(entry) {
|
|
2571
|
+
const message = parseCopilotJsonlLine(JSON.stringify(entry));
|
|
2572
|
+
if (!message) return emptySearchDocument();
|
|
2573
|
+
const tools = (message.metadata?.toolUseBlocks ?? []).map((block) => capToolPayload(block.input)).filter(Boolean).join(SEP);
|
|
2574
|
+
return { text: message.text, thinking: "", tools };
|
|
2575
|
+
}
|
|
2283
2576
|
function extractCursorDelta(entry) {
|
|
2284
2577
|
const msg = entry.message;
|
|
2285
2578
|
const content = msg?.content ?? entry.content;
|
|
@@ -2420,8 +2713,8 @@ function resolveTier(tierName, customTiers) {
|
|
|
2420
2713
|
|
|
2421
2714
|
// src/persistent/db.ts
|
|
2422
2715
|
var import_better_sqlite3 = __toESM(require("better-sqlite3"), 1);
|
|
2423
|
-
var
|
|
2424
|
-
var
|
|
2716
|
+
var import_fs11 = require("fs");
|
|
2717
|
+
var import_path10 = require("path");
|
|
2425
2718
|
|
|
2426
2719
|
// src/persistent/schema.ts
|
|
2427
2720
|
var SCHEMA_VERSION = 10;
|
|
@@ -2746,7 +3039,7 @@ function hasColumn(db, table, column) {
|
|
|
2746
3039
|
// src/persistent/db.ts
|
|
2747
3040
|
function openDatabase(dbPath) {
|
|
2748
3041
|
if (dbPath !== ":memory:") {
|
|
2749
|
-
(0,
|
|
3042
|
+
(0, import_fs11.mkdirSync)((0, import_path10.dirname)(dbPath), { recursive: true });
|
|
2750
3043
|
}
|
|
2751
3044
|
let db;
|
|
2752
3045
|
try {
|
|
@@ -2790,8 +3083,8 @@ function nativeBindingError(err, dbPath) {
|
|
|
2790
3083
|
}
|
|
2791
3084
|
|
|
2792
3085
|
// src/persistent/dir-watermark.ts
|
|
2793
|
-
var
|
|
2794
|
-
var
|
|
3086
|
+
var import_promises8 = require("fs/promises");
|
|
3087
|
+
var import_path11 = require("path");
|
|
2795
3088
|
var FULL_RECONCILE_EVERY_N_SCANS = 20;
|
|
2796
3089
|
async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
2797
3090
|
const log = getLogger();
|
|
@@ -2807,7 +3100,7 @@ async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
|
2807
3100
|
for (const projectDir of resolved.entries) {
|
|
2808
3101
|
let dirStat;
|
|
2809
3102
|
try {
|
|
2810
|
-
dirStat = await (0,
|
|
3103
|
+
dirStat = await (0, import_promises8.stat)(projectDir);
|
|
2811
3104
|
} catch {
|
|
2812
3105
|
scannedDirs.remove(projectDir);
|
|
2813
3106
|
continue;
|
|
@@ -2845,7 +3138,7 @@ async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
|
|
|
2845
3138
|
async function resolveProjectDirs(projectsDir, scannedDirs) {
|
|
2846
3139
|
let rootStat;
|
|
2847
3140
|
try {
|
|
2848
|
-
rootStat = await (0,
|
|
3141
|
+
rootStat = await (0, import_promises8.stat)(projectsDir);
|
|
2849
3142
|
} catch {
|
|
2850
3143
|
return null;
|
|
2851
3144
|
}
|
|
@@ -2855,17 +3148,17 @@ async function resolveProjectDirs(projectsDir, scannedDirs) {
|
|
|
2855
3148
|
}
|
|
2856
3149
|
let entries;
|
|
2857
3150
|
try {
|
|
2858
|
-
entries = (await (0,
|
|
3151
|
+
entries = (await (0, import_promises8.readdir)(projectsDir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => joinPath(projectsDir, e.name));
|
|
2859
3152
|
} catch {
|
|
2860
3153
|
return null;
|
|
2861
3154
|
}
|
|
2862
3155
|
return { entries, commitRoot: { mtimeMs: rootStat.mtimeMs } };
|
|
2863
3156
|
}
|
|
2864
3157
|
function dirnameOf(filePath) {
|
|
2865
|
-
return (0,
|
|
3158
|
+
return (0, import_path11.dirname)(filePath);
|
|
2866
3159
|
}
|
|
2867
3160
|
function joinPath(dir, name) {
|
|
2868
|
-
return (0,
|
|
3161
|
+
return (0, import_path11.join)(dir, name);
|
|
2869
3162
|
}
|
|
2870
3163
|
|
|
2871
3164
|
// src/persistent/repositories/checkpoints.repo.ts
|
|
@@ -2945,7 +3238,7 @@ function toCheckpoint(row) {
|
|
|
2945
3238
|
}
|
|
2946
3239
|
|
|
2947
3240
|
// src/persistent/repositories/conversation-files.repo.ts
|
|
2948
|
-
var
|
|
3241
|
+
var import_path12 = require("path");
|
|
2949
3242
|
var ConversationFilesRepo = class {
|
|
2950
3243
|
constructor(db) {
|
|
2951
3244
|
this.db = db;
|
|
@@ -2968,7 +3261,7 @@ var ConversationFilesRepo = class {
|
|
|
2968
3261
|
const info = this.db.prepare(
|
|
2969
3262
|
`INSERT INTO conversation_files (absolute_path, parent_dir, file_name, account)
|
|
2970
3263
|
VALUES (?, ?, ?, ?)`
|
|
2971
|
-
).run(canonical, (0,
|
|
3264
|
+
).run(canonical, (0, import_path12.dirname)(canonical), (0, import_path12.basename)(canonical), account);
|
|
2972
3265
|
return Number(info.lastInsertRowid);
|
|
2973
3266
|
}
|
|
2974
3267
|
// Advance the cursor + persisted reducer state after a successful index pass.
|
|
@@ -3485,6 +3778,12 @@ var PersistentEngine = class {
|
|
|
3485
3778
|
discovered.push({ ...f, provider: cursor });
|
|
3486
3779
|
}
|
|
3487
3780
|
}
|
|
3781
|
+
const copilot = new CopilotProvider();
|
|
3782
|
+
if (enabled.includes(COPILOT_PROVIDER) && (options.copilotRoots?.length ?? 0) > 0) {
|
|
3783
|
+
for (const f of await copilot.discover(options.copilotRoots)) {
|
|
3784
|
+
discovered.push({ ...f, provider: copilot });
|
|
3785
|
+
}
|
|
3786
|
+
}
|
|
3488
3787
|
let scanned = 0;
|
|
3489
3788
|
const gitBranchMemo = /* @__PURE__ */ new Map();
|
|
3490
3789
|
const resolveGitBranch = (projectPath) => {
|
|
@@ -3526,6 +3825,9 @@ var PersistentEngine = class {
|
|
|
3526
3825
|
if (enabled.includes(CURSOR_PROVIDER) && (options.cursorRoots?.length ?? 0) > 0) {
|
|
3527
3826
|
coveredAccounts.add("cursor");
|
|
3528
3827
|
}
|
|
3828
|
+
if (enabled.includes(COPILOT_PROVIDER) && (options.copilotRoots?.length ?? 0) > 0) {
|
|
3829
|
+
coveredAccounts.add("copilot");
|
|
3830
|
+
}
|
|
3529
3831
|
const seen = new Set(discovered.map((d) => canonicalPath(d.filePath)));
|
|
3530
3832
|
for (const path of this.files.activePathsByAccounts([...coveredAccounts])) {
|
|
3531
3833
|
if (!seen.has(path)) this.markDeleted(path);
|
|
@@ -3545,8 +3847,8 @@ var PersistentEngine = class {
|
|
|
3545
3847
|
const tier = resolveTier(tierName, customTiers);
|
|
3546
3848
|
const filePath = canonicalPath(rawFilePath);
|
|
3547
3849
|
const existing = this.files.getByPath(filePath);
|
|
3548
|
-
const { change, stat:
|
|
3549
|
-
if (change === "vanished" || !
|
|
3850
|
+
const { change, stat: stat6 } = classify(filePath, existing);
|
|
3851
|
+
if (change === "vanished" || !stat6) {
|
|
3550
3852
|
this.markDeleted(filePath);
|
|
3551
3853
|
return { meta: null, change: "vanished" };
|
|
3552
3854
|
}
|
|
@@ -3559,7 +3861,7 @@ var PersistentEngine = class {
|
|
|
3559
3861
|
filePath,
|
|
3560
3862
|
account,
|
|
3561
3863
|
tier,
|
|
3562
|
-
|
|
3864
|
+
stat6,
|
|
3563
3865
|
resolveGitBranch
|
|
3564
3866
|
);
|
|
3565
3867
|
return { meta: meta2, change };
|
|
@@ -3585,7 +3887,7 @@ var PersistentEngine = class {
|
|
|
3585
3887
|
return { meta: null, change };
|
|
3586
3888
|
}
|
|
3587
3889
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
3588
|
-
const fp =
|
|
3890
|
+
const fp = stat6.size > 0 ? fingerprint(filePath, stat6.size) : null;
|
|
3589
3891
|
const fileId = this.files.ensure(filePath, account);
|
|
3590
3892
|
const base = resume ? this.fts.readDocument(fileId) ?? emptySearchDocument() : emptySearchDocument();
|
|
3591
3893
|
const searchDoc = appendSearchDelta(base, searchDelta);
|
|
@@ -3596,8 +3898,8 @@ var PersistentEngine = class {
|
|
|
3596
3898
|
}
|
|
3597
3899
|
if (!resume) this.checkpoints.remove(filePath);
|
|
3598
3900
|
this.files.updateCursor(fileId, {
|
|
3599
|
-
sizeBytes:
|
|
3600
|
-
mtimeMs:
|
|
3901
|
+
sizeBytes: stat6.size,
|
|
3902
|
+
mtimeMs: stat6.mtimeMs,
|
|
3601
3903
|
// Advance only to the last fully-parsed line; a trailing partial line
|
|
3602
3904
|
// is left for the next pass.
|
|
3603
3905
|
offset: result.newOffset,
|
|
@@ -3614,8 +3916,8 @@ var PersistentEngine = class {
|
|
|
3614
3916
|
buildSidecar(
|
|
3615
3917
|
meta,
|
|
3616
3918
|
{
|
|
3617
|
-
sizeBytes:
|
|
3618
|
-
mtimeMs:
|
|
3919
|
+
sizeBytes: stat6.size,
|
|
3920
|
+
mtimeMs: stat6.mtimeMs,
|
|
3619
3921
|
offset: result.newOffset,
|
|
3620
3922
|
line: result.newLine
|
|
3621
3923
|
},
|
|
@@ -3634,7 +3936,7 @@ var PersistentEngine = class {
|
|
|
3634
3936
|
// the Threadbase path uses. The cursor records size/mtime (and offset = size)
|
|
3635
3937
|
// so the next pass classifies an unchanged file as "unchanged" and skips it;
|
|
3636
3938
|
// any change reparses from 0 again. No reducer_state is persisted.
|
|
3637
|
-
async indexFileWithProvider(provider, filePath, account, tier,
|
|
3939
|
+
async indexFileWithProvider(provider, filePath, account, tier, stat6, resolveGitBranch) {
|
|
3638
3940
|
const log = getLogger();
|
|
3639
3941
|
let searchDoc = emptySearchDocument();
|
|
3640
3942
|
const meta = await parseMetaWithProvider(provider, filePath, account, tier, (entry) => {
|
|
@@ -3647,7 +3949,7 @@ var PersistentEngine = class {
|
|
|
3647
3949
|
if (meta.gitBranch === null && meta.projectPath) {
|
|
3648
3950
|
meta.gitBranch = resolveGitBranch(meta.projectPath);
|
|
3649
3951
|
}
|
|
3650
|
-
const fp =
|
|
3952
|
+
const fp = stat6.size > 0 ? fingerprint(filePath, stat6.size) : null;
|
|
3651
3953
|
const fileId = this.files.ensure(filePath, account);
|
|
3652
3954
|
const upsert = this.db.transaction(() => {
|
|
3653
3955
|
this.conversations.upsert(fileId, meta, meta.messageCount);
|
|
@@ -3656,11 +3958,11 @@ var PersistentEngine = class {
|
|
|
3656
3958
|
}
|
|
3657
3959
|
this.checkpoints.remove(filePath);
|
|
3658
3960
|
this.files.updateCursor(fileId, {
|
|
3659
|
-
sizeBytes:
|
|
3660
|
-
mtimeMs:
|
|
3961
|
+
sizeBytes: stat6.size,
|
|
3962
|
+
mtimeMs: stat6.mtimeMs,
|
|
3661
3963
|
// offset = size marks the file fully consumed (non-zero so the next pass
|
|
3662
3964
|
// can classify it "unchanged"); no resumable reducer state is kept.
|
|
3663
|
-
offset:
|
|
3965
|
+
offset: stat6.size,
|
|
3664
3966
|
line: 0,
|
|
3665
3967
|
reducerState: null,
|
|
3666
3968
|
fingerprint: fp,
|
|
@@ -3726,8 +4028,8 @@ var PersistentEngine = class {
|
|
|
3726
4028
|
const meta = this.conversations.getByIdOrSession(id);
|
|
3727
4029
|
if (!meta) return null;
|
|
3728
4030
|
const filePath = meta.filePath;
|
|
3729
|
-
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER) {
|
|
3730
|
-
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : await parseCursorConversation(filePath, meta.account);
|
|
4031
|
+
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
|
|
4032
|
+
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
|
|
3731
4033
|
if (!conversation) return null;
|
|
3732
4034
|
const { messages } = conversation;
|
|
3733
4035
|
const total2 = messages.length;
|
|
@@ -3742,7 +4044,7 @@ var PersistentEngine = class {
|
|
|
3742
4044
|
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
3743
4045
|
if (floor) {
|
|
3744
4046
|
try {
|
|
3745
|
-
if (floor.byteOffset > (0,
|
|
4047
|
+
if (floor.byteOffset > (0, import_fs12.statSync)(filePath).size) floor = null;
|
|
3746
4048
|
} catch {
|
|
3747
4049
|
}
|
|
3748
4050
|
}
|
|
@@ -3881,7 +4183,7 @@ function defaultDbPath() {
|
|
|
3881
4183
|
"TB_SCANNER_DB must be set under a test runner. Refusing to open the default index at ~/.config/threadbase-scanner/index.db, because test fixtures written there are indistinguishable from real conversations and corrupt every measurement of the index. Point it at a temp directory in your test setup, or pass persistent.dbPath explicitly."
|
|
3882
4184
|
);
|
|
3883
4185
|
}
|
|
3884
|
-
return (0,
|
|
4186
|
+
return (0, import_path13.join)((0, import_os2.homedir)(), ".config", "threadbase-scanner", "index.db");
|
|
3885
4187
|
}
|
|
3886
4188
|
function capForMemory(doc, max) {
|
|
3887
4189
|
const combined = combineSearchContent(doc);
|
|
@@ -4060,7 +4362,7 @@ var ConversationScanner = class {
|
|
|
4060
4362
|
const cached = statCache.get(filePath);
|
|
4061
4363
|
if (cached) {
|
|
4062
4364
|
try {
|
|
4063
|
-
const s = (0,
|
|
4365
|
+
const s = (0, import_fs13.statSync)(filePath);
|
|
4064
4366
|
if (s.mtimeMs === cached.stat.mtimeMs && s.size === cached.stat.size) {
|
|
4065
4367
|
return {
|
|
4066
4368
|
meta: cached.meta,
|
|
@@ -4218,12 +4520,12 @@ var ConversationScanner = class {
|
|
|
4218
4520
|
}
|
|
4219
4521
|
log.debug({ id, filePath: meta.filePath }, "getConversation: cache miss, parsing");
|
|
4220
4522
|
try {
|
|
4221
|
-
if (this.persistent && meta.provider !== CODEX_CLI_PROVIDER && meta.provider !== CURSOR_PROVIDER) {
|
|
4523
|
+
if (this.persistent && meta.provider !== CODEX_CLI_PROVIDER && meta.provider !== CURSOR_PROVIDER && meta.provider !== COPILOT_PROVIDER) {
|
|
4222
4524
|
const parsed = await parseConversationResumable(meta.filePath, meta.account);
|
|
4223
4525
|
if (parsed) this.conversationLRU.set(cid, parsed);
|
|
4224
4526
|
return parsed?.conversation ?? null;
|
|
4225
4527
|
}
|
|
4226
|
-
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
4528
|
+
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(meta.filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(meta.filePath, meta.account) : meta.provider === COPILOT_PROVIDER ? await parseCopilotConversation(meta.filePath, meta.account) : await parseConversation(meta.filePath, meta.account);
|
|
4227
4529
|
if (conversation) {
|
|
4228
4530
|
this.conversationLRU.set(cid, { conversation });
|
|
4229
4531
|
}
|
|
@@ -4277,7 +4579,7 @@ var ConversationScanner = class {
|
|
|
4277
4579
|
// fallback, matching refreshFile.
|
|
4278
4580
|
async parseSingleFilePage(filePath, account, options) {
|
|
4279
4581
|
const provider = await this.resolveProviderForFile(filePath, null);
|
|
4280
|
-
const conversation = provider?.name === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, account ?? "default") : provider?.name === CURSOR_PROVIDER ? await parseCursorConversation(filePath, account ?? "default") : await parseConversation(filePath, account ?? "default");
|
|
4582
|
+
const conversation = provider?.name === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, account ?? "default") : provider?.name === CURSOR_PROVIDER ? await parseCursorConversation(filePath, account ?? "default") : provider?.name === COPILOT_PROVIDER ? await parseCopilotConversation(filePath, account ?? "default") : await parseConversation(filePath, account ?? "default");
|
|
4281
4583
|
if (!conversation) return null;
|
|
4282
4584
|
const { messages } = conversation;
|
|
4283
4585
|
const total = messages.length;
|
|
@@ -4587,6 +4889,12 @@ var ConversationScanner = class {
|
|
|
4587
4889
|
work.push({ ...f, provider });
|
|
4588
4890
|
}
|
|
4589
4891
|
}
|
|
4892
|
+
if (enabled.includes(COPILOT_PROVIDER) && (options.copilotRoots?.length ?? 0) > 0) {
|
|
4893
|
+
const provider = new CopilotProvider();
|
|
4894
|
+
for (const f of await provider.discover(options.copilotRoots)) {
|
|
4895
|
+
work.push({ ...f, provider });
|
|
4896
|
+
}
|
|
4897
|
+
}
|
|
4590
4898
|
return work;
|
|
4591
4899
|
}
|
|
4592
4900
|
// Resolve which provider should (re)parse a file. Stored metadata wins; for a
|
|
@@ -4598,16 +4906,17 @@ var ConversationScanner = class {
|
|
|
4598
4906
|
if (providerMatches(previous?.provider, CURSOR_PROVIDER)) {
|
|
4599
4907
|
return new CursorProvider();
|
|
4600
4908
|
}
|
|
4909
|
+
if (previous?.provider === COPILOT_PROVIDER) return new CopilotProvider();
|
|
4601
4910
|
if (previous?.provider) return void 0;
|
|
4602
4911
|
let sample = "";
|
|
4603
4912
|
try {
|
|
4604
|
-
const fd = (0,
|
|
4913
|
+
const fd = (0, import_fs13.openSync)(filePath, "r");
|
|
4605
4914
|
try {
|
|
4606
4915
|
const buf = Buffer.alloc(8192);
|
|
4607
|
-
const n = (0,
|
|
4916
|
+
const n = (0, import_fs13.readSync)(fd, buf, 0, buf.length, 0);
|
|
4608
4917
|
sample = buf.subarray(0, n).toString("utf8");
|
|
4609
4918
|
} finally {
|
|
4610
|
-
(0,
|
|
4919
|
+
(0, import_fs13.closeSync)(fd);
|
|
4611
4920
|
}
|
|
4612
4921
|
} catch {
|
|
4613
4922
|
return void 0;
|
|
@@ -4616,6 +4925,8 @@ var ConversationScanner = class {
|
|
|
4616
4925
|
if (cursor.canParse(filePath, sample)) return cursor;
|
|
4617
4926
|
const codex = new CodexCliProvider();
|
|
4618
4927
|
if (codex.canParse(filePath, sample)) return codex;
|
|
4928
|
+
const copilot = new CopilotProvider();
|
|
4929
|
+
if (copilot.canParse(filePath, sample)) return copilot;
|
|
4619
4930
|
return void 0;
|
|
4620
4931
|
}
|
|
4621
4932
|
addToSessionIndex(meta) {
|
|
@@ -4725,9 +5036,11 @@ async function getConversation(id, options, scanner) {
|
|
|
4725
5036
|
0 && (module.exports = {
|
|
4726
5037
|
CLAUDE_CODE_PROVIDER,
|
|
4727
5038
|
CODEX_CLI_PROVIDER,
|
|
5039
|
+
COPILOT_PROVIDER,
|
|
4728
5040
|
CURSOR_PROVIDER,
|
|
4729
5041
|
CodexCliProvider,
|
|
4730
5042
|
ConversationScanner,
|
|
5043
|
+
CopilotProvider,
|
|
4731
5044
|
CursorProvider,
|
|
4732
5045
|
DEFAULT_TIERS,
|
|
4733
5046
|
LEGACY_CURSOR_PROVIDER,
|
|
@@ -4752,6 +5065,7 @@ async function getConversation(id, options, scanner) {
|
|
|
4752
5065
|
isCodexInjectedContext,
|
|
4753
5066
|
loadProfiles,
|
|
4754
5067
|
parseCodexJsonlLine,
|
|
5068
|
+
parseCopilotJsonlLine,
|
|
4755
5069
|
parseCursorJsonlLine,
|
|
4756
5070
|
parseJsonlLine,
|
|
4757
5071
|
providerMatches,
|