hacklab 0.22.2 → 26.921.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/README.md +23 -430
  2. package/dist/index.js +132 -23
  3. package/package.json +20 -42
  4. package/LICENSE +0 -21
  5. package/dist/agent-handoff.d.ts +0 -51
  6. package/dist/agent-handoff.d.ts.map +0 -1
  7. package/dist/agent-handoff.js +0 -91
  8. package/dist/agent-handoff.js.map +0 -1
  9. package/dist/api-client.d.ts +0 -18
  10. package/dist/api-client.d.ts.map +0 -1
  11. package/dist/api-client.js +0 -40
  12. package/dist/api-client.js.map +0 -1
  13. package/dist/belt.d.ts +0 -25
  14. package/dist/belt.d.ts.map +0 -1
  15. package/dist/belt.js +0 -113
  16. package/dist/belt.js.map +0 -1
  17. package/dist/card.d.ts +0 -66
  18. package/dist/card.d.ts.map +0 -1
  19. package/dist/card.js +0 -151
  20. package/dist/card.js.map +0 -1
  21. package/dist/cli.d.ts +0 -2
  22. package/dist/cli.d.ts.map +0 -1
  23. package/dist/cli.js +0 -238
  24. package/dist/cli.js.map +0 -1
  25. package/dist/commands/book.d.ts +0 -14
  26. package/dist/commands/book.d.ts.map +0 -1
  27. package/dist/commands/book.js +0 -90
  28. package/dist/commands/book.js.map +0 -1
  29. package/dist/commands/chat.d.ts +0 -36
  30. package/dist/commands/chat.d.ts.map +0 -1
  31. package/dist/commands/chat.js +0 -642
  32. package/dist/commands/chat.js.map +0 -1
  33. package/dist/commands/config.d.ts +0 -2
  34. package/dist/commands/config.d.ts.map +0 -1
  35. package/dist/commands/config.js +0 -84
  36. package/dist/commands/config.js.map +0 -1
  37. package/dist/commands/daemon.d.ts +0 -19
  38. package/dist/commands/daemon.d.ts.map +0 -1
  39. package/dist/commands/daemon.js +0 -72
  40. package/dist/commands/daemon.js.map +0 -1
  41. package/dist/commands/drop.d.ts +0 -12
  42. package/dist/commands/drop.d.ts.map +0 -1
  43. package/dist/commands/drop.js +0 -103
  44. package/dist/commands/drop.js.map +0 -1
  45. package/dist/commands/drops.d.ts +0 -9
  46. package/dist/commands/drops.d.ts.map +0 -1
  47. package/dist/commands/drops.js +0 -96
  48. package/dist/commands/drops.js.map +0 -1
  49. package/dist/commands/essay.d.ts +0 -40
  50. package/dist/commands/essay.d.ts.map +0 -1
  51. package/dist/commands/essay.js +0 -651
  52. package/dist/commands/essay.js.map +0 -1
  53. package/dist/commands/hackathon.d.ts +0 -2
  54. package/dist/commands/hackathon.d.ts.map +0 -1
  55. package/dist/commands/hackathon.js +0 -893
  56. package/dist/commands/hackathon.js.map +0 -1
  57. package/dist/commands/hacker.d.ts +0 -2
  58. package/dist/commands/hacker.d.ts.map +0 -1
  59. package/dist/commands/hacker.js +0 -87
  60. package/dist/commands/hacker.js.map +0 -1
  61. package/dist/commands/jobs.d.ts +0 -18
  62. package/dist/commands/jobs.d.ts.map +0 -1
  63. package/dist/commands/jobs.js +0 -138
  64. package/dist/commands/jobs.js.map +0 -1
  65. package/dist/commands/keys.d.ts +0 -2
  66. package/dist/commands/keys.d.ts.map +0 -1
  67. package/dist/commands/keys.js +0 -138
  68. package/dist/commands/keys.js.map +0 -1
  69. package/dist/commands/login.d.ts +0 -83
  70. package/dist/commands/login.d.ts.map +0 -1
  71. package/dist/commands/login.js +0 -236
  72. package/dist/commands/login.js.map +0 -1
  73. package/dist/commands/logout.d.ts +0 -2
  74. package/dist/commands/logout.d.ts.map +0 -1
  75. package/dist/commands/logout.js +0 -22
  76. package/dist/commands/logout.js.map +0 -1
  77. package/dist/commands/org-jobs.d.ts +0 -50
  78. package/dist/commands/org-jobs.d.ts.map +0 -1
  79. package/dist/commands/org-jobs.js +0 -383
  80. package/dist/commands/org-jobs.js.map +0 -1
  81. package/dist/commands/org.d.ts +0 -67
  82. package/dist/commands/org.d.ts.map +0 -1
  83. package/dist/commands/org.js +0 -1244
  84. package/dist/commands/org.js.map +0 -1
  85. package/dist/commands/ping.d.ts +0 -2
  86. package/dist/commands/ping.d.ts.map +0 -1
  87. package/dist/commands/ping.js +0 -50
  88. package/dist/commands/ping.js.map +0 -1
  89. package/dist/commands/profile.d.ts +0 -72
  90. package/dist/commands/profile.d.ts.map +0 -1
  91. package/dist/commands/profile.js +0 -672
  92. package/dist/commands/profile.js.map +0 -1
  93. package/dist/commands/project.d.ts +0 -2
  94. package/dist/commands/project.d.ts.map +0 -1
  95. package/dist/commands/project.js +0 -583
  96. package/dist/commands/project.js.map +0 -1
  97. package/dist/commands/referral.d.ts +0 -6
  98. package/dist/commands/referral.d.ts.map +0 -1
  99. package/dist/commands/referral.js +0 -93
  100. package/dist/commands/referral.js.map +0 -1
  101. package/dist/commands/rtfm.d.ts +0 -9
  102. package/dist/commands/rtfm.d.ts.map +0 -1
  103. package/dist/commands/rtfm.js +0 -114
  104. package/dist/commands/rtfm.js.map +0 -1
  105. package/dist/commands/scan.d.ts +0 -13
  106. package/dist/commands/scan.d.ts.map +0 -1
  107. package/dist/commands/scan.js +0 -286
  108. package/dist/commands/scan.js.map +0 -1
  109. package/dist/commands/scout.d.ts +0 -18
  110. package/dist/commands/scout.d.ts.map +0 -1
  111. package/dist/commands/scout.js +0 -232
  112. package/dist/commands/scout.js.map +0 -1
  113. package/dist/commands/setup-rail.d.ts +0 -58
  114. package/dist/commands/setup-rail.d.ts.map +0 -1
  115. package/dist/commands/setup-rail.js +0 -135
  116. package/dist/commands/setup-rail.js.map +0 -1
  117. package/dist/commands/setup.d.ts +0 -29
  118. package/dist/commands/setup.d.ts.map +0 -1
  119. package/dist/commands/setup.js +0 -442
  120. package/dist/commands/setup.js.map +0 -1
  121. package/dist/commands/sync.d.ts +0 -9
  122. package/dist/commands/sync.d.ts.map +0 -1
  123. package/dist/commands/sync.js +0 -345
  124. package/dist/commands/sync.js.map +0 -1
  125. package/dist/commands/update.d.ts +0 -2
  126. package/dist/commands/update.d.ts.map +0 -1
  127. package/dist/commands/update.js +0 -92
  128. package/dist/commands/update.js.map +0 -1
  129. package/dist/commands/whoami.d.ts +0 -2
  130. package/dist/commands/whoami.d.ts.map +0 -1
  131. package/dist/commands/whoami.js +0 -41
  132. package/dist/commands/whoami.js.map +0 -1
  133. package/dist/config.d.ts +0 -78
  134. package/dist/config.d.ts.map +0 -1
  135. package/dist/config.js +0 -93
  136. package/dist/config.js.map +0 -1
  137. package/dist/daily-sync.d.ts +0 -150
  138. package/dist/daily-sync.d.ts.map +0 -1
  139. package/dist/daily-sync.js +0 -665
  140. package/dist/daily-sync.js.map +0 -1
  141. package/dist/dossier.d.ts +0 -73
  142. package/dist/dossier.d.ts.map +0 -1
  143. package/dist/dossier.js +0 -141
  144. package/dist/dossier.js.map +0 -1
  145. package/dist/field-normalize.d.ts +0 -4
  146. package/dist/field-normalize.d.ts.map +0 -1
  147. package/dist/field-normalize.js +0 -8
  148. package/dist/field-normalize.js.map +0 -1
  149. package/dist/index.d.ts +0 -3
  150. package/dist/index.d.ts.map +0 -1
  151. package/dist/index.js.map +0 -1
  152. package/dist/job-fields.d.ts +0 -47
  153. package/dist/job-fields.d.ts.map +0 -1
  154. package/dist/job-fields.js +0 -148
  155. package/dist/job-fields.js.map +0 -1
  156. package/dist/machine.d.ts +0 -15
  157. package/dist/machine.d.ts.map +0 -1
  158. package/dist/machine.js +0 -64
  159. package/dist/machine.js.map +0 -1
  160. package/dist/org-fields.d.ts +0 -57
  161. package/dist/org-fields.d.ts.map +0 -1
  162. package/dist/org-fields.js +0 -167
  163. package/dist/org-fields.js.map +0 -1
  164. package/dist/posthog.d.ts +0 -38
  165. package/dist/posthog.d.ts.map +0 -1
  166. package/dist/posthog.js +0 -224
  167. package/dist/posthog.js.map +0 -1
  168. package/dist/project-fields.d.ts +0 -40
  169. package/dist/project-fields.d.ts.map +0 -1
  170. package/dist/project-fields.js +0 -79
  171. package/dist/project-fields.js.map +0 -1
  172. package/dist/prompt-consent.d.ts +0 -73
  173. package/dist/prompt-consent.d.ts.map +0 -1
  174. package/dist/prompt-consent.js +0 -191
  175. package/dist/prompt-consent.js.map +0 -1
  176. package/dist/prompt-stats.d.ts +0 -226
  177. package/dist/prompt-stats.d.ts.map +0 -1
  178. package/dist/prompt-stats.js +0 -561
  179. package/dist/prompt-stats.js.map +0 -1
  180. package/dist/referral.d.ts +0 -14
  181. package/dist/referral.d.ts.map +0 -1
  182. package/dist/referral.js +0 -28
  183. package/dist/referral.js.map +0 -1
  184. package/dist/registry.d.ts +0 -15
  185. package/dist/registry.d.ts.map +0 -1
  186. package/dist/registry.js +0 -180
  187. package/dist/registry.js.map +0 -1
  188. package/dist/resolve-command.d.ts +0 -20
  189. package/dist/resolve-command.d.ts.map +0 -1
  190. package/dist/resolve-command.js +0 -21
  191. package/dist/resolve-command.js.map +0 -1
  192. package/dist/scanners/antigravity.d.ts +0 -41
  193. package/dist/scanners/antigravity.d.ts.map +0 -1
  194. package/dist/scanners/antigravity.js +0 -210
  195. package/dist/scanners/antigravity.js.map +0 -1
  196. package/dist/scanners/github-copilot-telemetry.d.ts +0 -23
  197. package/dist/scanners/github-copilot-telemetry.d.ts.map +0 -1
  198. package/dist/scanners/github-copilot-telemetry.js +0 -220
  199. package/dist/scanners/github-copilot-telemetry.js.map +0 -1
  200. package/dist/scanners/github-copilot.d.ts +0 -68
  201. package/dist/scanners/github-copilot.d.ts.map +0 -1
  202. package/dist/scanners/github-copilot.js +0 -262
  203. package/dist/scanners/github-copilot.js.map +0 -1
  204. package/dist/scanners/incremental.d.ts +0 -282
  205. package/dist/scanners/incremental.d.ts.map +0 -1
  206. package/dist/scanners/incremental.js +0 -788
  207. package/dist/scanners/incremental.js.map +0 -1
  208. package/dist/scanners/index.d.ts +0 -116
  209. package/dist/scanners/index.d.ts.map +0 -1
  210. package/dist/scanners/index.js +0 -726
  211. package/dist/scanners/index.js.map +0 -1
  212. package/dist/scanners/sqlite.d.ts +0 -18
  213. package/dist/scanners/sqlite.d.ts.map +0 -1
  214. package/dist/scanners/sqlite.js +0 -63
  215. package/dist/scanners/sqlite.js.map +0 -1
  216. package/dist/scanners/util.d.ts +0 -112
  217. package/dist/scanners/util.d.ts.map +0 -1
  218. package/dist/scanners/util.js +0 -167
  219. package/dist/scanners/util.js.map +0 -1
  220. package/dist/session.d.ts +0 -106
  221. package/dist/session.d.ts.map +0 -1
  222. package/dist/session.js +0 -191
  223. package/dist/session.js.map +0 -1
  224. package/dist/share-card-colors.d.ts +0 -19
  225. package/dist/share-card-colors.d.ts.map +0 -1
  226. package/dist/share-card-colors.js +0 -36
  227. package/dist/share-card-colors.js.map +0 -1
  228. package/dist/share-card.d.ts +0 -32
  229. package/dist/share-card.d.ts.map +0 -1
  230. package/dist/share-card.js +0 -420
  231. package/dist/share-card.js.map +0 -1
  232. package/dist/share.d.ts +0 -17
  233. package/dist/share.d.ts.map +0 -1
  234. package/dist/share.js +0 -77
  235. package/dist/share.js.map +0 -1
  236. package/dist/sync.d.ts +0 -100
  237. package/dist/sync.d.ts.map +0 -1
  238. package/dist/sync.js +0 -304
  239. package/dist/sync.js.map +0 -1
  240. package/dist/ui.d.ts +0 -33
  241. package/dist/ui.d.ts.map +0 -1
  242. package/dist/ui.js +0 -127
  243. package/dist/ui.js.map +0 -1
  244. package/dist/utils/npmGlobal.d.ts +0 -30
  245. package/dist/utils/npmGlobal.d.ts.map +0 -1
  246. package/dist/utils/npmGlobal.js +0 -69
  247. package/dist/utils/npmGlobal.js.map +0 -1
  248. package/dist/utils/openBrowser.d.ts +0 -14
  249. package/dist/utils/openBrowser.d.ts.map +0 -1
  250. package/dist/utils/openBrowser.js +0 -47
  251. package/dist/utils/openBrowser.js.map +0 -1
  252. package/dist/utils/updateCheck.d.ts +0 -16
  253. package/dist/utils/updateCheck.d.ts.map +0 -1
  254. package/dist/utils/updateCheck.js +0 -111
  255. package/dist/utils/updateCheck.js.map +0 -1
  256. package/dist/utils/waitForEnter.d.ts +0 -12
  257. package/dist/utils/waitForEnter.d.ts.map +0 -1
  258. package/dist/utils/waitForEnter.js +0 -43
  259. package/dist/utils/waitForEnter.js.map +0 -1
@@ -1,788 +0,0 @@
1
- import { mkdir, open, readFile, stat, writeFile } from 'node:fs/promises';
2
- import { dirname, join } from 'node:path';
3
- import { addPromptToActivity, countWords, emptyPromptActivity, IDE_PROMPT_SOURCES, mergePromptActivities, parsePromptLine, } from '../prompt-stats.js';
4
- import { getSessionPath } from '../session.js';
5
- import { antigravityTokenFiles, scanAntigravity } from './antigravity.js';
6
- import { githubCopilotTokenFiles, scanGitHubCopilot } from './github-copilot.js';
7
- import { claudeCodeFiles, codexDateForFile, codexFiles, codexFileTotals, dateDaysAgo, grokLogFiles, hermesDbPath, openclawFiles, opencodeDbPath, PROMPT_ACTIVITY_DATE_CAP, PROMPT_ACTIVITY_SESSION_CAP, parseClaudeCodeLine, parseGrokLine, parseOpenclawLine, scanHermes, scanOpenCode, toDateStr, } from './index.js';
8
- // The incremental half of the scanners: what `hacklab sync --tick` runs every
9
- // minute instead of the full stateless re-scan the daily job does.
10
- //
11
- // The idea is a tail-follow. Everything the last scan already accounted for is
12
- // kept in scan-state.json — per-file read offsets plus the aggregates those
13
- // bytes produced — so a tick only has to stat the log tree and parse the bytes
14
- // that were appended since. Lines are parsed by the exact same functions the
15
- // full scanners use (parseClaudeCodeLine / parseOpenclawLine / codexFileTotals),
16
- // so the tail can be added to aggregates the full scan produced without drift.
17
- //
18
- // Anything that makes the tail-follow unsound (a file shrank, or one we were
19
- // following disappeared) drops that harness back to a full re-read, because we
20
- // don't track which file contributed which tokens. And the daily `sync --quiet`
21
- // rebuilds the whole state from its full scan, so drift can never last a day.
22
- /**
23
- * Bumping this makes `loadScanState` return null, which re-reads every log
24
- * from scratch and re-derives the aggregates.
25
- *
26
- * 3: the prompt scanner learned to skip harness-written `user` entries
27
- * (PROMPT_SCANNER_VERSION 2). The server is wiping the prompt tables once that
28
- * ships, and `replacePromptActivity` only re-sends sessions and dates whose
29
- * numbers *moved* — so a day that happened to contain no harness noise would
30
- * stay identical locally and never be re-uploaded after the wipe. A cold
31
- * rebuild marks every session and date within local retention dirty exactly
32
- * once, and they all go back out.
33
- *
34
- * 4: the same wipe, for the machines 3 couldn't save. The wipe landed
35
- * 2026-09-10, and a machine that had already upgraded to 0.21.0 did its one
36
- * cold rebuild *before* that — it re-uploaded everything, cleared every dirty
37
- * flag, and then watched the server delete the lot. Nothing on that machine
38
- * moves afterwards, so `replacePromptActivity` re-sends nothing and the history
39
- * is simply gone. This bump is the one-shot rescue: another cold rebuild, after
40
- * the wipe this time. For anyone still on an older CLI it costs one extra
41
- * rescan and nothing else.
42
- *
43
- * 5: prompt activity is partitioned by source, so rebuilding a Claude log or
44
- * rotating an IDE transcript cannot erase or double-count another harness.
45
- *
46
- * Token totals are unaffected: they travel as cumulative absolutes that the
47
- * server diffs against its own per-machine snapshot, and `state.uploaded` is
48
- * only the tick's "nothing moved" short-circuit — so a rebuild re-baselines
49
- * rather than double-counting.
50
- */
51
- export const SCAN_STATE_VERSION = 5;
52
- /** How long a session is kept in the state after its last prompt. */
53
- export const PROMPT_SESSION_RETENTION_DAYS = 45;
54
- /** How long a per-day prompt tally is kept. Matches the wire's date cap. */
55
- export const PROMPT_DAILY_RETENTION_DAYS = PROMPT_ACTIVITY_DATE_CAP;
56
- /** Lives next to the session file, so it honors HACKLAB_SESSION_PATH. */
57
- export function scanStatePath() {
58
- return join(dirname(getSessionPath()), 'scan-state.json');
59
- }
60
- export function emptyHarness() {
61
- return { files: {}, daily: {}, models: {} };
62
- }
63
- export function emptyPromptState() {
64
- return { ...emptyPromptActivity(), dirtySessions: [], dirtyDates: [] };
65
- }
66
- export function emptyState() {
67
- return {
68
- version: SCAN_STATE_VERSION,
69
- harnesses: {},
70
- dirty: [],
71
- prompts: emptyPromptState(),
72
- promptSources: {},
73
- promptFiles: {},
74
- uploaded: { toolTotals: {}, modelTotals: {} },
75
- };
76
- }
77
- function harness(state, tool) {
78
- const existing = state.harnesses[tool];
79
- if (existing)
80
- return existing;
81
- const fresh = emptyHarness();
82
- state.harnesses[tool] = fresh;
83
- return fresh;
84
- }
85
- /**
86
- * The saved state, or null when there isn't a usable one — missing, corrupt, or
87
- * written by an older layout. Null means "re-read everything from scratch";
88
- * callers must treat that rebuild as a baseline, not as new tokens.
89
- */
90
- export async function loadScanState() {
91
- try {
92
- const parsed = JSON.parse(await readFile(scanStatePath(), 'utf8'));
93
- if (parsed?.version !== SCAN_STATE_VERSION)
94
- return null;
95
- if (!parsed.harnesses || typeof parsed.harnesses !== 'object')
96
- return null;
97
- parsed.dirty ??= [];
98
- parsed.prompts ??= emptyPromptState();
99
- parsed.uploaded ??= { toolTotals: {}, modelTotals: {} };
100
- return parsed;
101
- }
102
- catch {
103
- return null;
104
- }
105
- }
106
- /**
107
- * Drop prompt rows that have aged out, dirty or not.
108
- *
109
- * Dropping a dirty row means it is never uploaded, which is the right trade at
110
- * this age: a session or a day that has sat unsent for its whole retention
111
- * window is not coming back, and the alternative is a dirty list that grows
112
- * forever on a machine that never consented (the tick still counts prompts at
113
- * the `none` tier — it just never sends them).
114
- */
115
- function prunePromptAggregate(prompts) {
116
- const sessionCutoff = dateDaysAgo(PROMPT_SESSION_RETENTION_DAYS);
117
- for (const [id, session] of Object.entries(prompts.sessions)) {
118
- if (session.lastActiveAt.slice(0, 10) < sessionCutoff) {
119
- delete prompts.sessions[id];
120
- }
121
- }
122
- const dateCutoff = dateDaysAgo(PROMPT_DAILY_RETENTION_DAYS);
123
- for (const date of Object.keys(prompts.daily)) {
124
- if (date < dateCutoff)
125
- delete prompts.daily[date];
126
- }
127
- }
128
- function prunePrompts(prompts) {
129
- prunePromptAggregate(prompts);
130
- prompts.dirtySessions = prompts.dirtySessions.filter((id) => prompts.sessions[id] !== undefined);
131
- prompts.dirtyDates = prompts.dirtyDates.filter((date) => prompts.daily[date] !== undefined);
132
- }
133
- /** Persist the state, pruning prompt rows that aged out of their retention
134
- * window (they'd otherwise accumulate forever). Best-effort. */
135
- export async function saveScanState(state) {
136
- prunePrompts(state.prompts);
137
- for (const source of Object.values(state.promptSources)) {
138
- prunePromptAggregate(source);
139
- }
140
- try {
141
- await mkdir(dirname(scanStatePath()), { recursive: true });
142
- await writeFile(scanStatePath(), `${JSON.stringify(state)}\n`, 'utf8');
143
- }
144
- catch {
145
- // a state we can't persist just means the next tick re-reads everything
146
- }
147
- }
148
- // ---- aggregate bookkeeping -------------------------------------------------
149
- function addDaily(h, date, model, tokens, messages, dirty) {
150
- const key = `${date}|${model}`;
151
- const existing = h.daily[key];
152
- if (existing) {
153
- existing.tokens += tokens;
154
- existing.messages += messages;
155
- }
156
- else {
157
- h.daily[key] = { tokens, messages };
158
- }
159
- if (model)
160
- h.models[model] = (h.models[model] ?? 0) + tokens;
161
- dirty.add(date);
162
- }
163
- /** A full scanner's result as the maps the state stores. */
164
- export function aggregatesOfResult(result) {
165
- const daily = {};
166
- for (const entry of result.daily) {
167
- daily[`${entry.date}|${entry.model ?? ''}`] = {
168
- tokens: entry.tokens,
169
- messages: entry.messages ?? 0,
170
- };
171
- }
172
- return { daily, models: { ...result.models } };
173
- }
174
- function aggregatesOf(h) {
175
- return { daily: structuredClone(h.daily), models: { ...h.models } };
176
- }
177
- /**
178
- * Mark every date whose rows moved between two versions of a harness's
179
- * aggregates. Used whenever a harness is re-read rather than tailed (a SQLite
180
- * requery, or a JSONL harness whose tail-follow was invalidated): most dates
181
- * come back identical, so diffing keeps the upload down to what actually
182
- * changed instead of re-sending the user's whole history.
183
- */
184
- function markMovedDates(before, after, dirty) {
185
- const compare = (a, b) => {
186
- for (const key of new Set([...Object.keys(a), ...Object.keys(b)])) {
187
- const x = a[key];
188
- const y = b[key];
189
- if (!x || !y || x.tokens !== y.tokens || x.messages !== y.messages) {
190
- dirty.add(key.split('|')[0] ?? '');
191
- }
192
- }
193
- };
194
- compare(before.daily, after.daily);
195
- }
196
- /** Swap in freshly scanned aggregates, dirtying the dates that moved. */
197
- function replaceAggregates(h, next, dirty) {
198
- markMovedDates({ daily: h.daily, models: h.models }, next, dirty);
199
- h.daily = next.daily;
200
- h.models = next.models;
201
- }
202
- /**
203
- * Swap in a freshly scanned prompt aggregate, dirtying every session and date
204
- * whose numbers moved. Same idea as `markMovedDates`: a daily full scan mostly
205
- * reproduces what the ticks already sent, and only the difference is worth
206
- * putting back on the wire.
207
- */
208
- function replacePromptActivity(prompts, next) {
209
- const dirtySessions = new Set(prompts.dirtySessions);
210
- for (const id of new Set([
211
- ...Object.keys(prompts.sessions),
212
- ...Object.keys(next.sessions),
213
- ])) {
214
- const before = prompts.sessions[id];
215
- const after = next.sessions[id];
216
- if (!after)
217
- continue;
218
- if (!before ||
219
- before.promptCount !== after.promptCount ||
220
- before.startedAt !== after.startedAt ||
221
- before.lastActiveAt !== after.lastActiveAt) {
222
- dirtySessions.add(id);
223
- }
224
- }
225
- const dirtyDates = new Set(prompts.dirtyDates);
226
- for (const date of new Set([
227
- ...Object.keys(prompts.daily),
228
- ...Object.keys(next.daily),
229
- ])) {
230
- const before = prompts.daily[date];
231
- const after = next.daily[date];
232
- if (!after)
233
- continue;
234
- if (!before ||
235
- before.prompts !== after.prompts ||
236
- before.words !== after.words) {
237
- dirtyDates.add(date);
238
- }
239
- }
240
- prompts.sessions = next.sessions;
241
- prompts.daily = next.daily;
242
- // A session or date the full scan can no longer see has nothing left to say.
243
- prompts.dirtySessions = [...dirtySessions].filter((id) => next.sessions[id] !== undefined);
244
- prompts.dirtyDates = [...dirtyDates]
245
- .filter((date) => next.daily[date] !== undefined)
246
- .sort();
247
- }
248
- // ---- reading the tail ------------------------------------------------------
249
- async function statOrNull(path) {
250
- try {
251
- return await stat(path);
252
- }
253
- catch {
254
- return null;
255
- }
256
- }
257
- /**
258
- * Read `[from, to)` and hand back only whole lines. A harness may be mid-append
259
- * when we look, so the trailing partial line is left for the next tick — that's
260
- * what the returned offset points at.
261
- */
262
- export async function readCompleteLines(path, from, to) {
263
- if (to <= from)
264
- return { text: '', offset: from };
265
- const handle = await open(path, 'r');
266
- try {
267
- const buf = Buffer.alloc(to - from);
268
- const { bytesRead } = await handle.read(buf, 0, buf.length, from);
269
- const chunk = buf.subarray(0, bytesRead);
270
- // Byte-wise, not string-wise: the chunk can end mid-codepoint.
271
- const lastNewline = chunk.lastIndexOf(0x0a);
272
- if (lastNewline < 0)
273
- return { text: '', offset: from };
274
- return {
275
- text: chunk.subarray(0, lastNewline).toString('utf8'),
276
- offset: from + lastNewline + 1,
277
- };
278
- }
279
- finally {
280
- await handle.close();
281
- }
282
- }
283
- /** The real harnesses. Injectable so the tick can be tested against a tmp dir. */
284
- export function defaultSources() {
285
- return {
286
- jsonl: [
287
- {
288
- tool: 'claude_code',
289
- files: claudeCodeFiles,
290
- parse: parseClaudeCodeLine,
291
- parsePrompt: parsePromptLine,
292
- },
293
- { tool: 'openclaw', files: openclawFiles, parse: parseOpenclawLine },
294
- { tool: 'grok', files: grokLogFiles, parse: parseGrokLine },
295
- ],
296
- codex: { files: codexFiles, dateFor: codexDateForFile },
297
- sqlite: [
298
- { tool: 'hermes', dbPath: hermesDbPath, scan: scanHermes },
299
- { tool: 'opencode', dbPath: opencodeDbPath, scan: scanOpenCode },
300
- ],
301
- prompts: IDE_PROMPT_SOURCES,
302
- snapshots: [
303
- {
304
- tool: 'github_copilot',
305
- files: githubCopilotTokenFiles,
306
- scan: scanGitHubCopilot,
307
- },
308
- {
309
- tool: 'antigravity',
310
- files: antigravityTokenFiles,
311
- scan: scanAntigravity,
312
- },
313
- ],
314
- };
315
- }
316
- async function statFiles(paths) {
317
- const out = [];
318
- for (const path of paths) {
319
- const s = await statOrNull(path);
320
- if (s)
321
- out.push({ path, size: s.size, mtimeMs: s.mtimeMs, mtime: s.mtime });
322
- }
323
- return out;
324
- }
325
- /**
326
- * Can we still trust this harness's aggregates? Not if a file we were following
327
- * lost bytes or vanished: we know what those files contributed in total but not
328
- * per file, so there's nothing to subtract and the only repair is a re-read.
329
- */
330
- function tailInvalidated(h, files) {
331
- const present = new Map(files.map((f) => [f.path, f]));
332
- for (const [path, state] of Object.entries(h.files)) {
333
- const now = present.get(path);
334
- if (!now || now.size < state.offset)
335
- return true;
336
- }
337
- return false;
338
- }
339
- function resetHarness(h) {
340
- h.files = {};
341
- if (h.codexTotals)
342
- h.codexTotals = {};
343
- h.daily = {};
344
- h.models = {};
345
- }
346
- async function tickJsonl(state, src, dirty) {
347
- const h = harness(state, src.tool);
348
- const files = await statFiles(await src.files());
349
- const invalidated = tailInvalidated(h, files);
350
- // On a re-read, collect into a throwaway set and diff at the end: most of the
351
- // rebuilt aggregate is identical to what we had, and only the difference is
352
- // worth re-uploading.
353
- const before = invalidated ? aggregatesOf(h) : null;
354
- const target = invalidated ? new Set() : dirty;
355
- if (invalidated)
356
- resetHarness(h);
357
- // Keep each source independent: a rewritten Claude transcript must not
358
- // discard Copilot/Antigravity activity on the same date.
359
- if (src.parsePrompt && (invalidated || !state.promptSources[src.tool])) {
360
- state.promptSources[src.tool] = emptyPromptActivity();
361
- }
362
- const prompts = state.promptSources[src.tool];
363
- let changed = invalidated;
364
- for (const file of files) {
365
- const prev = h.files[file.path];
366
- if (prev && prev.size === file.size && prev.mtimeMs === file.mtimeMs) {
367
- continue;
368
- }
369
- const from = prev?.offset ?? 0;
370
- const { text, offset } = await readCompleteLines(file.path, from, file.size).catch(() => ({ text: '', offset: from }));
371
- let fallbackDate = null;
372
- for (const line of text.split('\n')) {
373
- const prompt = src.parsePrompt?.(line);
374
- if (prompt && prompts) {
375
- addPromptToActivity(prompts, prompt);
376
- }
377
- const usage = src.parse(line);
378
- if (!usage)
379
- continue;
380
- let date = usage.date;
381
- if (!date) {
382
- fallbackDate ??= toDateStr(file.mtime);
383
- date = fallbackDate;
384
- }
385
- addDaily(h, date, usage.model, usage.tokens, 1, target);
386
- }
387
- h.files[file.path] = { size: file.size, mtimeMs: file.mtimeMs, offset };
388
- changed = true;
389
- }
390
- if (before)
391
- markMovedDates(before, aggregatesOf(h), dirty);
392
- if (src.parsePrompt && changed) {
393
- replacePromptActivity(state.prompts, mergePromptActivities(Object.values(state.promptSources)));
394
- }
395
- return changed;
396
- }
397
- async function tickCodex(state, src, dirty) {
398
- const h = harness(state, 'codex');
399
- h.codexTotals ??= {};
400
- const files = await statFiles(await src.files());
401
- const invalidated = tailInvalidated(h, files);
402
- const before = invalidated ? aggregatesOf(h) : null;
403
- const target = invalidated ? new Set() : dirty;
404
- if (invalidated) {
405
- resetHarness(h);
406
- h.codexTotals = {};
407
- }
408
- let changed = invalidated;
409
- for (const file of files) {
410
- const prev = h.files[file.path];
411
- if (prev && prev.size === file.size && prev.mtimeMs === file.mtimeMs) {
412
- continue;
413
- }
414
- const from = prev?.offset ?? 0;
415
- const { text, offset } = await readCompleteLines(file.path, from, file.size).catch(() => ({ text: '', offset: from }));
416
- const { maxTotal, model } = codexFileTotals(text);
417
- const stored = h.codexTotals[file.path];
418
- const date = stored?.date ?? src.dateFor(file.path) ?? toDateStr(file.mtime);
419
- const storedMax = stored?.maxTotal ?? 0;
420
- const nextMax = Math.max(storedMax, maxTotal);
421
- // A session that switches model mid-file leaves the earlier tokens under the
422
- // old model — the full scan credits them all to the last model seen. The
423
- // daily repair pass reconciles that.
424
- const nextModel = model || stored?.model || '';
425
- if (nextMax > storedMax) {
426
- // Running totals: only the growth is new. `messages` counts the session
427
- // once, the way the full scan does (one addDaily per file).
428
- addDaily(h, date, nextModel, nextMax - storedMax, storedMax === 0 ? 1 : 0, target);
429
- }
430
- h.codexTotals[file.path] = { maxTotal: nextMax, model: nextModel, date };
431
- h.files[file.path] = { size: file.size, mtimeMs: file.mtimeMs, offset };
432
- changed = true;
433
- }
434
- if (before)
435
- markMovedDates(before, aggregatesOf(h), dirty);
436
- return changed;
437
- }
438
- /**
439
- * SQLite-backed harnesses can't be tailed — a write lands anywhere in the file —
440
- * but their DBs are small, so the incremental strategy is to fingerprint the db
441
- * and its -wal and re-run the harness's normal query only when one moved.
442
- */
443
- async function tickSqlite(state, src, dirty) {
444
- const h = harness(state, src.tool);
445
- const path = src.dbPath();
446
- const db = await statOrNull(path);
447
- const wal = await statOrNull(`${path}-wal`);
448
- const dbMtimeMs = db?.mtimeMs ?? 0;
449
- const walMtimeMs = wal?.mtimeMs ?? 0;
450
- if (h.dbMtimeMs === dbMtimeMs && h.walMtimeMs === walMtimeMs)
451
- return false;
452
- replaceAggregates(h, aggregatesOfResult(await src.scan()), dirty);
453
- h.dbMtimeMs = dbMtimeMs;
454
- h.walMtimeMs = walMtimeMs;
455
- return true;
456
- }
457
- /** Cumulative session records need replacement, never additive tail parsing. */
458
- async function tickSnapshot(state, source, dirty) {
459
- const files = await statFiles(await source.files());
460
- const h = harness(state, source.tool);
461
- if (Object.keys(h.files).length === files.length &&
462
- files.every((file) => h.files[file.path]?.size === file.size &&
463
- h.files[file.path]?.mtimeMs === file.mtimeMs))
464
- return false;
465
- replaceAggregates(h, aggregatesOfResult(await source.scan()), dirty);
466
- h.files = Object.fromEntries(files.map((file) => [
467
- file.path,
468
- { size: file.size, mtimeMs: file.mtimeMs, offset: 0 },
469
- ]));
470
- return true;
471
- }
472
- async function tickPromptSnapshot(state, source) {
473
- const files = await statFiles(await source.files());
474
- const previous = state.promptFiles[source.id];
475
- if (previous &&
476
- Object.keys(previous).length === files.length &&
477
- files.every((file) => previous[file.path]?.size === file.size &&
478
- previous[file.path]?.mtimeMs === file.mtimeMs))
479
- return false;
480
- const activity = emptyPromptActivity();
481
- for (const prompt of await source.scan()) {
482
- if (!prompt.timestamp)
483
- continue;
484
- const words = countWords(prompt.text);
485
- if (words > 0) {
486
- addPromptToActivity(activity, {
487
- sessionId: prompt.sessionId,
488
- timestamp: prompt.timestamp,
489
- words,
490
- });
491
- }
492
- }
493
- state.promptSources[source.id] = activity;
494
- state.promptFiles[source.id] = Object.fromEntries(files.map((file) => [
495
- file.path,
496
- { size: file.size, mtimeMs: file.mtimeMs, offset: 0 },
497
- ]));
498
- replacePromptActivity(state.prompts, mergePromptActivities(Object.values(state.promptSources)));
499
- return true;
500
- }
501
- /**
502
- * One incremental pass over every harness. `prev` is the saved state, or null
503
- * for a cold start — in which case this reads everything and marks no *token*
504
- * date dirty: those tokens are already on the server (the daily job put them
505
- * there), and re-uploading every date would only invite the reconcile to prune
506
- * rows this scan can't see (Cursor's, above all).
507
- *
508
- * Prompt rows go the other way on a cold start: everything found is marked
509
- * dirty. They're idempotent GREATEST upserts, so a resend costs nothing, and a
510
- * cold start (fresh install, or a state file this version can't read) is
511
- * exactly the case where we have no evidence any of it ever reached the server.
512
- */
513
- export async function runTick(prev, sources = defaultSources(), options = {}) {
514
- const cold = prev === null;
515
- const state = prev ?? emptyState();
516
- const dirty = new Set(state.dirty);
517
- let changed = cold;
518
- for (const src of sources.jsonl) {
519
- if (await tickJsonl(state, src, dirty))
520
- changed = true;
521
- }
522
- if (await tickCodex(state, sources.codex, dirty))
523
- changed = true;
524
- for (const src of sources.sqlite) {
525
- if (await tickSqlite(state, src, dirty))
526
- changed = true;
527
- }
528
- for (const source of sources.snapshots ?? []) {
529
- if (await tickSnapshot(state, source, dirty))
530
- changed = true;
531
- }
532
- if (options.promptActivity !== false) {
533
- for (const source of sources.prompts ?? []) {
534
- if (await tickPromptSnapshot(state, source))
535
- changed = true;
536
- }
537
- }
538
- if (cold) {
539
- state.dirty = [];
540
- state.prompts.dirtySessions = Object.keys(state.prompts.sessions);
541
- state.prompts.dirtyDates = Object.keys(state.prompts.daily).sort();
542
- }
543
- else {
544
- state.dirty = [...dirty].sort();
545
- }
546
- return { state, changed };
547
- }
548
- // ---- payload ---------------------------------------------------------------
549
- /** Cumulative per-tool and per-model totals for this machine. */
550
- export function cumulativeTotals(state) {
551
- const toolTotals = {};
552
- const modelTotals = {};
553
- for (const [tool, h] of Object.entries(state.harnesses)) {
554
- let sum = 0;
555
- for (const value of Object.values(h.daily))
556
- sum += value.tokens;
557
- toolTotals[tool] = sum;
558
- for (const [model, tokens] of Object.entries(h.models)) {
559
- modelTotals[model] = (modelTotals[model] ?? 0) + tokens;
560
- }
561
- }
562
- return { toolTotals, modelTotals };
563
- }
564
- export function sameTotals(a, b) {
565
- const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
566
- for (const key of keys)
567
- if ((a[key] ?? 0) !== (b[key] ?? 0))
568
- return false;
569
- return true;
570
- }
571
- /**
572
- * Is there prompt activity waiting to go out?
573
- *
574
- * A dirty id whose row is gone (pruned between the mark and the send) doesn't
575
- * count: it would have the tick POST an empty block every minute forever, since
576
- * there'd be nothing for `markUploaded` to clear.
577
- */
578
- export function hasPromptActivity(state) {
579
- const { prompts } = state;
580
- return (prompts.dirtySessions.some((id) => prompts.sessions[id] !== undefined) ||
581
- prompts.dirtyDates.some((date) => prompts.daily[date] !== undefined));
582
- }
583
- /**
584
- * Stake a claim on the outstanding prompt rows: the ones this payload carries
585
- * are recorded in `prompts.sent`, and `markUploaded` clears exactly those. Rows
586
- * a cap held back stay dirty and go out next tick.
587
- *
588
- * Most-recently-active first, so a machine with a long backlog syncs the work
589
- * the user actually cares about before the archaeology.
590
- */
591
- function takePromptActivity(state) {
592
- const { prompts } = state;
593
- const sessions = prompts.dirtySessions
594
- .flatMap((sessionId) => {
595
- const session = prompts.sessions[sessionId];
596
- return session ? [{ sessionId, ...session }] : [];
597
- })
598
- .sort((a, b) => b.lastActiveAt.localeCompare(a.lastActiveAt))
599
- .slice(0, PROMPT_ACTIVITY_SESSION_CAP);
600
- const dates = [...prompts.dirtyDates]
601
- .sort((a, b) => b.localeCompare(a))
602
- .slice(0, PROMPT_ACTIVITY_DATE_CAP);
603
- const dailyPrompts = dates.flatMap((date) => {
604
- const day = prompts.daily[date];
605
- return day ? [{ date, ...day }] : [];
606
- });
607
- prompts.sent = { sessions: sessions.map((s) => s.sessionId), dates };
608
- if (sessions.length === 0 && dailyPrompts.length === 0)
609
- return undefined;
610
- return { sessions, dailyPrompts };
611
- }
612
- /**
613
- * What a tick uploads: cumulative tool/model totals (the server diffs those
614
- * against its own per-machine snapshot) plus the daily rows for the dirty dates
615
- * only, and — under the `stats` or `full` tier — the outstanding prompt
616
- * activity.
617
- *
618
- * Every harness's rows go out for a dirty date, including Cursor's — which the
619
- * tick never scans, and carries over from the last full scan instead. The
620
- * server reconciles a date by deleting this machine's rows for it that the
621
- * payload didn't mention, so a date has to be reported complete or the harnesses
622
- * left out of it lose that day.
623
- */
624
- export function tickPayload(state, opts = {}) {
625
- const dirty = new Set(state.dirty);
626
- const dailyTotals = [];
627
- for (const [tool, h] of Object.entries(state.harnesses)) {
628
- for (const [key, value] of Object.entries(h.daily)) {
629
- const [date = '', model = ''] = key.split('|');
630
- if (!dirty.has(date) || value.tokens <= 0)
631
- continue;
632
- dailyTotals.push({
633
- date,
634
- tool,
635
- tokens: value.tokens,
636
- messages: value.messages,
637
- model: model || undefined,
638
- });
639
- }
640
- }
641
- // Claim the prompt rows this payload carries, or drop a claim a previous
642
- // payload left behind: a stale one would have `markUploaded` clear rows this
643
- // upload never mentioned (a tier turned down to `none` between two ticks).
644
- state.prompts.sent = undefined;
645
- const promptActivity = opts.promptActivity
646
- ? takePromptActivity(state)
647
- : undefined;
648
- const { toolTotals, modelTotals } = cumulativeTotals(state);
649
- return {
650
- toolTotals,
651
- dailyTotals,
652
- modelTotals,
653
- grandTotal: Object.values(toolTotals).reduce((a, b) => a + b, 0),
654
- cursorStats: null,
655
- cursorScanStatus: { source: 'none' },
656
- ...(promptActivity ? { promptActivity } : {}),
657
- };
658
- }
659
- /** Called on a 200: whatever that payload carried is on the server now. */
660
- export function markUploaded(state) {
661
- state.dirty = [];
662
- const sent = state.prompts.sent;
663
- if (sent) {
664
- const sessions = new Set(sent.sessions);
665
- const dates = new Set(sent.dates);
666
- state.prompts.dirtySessions = state.prompts.dirtySessions.filter((id) => !sessions.has(id));
667
- state.prompts.dirtyDates = state.prompts.dirtyDates.filter((date) => !dates.has(date));
668
- state.prompts.sent = undefined;
669
- }
670
- state.uploaded = cumulativeTotals(state);
671
- state.lastError = undefined;
672
- state.nextAllowedAt = undefined;
673
- }
674
- // ---- the daily repair pass -------------------------------------------------
675
- async function snapshotFileCursors(h, paths) {
676
- for (const file of await statFiles(paths)) {
677
- // Offsets are set to EOF *after* the full scan read the file, so a file that
678
- // grew mid-scan is re-read from the end rather than double-counted — the
679
- // next repair picks up whatever slipped through.
680
- h.files[file.path] = {
681
- size: file.size,
682
- mtimeMs: file.mtimeMs,
683
- offset: file.size,
684
- };
685
- }
686
- }
687
- /**
688
- * Rebuild the whole state from a full scan's results — the self-healing half of
689
- * the design. Whatever the tick's tail-follow got wrong (a rotated log, a
690
- * mid-file model switch, a Codex date guessed from an mtime) is corrected here,
691
- * so incremental drift can never outlive a day. Best-effort: a state we failed
692
- * to rebuild costs a tick, never the sync that called us.
693
- *
694
- * Staged rather than persisted outright, because the full-scan paths carry
695
- * prompt activity too — a machine with no daemon has no tick to drain it. The
696
- * rebuild diffs the scan's aggregate against what was already sent, claims the
697
- * difference for this upload, and only `commit` (on a 200) clears it. A failed
698
- * POST therefore leaves every row dirty for the next run, which the server's
699
- * GREATEST upserts make free to resend.
700
- *
701
- * `prompts` says whether this caller has any authority over the prompt state —
702
- * see `StagedPrompts`. Only a caller that actually resolved the consent tier
703
- * gets to rewrite it, and only such a caller claims prompt rows for the upload.
704
- */
705
- export async function stageFullScan(results, prompts = 'untouched', sources = defaultSources()) {
706
- const nothingStaged = {
707
- commit: async () => {
708
- // Nothing was staged, so there is nothing to record as delivered.
709
- },
710
- };
711
- try {
712
- const previous = await loadScanState();
713
- const state = emptyState();
714
- state.prompts = previous?.prompts ?? emptyPromptState();
715
- state.promptSources = previous?.promptSources ?? {};
716
- state.promptFiles = previous?.promptFiles ?? {};
717
- // Nothing is claimed as in-flight any more: whatever the last upload sent
718
- // was either acked (and cleared) or lost. A caller with authority re-derives
719
- // the claim below; one without leaves the rows dirty for the tick.
720
- state.prompts.sent = undefined;
721
- if (prompts !== 'untouched') {
722
- state.promptSources = prompts.scanned
723
- ? (prompts.scanned.sources ?? {
724
- claude_code: mergePromptActivities([prompts.scanned]),
725
- })
726
- : {};
727
- // A fresh scan's partitions are authoritative. Recheck IDE snapshots on
728
- // the next tick rather than advancing a cursor past concurrent writes.
729
- state.promptFiles = {};
730
- replacePromptActivity(state.prompts, prompts.scanned ?? emptyPromptActivity());
731
- }
732
- for (const result of results) {
733
- replaceAggregates(harness(state, result.tool), aggregatesOfResult(result), new Set());
734
- }
735
- for (const src of sources.jsonl) {
736
- await snapshotFileCursors(harness(state, src.tool), await src.files());
737
- }
738
- // Codex has to be re-read: its per-file running totals aren't recoverable
739
- // from the aggregate, and without them the next tick would count each file's
740
- // whole total a second time.
741
- const codex = harness(state, 'codex');
742
- codex.codexTotals = {};
743
- const codexPaths = await sources.codex.files();
744
- await snapshotFileCursors(codex, codexPaths);
745
- for (const path of codexPaths) {
746
- const cursor = codex.files[path];
747
- if (!cursor)
748
- continue;
749
- const content = await readFile(path, 'utf8').catch(() => '');
750
- const { maxTotal, model } = codexFileTotals(content);
751
- if (maxTotal <= 0)
752
- continue;
753
- codex.codexTotals[path] = {
754
- maxTotal,
755
- model,
756
- date: sources.codex.dateFor(path) ?? toDateStr(new Date(cursor.mtimeMs)),
757
- };
758
- }
759
- for (const src of sources.sqlite) {
760
- const h = harness(state, src.tool);
761
- const path = src.dbPath();
762
- h.dbMtimeMs = (await statOrNull(path))?.mtimeMs ?? 0;
763
- h.walMtimeMs = (await statOrNull(`${path}-wal`))?.mtimeMs ?? 0;
764
- }
765
- // Only a caller that knows the tier may put prompt rows on the wire. For
766
- // anyone else `sent` stays empty, so `commit` clears no prompt rows either.
767
- const claimed = prompts === 'untouched' ? undefined : takePromptActivity(state);
768
- return {
769
- ...(claimed ? { promptActivity: claimed } : {}),
770
- commit: async () => {
771
- try {
772
- // The full token payload just went out, and the server took the
773
- // prompt rows this upload claimed — markUploaded records both.
774
- markUploaded(state);
775
- await saveScanState(state);
776
- }
777
- catch {
778
- // never fail the sync that called us
779
- }
780
- },
781
- };
782
- }
783
- catch {
784
- // never fail the sync that called us
785
- return nothingStaged;
786
- }
787
- }
788
- //# sourceMappingURL=incremental.js.map