hacklab 0.22.3 → 26.921.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -430
- package/dist/index.js +132 -23
- package/package.json +20 -42
- package/LICENSE +0 -21
- package/dist/agent-handoff.d.ts +0 -51
- package/dist/agent-handoff.d.ts.map +0 -1
- package/dist/agent-handoff.js +0 -91
- package/dist/agent-handoff.js.map +0 -1
- package/dist/api-client.d.ts +0 -18
- package/dist/api-client.d.ts.map +0 -1
- package/dist/api-client.js +0 -40
- package/dist/api-client.js.map +0 -1
- package/dist/belt.d.ts +0 -25
- package/dist/belt.d.ts.map +0 -1
- package/dist/belt.js +0 -113
- package/dist/belt.js.map +0 -1
- package/dist/card.d.ts +0 -66
- package/dist/card.d.ts.map +0 -1
- package/dist/card.js +0 -151
- package/dist/card.js.map +0 -1
- package/dist/cli.d.ts +0 -2
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -238
- package/dist/cli.js.map +0 -1
- package/dist/commands/book.d.ts +0 -14
- package/dist/commands/book.d.ts.map +0 -1
- package/dist/commands/book.js +0 -90
- package/dist/commands/book.js.map +0 -1
- package/dist/commands/chat.d.ts +0 -36
- package/dist/commands/chat.d.ts.map +0 -1
- package/dist/commands/chat.js +0 -642
- package/dist/commands/chat.js.map +0 -1
- package/dist/commands/config.d.ts +0 -2
- package/dist/commands/config.d.ts.map +0 -1
- package/dist/commands/config.js +0 -84
- package/dist/commands/config.js.map +0 -1
- package/dist/commands/daemon.d.ts +0 -19
- package/dist/commands/daemon.d.ts.map +0 -1
- package/dist/commands/daemon.js +0 -72
- package/dist/commands/daemon.js.map +0 -1
- package/dist/commands/drop.d.ts +0 -12
- package/dist/commands/drop.d.ts.map +0 -1
- package/dist/commands/drop.js +0 -103
- package/dist/commands/drop.js.map +0 -1
- package/dist/commands/drops.d.ts +0 -9
- package/dist/commands/drops.d.ts.map +0 -1
- package/dist/commands/drops.js +0 -96
- package/dist/commands/drops.js.map +0 -1
- package/dist/commands/essay.d.ts +0 -40
- package/dist/commands/essay.d.ts.map +0 -1
- package/dist/commands/essay.js +0 -651
- package/dist/commands/essay.js.map +0 -1
- package/dist/commands/hackathon.d.ts +0 -2
- package/dist/commands/hackathon.d.ts.map +0 -1
- package/dist/commands/hackathon.js +0 -893
- package/dist/commands/hackathon.js.map +0 -1
- package/dist/commands/hacker.d.ts +0 -2
- package/dist/commands/hacker.d.ts.map +0 -1
- package/dist/commands/hacker.js +0 -87
- package/dist/commands/hacker.js.map +0 -1
- package/dist/commands/jobs.d.ts +0 -18
- package/dist/commands/jobs.d.ts.map +0 -1
- package/dist/commands/jobs.js +0 -138
- package/dist/commands/jobs.js.map +0 -1
- package/dist/commands/keys.d.ts +0 -2
- package/dist/commands/keys.d.ts.map +0 -1
- package/dist/commands/keys.js +0 -138
- package/dist/commands/keys.js.map +0 -1
- package/dist/commands/login.d.ts +0 -83
- package/dist/commands/login.d.ts.map +0 -1
- package/dist/commands/login.js +0 -236
- package/dist/commands/login.js.map +0 -1
- package/dist/commands/logout.d.ts +0 -2
- package/dist/commands/logout.d.ts.map +0 -1
- package/dist/commands/logout.js +0 -22
- package/dist/commands/logout.js.map +0 -1
- package/dist/commands/org-jobs.d.ts +0 -50
- package/dist/commands/org-jobs.d.ts.map +0 -1
- package/dist/commands/org-jobs.js +0 -383
- package/dist/commands/org-jobs.js.map +0 -1
- package/dist/commands/org.d.ts +0 -67
- package/dist/commands/org.d.ts.map +0 -1
- package/dist/commands/org.js +0 -1244
- package/dist/commands/org.js.map +0 -1
- package/dist/commands/ping.d.ts +0 -2
- package/dist/commands/ping.d.ts.map +0 -1
- package/dist/commands/ping.js +0 -50
- package/dist/commands/ping.js.map +0 -1
- package/dist/commands/profile.d.ts +0 -72
- package/dist/commands/profile.d.ts.map +0 -1
- package/dist/commands/profile.js +0 -672
- package/dist/commands/profile.js.map +0 -1
- package/dist/commands/project.d.ts +0 -2
- package/dist/commands/project.d.ts.map +0 -1
- package/dist/commands/project.js +0 -583
- package/dist/commands/project.js.map +0 -1
- package/dist/commands/referral.d.ts +0 -6
- package/dist/commands/referral.d.ts.map +0 -1
- package/dist/commands/referral.js +0 -93
- package/dist/commands/referral.js.map +0 -1
- package/dist/commands/rtfm.d.ts +0 -9
- package/dist/commands/rtfm.d.ts.map +0 -1
- package/dist/commands/rtfm.js +0 -114
- package/dist/commands/rtfm.js.map +0 -1
- package/dist/commands/scan.d.ts +0 -13
- package/dist/commands/scan.d.ts.map +0 -1
- package/dist/commands/scan.js +0 -286
- package/dist/commands/scan.js.map +0 -1
- package/dist/commands/scout.d.ts +0 -18
- package/dist/commands/scout.d.ts.map +0 -1
- package/dist/commands/scout.js +0 -232
- package/dist/commands/scout.js.map +0 -1
- package/dist/commands/setup-rail.d.ts +0 -58
- package/dist/commands/setup-rail.d.ts.map +0 -1
- package/dist/commands/setup-rail.js +0 -135
- package/dist/commands/setup-rail.js.map +0 -1
- package/dist/commands/setup.d.ts +0 -29
- package/dist/commands/setup.d.ts.map +0 -1
- package/dist/commands/setup.js +0 -442
- package/dist/commands/setup.js.map +0 -1
- package/dist/commands/sync.d.ts +0 -9
- package/dist/commands/sync.d.ts.map +0 -1
- package/dist/commands/sync.js +0 -345
- package/dist/commands/sync.js.map +0 -1
- package/dist/commands/update.d.ts +0 -2
- package/dist/commands/update.d.ts.map +0 -1
- package/dist/commands/update.js +0 -92
- package/dist/commands/update.js.map +0 -1
- package/dist/commands/whoami.d.ts +0 -2
- package/dist/commands/whoami.d.ts.map +0 -1
- package/dist/commands/whoami.js +0 -41
- package/dist/commands/whoami.js.map +0 -1
- package/dist/config.d.ts +0 -78
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -93
- package/dist/config.js.map +0 -1
- package/dist/daily-sync.d.ts +0 -150
- package/dist/daily-sync.d.ts.map +0 -1
- package/dist/daily-sync.js +0 -665
- package/dist/daily-sync.js.map +0 -1
- package/dist/dossier.d.ts +0 -73
- package/dist/dossier.d.ts.map +0 -1
- package/dist/dossier.js +0 -141
- package/dist/dossier.js.map +0 -1
- package/dist/field-normalize.d.ts +0 -4
- package/dist/field-normalize.d.ts.map +0 -1
- package/dist/field-normalize.js +0 -8
- package/dist/field-normalize.js.map +0 -1
- package/dist/index.d.ts +0 -3
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/job-fields.d.ts +0 -47
- package/dist/job-fields.d.ts.map +0 -1
- package/dist/job-fields.js +0 -148
- package/dist/job-fields.js.map +0 -1
- package/dist/machine.d.ts +0 -15
- package/dist/machine.d.ts.map +0 -1
- package/dist/machine.js +0 -64
- package/dist/machine.js.map +0 -1
- package/dist/org-fields.d.ts +0 -57
- package/dist/org-fields.d.ts.map +0 -1
- package/dist/org-fields.js +0 -167
- package/dist/org-fields.js.map +0 -1
- package/dist/posthog.d.ts +0 -38
- package/dist/posthog.d.ts.map +0 -1
- package/dist/posthog.js +0 -224
- package/dist/posthog.js.map +0 -1
- package/dist/project-fields.d.ts +0 -40
- package/dist/project-fields.d.ts.map +0 -1
- package/dist/project-fields.js +0 -79
- package/dist/project-fields.js.map +0 -1
- package/dist/prompt-consent.d.ts +0 -73
- package/dist/prompt-consent.d.ts.map +0 -1
- package/dist/prompt-consent.js +0 -191
- package/dist/prompt-consent.js.map +0 -1
- package/dist/prompt-stats.d.ts +0 -226
- package/dist/prompt-stats.d.ts.map +0 -1
- package/dist/prompt-stats.js +0 -561
- package/dist/prompt-stats.js.map +0 -1
- package/dist/referral.d.ts +0 -14
- package/dist/referral.d.ts.map +0 -1
- package/dist/referral.js +0 -28
- package/dist/referral.js.map +0 -1
- package/dist/registry.d.ts +0 -15
- package/dist/registry.d.ts.map +0 -1
- package/dist/registry.js +0 -180
- package/dist/registry.js.map +0 -1
- package/dist/resolve-command.d.ts +0 -20
- package/dist/resolve-command.d.ts.map +0 -1
- package/dist/resolve-command.js +0 -21
- package/dist/resolve-command.js.map +0 -1
- package/dist/scanners/antigravity.d.ts +0 -41
- package/dist/scanners/antigravity.d.ts.map +0 -1
- package/dist/scanners/antigravity.js +0 -210
- package/dist/scanners/antigravity.js.map +0 -1
- package/dist/scanners/github-copilot-telemetry.d.ts +0 -23
- package/dist/scanners/github-copilot-telemetry.d.ts.map +0 -1
- package/dist/scanners/github-copilot-telemetry.js +0 -220
- package/dist/scanners/github-copilot-telemetry.js.map +0 -1
- package/dist/scanners/github-copilot.d.ts +0 -68
- package/dist/scanners/github-copilot.d.ts.map +0 -1
- package/dist/scanners/github-copilot.js +0 -262
- package/dist/scanners/github-copilot.js.map +0 -1
- package/dist/scanners/incremental.d.ts +0 -282
- package/dist/scanners/incremental.d.ts.map +0 -1
- package/dist/scanners/incremental.js +0 -788
- package/dist/scanners/incremental.js.map +0 -1
- package/dist/scanners/index.d.ts +0 -116
- package/dist/scanners/index.d.ts.map +0 -1
- package/dist/scanners/index.js +0 -726
- package/dist/scanners/index.js.map +0 -1
- package/dist/scanners/sqlite.d.ts +0 -18
- package/dist/scanners/sqlite.d.ts.map +0 -1
- package/dist/scanners/sqlite.js +0 -63
- package/dist/scanners/sqlite.js.map +0 -1
- package/dist/scanners/util.d.ts +0 -112
- package/dist/scanners/util.d.ts.map +0 -1
- package/dist/scanners/util.js +0 -167
- package/dist/scanners/util.js.map +0 -1
- package/dist/session.d.ts +0 -106
- package/dist/session.d.ts.map +0 -1
- package/dist/session.js +0 -191
- package/dist/session.js.map +0 -1
- package/dist/share-card-colors.d.ts +0 -19
- package/dist/share-card-colors.d.ts.map +0 -1
- package/dist/share-card-colors.js +0 -36
- package/dist/share-card-colors.js.map +0 -1
- package/dist/share-card.d.ts +0 -31
- package/dist/share-card.d.ts.map +0 -1
- package/dist/share-card.js +0 -380
- package/dist/share-card.js.map +0 -1
- package/dist/share.d.ts +0 -17
- package/dist/share.d.ts.map +0 -1
- package/dist/share.js +0 -75
- package/dist/share.js.map +0 -1
- package/dist/sync.d.ts +0 -100
- package/dist/sync.d.ts.map +0 -1
- package/dist/sync.js +0 -304
- package/dist/sync.js.map +0 -1
- package/dist/ui.d.ts +0 -33
- package/dist/ui.d.ts.map +0 -1
- package/dist/ui.js +0 -127
- package/dist/ui.js.map +0 -1
- package/dist/utils/npmGlobal.d.ts +0 -30
- package/dist/utils/npmGlobal.d.ts.map +0 -1
- package/dist/utils/npmGlobal.js +0 -69
- package/dist/utils/npmGlobal.js.map +0 -1
- package/dist/utils/openBrowser.d.ts +0 -14
- package/dist/utils/openBrowser.d.ts.map +0 -1
- package/dist/utils/openBrowser.js +0 -47
- package/dist/utils/openBrowser.js.map +0 -1
- package/dist/utils/updateCheck.d.ts +0 -16
- package/dist/utils/updateCheck.d.ts.map +0 -1
- package/dist/utils/updateCheck.js +0 -111
- package/dist/utils/updateCheck.js.map +0 -1
- package/dist/utils/waitForEnter.d.ts +0 -12
- package/dist/utils/waitForEnter.d.ts.map +0 -1
- package/dist/utils/waitForEnter.js +0 -43
- package/dist/utils/waitForEnter.js.map +0 -1
|
@@ -1,788 +0,0 @@
|
|
|
1
|
-
import { mkdir, open, readFile, stat, writeFile } from 'node:fs/promises';
|
|
2
|
-
import { dirname, join } from 'node:path';
|
|
3
|
-
import { addPromptToActivity, countWords, emptyPromptActivity, IDE_PROMPT_SOURCES, mergePromptActivities, parsePromptLine, } from '../prompt-stats.js';
|
|
4
|
-
import { getSessionPath } from '../session.js';
|
|
5
|
-
import { antigravityTokenFiles, scanAntigravity } from './antigravity.js';
|
|
6
|
-
import { githubCopilotTokenFiles, scanGitHubCopilot } from './github-copilot.js';
|
|
7
|
-
import { claudeCodeFiles, codexDateForFile, codexFiles, codexFileTotals, dateDaysAgo, grokLogFiles, hermesDbPath, openclawFiles, opencodeDbPath, PROMPT_ACTIVITY_DATE_CAP, PROMPT_ACTIVITY_SESSION_CAP, parseClaudeCodeLine, parseGrokLine, parseOpenclawLine, scanHermes, scanOpenCode, toDateStr, } from './index.js';
|
|
8
|
-
// The incremental half of the scanners: what `hacklab sync --tick` runs every
|
|
9
|
-
// minute instead of the full stateless re-scan the daily job does.
|
|
10
|
-
//
|
|
11
|
-
// The idea is a tail-follow. Everything the last scan already accounted for is
|
|
12
|
-
// kept in scan-state.json — per-file read offsets plus the aggregates those
|
|
13
|
-
// bytes produced — so a tick only has to stat the log tree and parse the bytes
|
|
14
|
-
// that were appended since. Lines are parsed by the exact same functions the
|
|
15
|
-
// full scanners use (parseClaudeCodeLine / parseOpenclawLine / codexFileTotals),
|
|
16
|
-
// so the tail can be added to aggregates the full scan produced without drift.
|
|
17
|
-
//
|
|
18
|
-
// Anything that makes the tail-follow unsound (a file shrank, or one we were
|
|
19
|
-
// following disappeared) drops that harness back to a full re-read, because we
|
|
20
|
-
// don't track which file contributed which tokens. And the daily `sync --quiet`
|
|
21
|
-
// rebuilds the whole state from its full scan, so drift can never last a day.
|
|
22
|
-
/**
|
|
23
|
-
* Bumping this makes `loadScanState` return null, which re-reads every log
|
|
24
|
-
* from scratch and re-derives the aggregates.
|
|
25
|
-
*
|
|
26
|
-
* 3: the prompt scanner learned to skip harness-written `user` entries
|
|
27
|
-
* (PROMPT_SCANNER_VERSION 2). The server is wiping the prompt tables once that
|
|
28
|
-
* ships, and `replacePromptActivity` only re-sends sessions and dates whose
|
|
29
|
-
* numbers *moved* — so a day that happened to contain no harness noise would
|
|
30
|
-
* stay identical locally and never be re-uploaded after the wipe. A cold
|
|
31
|
-
* rebuild marks every session and date within local retention dirty exactly
|
|
32
|
-
* once, and they all go back out.
|
|
33
|
-
*
|
|
34
|
-
* 4: the same wipe, for the machines 3 couldn't save. The wipe landed
|
|
35
|
-
* 2026-09-10, and a machine that had already upgraded to 0.21.0 did its one
|
|
36
|
-
* cold rebuild *before* that — it re-uploaded everything, cleared every dirty
|
|
37
|
-
* flag, and then watched the server delete the lot. Nothing on that machine
|
|
38
|
-
* moves afterwards, so `replacePromptActivity` re-sends nothing and the history
|
|
39
|
-
* is simply gone. This bump is the one-shot rescue: another cold rebuild, after
|
|
40
|
-
* the wipe this time. For anyone still on an older CLI it costs one extra
|
|
41
|
-
* rescan and nothing else.
|
|
42
|
-
*
|
|
43
|
-
* 5: prompt activity is partitioned by source, so rebuilding a Claude log or
|
|
44
|
-
* rotating an IDE transcript cannot erase or double-count another harness.
|
|
45
|
-
*
|
|
46
|
-
* Token totals are unaffected: they travel as cumulative absolutes that the
|
|
47
|
-
* server diffs against its own per-machine snapshot, and `state.uploaded` is
|
|
48
|
-
* only the tick's "nothing moved" short-circuit — so a rebuild re-baselines
|
|
49
|
-
* rather than double-counting.
|
|
50
|
-
*/
|
|
51
|
-
export const SCAN_STATE_VERSION = 5;
|
|
52
|
-
/** How long a session is kept in the state after its last prompt. */
|
|
53
|
-
export const PROMPT_SESSION_RETENTION_DAYS = 45;
|
|
54
|
-
/** How long a per-day prompt tally is kept. Matches the wire's date cap. */
|
|
55
|
-
export const PROMPT_DAILY_RETENTION_DAYS = PROMPT_ACTIVITY_DATE_CAP;
|
|
56
|
-
/** Lives next to the session file, so it honors HACKLAB_SESSION_PATH. */
|
|
57
|
-
export function scanStatePath() {
|
|
58
|
-
return join(dirname(getSessionPath()), 'scan-state.json');
|
|
59
|
-
}
|
|
60
|
-
export function emptyHarness() {
|
|
61
|
-
return { files: {}, daily: {}, models: {} };
|
|
62
|
-
}
|
|
63
|
-
export function emptyPromptState() {
|
|
64
|
-
return { ...emptyPromptActivity(), dirtySessions: [], dirtyDates: [] };
|
|
65
|
-
}
|
|
66
|
-
export function emptyState() {
|
|
67
|
-
return {
|
|
68
|
-
version: SCAN_STATE_VERSION,
|
|
69
|
-
harnesses: {},
|
|
70
|
-
dirty: [],
|
|
71
|
-
prompts: emptyPromptState(),
|
|
72
|
-
promptSources: {},
|
|
73
|
-
promptFiles: {},
|
|
74
|
-
uploaded: { toolTotals: {}, modelTotals: {} },
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
function harness(state, tool) {
|
|
78
|
-
const existing = state.harnesses[tool];
|
|
79
|
-
if (existing)
|
|
80
|
-
return existing;
|
|
81
|
-
const fresh = emptyHarness();
|
|
82
|
-
state.harnesses[tool] = fresh;
|
|
83
|
-
return fresh;
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* The saved state, or null when there isn't a usable one — missing, corrupt, or
|
|
87
|
-
* written by an older layout. Null means "re-read everything from scratch";
|
|
88
|
-
* callers must treat that rebuild as a baseline, not as new tokens.
|
|
89
|
-
*/
|
|
90
|
-
export async function loadScanState() {
|
|
91
|
-
try {
|
|
92
|
-
const parsed = JSON.parse(await readFile(scanStatePath(), 'utf8'));
|
|
93
|
-
if (parsed?.version !== SCAN_STATE_VERSION)
|
|
94
|
-
return null;
|
|
95
|
-
if (!parsed.harnesses || typeof parsed.harnesses !== 'object')
|
|
96
|
-
return null;
|
|
97
|
-
parsed.dirty ??= [];
|
|
98
|
-
parsed.prompts ??= emptyPromptState();
|
|
99
|
-
parsed.uploaded ??= { toolTotals: {}, modelTotals: {} };
|
|
100
|
-
return parsed;
|
|
101
|
-
}
|
|
102
|
-
catch {
|
|
103
|
-
return null;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Drop prompt rows that have aged out, dirty or not.
|
|
108
|
-
*
|
|
109
|
-
* Dropping a dirty row means it is never uploaded, which is the right trade at
|
|
110
|
-
* this age: a session or a day that has sat unsent for its whole retention
|
|
111
|
-
* window is not coming back, and the alternative is a dirty list that grows
|
|
112
|
-
* forever on a machine that never consented (the tick still counts prompts at
|
|
113
|
-
* the `none` tier — it just never sends them).
|
|
114
|
-
*/
|
|
115
|
-
function prunePromptAggregate(prompts) {
|
|
116
|
-
const sessionCutoff = dateDaysAgo(PROMPT_SESSION_RETENTION_DAYS);
|
|
117
|
-
for (const [id, session] of Object.entries(prompts.sessions)) {
|
|
118
|
-
if (session.lastActiveAt.slice(0, 10) < sessionCutoff) {
|
|
119
|
-
delete prompts.sessions[id];
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
const dateCutoff = dateDaysAgo(PROMPT_DAILY_RETENTION_DAYS);
|
|
123
|
-
for (const date of Object.keys(prompts.daily)) {
|
|
124
|
-
if (date < dateCutoff)
|
|
125
|
-
delete prompts.daily[date];
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
function prunePrompts(prompts) {
|
|
129
|
-
prunePromptAggregate(prompts);
|
|
130
|
-
prompts.dirtySessions = prompts.dirtySessions.filter((id) => prompts.sessions[id] !== undefined);
|
|
131
|
-
prompts.dirtyDates = prompts.dirtyDates.filter((date) => prompts.daily[date] !== undefined);
|
|
132
|
-
}
|
|
133
|
-
/** Persist the state, pruning prompt rows that aged out of their retention
|
|
134
|
-
* window (they'd otherwise accumulate forever). Best-effort. */
|
|
135
|
-
export async function saveScanState(state) {
|
|
136
|
-
prunePrompts(state.prompts);
|
|
137
|
-
for (const source of Object.values(state.promptSources)) {
|
|
138
|
-
prunePromptAggregate(source);
|
|
139
|
-
}
|
|
140
|
-
try {
|
|
141
|
-
await mkdir(dirname(scanStatePath()), { recursive: true });
|
|
142
|
-
await writeFile(scanStatePath(), `${JSON.stringify(state)}\n`, 'utf8');
|
|
143
|
-
}
|
|
144
|
-
catch {
|
|
145
|
-
// a state we can't persist just means the next tick re-reads everything
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
// ---- aggregate bookkeeping -------------------------------------------------
|
|
149
|
-
function addDaily(h, date, model, tokens, messages, dirty) {
|
|
150
|
-
const key = `${date}|${model}`;
|
|
151
|
-
const existing = h.daily[key];
|
|
152
|
-
if (existing) {
|
|
153
|
-
existing.tokens += tokens;
|
|
154
|
-
existing.messages += messages;
|
|
155
|
-
}
|
|
156
|
-
else {
|
|
157
|
-
h.daily[key] = { tokens, messages };
|
|
158
|
-
}
|
|
159
|
-
if (model)
|
|
160
|
-
h.models[model] = (h.models[model] ?? 0) + tokens;
|
|
161
|
-
dirty.add(date);
|
|
162
|
-
}
|
|
163
|
-
/** A full scanner's result as the maps the state stores. */
|
|
164
|
-
export function aggregatesOfResult(result) {
|
|
165
|
-
const daily = {};
|
|
166
|
-
for (const entry of result.daily) {
|
|
167
|
-
daily[`${entry.date}|${entry.model ?? ''}`] = {
|
|
168
|
-
tokens: entry.tokens,
|
|
169
|
-
messages: entry.messages ?? 0,
|
|
170
|
-
};
|
|
171
|
-
}
|
|
172
|
-
return { daily, models: { ...result.models } };
|
|
173
|
-
}
|
|
174
|
-
function aggregatesOf(h) {
|
|
175
|
-
return { daily: structuredClone(h.daily), models: { ...h.models } };
|
|
176
|
-
}
|
|
177
|
-
/**
|
|
178
|
-
* Mark every date whose rows moved between two versions of a harness's
|
|
179
|
-
* aggregates. Used whenever a harness is re-read rather than tailed (a SQLite
|
|
180
|
-
* requery, or a JSONL harness whose tail-follow was invalidated): most dates
|
|
181
|
-
* come back identical, so diffing keeps the upload down to what actually
|
|
182
|
-
* changed instead of re-sending the user's whole history.
|
|
183
|
-
*/
|
|
184
|
-
function markMovedDates(before, after, dirty) {
|
|
185
|
-
const compare = (a, b) => {
|
|
186
|
-
for (const key of new Set([...Object.keys(a), ...Object.keys(b)])) {
|
|
187
|
-
const x = a[key];
|
|
188
|
-
const y = b[key];
|
|
189
|
-
if (!x || !y || x.tokens !== y.tokens || x.messages !== y.messages) {
|
|
190
|
-
dirty.add(key.split('|')[0] ?? '');
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
};
|
|
194
|
-
compare(before.daily, after.daily);
|
|
195
|
-
}
|
|
196
|
-
/** Swap in freshly scanned aggregates, dirtying the dates that moved. */
|
|
197
|
-
function replaceAggregates(h, next, dirty) {
|
|
198
|
-
markMovedDates({ daily: h.daily, models: h.models }, next, dirty);
|
|
199
|
-
h.daily = next.daily;
|
|
200
|
-
h.models = next.models;
|
|
201
|
-
}
|
|
202
|
-
/**
|
|
203
|
-
* Swap in a freshly scanned prompt aggregate, dirtying every session and date
|
|
204
|
-
* whose numbers moved. Same idea as `markMovedDates`: a daily full scan mostly
|
|
205
|
-
* reproduces what the ticks already sent, and only the difference is worth
|
|
206
|
-
* putting back on the wire.
|
|
207
|
-
*/
|
|
208
|
-
function replacePromptActivity(prompts, next) {
|
|
209
|
-
const dirtySessions = new Set(prompts.dirtySessions);
|
|
210
|
-
for (const id of new Set([
|
|
211
|
-
...Object.keys(prompts.sessions),
|
|
212
|
-
...Object.keys(next.sessions),
|
|
213
|
-
])) {
|
|
214
|
-
const before = prompts.sessions[id];
|
|
215
|
-
const after = next.sessions[id];
|
|
216
|
-
if (!after)
|
|
217
|
-
continue;
|
|
218
|
-
if (!before ||
|
|
219
|
-
before.promptCount !== after.promptCount ||
|
|
220
|
-
before.startedAt !== after.startedAt ||
|
|
221
|
-
before.lastActiveAt !== after.lastActiveAt) {
|
|
222
|
-
dirtySessions.add(id);
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
const dirtyDates = new Set(prompts.dirtyDates);
|
|
226
|
-
for (const date of new Set([
|
|
227
|
-
...Object.keys(prompts.daily),
|
|
228
|
-
...Object.keys(next.daily),
|
|
229
|
-
])) {
|
|
230
|
-
const before = prompts.daily[date];
|
|
231
|
-
const after = next.daily[date];
|
|
232
|
-
if (!after)
|
|
233
|
-
continue;
|
|
234
|
-
if (!before ||
|
|
235
|
-
before.prompts !== after.prompts ||
|
|
236
|
-
before.words !== after.words) {
|
|
237
|
-
dirtyDates.add(date);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
prompts.sessions = next.sessions;
|
|
241
|
-
prompts.daily = next.daily;
|
|
242
|
-
// A session or date the full scan can no longer see has nothing left to say.
|
|
243
|
-
prompts.dirtySessions = [...dirtySessions].filter((id) => next.sessions[id] !== undefined);
|
|
244
|
-
prompts.dirtyDates = [...dirtyDates]
|
|
245
|
-
.filter((date) => next.daily[date] !== undefined)
|
|
246
|
-
.sort();
|
|
247
|
-
}
|
|
248
|
-
// ---- reading the tail ------------------------------------------------------
|
|
249
|
-
async function statOrNull(path) {
|
|
250
|
-
try {
|
|
251
|
-
return await stat(path);
|
|
252
|
-
}
|
|
253
|
-
catch {
|
|
254
|
-
return null;
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
/**
|
|
258
|
-
* Read `[from, to)` and hand back only whole lines. A harness may be mid-append
|
|
259
|
-
* when we look, so the trailing partial line is left for the next tick — that's
|
|
260
|
-
* what the returned offset points at.
|
|
261
|
-
*/
|
|
262
|
-
export async function readCompleteLines(path, from, to) {
|
|
263
|
-
if (to <= from)
|
|
264
|
-
return { text: '', offset: from };
|
|
265
|
-
const handle = await open(path, 'r');
|
|
266
|
-
try {
|
|
267
|
-
const buf = Buffer.alloc(to - from);
|
|
268
|
-
const { bytesRead } = await handle.read(buf, 0, buf.length, from);
|
|
269
|
-
const chunk = buf.subarray(0, bytesRead);
|
|
270
|
-
// Byte-wise, not string-wise: the chunk can end mid-codepoint.
|
|
271
|
-
const lastNewline = chunk.lastIndexOf(0x0a);
|
|
272
|
-
if (lastNewline < 0)
|
|
273
|
-
return { text: '', offset: from };
|
|
274
|
-
return {
|
|
275
|
-
text: chunk.subarray(0, lastNewline).toString('utf8'),
|
|
276
|
-
offset: from + lastNewline + 1,
|
|
277
|
-
};
|
|
278
|
-
}
|
|
279
|
-
finally {
|
|
280
|
-
await handle.close();
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
/** The real harnesses. Injectable so the tick can be tested against a tmp dir. */
|
|
284
|
-
export function defaultSources() {
|
|
285
|
-
return {
|
|
286
|
-
jsonl: [
|
|
287
|
-
{
|
|
288
|
-
tool: 'claude_code',
|
|
289
|
-
files: claudeCodeFiles,
|
|
290
|
-
parse: parseClaudeCodeLine,
|
|
291
|
-
parsePrompt: parsePromptLine,
|
|
292
|
-
},
|
|
293
|
-
{ tool: 'openclaw', files: openclawFiles, parse: parseOpenclawLine },
|
|
294
|
-
{ tool: 'grok', files: grokLogFiles, parse: parseGrokLine },
|
|
295
|
-
],
|
|
296
|
-
codex: { files: codexFiles, dateFor: codexDateForFile },
|
|
297
|
-
sqlite: [
|
|
298
|
-
{ tool: 'hermes', dbPath: hermesDbPath, scan: scanHermes },
|
|
299
|
-
{ tool: 'opencode', dbPath: opencodeDbPath, scan: scanOpenCode },
|
|
300
|
-
],
|
|
301
|
-
prompts: IDE_PROMPT_SOURCES,
|
|
302
|
-
snapshots: [
|
|
303
|
-
{
|
|
304
|
-
tool: 'github_copilot',
|
|
305
|
-
files: githubCopilotTokenFiles,
|
|
306
|
-
scan: scanGitHubCopilot,
|
|
307
|
-
},
|
|
308
|
-
{
|
|
309
|
-
tool: 'antigravity',
|
|
310
|
-
files: antigravityTokenFiles,
|
|
311
|
-
scan: scanAntigravity,
|
|
312
|
-
},
|
|
313
|
-
],
|
|
314
|
-
};
|
|
315
|
-
}
|
|
316
|
-
async function statFiles(paths) {
|
|
317
|
-
const out = [];
|
|
318
|
-
for (const path of paths) {
|
|
319
|
-
const s = await statOrNull(path);
|
|
320
|
-
if (s)
|
|
321
|
-
out.push({ path, size: s.size, mtimeMs: s.mtimeMs, mtime: s.mtime });
|
|
322
|
-
}
|
|
323
|
-
return out;
|
|
324
|
-
}
|
|
325
|
-
/**
|
|
326
|
-
* Can we still trust this harness's aggregates? Not if a file we were following
|
|
327
|
-
* lost bytes or vanished: we know what those files contributed in total but not
|
|
328
|
-
* per file, so there's nothing to subtract and the only repair is a re-read.
|
|
329
|
-
*/
|
|
330
|
-
function tailInvalidated(h, files) {
|
|
331
|
-
const present = new Map(files.map((f) => [f.path, f]));
|
|
332
|
-
for (const [path, state] of Object.entries(h.files)) {
|
|
333
|
-
const now = present.get(path);
|
|
334
|
-
if (!now || now.size < state.offset)
|
|
335
|
-
return true;
|
|
336
|
-
}
|
|
337
|
-
return false;
|
|
338
|
-
}
|
|
339
|
-
function resetHarness(h) {
|
|
340
|
-
h.files = {};
|
|
341
|
-
if (h.codexTotals)
|
|
342
|
-
h.codexTotals = {};
|
|
343
|
-
h.daily = {};
|
|
344
|
-
h.models = {};
|
|
345
|
-
}
|
|
346
|
-
async function tickJsonl(state, src, dirty) {
|
|
347
|
-
const h = harness(state, src.tool);
|
|
348
|
-
const files = await statFiles(await src.files());
|
|
349
|
-
const invalidated = tailInvalidated(h, files);
|
|
350
|
-
// On a re-read, collect into a throwaway set and diff at the end: most of the
|
|
351
|
-
// rebuilt aggregate is identical to what we had, and only the difference is
|
|
352
|
-
// worth re-uploading.
|
|
353
|
-
const before = invalidated ? aggregatesOf(h) : null;
|
|
354
|
-
const target = invalidated ? new Set() : dirty;
|
|
355
|
-
if (invalidated)
|
|
356
|
-
resetHarness(h);
|
|
357
|
-
// Keep each source independent: a rewritten Claude transcript must not
|
|
358
|
-
// discard Copilot/Antigravity activity on the same date.
|
|
359
|
-
if (src.parsePrompt && (invalidated || !state.promptSources[src.tool])) {
|
|
360
|
-
state.promptSources[src.tool] = emptyPromptActivity();
|
|
361
|
-
}
|
|
362
|
-
const prompts = state.promptSources[src.tool];
|
|
363
|
-
let changed = invalidated;
|
|
364
|
-
for (const file of files) {
|
|
365
|
-
const prev = h.files[file.path];
|
|
366
|
-
if (prev && prev.size === file.size && prev.mtimeMs === file.mtimeMs) {
|
|
367
|
-
continue;
|
|
368
|
-
}
|
|
369
|
-
const from = prev?.offset ?? 0;
|
|
370
|
-
const { text, offset } = await readCompleteLines(file.path, from, file.size).catch(() => ({ text: '', offset: from }));
|
|
371
|
-
let fallbackDate = null;
|
|
372
|
-
for (const line of text.split('\n')) {
|
|
373
|
-
const prompt = src.parsePrompt?.(line);
|
|
374
|
-
if (prompt && prompts) {
|
|
375
|
-
addPromptToActivity(prompts, prompt);
|
|
376
|
-
}
|
|
377
|
-
const usage = src.parse(line);
|
|
378
|
-
if (!usage)
|
|
379
|
-
continue;
|
|
380
|
-
let date = usage.date;
|
|
381
|
-
if (!date) {
|
|
382
|
-
fallbackDate ??= toDateStr(file.mtime);
|
|
383
|
-
date = fallbackDate;
|
|
384
|
-
}
|
|
385
|
-
addDaily(h, date, usage.model, usage.tokens, 1, target);
|
|
386
|
-
}
|
|
387
|
-
h.files[file.path] = { size: file.size, mtimeMs: file.mtimeMs, offset };
|
|
388
|
-
changed = true;
|
|
389
|
-
}
|
|
390
|
-
if (before)
|
|
391
|
-
markMovedDates(before, aggregatesOf(h), dirty);
|
|
392
|
-
if (src.parsePrompt && changed) {
|
|
393
|
-
replacePromptActivity(state.prompts, mergePromptActivities(Object.values(state.promptSources)));
|
|
394
|
-
}
|
|
395
|
-
return changed;
|
|
396
|
-
}
|
|
397
|
-
async function tickCodex(state, src, dirty) {
|
|
398
|
-
const h = harness(state, 'codex');
|
|
399
|
-
h.codexTotals ??= {};
|
|
400
|
-
const files = await statFiles(await src.files());
|
|
401
|
-
const invalidated = tailInvalidated(h, files);
|
|
402
|
-
const before = invalidated ? aggregatesOf(h) : null;
|
|
403
|
-
const target = invalidated ? new Set() : dirty;
|
|
404
|
-
if (invalidated) {
|
|
405
|
-
resetHarness(h);
|
|
406
|
-
h.codexTotals = {};
|
|
407
|
-
}
|
|
408
|
-
let changed = invalidated;
|
|
409
|
-
for (const file of files) {
|
|
410
|
-
const prev = h.files[file.path];
|
|
411
|
-
if (prev && prev.size === file.size && prev.mtimeMs === file.mtimeMs) {
|
|
412
|
-
continue;
|
|
413
|
-
}
|
|
414
|
-
const from = prev?.offset ?? 0;
|
|
415
|
-
const { text, offset } = await readCompleteLines(file.path, from, file.size).catch(() => ({ text: '', offset: from }));
|
|
416
|
-
const { maxTotal, model } = codexFileTotals(text);
|
|
417
|
-
const stored = h.codexTotals[file.path];
|
|
418
|
-
const date = stored?.date ?? src.dateFor(file.path) ?? toDateStr(file.mtime);
|
|
419
|
-
const storedMax = stored?.maxTotal ?? 0;
|
|
420
|
-
const nextMax = Math.max(storedMax, maxTotal);
|
|
421
|
-
// A session that switches model mid-file leaves the earlier tokens under the
|
|
422
|
-
// old model — the full scan credits them all to the last model seen. The
|
|
423
|
-
// daily repair pass reconciles that.
|
|
424
|
-
const nextModel = model || stored?.model || '';
|
|
425
|
-
if (nextMax > storedMax) {
|
|
426
|
-
// Running totals: only the growth is new. `messages` counts the session
|
|
427
|
-
// once, the way the full scan does (one addDaily per file).
|
|
428
|
-
addDaily(h, date, nextModel, nextMax - storedMax, storedMax === 0 ? 1 : 0, target);
|
|
429
|
-
}
|
|
430
|
-
h.codexTotals[file.path] = { maxTotal: nextMax, model: nextModel, date };
|
|
431
|
-
h.files[file.path] = { size: file.size, mtimeMs: file.mtimeMs, offset };
|
|
432
|
-
changed = true;
|
|
433
|
-
}
|
|
434
|
-
if (before)
|
|
435
|
-
markMovedDates(before, aggregatesOf(h), dirty);
|
|
436
|
-
return changed;
|
|
437
|
-
}
|
|
438
|
-
/**
|
|
439
|
-
* SQLite-backed harnesses can't be tailed — a write lands anywhere in the file —
|
|
440
|
-
* but their DBs are small, so the incremental strategy is to fingerprint the db
|
|
441
|
-
* and its -wal and re-run the harness's normal query only when one moved.
|
|
442
|
-
*/
|
|
443
|
-
async function tickSqlite(state, src, dirty) {
|
|
444
|
-
const h = harness(state, src.tool);
|
|
445
|
-
const path = src.dbPath();
|
|
446
|
-
const db = await statOrNull(path);
|
|
447
|
-
const wal = await statOrNull(`${path}-wal`);
|
|
448
|
-
const dbMtimeMs = db?.mtimeMs ?? 0;
|
|
449
|
-
const walMtimeMs = wal?.mtimeMs ?? 0;
|
|
450
|
-
if (h.dbMtimeMs === dbMtimeMs && h.walMtimeMs === walMtimeMs)
|
|
451
|
-
return false;
|
|
452
|
-
replaceAggregates(h, aggregatesOfResult(await src.scan()), dirty);
|
|
453
|
-
h.dbMtimeMs = dbMtimeMs;
|
|
454
|
-
h.walMtimeMs = walMtimeMs;
|
|
455
|
-
return true;
|
|
456
|
-
}
|
|
457
|
-
/** Cumulative session records need replacement, never additive tail parsing. */
|
|
458
|
-
async function tickSnapshot(state, source, dirty) {
|
|
459
|
-
const files = await statFiles(await source.files());
|
|
460
|
-
const h = harness(state, source.tool);
|
|
461
|
-
if (Object.keys(h.files).length === files.length &&
|
|
462
|
-
files.every((file) => h.files[file.path]?.size === file.size &&
|
|
463
|
-
h.files[file.path]?.mtimeMs === file.mtimeMs))
|
|
464
|
-
return false;
|
|
465
|
-
replaceAggregates(h, aggregatesOfResult(await source.scan()), dirty);
|
|
466
|
-
h.files = Object.fromEntries(files.map((file) => [
|
|
467
|
-
file.path,
|
|
468
|
-
{ size: file.size, mtimeMs: file.mtimeMs, offset: 0 },
|
|
469
|
-
]));
|
|
470
|
-
return true;
|
|
471
|
-
}
|
|
472
|
-
async function tickPromptSnapshot(state, source) {
|
|
473
|
-
const files = await statFiles(await source.files());
|
|
474
|
-
const previous = state.promptFiles[source.id];
|
|
475
|
-
if (previous &&
|
|
476
|
-
Object.keys(previous).length === files.length &&
|
|
477
|
-
files.every((file) => previous[file.path]?.size === file.size &&
|
|
478
|
-
previous[file.path]?.mtimeMs === file.mtimeMs))
|
|
479
|
-
return false;
|
|
480
|
-
const activity = emptyPromptActivity();
|
|
481
|
-
for (const prompt of await source.scan()) {
|
|
482
|
-
if (!prompt.timestamp)
|
|
483
|
-
continue;
|
|
484
|
-
const words = countWords(prompt.text);
|
|
485
|
-
if (words > 0) {
|
|
486
|
-
addPromptToActivity(activity, {
|
|
487
|
-
sessionId: prompt.sessionId,
|
|
488
|
-
timestamp: prompt.timestamp,
|
|
489
|
-
words,
|
|
490
|
-
});
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
state.promptSources[source.id] = activity;
|
|
494
|
-
state.promptFiles[source.id] = Object.fromEntries(files.map((file) => [
|
|
495
|
-
file.path,
|
|
496
|
-
{ size: file.size, mtimeMs: file.mtimeMs, offset: 0 },
|
|
497
|
-
]));
|
|
498
|
-
replacePromptActivity(state.prompts, mergePromptActivities(Object.values(state.promptSources)));
|
|
499
|
-
return true;
|
|
500
|
-
}
|
|
501
|
-
/**
|
|
502
|
-
* One incremental pass over every harness. `prev` is the saved state, or null
|
|
503
|
-
* for a cold start — in which case this reads everything and marks no *token*
|
|
504
|
-
* date dirty: those tokens are already on the server (the daily job put them
|
|
505
|
-
* there), and re-uploading every date would only invite the reconcile to prune
|
|
506
|
-
* rows this scan can't see (Cursor's, above all).
|
|
507
|
-
*
|
|
508
|
-
* Prompt rows go the other way on a cold start: everything found is marked
|
|
509
|
-
* dirty. They're idempotent GREATEST upserts, so a resend costs nothing, and a
|
|
510
|
-
* cold start (fresh install, or a state file this version can't read) is
|
|
511
|
-
* exactly the case where we have no evidence any of it ever reached the server.
|
|
512
|
-
*/
|
|
513
|
-
export async function runTick(prev, sources = defaultSources(), options = {}) {
|
|
514
|
-
const cold = prev === null;
|
|
515
|
-
const state = prev ?? emptyState();
|
|
516
|
-
const dirty = new Set(state.dirty);
|
|
517
|
-
let changed = cold;
|
|
518
|
-
for (const src of sources.jsonl) {
|
|
519
|
-
if (await tickJsonl(state, src, dirty))
|
|
520
|
-
changed = true;
|
|
521
|
-
}
|
|
522
|
-
if (await tickCodex(state, sources.codex, dirty))
|
|
523
|
-
changed = true;
|
|
524
|
-
for (const src of sources.sqlite) {
|
|
525
|
-
if (await tickSqlite(state, src, dirty))
|
|
526
|
-
changed = true;
|
|
527
|
-
}
|
|
528
|
-
for (const source of sources.snapshots ?? []) {
|
|
529
|
-
if (await tickSnapshot(state, source, dirty))
|
|
530
|
-
changed = true;
|
|
531
|
-
}
|
|
532
|
-
if (options.promptActivity !== false) {
|
|
533
|
-
for (const source of sources.prompts ?? []) {
|
|
534
|
-
if (await tickPromptSnapshot(state, source))
|
|
535
|
-
changed = true;
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
if (cold) {
|
|
539
|
-
state.dirty = [];
|
|
540
|
-
state.prompts.dirtySessions = Object.keys(state.prompts.sessions);
|
|
541
|
-
state.prompts.dirtyDates = Object.keys(state.prompts.daily).sort();
|
|
542
|
-
}
|
|
543
|
-
else {
|
|
544
|
-
state.dirty = [...dirty].sort();
|
|
545
|
-
}
|
|
546
|
-
return { state, changed };
|
|
547
|
-
}
|
|
548
|
-
// ---- payload ---------------------------------------------------------------
|
|
549
|
-
/** Cumulative per-tool and per-model totals for this machine. */
|
|
550
|
-
export function cumulativeTotals(state) {
|
|
551
|
-
const toolTotals = {};
|
|
552
|
-
const modelTotals = {};
|
|
553
|
-
for (const [tool, h] of Object.entries(state.harnesses)) {
|
|
554
|
-
let sum = 0;
|
|
555
|
-
for (const value of Object.values(h.daily))
|
|
556
|
-
sum += value.tokens;
|
|
557
|
-
toolTotals[tool] = sum;
|
|
558
|
-
for (const [model, tokens] of Object.entries(h.models)) {
|
|
559
|
-
modelTotals[model] = (modelTotals[model] ?? 0) + tokens;
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
return { toolTotals, modelTotals };
|
|
563
|
-
}
|
|
564
|
-
export function sameTotals(a, b) {
|
|
565
|
-
const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
|
|
566
|
-
for (const key of keys)
|
|
567
|
-
if ((a[key] ?? 0) !== (b[key] ?? 0))
|
|
568
|
-
return false;
|
|
569
|
-
return true;
|
|
570
|
-
}
|
|
571
|
-
/**
|
|
572
|
-
* Is there prompt activity waiting to go out?
|
|
573
|
-
*
|
|
574
|
-
* A dirty id whose row is gone (pruned between the mark and the send) doesn't
|
|
575
|
-
* count: it would have the tick POST an empty block every minute forever, since
|
|
576
|
-
* there'd be nothing for `markUploaded` to clear.
|
|
577
|
-
*/
|
|
578
|
-
export function hasPromptActivity(state) {
|
|
579
|
-
const { prompts } = state;
|
|
580
|
-
return (prompts.dirtySessions.some((id) => prompts.sessions[id] !== undefined) ||
|
|
581
|
-
prompts.dirtyDates.some((date) => prompts.daily[date] !== undefined));
|
|
582
|
-
}
|
|
583
|
-
/**
|
|
584
|
-
* Stake a claim on the outstanding prompt rows: the ones this payload carries
|
|
585
|
-
* are recorded in `prompts.sent`, and `markUploaded` clears exactly those. Rows
|
|
586
|
-
* a cap held back stay dirty and go out next tick.
|
|
587
|
-
*
|
|
588
|
-
* Most-recently-active first, so a machine with a long backlog syncs the work
|
|
589
|
-
* the user actually cares about before the archaeology.
|
|
590
|
-
*/
|
|
591
|
-
function takePromptActivity(state) {
|
|
592
|
-
const { prompts } = state;
|
|
593
|
-
const sessions = prompts.dirtySessions
|
|
594
|
-
.flatMap((sessionId) => {
|
|
595
|
-
const session = prompts.sessions[sessionId];
|
|
596
|
-
return session ? [{ sessionId, ...session }] : [];
|
|
597
|
-
})
|
|
598
|
-
.sort((a, b) => b.lastActiveAt.localeCompare(a.lastActiveAt))
|
|
599
|
-
.slice(0, PROMPT_ACTIVITY_SESSION_CAP);
|
|
600
|
-
const dates = [...prompts.dirtyDates]
|
|
601
|
-
.sort((a, b) => b.localeCompare(a))
|
|
602
|
-
.slice(0, PROMPT_ACTIVITY_DATE_CAP);
|
|
603
|
-
const dailyPrompts = dates.flatMap((date) => {
|
|
604
|
-
const day = prompts.daily[date];
|
|
605
|
-
return day ? [{ date, ...day }] : [];
|
|
606
|
-
});
|
|
607
|
-
prompts.sent = { sessions: sessions.map((s) => s.sessionId), dates };
|
|
608
|
-
if (sessions.length === 0 && dailyPrompts.length === 0)
|
|
609
|
-
return undefined;
|
|
610
|
-
return { sessions, dailyPrompts };
|
|
611
|
-
}
|
|
612
|
-
/**
|
|
613
|
-
* What a tick uploads: cumulative tool/model totals (the server diffs those
|
|
614
|
-
* against its own per-machine snapshot) plus the daily rows for the dirty dates
|
|
615
|
-
* only, and — under the `stats` or `full` tier — the outstanding prompt
|
|
616
|
-
* activity.
|
|
617
|
-
*
|
|
618
|
-
* Every harness's rows go out for a dirty date, including Cursor's — which the
|
|
619
|
-
* tick never scans, and carries over from the last full scan instead. The
|
|
620
|
-
* server reconciles a date by deleting this machine's rows for it that the
|
|
621
|
-
* payload didn't mention, so a date has to be reported complete or the harnesses
|
|
622
|
-
* left out of it lose that day.
|
|
623
|
-
*/
|
|
624
|
-
export function tickPayload(state, opts = {}) {
|
|
625
|
-
const dirty = new Set(state.dirty);
|
|
626
|
-
const dailyTotals = [];
|
|
627
|
-
for (const [tool, h] of Object.entries(state.harnesses)) {
|
|
628
|
-
for (const [key, value] of Object.entries(h.daily)) {
|
|
629
|
-
const [date = '', model = ''] = key.split('|');
|
|
630
|
-
if (!dirty.has(date) || value.tokens <= 0)
|
|
631
|
-
continue;
|
|
632
|
-
dailyTotals.push({
|
|
633
|
-
date,
|
|
634
|
-
tool,
|
|
635
|
-
tokens: value.tokens,
|
|
636
|
-
messages: value.messages,
|
|
637
|
-
model: model || undefined,
|
|
638
|
-
});
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
// Claim the prompt rows this payload carries, or drop a claim a previous
|
|
642
|
-
// payload left behind: a stale one would have `markUploaded` clear rows this
|
|
643
|
-
// upload never mentioned (a tier turned down to `none` between two ticks).
|
|
644
|
-
state.prompts.sent = undefined;
|
|
645
|
-
const promptActivity = opts.promptActivity
|
|
646
|
-
? takePromptActivity(state)
|
|
647
|
-
: undefined;
|
|
648
|
-
const { toolTotals, modelTotals } = cumulativeTotals(state);
|
|
649
|
-
return {
|
|
650
|
-
toolTotals,
|
|
651
|
-
dailyTotals,
|
|
652
|
-
modelTotals,
|
|
653
|
-
grandTotal: Object.values(toolTotals).reduce((a, b) => a + b, 0),
|
|
654
|
-
cursorStats: null,
|
|
655
|
-
cursorScanStatus: { source: 'none' },
|
|
656
|
-
...(promptActivity ? { promptActivity } : {}),
|
|
657
|
-
};
|
|
658
|
-
}
|
|
659
|
-
/** Called on a 200: whatever that payload carried is on the server now. */
|
|
660
|
-
export function markUploaded(state) {
|
|
661
|
-
state.dirty = [];
|
|
662
|
-
const sent = state.prompts.sent;
|
|
663
|
-
if (sent) {
|
|
664
|
-
const sessions = new Set(sent.sessions);
|
|
665
|
-
const dates = new Set(sent.dates);
|
|
666
|
-
state.prompts.dirtySessions = state.prompts.dirtySessions.filter((id) => !sessions.has(id));
|
|
667
|
-
state.prompts.dirtyDates = state.prompts.dirtyDates.filter((date) => !dates.has(date));
|
|
668
|
-
state.prompts.sent = undefined;
|
|
669
|
-
}
|
|
670
|
-
state.uploaded = cumulativeTotals(state);
|
|
671
|
-
state.lastError = undefined;
|
|
672
|
-
state.nextAllowedAt = undefined;
|
|
673
|
-
}
|
|
674
|
-
// ---- the daily repair pass -------------------------------------------------
|
|
675
|
-
async function snapshotFileCursors(h, paths) {
|
|
676
|
-
for (const file of await statFiles(paths)) {
|
|
677
|
-
// Offsets are set to EOF *after* the full scan read the file, so a file that
|
|
678
|
-
// grew mid-scan is re-read from the end rather than double-counted — the
|
|
679
|
-
// next repair picks up whatever slipped through.
|
|
680
|
-
h.files[file.path] = {
|
|
681
|
-
size: file.size,
|
|
682
|
-
mtimeMs: file.mtimeMs,
|
|
683
|
-
offset: file.size,
|
|
684
|
-
};
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
/**
|
|
688
|
-
* Rebuild the whole state from a full scan's results — the self-healing half of
|
|
689
|
-
* the design. Whatever the tick's tail-follow got wrong (a rotated log, a
|
|
690
|
-
* mid-file model switch, a Codex date guessed from an mtime) is corrected here,
|
|
691
|
-
* so incremental drift can never outlive a day. Best-effort: a state we failed
|
|
692
|
-
* to rebuild costs a tick, never the sync that called us.
|
|
693
|
-
*
|
|
694
|
-
* Staged rather than persisted outright, because the full-scan paths carry
|
|
695
|
-
* prompt activity too — a machine with no daemon has no tick to drain it. The
|
|
696
|
-
* rebuild diffs the scan's aggregate against what was already sent, claims the
|
|
697
|
-
* difference for this upload, and only `commit` (on a 200) clears it. A failed
|
|
698
|
-
* POST therefore leaves every row dirty for the next run, which the server's
|
|
699
|
-
* GREATEST upserts make free to resend.
|
|
700
|
-
*
|
|
701
|
-
* `prompts` says whether this caller has any authority over the prompt state —
|
|
702
|
-
* see `StagedPrompts`. Only a caller that actually resolved the consent tier
|
|
703
|
-
* gets to rewrite it, and only such a caller claims prompt rows for the upload.
|
|
704
|
-
*/
|
|
705
|
-
export async function stageFullScan(results, prompts = 'untouched', sources = defaultSources()) {
|
|
706
|
-
const nothingStaged = {
|
|
707
|
-
commit: async () => {
|
|
708
|
-
// Nothing was staged, so there is nothing to record as delivered.
|
|
709
|
-
},
|
|
710
|
-
};
|
|
711
|
-
try {
|
|
712
|
-
const previous = await loadScanState();
|
|
713
|
-
const state = emptyState();
|
|
714
|
-
state.prompts = previous?.prompts ?? emptyPromptState();
|
|
715
|
-
state.promptSources = previous?.promptSources ?? {};
|
|
716
|
-
state.promptFiles = previous?.promptFiles ?? {};
|
|
717
|
-
// Nothing is claimed as in-flight any more: whatever the last upload sent
|
|
718
|
-
// was either acked (and cleared) or lost. A caller with authority re-derives
|
|
719
|
-
// the claim below; one without leaves the rows dirty for the tick.
|
|
720
|
-
state.prompts.sent = undefined;
|
|
721
|
-
if (prompts !== 'untouched') {
|
|
722
|
-
state.promptSources = prompts.scanned
|
|
723
|
-
? (prompts.scanned.sources ?? {
|
|
724
|
-
claude_code: mergePromptActivities([prompts.scanned]),
|
|
725
|
-
})
|
|
726
|
-
: {};
|
|
727
|
-
// A fresh scan's partitions are authoritative. Recheck IDE snapshots on
|
|
728
|
-
// the next tick rather than advancing a cursor past concurrent writes.
|
|
729
|
-
state.promptFiles = {};
|
|
730
|
-
replacePromptActivity(state.prompts, prompts.scanned ?? emptyPromptActivity());
|
|
731
|
-
}
|
|
732
|
-
for (const result of results) {
|
|
733
|
-
replaceAggregates(harness(state, result.tool), aggregatesOfResult(result), new Set());
|
|
734
|
-
}
|
|
735
|
-
for (const src of sources.jsonl) {
|
|
736
|
-
await snapshotFileCursors(harness(state, src.tool), await src.files());
|
|
737
|
-
}
|
|
738
|
-
// Codex has to be re-read: its per-file running totals aren't recoverable
|
|
739
|
-
// from the aggregate, and without them the next tick would count each file's
|
|
740
|
-
// whole total a second time.
|
|
741
|
-
const codex = harness(state, 'codex');
|
|
742
|
-
codex.codexTotals = {};
|
|
743
|
-
const codexPaths = await sources.codex.files();
|
|
744
|
-
await snapshotFileCursors(codex, codexPaths);
|
|
745
|
-
for (const path of codexPaths) {
|
|
746
|
-
const cursor = codex.files[path];
|
|
747
|
-
if (!cursor)
|
|
748
|
-
continue;
|
|
749
|
-
const content = await readFile(path, 'utf8').catch(() => '');
|
|
750
|
-
const { maxTotal, model } = codexFileTotals(content);
|
|
751
|
-
if (maxTotal <= 0)
|
|
752
|
-
continue;
|
|
753
|
-
codex.codexTotals[path] = {
|
|
754
|
-
maxTotal,
|
|
755
|
-
model,
|
|
756
|
-
date: sources.codex.dateFor(path) ?? toDateStr(new Date(cursor.mtimeMs)),
|
|
757
|
-
};
|
|
758
|
-
}
|
|
759
|
-
for (const src of sources.sqlite) {
|
|
760
|
-
const h = harness(state, src.tool);
|
|
761
|
-
const path = src.dbPath();
|
|
762
|
-
h.dbMtimeMs = (await statOrNull(path))?.mtimeMs ?? 0;
|
|
763
|
-
h.walMtimeMs = (await statOrNull(`${path}-wal`))?.mtimeMs ?? 0;
|
|
764
|
-
}
|
|
765
|
-
// Only a caller that knows the tier may put prompt rows on the wire. For
|
|
766
|
-
// anyone else `sent` stays empty, so `commit` clears no prompt rows either.
|
|
767
|
-
const claimed = prompts === 'untouched' ? undefined : takePromptActivity(state);
|
|
768
|
-
return {
|
|
769
|
-
...(claimed ? { promptActivity: claimed } : {}),
|
|
770
|
-
commit: async () => {
|
|
771
|
-
try {
|
|
772
|
-
// The full token payload just went out, and the server took the
|
|
773
|
-
// prompt rows this upload claimed — markUploaded records both.
|
|
774
|
-
markUploaded(state);
|
|
775
|
-
await saveScanState(state);
|
|
776
|
-
}
|
|
777
|
-
catch {
|
|
778
|
-
// never fail the sync that called us
|
|
779
|
-
}
|
|
780
|
-
},
|
|
781
|
-
};
|
|
782
|
-
}
|
|
783
|
-
catch {
|
|
784
|
-
// never fail the sync that called us
|
|
785
|
-
return nothingStaged;
|
|
786
|
-
}
|
|
787
|
-
}
|
|
788
|
-
//# sourceMappingURL=incremental.js.map
|