@ipv9/tokentracker-cli 0.39.37 → 0.39.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -9
- package/dashboard/dist/assets/{Card-DKmo-v8a.js → Card-BbRYRVXf.js} +1 -1
- package/dashboard/dist/assets/DashboardPage-DYKHv0m3.js +60 -0
- package/dashboard/dist/assets/{FadeIn-Cf30rPTW.js → FadeIn-CjAacJ5t.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-D3CL8XP0.js → IpCheckPage-BGALK8Vy.js} +1 -1
- package/dashboard/dist/assets/{LimitsPage-BqST7DxD.js → LimitsPage-LkZGPYvg.js} +2 -2
- package/dashboard/dist/assets/{LocalOnlyNotice-CqHzPooT.js → LocalOnlyNotice-Dxo2vFMQ.js} +1 -1
- package/dashboard/dist/assets/PopoverPopup-DP9fwdtl.js +1 -0
- package/dashboard/dist/assets/Select-DPE53NmM.js +1 -0
- package/dashboard/dist/assets/SelectItemText-D69v77is.js +12 -0
- package/dashboard/dist/assets/SettingsPage-Dxzqpi02.js +1 -0
- package/dashboard/dist/assets/SkillsPage-Bo7e39AB.js +1 -0
- package/dashboard/dist/assets/WidgetsPage-HHcIr_Eq.js +1 -0
- package/dashboard/dist/assets/WrappedPage-C-_jLXXU.js +1 -0
- package/dashboard/dist/assets/{arrow-up-right-BJzUcPig.js → arrow-up-right-BJvJHUGj.js} +1 -1
- package/dashboard/dist/assets/{download-CsXweDLj.js → download-CnzY_EgN.js} +1 -1
- package/dashboard/dist/assets/format-1w-q5n2f.js +1 -0
- package/dashboard/dist/assets/main-1kxAVM3m.css +1 -0
- package/dashboard/dist/assets/main-3gBEI6Jl.js +694 -0
- package/dashboard/dist/assets/mock-data-CJOMGcL9.js +1 -0
- package/dashboard/dist/assets/use-limits-display-prefs-D1nT9-mr.js +1 -0
- package/dashboard/dist/assets/use-native-settings-Dh6NzqjY.js +1 -0
- package/dashboard/dist/assets/use-usage-limits-BXR6lzGf.js +1 -0
- package/dashboard/dist/assets/useCurrency-Hhs5jbWw.js +1 -0
- package/dashboard/dist/index.html +16 -28
- package/dashboard/dist/llms.txt +0 -2
- package/dashboard/dist/sitemap.xml +0 -12
- package/package.json +7 -4
- package/src/cli.js +4 -13
- package/src/commands/doctor.js +1 -3
- package/src/commands/init.js +63 -53
- package/src/commands/serve.js +2 -50
- package/src/commands/status.js +1 -55
- package/src/commands/sync.js +134 -227
- package/src/lib/{browser-auth.js → browser-open.js} +3 -149
- package/src/lib/claude-categorizer.js +204 -32
- package/src/lib/codex-context-breakdown.js +83 -7
- package/src/lib/codex-rollout-parser.js +52 -10
- package/src/lib/diagnostics.js +1 -49
- package/src/lib/doctor.js +12 -87
- package/src/lib/file-identity.js +69 -0
- package/src/lib/init-flow.js +6 -3
- package/src/lib/local-api.js +6 -364
- package/src/lib/pricing/curated-overrides.json +1 -1
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +392 -144
- package/src/lib/runtime-config.js +0 -32
- package/src/lib/sqlite-reader.js +83 -8
- package/dashboard/dist/assets/ActivityHeatmap-DPZ_Aj6E.js +0 -42
- package/dashboard/dist/assets/DashboardPage-C9Q5HCDh.js +0 -19
- package/dashboard/dist/assets/DevicePage-DhQFtROo.js +0 -1
- package/dashboard/dist/assets/DialogTitle-DduJCLiL.js +0 -1
- package/dashboard/dist/assets/InsforgeUserHeaderControls-CDqYVWBQ.js +0 -1
- package/dashboard/dist/assets/LandingPage-9345o0vr.js +0 -4356
- package/dashboard/dist/assets/LeaderboardAvatar-DNORVh3n.js +0 -1
- package/dashboard/dist/assets/LeaderboardPage-DBsdCCzY.js +0 -6
- package/dashboard/dist/assets/LeaderboardProfileModal-BuYdBVh_.js +0 -72
- package/dashboard/dist/assets/LeaderboardProfilePage-D4Fe1LOk.js +0 -1
- package/dashboard/dist/assets/LoginPage-BmcybmeK.js +0 -1
- package/dashboard/dist/assets/PopoverPopup-CDL7kqy5.js +0 -1
- package/dashboard/dist/assets/Select-BkzhG41t.js +0 -1
- package/dashboard/dist/assets/SelectItemText-BmRzs9ud.js +0 -1
- package/dashboard/dist/assets/SettingsPage-gXQl_o6s.js +0 -1
- package/dashboard/dist/assets/SkillsPage-BRXYEsKm.js +0 -1
- package/dashboard/dist/assets/WidgetsPage-CLHLz2UW.js +0 -1
- package/dashboard/dist/assets/WrappedPage-p4ZFwQjO.js +0 -1
- package/dashboard/dist/assets/index-BeoRn2gJ.js +0 -2
- package/dashboard/dist/assets/info-CuLr7ZIa.js +0 -1
- package/dashboard/dist/assets/main-CPGI-smk.js +0 -1004
- package/dashboard/dist/assets/main-DAcSUW2t.css +0 -1
- package/dashboard/dist/assets/use-limits-display-prefs-DOB09__q.js +0 -1
- package/dashboard/dist/assets/use-native-settings-JSylS2xS.js +0 -1
- package/dashboard/dist/assets/use-spring-qjUEvMQX.js +0 -1
- package/dashboard/dist/assets/use-usage-limits-B0bzPxNR.js +0 -1
- package/dashboard/dist/assets/useCurrency-BRorShq7.js +0 -1
- package/dashboard/dist/assets/useOpenInteractionType-DPP-6ZsL.js +0 -12
- package/dashboard/dist/share.html +0 -140
- package/src/commands/device-login.js +0 -161
- package/src/lib/upload-throttle.js +0 -148
package/src/commands/serve.js
CHANGED
|
@@ -2,7 +2,6 @@ const http = require("node:http");
|
|
|
2
2
|
const os = require("node:os");
|
|
3
3
|
const path = require("node:path");
|
|
4
4
|
const fssync = require("node:fs");
|
|
5
|
-
const cp = require("node:child_process");
|
|
6
5
|
|
|
7
6
|
const { resolveTrackerPaths } = require("../lib/tracker-paths");
|
|
8
7
|
const { createLocalApiHandler, resolveQueuePath } = require("../lib/local-api");
|
|
@@ -12,7 +11,7 @@ const {
|
|
|
12
11
|
} = require("../lib/local-data-preflight");
|
|
13
12
|
const { ensurePricingLoaded } = require("../lib/pricing");
|
|
14
13
|
const { serveStaticFile } = require("../lib/static-server");
|
|
15
|
-
const { openInBrowser } = require("../lib/browser-
|
|
14
|
+
const { openInBrowser } = require("../lib/browser-open");
|
|
16
15
|
|
|
17
16
|
const DEFAULT_PORT = 7680;
|
|
18
17
|
const DEFAULT_MAX_PORT_ATTEMPTS = 20;
|
|
@@ -20,7 +19,7 @@ const NPM_PACKAGE_NAME = "@ipv9/tokentracker-cli";
|
|
|
20
19
|
const LOCAL_BIND_HOST = "127.0.0.1";
|
|
21
20
|
|
|
22
21
|
function buildPortInUseHint(port) {
|
|
23
|
-
return `Port ${port} is
|
|
22
|
+
return `Port ${port} is unavailable. Try: npx ${NPM_PACKAGE_NAME} serve --port ${port + 1}\n`;
|
|
24
23
|
}
|
|
25
24
|
|
|
26
25
|
function isPortUnavailableError(error) {
|
|
@@ -155,7 +154,6 @@ async function cmdServe(argv) {
|
|
|
155
154
|
try {
|
|
156
155
|
port = await listenOnAvailablePort(server, opts.port, {
|
|
157
156
|
allowFallback: !opts.portExplicit,
|
|
158
|
-
ensurePortFreeFn: opts.portExplicit ? ensurePortFree : null,
|
|
159
157
|
onRetry: (failedPort) => {
|
|
160
158
|
process.stdout.write(`Port ${failedPort} unavailable, trying ${failedPort + 1}...\n`);
|
|
161
159
|
},
|
|
@@ -208,47 +206,6 @@ async function cmdServe(argv) {
|
|
|
208
206
|
await new Promise(() => {});
|
|
209
207
|
}
|
|
210
208
|
|
|
211
|
-
function findPidOnPort(port) {
|
|
212
|
-
try {
|
|
213
|
-
const out = cp.execFileSync("lsof", ["-ti", `tcp:${port}`], { encoding: "utf8", timeout: 5000 });
|
|
214
|
-
const pids = out.trim().split(/\s+/).map(Number).filter((n) => Number.isFinite(n) && n > 0);
|
|
215
|
-
return pids;
|
|
216
|
-
} catch (_e) {
|
|
217
|
-
return [];
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
async function ensurePortFree(port) {
|
|
222
|
-
const pids = findPidOnPort(port);
|
|
223
|
-
if (pids.length === 0) return;
|
|
224
|
-
|
|
225
|
-
// Don't kill ourselves
|
|
226
|
-
const self = process.pid;
|
|
227
|
-
const targets = pids.filter((p) => p !== self);
|
|
228
|
-
if (targets.length === 0) return;
|
|
229
|
-
|
|
230
|
-
process.stdout.write(`Stopping previous server on port ${port} (pid ${targets.join(", ")})...\n`);
|
|
231
|
-
for (const pid of targets) {
|
|
232
|
-
try {
|
|
233
|
-
process.kill(pid, "SIGTERM");
|
|
234
|
-
} catch (_e) {}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
// Wait briefly for port to free up
|
|
238
|
-
for (let i = 0; i < 10; i++) {
|
|
239
|
-
await new Promise((r) => setTimeout(r, 300));
|
|
240
|
-
if (findPidOnPort(port).length === 0) return;
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
// Force kill if still alive
|
|
244
|
-
for (const pid of targets) {
|
|
245
|
-
try {
|
|
246
|
-
process.kill(pid, "SIGKILL");
|
|
247
|
-
} catch (_e) {}
|
|
248
|
-
}
|
|
249
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
250
|
-
}
|
|
251
|
-
|
|
252
209
|
function listenOnce(server, port, host) {
|
|
253
210
|
return new Promise((resolve, reject) => {
|
|
254
211
|
let settled = false;
|
|
@@ -283,7 +240,6 @@ async function listenOnAvailablePort(
|
|
|
283
240
|
host = LOCAL_BIND_HOST,
|
|
284
241
|
allowFallback = false,
|
|
285
242
|
maxAttempts = DEFAULT_MAX_PORT_ATTEMPTS,
|
|
286
|
-
ensurePortFreeFn = null,
|
|
287
243
|
onRetry = null,
|
|
288
244
|
} = {},
|
|
289
245
|
) {
|
|
@@ -292,10 +248,6 @@ async function listenOnAvailablePort(
|
|
|
292
248
|
let lastError = null;
|
|
293
249
|
|
|
294
250
|
for (let i = 0; i < attempts && port < 65536; i++, port++) {
|
|
295
|
-
if (ensurePortFreeFn) {
|
|
296
|
-
await ensurePortFreeFn(port);
|
|
297
|
-
}
|
|
298
|
-
|
|
299
251
|
try {
|
|
300
252
|
await listenOnce(server, port, host);
|
|
301
253
|
return port;
|
package/src/commands/status.js
CHANGED
|
@@ -25,9 +25,6 @@ const {
|
|
|
25
25
|
describeCopilotOtelStatus,
|
|
26
26
|
readCopilotOauthToken,
|
|
27
27
|
} = require("../lib/usage-limits");
|
|
28
|
-
const {
|
|
29
|
-
normalizeState: normalizeUploadState,
|
|
30
|
-
} = require("../lib/upload-throttle");
|
|
31
28
|
const { collectTrackerDiagnostics } = require("../lib/diagnostics");
|
|
32
29
|
const { detectPassiveProviders, isPassiveModeActive } = require("../lib/passive-mode");
|
|
33
30
|
const { probeOpenclawHookState } = require("../lib/openclaw-hook");
|
|
@@ -68,15 +65,11 @@ async function cmdStatus(argv = []) {
|
|
|
68
65
|
|
|
69
66
|
const home = os.homedir();
|
|
70
67
|
const { trackerDir, binDir } = await resolveTrackerPaths({ home });
|
|
71
|
-
const configPath = path.join(trackerDir, "config.json");
|
|
72
68
|
const queuePath = path.join(trackerDir, "queue.jsonl");
|
|
73
|
-
const queueStatePath = path.join(trackerDir, "queue.state.json");
|
|
74
69
|
const cursorsPath = path.join(trackerDir, "cursors.json");
|
|
75
70
|
const notifySignalPath = path.join(trackerDir, "notify.signal");
|
|
76
71
|
const openclawSignalPath = path.join(trackerDir, "openclaw.signal");
|
|
77
72
|
const throttlePath = path.join(trackerDir, "sync.throttle");
|
|
78
|
-
const uploadThrottlePath = path.join(trackerDir, "upload.throttle.json");
|
|
79
|
-
const autoRetryPath = path.join(trackerDir, "auto.retry.json");
|
|
80
73
|
const codexHome = process.env.CODEX_HOME || path.join(home, ".codex");
|
|
81
74
|
const codexConfigPath = path.join(codexHome, "config.toml");
|
|
82
75
|
const codeHome = process.env.CODE_HOME || path.join(home, ".code");
|
|
@@ -99,16 +92,9 @@ async function cmdStatus(argv = []) {
|
|
|
99
92
|
const codebuddyHookCommand = buildHookCommand(notifyPath, "codebuddy");
|
|
100
93
|
const geminiHookCommand = buildGeminiHookCommand(notifyPath);
|
|
101
94
|
|
|
102
|
-
const config = await readJson(configPath);
|
|
103
95
|
const cursors = await readJson(cursorsPath);
|
|
104
|
-
const queueState = (await readJson(queueStatePath)) || { offset: 0 };
|
|
105
|
-
const uploadThrottle = normalizeUploadState(
|
|
106
|
-
await readJson(uploadThrottlePath),
|
|
107
|
-
);
|
|
108
|
-
const autoRetry = await readJson(autoRetryPath);
|
|
109
96
|
|
|
110
97
|
const queueSize = await safeStatSize(queuePath);
|
|
111
|
-
const pendingBytes = Math.max(0, queueSize - (queueState.offset || 0));
|
|
112
98
|
|
|
113
99
|
const lastNotify = (await safeReadText(notifySignalPath))?.trim() || null;
|
|
114
100
|
const lastOpenclawSync =
|
|
@@ -148,23 +134,6 @@ async function cmdStatus(argv = []) {
|
|
|
148
134
|
env: process.env,
|
|
149
135
|
});
|
|
150
136
|
|
|
151
|
-
const lastUpload = uploadThrottle.lastSuccessMs
|
|
152
|
-
? parseEpochMsToIso(uploadThrottle.lastSuccessMs)
|
|
153
|
-
: typeof queueState.updatedAt === "string"
|
|
154
|
-
? queueState.updatedAt
|
|
155
|
-
: null;
|
|
156
|
-
const nextUpload = parseEpochMsToIso(uploadThrottle.nextAllowedAtMs || null);
|
|
157
|
-
const backoffUntil = parseEpochMsToIso(uploadThrottle.backoffUntilMs || null);
|
|
158
|
-
const lastUploadError = uploadThrottle.lastError
|
|
159
|
-
? `${uploadThrottle.lastErrorAt || "unknown"} ${uploadThrottle.lastError}`
|
|
160
|
-
: null;
|
|
161
|
-
const autoRetryAt = parseEpochMsToIso(autoRetry?.retryAtMs || null);
|
|
162
|
-
const autoRetryLine = autoRetryAt
|
|
163
|
-
? `- Auto retry after: ${autoRetryAt} (${autoRetry?.reason || "scheduled"}, pending ${Number(
|
|
164
|
-
autoRetry?.pendingBytes || 0,
|
|
165
|
-
)} bytes)`
|
|
166
|
-
: null;
|
|
167
|
-
|
|
168
137
|
const subscriptions = await collectLocalSubscriptions({
|
|
169
138
|
home,
|
|
170
139
|
env: process.env,
|
|
@@ -277,22 +246,13 @@ async function cmdStatus(argv = []) {
|
|
|
277
246
|
if (opts.json || opts.light) {
|
|
278
247
|
const summary = {
|
|
279
248
|
generated_at: new Date().toISOString(),
|
|
280
|
-
base_url: config?.baseUrl || null,
|
|
281
|
-
device_token_set: Boolean(config?.deviceToken),
|
|
282
249
|
queue: {
|
|
283
|
-
pending_bytes: pendingBytes,
|
|
284
250
|
size_bytes: queueSize,
|
|
285
|
-
offset: queueState.offset || 0,
|
|
286
251
|
},
|
|
287
252
|
last_parse: cursors?.updatedAt || null,
|
|
288
253
|
last_notify: lastNotify || null,
|
|
289
254
|
last_openclaw_sync: lastOpenclawSync || null,
|
|
290
255
|
last_notify_spawn: lastNotifySpawn || null,
|
|
291
|
-
last_upload: lastUpload || null,
|
|
292
|
-
next_upload_after: nextUpload || null,
|
|
293
|
-
backoff_until: backoffUntil || null,
|
|
294
|
-
last_upload_error: lastUploadError || null,
|
|
295
|
-
auto_retry: autoRetry || null,
|
|
296
256
|
hooks: {
|
|
297
257
|
codex_notify: notifyConfigured,
|
|
298
258
|
every_code_notify: everyCodeConfigured,
|
|
@@ -371,18 +331,11 @@ async function cmdStatus(argv = []) {
|
|
|
371
331
|
process.stdout.write(
|
|
372
332
|
[
|
|
373
333
|
"Status:",
|
|
374
|
-
`-
|
|
375
|
-
`- Device token: ${config?.deviceToken ? "set" : "unset"}`,
|
|
376
|
-
`- Queue: ${pendingBytes} bytes pending`,
|
|
334
|
+
`- Queue: ${queueSize} bytes`,
|
|
377
335
|
`- Last parse: ${cursors?.updatedAt || "never"}`,
|
|
378
336
|
`- Last notify: ${lastNotify || "never"}`,
|
|
379
337
|
`- Last OpenClaw-triggered sync: ${lastOpenclawSync || "never"}`,
|
|
380
338
|
`- Last notify-triggered sync: ${lastNotifySpawn || "never"}`,
|
|
381
|
-
`- Last upload: ${lastUpload || "never"}`,
|
|
382
|
-
`- Next upload after: ${nextUpload || "never"}`,
|
|
383
|
-
`- Backoff until: ${backoffUntil || "never"}`,
|
|
384
|
-
lastUploadError ? `- Last upload error: ${lastUploadError}` : null,
|
|
385
|
-
autoRetryLine,
|
|
386
339
|
`- Codex notify: ${notifyConfigured ? JSON.stringify(codexNotify) : "unset"}`,
|
|
387
340
|
`- Every Code notify: ${everyCodeConfigured ? JSON.stringify(everyCodeNotify) : "unset"}`,
|
|
388
341
|
`- Claude hooks: ${claudeHookConfigured ? "set" : "unset"}`,
|
|
@@ -538,16 +491,9 @@ function renderLightTable(summary) {
|
|
|
538
491
|
const rows = [];
|
|
539
492
|
const push = (k, v) => rows.push([k, v == null || v === "" ? "—" : String(v)]);
|
|
540
493
|
|
|
541
|
-
push("Base URL", summary.base_url);
|
|
542
|
-
push("Device token", summary.device_token_set ? "set" : "unset");
|
|
543
|
-
push("Queue pending (bytes)", summary.queue.pending_bytes);
|
|
544
494
|
push("Queue size (bytes)", summary.queue.size_bytes);
|
|
545
495
|
push("Last parse", summary.last_parse);
|
|
546
496
|
push("Last notify", summary.last_notify);
|
|
547
|
-
push("Last upload", summary.last_upload);
|
|
548
|
-
push("Next upload after", summary.next_upload_after);
|
|
549
|
-
push("Backoff until", summary.backoff_until);
|
|
550
|
-
if (summary.last_upload_error) push("Last upload error", summary.last_upload_error);
|
|
551
497
|
|
|
552
498
|
for (const [name, state] of Object.entries(summary.hooks || {})) {
|
|
553
499
|
push(`Hook · ${name}`, state ? "set" : "unset");
|
package/src/commands/sync.js
CHANGED
|
@@ -62,13 +62,6 @@ const {
|
|
|
62
62
|
} = require("../lib/rollout");
|
|
63
63
|
const { computeClaudeGroundTruthBuckets } = require("../lib/claude-categorizer");
|
|
64
64
|
const { createProgress, renderBar, formatNumber, formatBytes } = require("../lib/progress");
|
|
65
|
-
const {
|
|
66
|
-
normalizeState: normalizeUploadState,
|
|
67
|
-
decideAutoUpload,
|
|
68
|
-
recordUploadFailure,
|
|
69
|
-
recordUploadSuccess,
|
|
70
|
-
parseRetryAfterMs,
|
|
71
|
-
} = require("../lib/upload-throttle");
|
|
72
65
|
const {
|
|
73
66
|
isCursorInstalled,
|
|
74
67
|
extractCursorSessionToken,
|
|
@@ -77,10 +70,16 @@ const {
|
|
|
77
70
|
} = require("../lib/cursor-config");
|
|
78
71
|
const { purgeProjectUsage } = require("../lib/project-usage-purge");
|
|
79
72
|
const { resolveTrackerPaths } = require("../lib/tracker-paths");
|
|
80
|
-
const { resolveRuntimeConfig } = require("../lib/runtime-config");
|
|
81
73
|
|
|
82
74
|
const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
|
|
83
75
|
const ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY = "rolloutCumulativeDeltaReparse_2026_05";
|
|
76
|
+
// #75: rebuild Codex/every-code aggregates that were accumulated with the
|
|
77
|
+
// pre-fix parser, which counted replayed subagent-fork parent history as child
|
|
78
|
+
// consumption (up to ~40x Cache Read inflation on the local fork corpus). The
|
|
79
|
+
// rolloutCumulativeDeltaReparse migration above already rebuilt these once, but
|
|
80
|
+
// with the still-buggy parser, so a fresh key is required to rebuild again now
|
|
81
|
+
// that the parser is fixed.
|
|
82
|
+
const CODEX_FORK_OVERCOUNT_MIGRATION_KEY = "codexForkOvercountRebuild_2026_07";
|
|
84
83
|
const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
|
|
85
84
|
const GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY = "grokAppendOnlyRepair_2026_05_v4";
|
|
86
85
|
const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
|
|
@@ -146,14 +145,11 @@ async function cmdSync(argv) {
|
|
|
146
145
|
const queueStatePath = path.join(trackerDir, "queue.state.json");
|
|
147
146
|
const projectQueuePath = path.join(trackerDir, "project.queue.jsonl");
|
|
148
147
|
const projectQueueStatePath = path.join(trackerDir, "project.queue.state.json");
|
|
149
|
-
const uploadThrottlePath = path.join(trackerDir, "upload.throttle.json");
|
|
150
148
|
const grokSignalPath = path.join(trackerDir, "grok-last-session.json");
|
|
151
149
|
const legacyGrokSignalPath = path.join(trackerDir, "tracker", "grok-last-session.json");
|
|
152
150
|
|
|
153
151
|
const config = await readJson(configPath);
|
|
154
152
|
const cursors = (await readJson(cursorsPath)) || { version: 1, files: {}, updatedAt: null };
|
|
155
|
-
const uploadThrottle = normalizeUploadState(await readJson(uploadThrottlePath));
|
|
156
|
-
let uploadThrottleState = uploadThrottle;
|
|
157
153
|
let grokHookSignal = null;
|
|
158
154
|
let grokHookSignalPath = null;
|
|
159
155
|
for (const candidate of [grokSignalPath, legacyGrokSignalPath]) {
|
|
@@ -199,6 +195,7 @@ async function cmdSync(argv) {
|
|
|
199
195
|
}
|
|
200
196
|
|
|
201
197
|
await migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rolloutFiles });
|
|
198
|
+
await migrateCodexForkOvercountRebuild({ cursors, queuePath, projectQueuePath, rolloutFiles });
|
|
202
199
|
|
|
203
200
|
const openclawFiles = openclawSignal?.sessionFile
|
|
204
201
|
? [{ path: openclawSignal.sessionFile, source: "openclaw" }]
|
|
@@ -928,69 +925,7 @@ async function cmdSync(argv) {
|
|
|
928
925
|
|
|
929
926
|
progress?.stop();
|
|
930
927
|
|
|
931
|
-
const runtime = resolveRuntimeConfig({ config: config || {}, env: process.env });
|
|
932
|
-
|
|
933
|
-
let uploadResult = { inserted: 0, skipped: 0 };
|
|
934
|
-
let uploadAttempted = false;
|
|
935
|
-
|
|
936
|
-
if (runtime.deviceToken && runtime.baseUrl) {
|
|
937
|
-
uploadAttempted = true;
|
|
938
|
-
try {
|
|
939
|
-
uploadResult = await drainQueueToCloud({
|
|
940
|
-
baseUrl: runtime.baseUrl,
|
|
941
|
-
deviceToken: runtime.deviceToken,
|
|
942
|
-
queuePath,
|
|
943
|
-
queueStatePath,
|
|
944
|
-
maxBatches: opts.drain ? 100 : 5,
|
|
945
|
-
batchSize: 200,
|
|
946
|
-
});
|
|
947
|
-
// Record success so the exponential backoff step resets — otherwise
|
|
948
|
-
// a single past failure keeps us pessimistically throttled forever.
|
|
949
|
-
uploadThrottleState = recordUploadSuccess({
|
|
950
|
-
nowMs: Date.now(),
|
|
951
|
-
state: uploadThrottleState,
|
|
952
|
-
});
|
|
953
|
-
await writeJson(uploadThrottlePath, uploadThrottleState);
|
|
954
|
-
} catch (e) {
|
|
955
|
-
// Persist a backoff on 429 / 5xx so the next auto-sync waits instead
|
|
956
|
-
// of retrying immediately and making the rate-limit worse. The
|
|
957
|
-
// throttle module already parses Retry-After when we surface it on
|
|
958
|
-
// the error object (drainQueueToCloud stamps err.status + err.retryAfterMs).
|
|
959
|
-
uploadThrottleState = recordUploadFailure({
|
|
960
|
-
nowMs: Date.now(),
|
|
961
|
-
state: uploadThrottleState,
|
|
962
|
-
error: e,
|
|
963
|
-
});
|
|
964
|
-
await writeJson(uploadThrottlePath, uploadThrottleState);
|
|
965
|
-
if (!opts.auto) {
|
|
966
|
-
process.stderr.write(`Upload error: ${e?.message || e}\n`);
|
|
967
|
-
}
|
|
968
|
-
}
|
|
969
|
-
}
|
|
970
928
|
|
|
971
|
-
const afterState = (await readJson(queueStatePath)) || { offset: 0 };
|
|
972
|
-
const queueSize = await safeStatSize(queuePath);
|
|
973
|
-
const projectAfterState = (await readJson(projectQueueStatePath)) || { offset: 0 };
|
|
974
|
-
const projectQueueSize = await safeStatSize(projectQueuePath);
|
|
975
|
-
const pendingBytes =
|
|
976
|
-
Math.max(0, queueSize - Number(afterState.offset || 0)) +
|
|
977
|
-
Math.max(0, projectQueueSize - Number(projectAfterState.offset || 0));
|
|
978
|
-
|
|
979
|
-
if (pendingBytes <= 0) {
|
|
980
|
-
await clearAutoRetry(trackerDir);
|
|
981
|
-
} else if (opts.auto && uploadAttempted) {
|
|
982
|
-
const retryAtMs = Number(uploadThrottleState?.nextAllowedAtMs || 0);
|
|
983
|
-
if (retryAtMs > Date.now()) {
|
|
984
|
-
await scheduleAutoRetry({
|
|
985
|
-
trackerDir,
|
|
986
|
-
retryAtMs,
|
|
987
|
-
reason: "backlog",
|
|
988
|
-
pendingBytes,
|
|
989
|
-
source: "auto-backlog",
|
|
990
|
-
autoRetryNoSpawn: runtime.autoRetryNoSpawn,
|
|
991
|
-
});
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
929
|
|
|
995
930
|
const totalParsed =
|
|
996
931
|
parseResult.filesProcessed +
|
|
@@ -1045,9 +980,6 @@ async function cmdSync(argv) {
|
|
|
1045
980
|
const summary = {
|
|
1046
981
|
totalParsed,
|
|
1047
982
|
totalBuckets,
|
|
1048
|
-
upload: uploadResult,
|
|
1049
|
-
uploadAttempted,
|
|
1050
|
-
pendingBytes,
|
|
1051
983
|
};
|
|
1052
984
|
|
|
1053
985
|
if (!opts.auto) {
|
|
@@ -1056,12 +988,6 @@ async function cmdSync(argv) {
|
|
|
1056
988
|
"Sync finished:",
|
|
1057
989
|
`- Parsed files: ${totalParsed}`,
|
|
1058
990
|
`- New 30-min buckets queued: ${totalBuckets}`,
|
|
1059
|
-
runtime.deviceToken
|
|
1060
|
-
? `- Uploaded: ${uploadResult.inserted} inserted, ${uploadResult.skipped} skipped`
|
|
1061
|
-
: "- Uploaded: skipped (no device token)",
|
|
1062
|
-
runtime.deviceToken && pendingBytes > 0 && !opts.drain
|
|
1063
|
-
? `- Remaining: ${formatBytes(pendingBytes)} pending (run sync again, or use --drain)`
|
|
1064
|
-
: null,
|
|
1065
991
|
"",
|
|
1066
992
|
]
|
|
1067
993
|
.filter(Boolean)
|
|
@@ -1102,10 +1028,12 @@ module.exports = {
|
|
|
1102
1028
|
cmdSync,
|
|
1103
1029
|
migrateCursorUnknownBuckets,
|
|
1104
1030
|
migrateRolloutCumulativeDeltaBuckets,
|
|
1031
|
+
migrateCodexForkOvercountRebuild,
|
|
1105
1032
|
reincludeClaudeMemObserverFiles,
|
|
1106
1033
|
repairGrokQueueFromSessionSnapshots,
|
|
1107
1034
|
CURSOR_UNKNOWN_MIGRATION_KEY,
|
|
1108
1035
|
ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY,
|
|
1036
|
+
CODEX_FORK_OVERCOUNT_MIGRATION_KEY,
|
|
1109
1037
|
CLAUDE_MEM_OBSERVER_REINCLUDE_KEY,
|
|
1110
1038
|
GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY,
|
|
1111
1039
|
};
|
|
@@ -1276,102 +1204,10 @@ function deriveAutoSkipReason({ decision, state }) {
|
|
|
1276
1204
|
return "throttled";
|
|
1277
1205
|
}
|
|
1278
1206
|
|
|
1279
|
-
async function scheduleAutoRetry({
|
|
1280
|
-
trackerDir,
|
|
1281
|
-
retryAtMs,
|
|
1282
|
-
reason,
|
|
1283
|
-
pendingBytes,
|
|
1284
|
-
source,
|
|
1285
|
-
autoRetryNoSpawn,
|
|
1286
|
-
}) {
|
|
1287
|
-
const retryMs = coerceRetryMs(retryAtMs);
|
|
1288
|
-
if (!retryMs) return { scheduled: false, retryAtMs: 0 };
|
|
1289
|
-
|
|
1290
|
-
const retryPath = path.join(trackerDir, AUTO_RETRY_FILENAME);
|
|
1291
|
-
const nowMs = Date.now();
|
|
1292
|
-
const existing = await readJson(retryPath);
|
|
1293
|
-
const existingMs = coerceRetryMs(existing?.retryAtMs);
|
|
1294
|
-
if (existingMs && existingMs >= retryMs - 1000) {
|
|
1295
|
-
return { scheduled: false, retryAtMs: existingMs };
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
|
-
const payload = {
|
|
1299
|
-
version: 1,
|
|
1300
|
-
retryAtMs: retryMs,
|
|
1301
|
-
retryAt: new Date(retryMs).toISOString(),
|
|
1302
|
-
reason: typeof reason === "string" && reason.length > 0 ? reason : "throttled",
|
|
1303
|
-
pendingBytes: Math.max(0, Number(pendingBytes || 0)),
|
|
1304
|
-
scheduledAt: new Date(nowMs).toISOString(),
|
|
1305
|
-
source: typeof source === "string" ? source : "auto",
|
|
1306
|
-
};
|
|
1307
|
-
|
|
1308
|
-
await writeJson(retryPath, payload);
|
|
1309
|
-
|
|
1310
|
-
const delayMs = Math.min(AUTO_RETRY_MAX_DELAY_MS, Math.max(0, retryMs - nowMs));
|
|
1311
|
-
if (delayMs <= 0) return { scheduled: false, retryAtMs: retryMs };
|
|
1312
|
-
if (autoRetryNoSpawn) {
|
|
1313
|
-
return { scheduled: false, retryAtMs: retryMs };
|
|
1314
|
-
}
|
|
1315
|
-
|
|
1316
|
-
spawnAutoRetryProcess({
|
|
1317
|
-
retryPath,
|
|
1318
|
-
trackerBinPath: path.join(trackerDir, "app", "bin", "tracker.js"),
|
|
1319
|
-
fallbackPkg: "@ipv9/tokentracker-cli",
|
|
1320
|
-
delayMs,
|
|
1321
|
-
});
|
|
1322
|
-
return { scheduled: true, retryAtMs: retryMs };
|
|
1323
|
-
}
|
|
1324
1207
|
|
|
1325
|
-
async function clearAutoRetry(trackerDir) {
|
|
1326
|
-
const retryPath = path.join(trackerDir, AUTO_RETRY_FILENAME);
|
|
1327
|
-
await fs.unlink(retryPath).catch(() => {});
|
|
1328
|
-
}
|
|
1329
1208
|
|
|
1330
|
-
function spawnAutoRetryProcess({ retryPath, trackerBinPath, fallbackPkg, delayMs }) {
|
|
1331
|
-
const script = buildAutoRetryScript({ retryPath, trackerBinPath, fallbackPkg, delayMs });
|
|
1332
|
-
try {
|
|
1333
|
-
const child = cp.spawn(process.execPath, ["-e", script], {
|
|
1334
|
-
detached: true,
|
|
1335
|
-
stdio: "ignore",
|
|
1336
|
-
env: process.env,
|
|
1337
|
-
});
|
|
1338
|
-
child.unref();
|
|
1339
|
-
} catch (_e) {}
|
|
1340
|
-
}
|
|
1341
1209
|
|
|
1342
|
-
function buildAutoRetryScript({ retryPath, trackerBinPath, fallbackPkg, delayMs }) {
|
|
1343
|
-
return (
|
|
1344
|
-
`'use strict';\n` +
|
|
1345
|
-
`const fs = require('node:fs');\n` +
|
|
1346
|
-
`const cp = require('node:child_process');\n` +
|
|
1347
|
-
`const retryPath = ${JSON.stringify(retryPath)};\n` +
|
|
1348
|
-
`const trackerBinPath = ${JSON.stringify(trackerBinPath)};\n` +
|
|
1349
|
-
`const fallbackPkg = ${JSON.stringify(fallbackPkg)};\n` +
|
|
1350
|
-
`const delayMs = ${Math.max(0, Math.floor(delayMs || 0))};\n` +
|
|
1351
|
-
`setTimeout(() => {\n` +
|
|
1352
|
-
` let retryAtMs = 0;\n` +
|
|
1353
|
-
` try {\n` +
|
|
1354
|
-
` const raw = fs.readFileSync(retryPath, 'utf8');\n` +
|
|
1355
|
-
` retryAtMs = Number(JSON.parse(raw).retryAtMs || 0);\n` +
|
|
1356
|
-
` } catch (_) {}\n` +
|
|
1357
|
-
` if (!retryAtMs || Date.now() + 1000 < retryAtMs) process.exit(0);\n` +
|
|
1358
|
-
` const argv = ['sync', '--auto', '--from-retry'];\n` +
|
|
1359
|
-
` const cmd = fs.existsSync(trackerBinPath)\n` +
|
|
1360
|
-
` ? [process.execPath, trackerBinPath, ...argv]\n` +
|
|
1361
|
-
` : ['npx', '--yes', fallbackPkg, ...argv];\n` +
|
|
1362
|
-
` try {\n` +
|
|
1363
|
-
` const child = cp.spawn(cmd[0], cmd.slice(1), { detached: true, stdio: 'ignore', env: process.env });\n` +
|
|
1364
|
-
` child.unref();\n` +
|
|
1365
|
-
` } catch (_) {}\n` +
|
|
1366
|
-
`}, delayMs);\n`
|
|
1367
|
-
);
|
|
1368
|
-
}
|
|
1369
1210
|
|
|
1370
|
-
function coerceRetryMs(v) {
|
|
1371
|
-
const n = Number(v);
|
|
1372
|
-
if (!Number.isFinite(n) || n <= 0) return 0;
|
|
1373
|
-
return Math.floor(n);
|
|
1374
|
-
}
|
|
1375
1211
|
|
|
1376
1212
|
async function writeOpenclawSignal(trackerDir) {
|
|
1377
1213
|
const openclawSignalPath = path.join(trackerDir, "openclaw.signal");
|
|
@@ -1382,63 +1218,10 @@ async function writeOpenclawSignal(trackerDir) {
|
|
|
1382
1218
|
}
|
|
1383
1219
|
}
|
|
1384
1220
|
|
|
1385
|
-
const AUTO_RETRY_FILENAME = "auto.retry.json";
|
|
1386
1221
|
const AUTO_RETRY_MAX_DELAY_MS = 2 * 60 * 60 * 1000;
|
|
1387
1222
|
|
|
1388
|
-
const INGEST_SLUG = "tokentracker-ingest";
|
|
1389
1223
|
const MAX_INGEST_BUCKETS = 500;
|
|
1390
1224
|
|
|
1391
|
-
async function drainQueueToCloud({ baseUrl, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200 }) {
|
|
1392
|
-
const state = (await readJson(queueStatePath)) || { offset: 0 };
|
|
1393
|
-
let offset = Number(state.offset || 0);
|
|
1394
|
-
let inserted = 0;
|
|
1395
|
-
let skipped = 0;
|
|
1396
|
-
|
|
1397
|
-
const queueSize = await safeStatSize(queuePath);
|
|
1398
|
-
const limit = Math.min(Math.max(1, Math.floor(Number(batchSize || 200))), MAX_INGEST_BUCKETS);
|
|
1399
|
-
|
|
1400
|
-
for (let batch = 0; batch < maxBatches; batch++) {
|
|
1401
|
-
if (offset >= queueSize) break;
|
|
1402
|
-
const result = await readQueueBatch(queuePath, offset, limit);
|
|
1403
|
-
if (result.buckets.length === 0) break;
|
|
1404
|
-
|
|
1405
|
-
const root = baseUrl.replace(/\/$/, "");
|
|
1406
|
-
const anonKey = process.env.TOKENTRACKER_INSFORGE_ANON_KEY || "";
|
|
1407
|
-
const headers = {
|
|
1408
|
-
"Content-Type": "application/json",
|
|
1409
|
-
Accept: "application/json",
|
|
1410
|
-
Authorization: `Bearer ${deviceToken}`,
|
|
1411
|
-
};
|
|
1412
|
-
if (anonKey) headers.apikey = anonKey;
|
|
1413
|
-
const res = await fetch(`${root}/functions/${INGEST_SLUG}`, {
|
|
1414
|
-
method: "POST",
|
|
1415
|
-
headers,
|
|
1416
|
-
body: JSON.stringify({ hourly: result.buckets }),
|
|
1417
|
-
});
|
|
1418
|
-
|
|
1419
|
-
const rawText = await res.text().catch(() => "");
|
|
1420
|
-
let data = {};
|
|
1421
|
-
try { data = JSON.parse(rawText); } catch { data = {}; }
|
|
1422
|
-
if (!res.ok) {
|
|
1423
|
-
const err = new Error(`HTTP ${res.status}: ${rawText.substring(0, 500)}`);
|
|
1424
|
-
err.status = res.status;
|
|
1425
|
-
const retryAfter = res.headers?.get?.("Retry-After") ?? null;
|
|
1426
|
-
const retryAfterMs = parseRetryAfterMs(retryAfter);
|
|
1427
|
-
if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
|
|
1428
|
-
throw err;
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
inserted += Number(data?.inserted || 0);
|
|
1432
|
-
skipped += Number(data?.skipped || 0);
|
|
1433
|
-
|
|
1434
|
-
offset = result.nextOffset;
|
|
1435
|
-
state.offset = offset;
|
|
1436
|
-
state.updatedAt = new Date().toISOString();
|
|
1437
|
-
await writeJson(queueStatePath, state);
|
|
1438
|
-
}
|
|
1439
|
-
|
|
1440
|
-
return { inserted, skipped };
|
|
1441
|
-
}
|
|
1442
1225
|
|
|
1443
1226
|
async function readQueueBatch(queuePath, startOffset, maxBuckets) {
|
|
1444
1227
|
const st = await fs.stat(queuePath).catch(() => null);
|
|
@@ -1922,6 +1705,130 @@ async function migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rollou
|
|
|
1922
1705
|
cursors.migrations[ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY] = new Date().toISOString();
|
|
1923
1706
|
}
|
|
1924
1707
|
|
|
1708
|
+
// One-time rebuild for issue #75. The prior parser counted a Codex subagent
|
|
1709
|
+
// fork's replayed parent token_count history as the child's consumption. After
|
|
1710
|
+
// the parser fix (PR #77) that only corrects go-forward parsing — already-
|
|
1711
|
+
// ingested files sit at offset = EOF and are never re-read — so their inflated
|
|
1712
|
+
// contribution stays baked into cursors.hourly / project aggregates and the
|
|
1713
|
+
// dashboard until this migration retracts it and forces a clean re-parse.
|
|
1714
|
+
//
|
|
1715
|
+
// Choreography mirrors migrateRolloutCumulativeDeltaBuckets (codex-scoped, so
|
|
1716
|
+
// no other provider is touched): (1) drop the per-file cursors so the same-run
|
|
1717
|
+
// sync re-parses from offset 0 with the fixed accounting; (2) delete the
|
|
1718
|
+
// inflated hourly buckets and append zero rows to queue.jsonl — the dashboard
|
|
1719
|
+
// keep-lasts per source|model|hour_start (local-api.js readQueueData), so the
|
|
1720
|
+
// re-parse's corrected row supersedes the zero, or the zero stands for a file
|
|
1721
|
+
// no longer on disk; (3) do the same for project aggregates, which are on by
|
|
1722
|
+
// default (sync passes projectQueuePath) and would otherwise double-count on
|
|
1723
|
+
// re-parse — readProjectQueueData also keep-lasts per project_key|source|
|
|
1724
|
+
// hour_start. Run-once via the migrations sentinel (the issue asks for a
|
|
1725
|
+
// one-time rebuild, not a repeatable command).
|
|
1726
|
+
//
|
|
1727
|
+
// Project aggregates are handled conservatively: the accumulator buckets are
|
|
1728
|
+
// reset (so files still on disk rebuild clean and their corrected row wins via
|
|
1729
|
+
// keep-last), but no zero rows are written. A project bucket whose file was
|
|
1730
|
+
// deleted or whose repo moved keeps its prior queue row rather than being
|
|
1731
|
+
// zeroed — we never destroy project history we can no longer recompute. Main-
|
|
1732
|
+
// dashboard (source) totals are always rebuilt regardless of project
|
|
1733
|
+
// resolution. See step (3).
|
|
1734
|
+
async function migrateCodexForkOvercountRebuild({
|
|
1735
|
+
cursors,
|
|
1736
|
+
queuePath,
|
|
1737
|
+
projectQueuePath,
|
|
1738
|
+
rolloutFiles,
|
|
1739
|
+
}) {
|
|
1740
|
+
if (!cursors || typeof cursors !== "object") return;
|
|
1741
|
+
cursors.migrations = cursors.migrations || {};
|
|
1742
|
+
if (cursors.migrations[CODEX_FORK_OVERCOUNT_MIGRATION_KEY]) return;
|
|
1743
|
+
|
|
1744
|
+
const rolloutPaths = new Set();
|
|
1745
|
+
for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
|
|
1746
|
+
const filePath = typeof entry === "string" ? entry : entry?.path;
|
|
1747
|
+
const source = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
|
|
1748
|
+
if (filePath && (source === "codex" || source === "every-code")) {
|
|
1749
|
+
rolloutPaths.add(filePath);
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
// (1) Drop per-file cursors -> clean re-parse from offset 0 with the fix.
|
|
1754
|
+
if (cursors.files && typeof cursors.files === "object") {
|
|
1755
|
+
for (const filePath of rolloutPaths) delete cursors.files[filePath];
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
// (2) Retract + delete inflated hourly codex/every-code buckets.
|
|
1759
|
+
const hourlyRetractions = [];
|
|
1760
|
+
const buckets = cursors.hourly?.buckets;
|
|
1761
|
+
if (buckets && typeof buckets === "object") {
|
|
1762
|
+
for (const key of Object.keys(buckets)) {
|
|
1763
|
+
const [source, model, ...hourParts] = key.split("|");
|
|
1764
|
+
if (source !== "codex" && source !== "every-code") continue;
|
|
1765
|
+
hourlyRetractions.push(
|
|
1766
|
+
JSON.stringify({
|
|
1767
|
+
source,
|
|
1768
|
+
model: model || "unknown",
|
|
1769
|
+
hour_start: hourParts.join("|"),
|
|
1770
|
+
input_tokens: 0,
|
|
1771
|
+
cached_input_tokens: 0,
|
|
1772
|
+
cache_creation_input_tokens: 0,
|
|
1773
|
+
output_tokens: 0,
|
|
1774
|
+
reasoning_output_tokens: 0,
|
|
1775
|
+
total_tokens: 0,
|
|
1776
|
+
billable_total_tokens: 0,
|
|
1777
|
+
conversation_count: 0,
|
|
1778
|
+
}),
|
|
1779
|
+
);
|
|
1780
|
+
delete buckets[key];
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
const groupQueued = cursors.hourly?.groupQueued;
|
|
1784
|
+
if (groupQueued && typeof groupQueued === "object") {
|
|
1785
|
+
for (const key of Object.keys(groupQueued)) {
|
|
1786
|
+
if (key.startsWith("codex|") || key.startsWith("every-code|")) {
|
|
1787
|
+
delete groupQueued[key];
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
if (hourlyRetractions.length > 0) {
|
|
1792
|
+
await ensureDir(path.dirname(queuePath));
|
|
1793
|
+
await fs.appendFile(queuePath, hourlyRetractions.join("\n") + "\n");
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
// (3) Reset the inflated project codex/every-code accumulator buckets, but
|
|
1797
|
+
// do NOT append zero rows to project.queue.jsonl. Deleting the in-memory
|
|
1798
|
+
// bucket is what prevents the double-count: the same-run re-parse of a file
|
|
1799
|
+
// still on disk starts that project bucket fresh from zero and appends a
|
|
1800
|
+
// corrected row, which the keep-last project reader (local-api.js
|
|
1801
|
+
// readProjectQueueData, per project_key|source|hour_start) picks over the old
|
|
1802
|
+
// inflated row. A bucket whose file has since been deleted or whose repo
|
|
1803
|
+
// moved is NOT re-parsed; leaving its prior queue row in place keeps that
|
|
1804
|
+
// (possibly fork-inflated) historical value visible rather than zeroing data
|
|
1805
|
+
// we can no longer recompute. Unlike the hourly path, project aggregates were
|
|
1806
|
+
// never retracted by the earlier delta migration, so a zero here would be a
|
|
1807
|
+
// brand-new, unrecoverable loss for orphaned files — hence the conservative
|
|
1808
|
+
// reset-accumulator-only approach.
|
|
1809
|
+
let projectReset = 0;
|
|
1810
|
+
const projectBuckets = cursors.projectHourly?.buckets;
|
|
1811
|
+
if (projectQueuePath && projectBuckets && typeof projectBuckets === "object") {
|
|
1812
|
+
for (const key of Object.keys(projectBuckets)) {
|
|
1813
|
+
const source = projectBuckets[key]?.source;
|
|
1814
|
+
if (source !== "codex" && source !== "every-code") continue;
|
|
1815
|
+
delete projectBuckets[key];
|
|
1816
|
+
projectReset += 1;
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
if (process.env.TOKENTRACKER_DEBUG) {
|
|
1821
|
+
process.stderr.write(
|
|
1822
|
+
`[codex-fork] #75 rebuild: reset ${rolloutPaths.size} codex/every-code file ` +
|
|
1823
|
+
`cursors, retracted ${hourlyRetractions.length} hourly bucket(s), reset ` +
|
|
1824
|
+
`${projectReset} project accumulator bucket(s) (no project rows zeroed); ` +
|
|
1825
|
+
`this sync re-parses files still on disk with the corrected accounting\n`,
|
|
1826
|
+
);
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
cursors.migrations[CODEX_FORK_OVERCOUNT_MIGRATION_KEY] = new Date().toISOString();
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1925
1832
|
// One-time repair migration: rebuild source=claude rows in queue.jsonl from
|
|
1926
1833
|
// the actual jsonl files using ccusage's algorithm (msgId+reqId global
|
|
1927
1834
|
// dedup). Earlier `reincludeClaudeMemObserverFiles` versions (v1/v2/v3) each
|