@ipv9/tokentracker-cli 0.39.38 → 0.39.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +43 -11
  2. package/dashboard/dist/assets/{Card-DD4s0V6T.js → Card-JS4gjUZf.js} +1 -1
  3. package/dashboard/dist/assets/DashboardPage-BoXhzJb1.js +60 -0
  4. package/dashboard/dist/assets/{FadeIn-CUSBK0jm.js → FadeIn-DvLfvwys.js} +1 -1
  5. package/dashboard/dist/assets/{IpCheckPage-PHF4lPax.js → IpCheckPage-Dcfc0Uts.js} +1 -1
  6. package/dashboard/dist/assets/LimitsPage-Vz-GVdi9.js +2 -0
  7. package/dashboard/dist/assets/{LocalOnlyNotice-DCmtS_j5.js → LocalOnlyNotice-8xF5couh.js} +1 -1
  8. package/dashboard/dist/assets/PopoverPopup-7HrUzeRx.js +1 -0
  9. package/dashboard/dist/assets/Select-9XDsaO_C.js +1 -0
  10. package/dashboard/dist/assets/SelectItemText-DE5Fs9p6.js +12 -0
  11. package/dashboard/dist/assets/SettingsPage-CMnfAptE.js +1 -0
  12. package/dashboard/dist/assets/SkillsPage-CbvLHN0L.js +1 -0
  13. package/dashboard/dist/assets/WidgetsPage-CeGQlyt7.js +1 -0
  14. package/dashboard/dist/assets/WrappedPage-Bux8qLUC.js +1 -0
  15. package/dashboard/dist/assets/{arrow-up-right-CjaBQnwI.js → arrow-up-right-B7rCoZdW.js} +1 -1
  16. package/dashboard/dist/assets/{download-DLBqkSwp.js → download-CtTYpbcx.js} +1 -1
  17. package/dashboard/dist/assets/format-BuQKK6kQ.js +1 -0
  18. package/dashboard/dist/assets/limitDisplay-WwuTdS_a.js +1 -0
  19. package/dashboard/dist/assets/main-Ck4SUH4m.js +696 -0
  20. package/dashboard/dist/assets/main-ZrWkoMlr.css +1 -0
  21. package/dashboard/dist/assets/mock-data-Bvbe8tYK.js +1 -0
  22. package/dashboard/dist/assets/use-limits-display-prefs-BKWYPgeA.js +1 -0
  23. package/dashboard/dist/assets/use-native-settings-D7Nj1dXR.js +1 -0
  24. package/dashboard/dist/assets/useCurrency-BX3pvyhh.js +1 -0
  25. package/dashboard/dist/index.html +15 -27
  26. package/dashboard/dist/llms.txt +0 -2
  27. package/dashboard/dist/sitemap.xml +0 -12
  28. package/package.json +7 -4
  29. package/src/cli.js +4 -13
  30. package/src/commands/doctor.js +1 -3
  31. package/src/commands/init.js +63 -53
  32. package/src/commands/serve.js +2 -50
  33. package/src/commands/status.js +1 -55
  34. package/src/commands/sync.js +134 -227
  35. package/src/lib/{browser-auth.js → browser-open.js} +3 -149
  36. package/src/lib/claude-categorizer.js +204 -32
  37. package/src/lib/codex-context-breakdown.js +83 -7
  38. package/src/lib/codex-rollout-parser.js +52 -10
  39. package/src/lib/diagnostics.js +1 -49
  40. package/src/lib/doctor.js +12 -87
  41. package/src/lib/file-identity.js +69 -0
  42. package/src/lib/init-flow.js +6 -3
  43. package/src/lib/local-api.js +6 -364
  44. package/src/lib/pricing/curated-overrides.json +1 -1
  45. package/src/lib/pricing/seed-snapshot.json +1 -1
  46. package/src/lib/rollout.js +392 -144
  47. package/src/lib/runtime-config.js +0 -32
  48. package/src/lib/sqlite-reader.js +83 -8
  49. package/dashboard/dist/assets/ActivityHeatmap-BrehqUIb.js +0 -42
  50. package/dashboard/dist/assets/DashboardPage-CyUShBRJ.js +0 -19
  51. package/dashboard/dist/assets/DevicePage-BA74Su1l.js +0 -1
  52. package/dashboard/dist/assets/DialogTitle-DoKsz5wi.js +0 -1
  53. package/dashboard/dist/assets/InsforgeUserHeaderControls-Bk_U2dHr.js +0 -1
  54. package/dashboard/dist/assets/LandingPage-Bbeu84Iu.js +0 -4356
  55. package/dashboard/dist/assets/LeaderboardAvatar-ByqeB6bM.js +0 -1
  56. package/dashboard/dist/assets/LeaderboardPage-BeWXF00S.js +0 -6
  57. package/dashboard/dist/assets/LeaderboardProfileModal-C4yvbZd_.js +0 -72
  58. package/dashboard/dist/assets/LeaderboardProfilePage-D3cbSh5o.js +0 -1
  59. package/dashboard/dist/assets/LimitsPage-dS9C9Vdt.js +0 -2
  60. package/dashboard/dist/assets/LoginPage-Ch3UQoWj.js +0 -1
  61. package/dashboard/dist/assets/PopoverPopup-DjG0gjlA.js +0 -1
  62. package/dashboard/dist/assets/Select-HzIjWJpE.js +0 -1
  63. package/dashboard/dist/assets/SelectItemText-f0tCwtvZ.js +0 -1
  64. package/dashboard/dist/assets/SettingsPage-DJOZAlZy.js +0 -1
  65. package/dashboard/dist/assets/SkillsPage-BQcyJIdx.js +0 -1
  66. package/dashboard/dist/assets/WidgetsPage-B_vxuV09.js +0 -1
  67. package/dashboard/dist/assets/WrappedPage-CZN9Y3Vz.js +0 -1
  68. package/dashboard/dist/assets/index-BeoRn2gJ.js +0 -2
  69. package/dashboard/dist/assets/info-BAIxlHIB.js +0 -1
  70. package/dashboard/dist/assets/main-DAcSUW2t.css +0 -1
  71. package/dashboard/dist/assets/main-pi07CfeC.js +0 -1004
  72. package/dashboard/dist/assets/use-limits-display-prefs-CMfASZsh.js +0 -1
  73. package/dashboard/dist/assets/use-native-settings-CfPik-Ao.js +0 -1
  74. package/dashboard/dist/assets/use-spring-wsmJQRqb.js +0 -1
  75. package/dashboard/dist/assets/use-usage-limits-Bt0Dp6kx.js +0 -1
  76. package/dashboard/dist/assets/useCurrency-CsaAMc3j.js +0 -1
  77. package/dashboard/dist/assets/useOpenInteractionType-CPzi5Z8M.js +0 -12
  78. package/dashboard/dist/share.html +0 -140
  79. package/src/commands/device-login.js +0 -161
  80. package/src/lib/upload-throttle.js +0 -148
@@ -62,13 +62,6 @@ const {
62
62
  } = require("../lib/rollout");
63
63
  const { computeClaudeGroundTruthBuckets } = require("../lib/claude-categorizer");
64
64
  const { createProgress, renderBar, formatNumber, formatBytes } = require("../lib/progress");
65
- const {
66
- normalizeState: normalizeUploadState,
67
- decideAutoUpload,
68
- recordUploadFailure,
69
- recordUploadSuccess,
70
- parseRetryAfterMs,
71
- } = require("../lib/upload-throttle");
72
65
  const {
73
66
  isCursorInstalled,
74
67
  extractCursorSessionToken,
@@ -77,10 +70,16 @@ const {
77
70
  } = require("../lib/cursor-config");
78
71
  const { purgeProjectUsage } = require("../lib/project-usage-purge");
79
72
  const { resolveTrackerPaths } = require("../lib/tracker-paths");
80
- const { resolveRuntimeConfig } = require("../lib/runtime-config");
81
73
 
82
74
  const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
83
75
  const ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY = "rolloutCumulativeDeltaReparse_2026_05";
76
+ // #75: rebuild Codex/every-code aggregates that were accumulated with the
77
+ // pre-fix parser, which counted replayed subagent-fork parent history as child
78
+ // consumption (up to ~40x Cache Read inflation on the local fork corpus). The
79
+ // rolloutCumulativeDeltaReparse migration above already rebuilt these once, but
80
+ // with the still-buggy parser, so a fresh key is required to rebuild again now
81
+ // that the parser is fixed.
82
+ const CODEX_FORK_OVERCOUNT_MIGRATION_KEY = "codexForkOvercountRebuild_2026_07";
84
83
  const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
85
84
  const GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY = "grokAppendOnlyRepair_2026_05_v4";
86
85
  const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
@@ -146,14 +145,11 @@ async function cmdSync(argv) {
146
145
  const queueStatePath = path.join(trackerDir, "queue.state.json");
147
146
  const projectQueuePath = path.join(trackerDir, "project.queue.jsonl");
148
147
  const projectQueueStatePath = path.join(trackerDir, "project.queue.state.json");
149
- const uploadThrottlePath = path.join(trackerDir, "upload.throttle.json");
150
148
  const grokSignalPath = path.join(trackerDir, "grok-last-session.json");
151
149
  const legacyGrokSignalPath = path.join(trackerDir, "tracker", "grok-last-session.json");
152
150
 
153
151
  const config = await readJson(configPath);
154
152
  const cursors = (await readJson(cursorsPath)) || { version: 1, files: {}, updatedAt: null };
155
- const uploadThrottle = normalizeUploadState(await readJson(uploadThrottlePath));
156
- let uploadThrottleState = uploadThrottle;
157
153
  let grokHookSignal = null;
158
154
  let grokHookSignalPath = null;
159
155
  for (const candidate of [grokSignalPath, legacyGrokSignalPath]) {
@@ -199,6 +195,7 @@ async function cmdSync(argv) {
199
195
  }
200
196
 
201
197
  await migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rolloutFiles });
198
+ await migrateCodexForkOvercountRebuild({ cursors, queuePath, projectQueuePath, rolloutFiles });
202
199
 
203
200
  const openclawFiles = openclawSignal?.sessionFile
204
201
  ? [{ path: openclawSignal.sessionFile, source: "openclaw" }]
@@ -928,69 +925,7 @@ async function cmdSync(argv) {
928
925
 
929
926
  progress?.stop();
930
927
 
931
- const runtime = resolveRuntimeConfig({ config: config || {}, env: process.env });
932
-
933
- let uploadResult = { inserted: 0, skipped: 0 };
934
- let uploadAttempted = false;
935
-
936
- if (runtime.deviceToken && runtime.baseUrl) {
937
- uploadAttempted = true;
938
- try {
939
- uploadResult = await drainQueueToCloud({
940
- baseUrl: runtime.baseUrl,
941
- deviceToken: runtime.deviceToken,
942
- queuePath,
943
- queueStatePath,
944
- maxBatches: opts.drain ? 100 : 5,
945
- batchSize: 200,
946
- });
947
- // Record success so the exponential backoff step resets — otherwise
948
- // a single past failure keeps us pessimistically throttled forever.
949
- uploadThrottleState = recordUploadSuccess({
950
- nowMs: Date.now(),
951
- state: uploadThrottleState,
952
- });
953
- await writeJson(uploadThrottlePath, uploadThrottleState);
954
- } catch (e) {
955
- // Persist a backoff on 429 / 5xx so the next auto-sync waits instead
956
- // of retrying immediately and making the rate-limit worse. The
957
- // throttle module already parses Retry-After when we surface it on
958
- // the error object (drainQueueToCloud stamps err.status + err.retryAfterMs).
959
- uploadThrottleState = recordUploadFailure({
960
- nowMs: Date.now(),
961
- state: uploadThrottleState,
962
- error: e,
963
- });
964
- await writeJson(uploadThrottlePath, uploadThrottleState);
965
- if (!opts.auto) {
966
- process.stderr.write(`Upload error: ${e?.message || e}\n`);
967
- }
968
- }
969
- }
970
928
 
971
- const afterState = (await readJson(queueStatePath)) || { offset: 0 };
972
- const queueSize = await safeStatSize(queuePath);
973
- const projectAfterState = (await readJson(projectQueueStatePath)) || { offset: 0 };
974
- const projectQueueSize = await safeStatSize(projectQueuePath);
975
- const pendingBytes =
976
- Math.max(0, queueSize - Number(afterState.offset || 0)) +
977
- Math.max(0, projectQueueSize - Number(projectAfterState.offset || 0));
978
-
979
- if (pendingBytes <= 0) {
980
- await clearAutoRetry(trackerDir);
981
- } else if (opts.auto && uploadAttempted) {
982
- const retryAtMs = Number(uploadThrottleState?.nextAllowedAtMs || 0);
983
- if (retryAtMs > Date.now()) {
984
- await scheduleAutoRetry({
985
- trackerDir,
986
- retryAtMs,
987
- reason: "backlog",
988
- pendingBytes,
989
- source: "auto-backlog",
990
- autoRetryNoSpawn: runtime.autoRetryNoSpawn,
991
- });
992
- }
993
- }
994
929
 
995
930
  const totalParsed =
996
931
  parseResult.filesProcessed +
@@ -1045,9 +980,6 @@ async function cmdSync(argv) {
1045
980
  const summary = {
1046
981
  totalParsed,
1047
982
  totalBuckets,
1048
- upload: uploadResult,
1049
- uploadAttempted,
1050
- pendingBytes,
1051
983
  };
1052
984
 
1053
985
  if (!opts.auto) {
@@ -1056,12 +988,6 @@ async function cmdSync(argv) {
1056
988
  "Sync finished:",
1057
989
  `- Parsed files: ${totalParsed}`,
1058
990
  `- New 30-min buckets queued: ${totalBuckets}`,
1059
- runtime.deviceToken
1060
- ? `- Uploaded: ${uploadResult.inserted} inserted, ${uploadResult.skipped} skipped`
1061
- : "- Uploaded: skipped (no device token)",
1062
- runtime.deviceToken && pendingBytes > 0 && !opts.drain
1063
- ? `- Remaining: ${formatBytes(pendingBytes)} pending (run sync again, or use --drain)`
1064
- : null,
1065
991
  "",
1066
992
  ]
1067
993
  .filter(Boolean)
@@ -1102,10 +1028,12 @@ module.exports = {
1102
1028
  cmdSync,
1103
1029
  migrateCursorUnknownBuckets,
1104
1030
  migrateRolloutCumulativeDeltaBuckets,
1031
+ migrateCodexForkOvercountRebuild,
1105
1032
  reincludeClaudeMemObserverFiles,
1106
1033
  repairGrokQueueFromSessionSnapshots,
1107
1034
  CURSOR_UNKNOWN_MIGRATION_KEY,
1108
1035
  ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY,
1036
+ CODEX_FORK_OVERCOUNT_MIGRATION_KEY,
1109
1037
  CLAUDE_MEM_OBSERVER_REINCLUDE_KEY,
1110
1038
  GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY,
1111
1039
  };
@@ -1276,102 +1204,10 @@ function deriveAutoSkipReason({ decision, state }) {
1276
1204
  return "throttled";
1277
1205
  }
1278
1206
 
1279
- async function scheduleAutoRetry({
1280
- trackerDir,
1281
- retryAtMs,
1282
- reason,
1283
- pendingBytes,
1284
- source,
1285
- autoRetryNoSpawn,
1286
- }) {
1287
- const retryMs = coerceRetryMs(retryAtMs);
1288
- if (!retryMs) return { scheduled: false, retryAtMs: 0 };
1289
-
1290
- const retryPath = path.join(trackerDir, AUTO_RETRY_FILENAME);
1291
- const nowMs = Date.now();
1292
- const existing = await readJson(retryPath);
1293
- const existingMs = coerceRetryMs(existing?.retryAtMs);
1294
- if (existingMs && existingMs >= retryMs - 1000) {
1295
- return { scheduled: false, retryAtMs: existingMs };
1296
- }
1297
-
1298
- const payload = {
1299
- version: 1,
1300
- retryAtMs: retryMs,
1301
- retryAt: new Date(retryMs).toISOString(),
1302
- reason: typeof reason === "string" && reason.length > 0 ? reason : "throttled",
1303
- pendingBytes: Math.max(0, Number(pendingBytes || 0)),
1304
- scheduledAt: new Date(nowMs).toISOString(),
1305
- source: typeof source === "string" ? source : "auto",
1306
- };
1307
-
1308
- await writeJson(retryPath, payload);
1309
-
1310
- const delayMs = Math.min(AUTO_RETRY_MAX_DELAY_MS, Math.max(0, retryMs - nowMs));
1311
- if (delayMs <= 0) return { scheduled: false, retryAtMs: retryMs };
1312
- if (autoRetryNoSpawn) {
1313
- return { scheduled: false, retryAtMs: retryMs };
1314
- }
1315
-
1316
- spawnAutoRetryProcess({
1317
- retryPath,
1318
- trackerBinPath: path.join(trackerDir, "app", "bin", "tracker.js"),
1319
- fallbackPkg: "@ipv9/tokentracker-cli",
1320
- delayMs,
1321
- });
1322
- return { scheduled: true, retryAtMs: retryMs };
1323
- }
1324
1207
 
1325
- async function clearAutoRetry(trackerDir) {
1326
- const retryPath = path.join(trackerDir, AUTO_RETRY_FILENAME);
1327
- await fs.unlink(retryPath).catch(() => {});
1328
- }
1329
1208
 
1330
- function spawnAutoRetryProcess({ retryPath, trackerBinPath, fallbackPkg, delayMs }) {
1331
- const script = buildAutoRetryScript({ retryPath, trackerBinPath, fallbackPkg, delayMs });
1332
- try {
1333
- const child = cp.spawn(process.execPath, ["-e", script], {
1334
- detached: true,
1335
- stdio: "ignore",
1336
- env: process.env,
1337
- });
1338
- child.unref();
1339
- } catch (_e) {}
1340
- }
1341
1209
 
1342
- function buildAutoRetryScript({ retryPath, trackerBinPath, fallbackPkg, delayMs }) {
1343
- return (
1344
- `'use strict';\n` +
1345
- `const fs = require('node:fs');\n` +
1346
- `const cp = require('node:child_process');\n` +
1347
- `const retryPath = ${JSON.stringify(retryPath)};\n` +
1348
- `const trackerBinPath = ${JSON.stringify(trackerBinPath)};\n` +
1349
- `const fallbackPkg = ${JSON.stringify(fallbackPkg)};\n` +
1350
- `const delayMs = ${Math.max(0, Math.floor(delayMs || 0))};\n` +
1351
- `setTimeout(() => {\n` +
1352
- ` let retryAtMs = 0;\n` +
1353
- ` try {\n` +
1354
- ` const raw = fs.readFileSync(retryPath, 'utf8');\n` +
1355
- ` retryAtMs = Number(JSON.parse(raw).retryAtMs || 0);\n` +
1356
- ` } catch (_) {}\n` +
1357
- ` if (!retryAtMs || Date.now() + 1000 < retryAtMs) process.exit(0);\n` +
1358
- ` const argv = ['sync', '--auto', '--from-retry'];\n` +
1359
- ` const cmd = fs.existsSync(trackerBinPath)\n` +
1360
- ` ? [process.execPath, trackerBinPath, ...argv]\n` +
1361
- ` : ['npx', '--yes', fallbackPkg, ...argv];\n` +
1362
- ` try {\n` +
1363
- ` const child = cp.spawn(cmd[0], cmd.slice(1), { detached: true, stdio: 'ignore', env: process.env });\n` +
1364
- ` child.unref();\n` +
1365
- ` } catch (_) {}\n` +
1366
- `}, delayMs);\n`
1367
- );
1368
- }
1369
1210
 
1370
- function coerceRetryMs(v) {
1371
- const n = Number(v);
1372
- if (!Number.isFinite(n) || n <= 0) return 0;
1373
- return Math.floor(n);
1374
- }
1375
1211
 
1376
1212
  async function writeOpenclawSignal(trackerDir) {
1377
1213
  const openclawSignalPath = path.join(trackerDir, "openclaw.signal");
@@ -1382,63 +1218,10 @@ async function writeOpenclawSignal(trackerDir) {
1382
1218
  }
1383
1219
  }
1384
1220
 
1385
- const AUTO_RETRY_FILENAME = "auto.retry.json";
1386
1221
  const AUTO_RETRY_MAX_DELAY_MS = 2 * 60 * 60 * 1000;
1387
1222
 
1388
- const INGEST_SLUG = "tokentracker-ingest";
1389
1223
  const MAX_INGEST_BUCKETS = 500;
1390
1224
 
1391
- async function drainQueueToCloud({ baseUrl, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200 }) {
1392
- const state = (await readJson(queueStatePath)) || { offset: 0 };
1393
- let offset = Number(state.offset || 0);
1394
- let inserted = 0;
1395
- let skipped = 0;
1396
-
1397
- const queueSize = await safeStatSize(queuePath);
1398
- const limit = Math.min(Math.max(1, Math.floor(Number(batchSize || 200))), MAX_INGEST_BUCKETS);
1399
-
1400
- for (let batch = 0; batch < maxBatches; batch++) {
1401
- if (offset >= queueSize) break;
1402
- const result = await readQueueBatch(queuePath, offset, limit);
1403
- if (result.buckets.length === 0) break;
1404
-
1405
- const root = baseUrl.replace(/\/$/, "");
1406
- const anonKey = process.env.TOKENTRACKER_INSFORGE_ANON_KEY || "";
1407
- const headers = {
1408
- "Content-Type": "application/json",
1409
- Accept: "application/json",
1410
- Authorization: `Bearer ${deviceToken}`,
1411
- };
1412
- if (anonKey) headers.apikey = anonKey;
1413
- const res = await fetch(`${root}/functions/${INGEST_SLUG}`, {
1414
- method: "POST",
1415
- headers,
1416
- body: JSON.stringify({ hourly: result.buckets }),
1417
- });
1418
-
1419
- const rawText = await res.text().catch(() => "");
1420
- let data = {};
1421
- try { data = JSON.parse(rawText); } catch { data = {}; }
1422
- if (!res.ok) {
1423
- const err = new Error(`HTTP ${res.status}: ${rawText.substring(0, 500)}`);
1424
- err.status = res.status;
1425
- const retryAfter = res.headers?.get?.("Retry-After") ?? null;
1426
- const retryAfterMs = parseRetryAfterMs(retryAfter);
1427
- if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
1428
- throw err;
1429
- }
1430
-
1431
- inserted += Number(data?.inserted || 0);
1432
- skipped += Number(data?.skipped || 0);
1433
-
1434
- offset = result.nextOffset;
1435
- state.offset = offset;
1436
- state.updatedAt = new Date().toISOString();
1437
- await writeJson(queueStatePath, state);
1438
- }
1439
-
1440
- return { inserted, skipped };
1441
- }
1442
1225
 
1443
1226
  async function readQueueBatch(queuePath, startOffset, maxBuckets) {
1444
1227
  const st = await fs.stat(queuePath).catch(() => null);
@@ -1922,6 +1705,130 @@ async function migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rollou
1922
1705
  cursors.migrations[ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY] = new Date().toISOString();
1923
1706
  }
1924
1707
 
1708
+ // One-time rebuild for issue #75. The prior parser counted a Codex subagent
1709
+ // fork's replayed parent token_count history as the child's consumption. After
1710
+ // the parser fix (PR #77) that only corrects go-forward parsing — already-
1711
+ // ingested files sit at offset = EOF and are never re-read — so their inflated
1712
+ // contribution stays baked into cursors.hourly / project aggregates and the
1713
+ // dashboard until this migration retracts it and forces a clean re-parse.
1714
+ //
1715
+ // Choreography mirrors migrateRolloutCumulativeDeltaBuckets (codex-scoped, so
1716
+ // no other provider is touched): (1) drop the per-file cursors so the same-run
1717
+ // sync re-parses from offset 0 with the fixed accounting; (2) delete the
1718
+ // inflated hourly buckets and append zero rows to queue.jsonl — the dashboard
1719
+ // keep-lasts per source|model|hour_start (local-api.js readQueueData), so the
1720
+ // re-parse's corrected row supersedes the zero, or the zero stands for a file
1721
+ // no longer on disk; (3) do the same for project aggregates, which are on by
1722
+ // default (sync passes projectQueuePath) and would otherwise double-count on
1723
+ // re-parse — readProjectQueueData also keep-lasts per project_key|source|
1724
+ // hour_start. Run-once via the migrations sentinel (the issue asks for a
1725
+ // one-time rebuild, not a repeatable command).
1726
+ //
1727
+ // Project aggregates are handled conservatively: the accumulator buckets are
1728
+ // reset (so files still on disk rebuild clean and their corrected row wins via
1729
+ // keep-last), but no zero rows are written. A project bucket whose file was
1730
+ // deleted or whose repo moved keeps its prior queue row rather than being
1731
+ // zeroed — we never destroy project history we can no longer recompute. Main-
1732
+ // dashboard (source) totals are always rebuilt regardless of project
1733
+ // resolution. See step (3).
1734
+ async function migrateCodexForkOvercountRebuild({
1735
+ cursors,
1736
+ queuePath,
1737
+ projectQueuePath,
1738
+ rolloutFiles,
1739
+ }) {
1740
+ if (!cursors || typeof cursors !== "object") return;
1741
+ cursors.migrations = cursors.migrations || {};
1742
+ if (cursors.migrations[CODEX_FORK_OVERCOUNT_MIGRATION_KEY]) return;
1743
+
1744
+ const rolloutPaths = new Set();
1745
+ for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
1746
+ const filePath = typeof entry === "string" ? entry : entry?.path;
1747
+ const source = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
1748
+ if (filePath && (source === "codex" || source === "every-code")) {
1749
+ rolloutPaths.add(filePath);
1750
+ }
1751
+ }
1752
+
1753
+ // (1) Drop per-file cursors -> clean re-parse from offset 0 with the fix.
1754
+ if (cursors.files && typeof cursors.files === "object") {
1755
+ for (const filePath of rolloutPaths) delete cursors.files[filePath];
1756
+ }
1757
+
1758
+ // (2) Retract + delete inflated hourly codex/every-code buckets.
1759
+ const hourlyRetractions = [];
1760
+ const buckets = cursors.hourly?.buckets;
1761
+ if (buckets && typeof buckets === "object") {
1762
+ for (const key of Object.keys(buckets)) {
1763
+ const [source, model, ...hourParts] = key.split("|");
1764
+ if (source !== "codex" && source !== "every-code") continue;
1765
+ hourlyRetractions.push(
1766
+ JSON.stringify({
1767
+ source,
1768
+ model: model || "unknown",
1769
+ hour_start: hourParts.join("|"),
1770
+ input_tokens: 0,
1771
+ cached_input_tokens: 0,
1772
+ cache_creation_input_tokens: 0,
1773
+ output_tokens: 0,
1774
+ reasoning_output_tokens: 0,
1775
+ total_tokens: 0,
1776
+ billable_total_tokens: 0,
1777
+ conversation_count: 0,
1778
+ }),
1779
+ );
1780
+ delete buckets[key];
1781
+ }
1782
+ }
1783
+ const groupQueued = cursors.hourly?.groupQueued;
1784
+ if (groupQueued && typeof groupQueued === "object") {
1785
+ for (const key of Object.keys(groupQueued)) {
1786
+ if (key.startsWith("codex|") || key.startsWith("every-code|")) {
1787
+ delete groupQueued[key];
1788
+ }
1789
+ }
1790
+ }
1791
+ if (hourlyRetractions.length > 0) {
1792
+ await ensureDir(path.dirname(queuePath));
1793
+ await fs.appendFile(queuePath, hourlyRetractions.join("\n") + "\n");
1794
+ }
1795
+
1796
+ // (3) Reset the inflated project codex/every-code accumulator buckets, but
1797
+ // do NOT append zero rows to project.queue.jsonl. Deleting the in-memory
1798
+ // bucket is what prevents the double-count: the same-run re-parse of a file
1799
+ // still on disk starts that project bucket fresh from zero and appends a
1800
+ // corrected row, which the keep-last project reader (local-api.js
1801
+ // readProjectQueueData, per project_key|source|hour_start) picks over the old
1802
+ // inflated row. A bucket whose file has since been deleted or whose repo
1803
+ // moved is NOT re-parsed; leaving its prior queue row in place keeps that
1804
+ // (possibly fork-inflated) historical value visible rather than zeroing data
1805
+ // we can no longer recompute. Unlike the hourly path, project aggregates were
1806
+ // never retracted by the earlier delta migration, so a zero here would be a
1807
+ // brand-new, unrecoverable loss for orphaned files — hence the conservative
1808
+ // reset-accumulator-only approach.
1809
+ let projectReset = 0;
1810
+ const projectBuckets = cursors.projectHourly?.buckets;
1811
+ if (projectQueuePath && projectBuckets && typeof projectBuckets === "object") {
1812
+ for (const key of Object.keys(projectBuckets)) {
1813
+ const source = projectBuckets[key]?.source;
1814
+ if (source !== "codex" && source !== "every-code") continue;
1815
+ delete projectBuckets[key];
1816
+ projectReset += 1;
1817
+ }
1818
+ }
1819
+
1820
+ if (process.env.TOKENTRACKER_DEBUG) {
1821
+ process.stderr.write(
1822
+ `[codex-fork] #75 rebuild: reset ${rolloutPaths.size} codex/every-code file ` +
1823
+ `cursors, retracted ${hourlyRetractions.length} hourly bucket(s), reset ` +
1824
+ `${projectReset} project accumulator bucket(s) (no project rows zeroed); ` +
1825
+ `this sync re-parses files still on disk with the corrected accounting\n`,
1826
+ );
1827
+ }
1828
+
1829
+ cursors.migrations[CODEX_FORK_OVERCOUNT_MIGRATION_KEY] = new Date().toISOString();
1830
+ }
1831
+
1925
1832
  // One-time repair migration: rebuild source=claude rows in queue.jsonl from
1926
1833
  // the actual jsonl files using ccusage's algorithm (msgId+reqId global
1927
1834
  // dedup). Earlier `reincludeClaudeMemObserverFiles` versions (v1/v2/v3) each
@@ -1,138 +1,8 @@
1
- const http = require("node:http");
2
- const crypto = require("node:crypto");
1
+ // Opens a URL in the user's browser. This module used to also run the cloud
2
+ // sign-in flow (a local callback server plus a redirect dance); that went away
3
+ // with the cloud, and the file was renamed so its name stops implying auth.
3
4
  const cp = require("node:child_process");
4
5
 
5
- const { DEFAULT_BASE_URL } = require("./runtime-config");
6
-
7
- async function beginBrowserAuth({ baseUrl, dashboardUrl, timeoutMs, open }) {
8
- const nonce = crypto.randomBytes(16).toString("hex");
9
- const callbackPath = `/tokentracker/callback/${nonce}`;
10
- const authUrl = dashboardUrl ? new URL("/", dashboardUrl) : new URL("/auth/sign-up", baseUrl);
11
- const postAuthRedirect = resolvePostAuthRedirect({ dashboardUrl, authUrl });
12
- const { callbackUrl, waitForCallback } = await startLocalCallbackServer({
13
- callbackPath,
14
- timeoutMs,
15
- redirectUrl: postAuthRedirect,
16
- });
17
- authUrl.searchParams.set("redirect", callbackUrl);
18
- if (dashboardUrl && baseUrl && baseUrl !== DEFAULT_BASE_URL) {
19
- authUrl.searchParams.set("base_url", baseUrl);
20
- }
21
-
22
- if (open !== false) openInBrowser(authUrl.toString());
23
-
24
- return { authUrl: authUrl.toString(), waitForCallback };
25
- }
26
-
27
- async function startLocalCallbackServer({ callbackPath, timeoutMs, redirectUrl }) {
28
- let resolved = false;
29
- let resolveResult;
30
- let rejectResult;
31
-
32
- const resultPromise = new Promise((resolve, reject) => {
33
- resolveResult = resolve;
34
- rejectResult = reject;
35
- });
36
-
37
- const server = http.createServer((req, res) => {
38
- if (resolved) {
39
- res.writeHead(409, { "Content-Type": "text/plain; charset=utf-8" });
40
- res.end("Already authenticated.\n");
41
- return;
42
- }
43
-
44
- const method = req.method || "GET";
45
- if (method !== "GET") {
46
- res.writeHead(405, { "Content-Type": "text/plain; charset=utf-8" });
47
- res.end("Method not allowed.\n");
48
- return;
49
- }
50
-
51
- const url = new URL(req.url || "/", "http://127.0.0.1");
52
- if (url.pathname !== callbackPath) {
53
- res.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" });
54
- res.end("Not found.\n");
55
- return;
56
- }
57
-
58
- const accessToken = url.searchParams.get("access_token") || "";
59
- if (!accessToken) {
60
- res.writeHead(400, { "Content-Type": "text/plain; charset=utf-8" });
61
- res.end("Missing access_token.\n");
62
- return;
63
- }
64
-
65
- resolved = true;
66
- if (redirectUrl) {
67
- res.writeHead(302, {
68
- Location: redirectUrl,
69
- "Content-Type": "text/html; charset=utf-8",
70
- });
71
- res.end(
72
- [
73
- "<!doctype html>",
74
- '<html><head><meta charset="utf-8"><title>TokenTracker</title></head>',
75
- "<body>",
76
- "<h2>Login succeeded</h2>",
77
- `<p>Redirecting to <a href="${redirectUrl}">dashboard</a>...</p>`,
78
- "</body></html>",
79
- ].join(""),
80
- );
81
- } else {
82
- res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
83
- res.end(
84
- [
85
- "<!doctype html>",
86
- '<html><head><meta charset="utf-8"><title>TokenTracker</title></head>',
87
- "<body>",
88
- "<h2>Login succeeded</h2>",
89
- "<p>You can close this tab and return to the CLI.</p>",
90
- "</body></html>",
91
- ].join(""),
92
- );
93
- }
94
-
95
- resolveResult({
96
- accessToken,
97
- userId: url.searchParams.get("user_id") || null,
98
- email: url.searchParams.get("email") || null,
99
- name: url.searchParams.get("name") || null,
100
- });
101
- });
102
-
103
- await new Promise((resolve, reject) => {
104
- server.once("error", reject);
105
- server.listen(0, "127.0.0.1", resolve);
106
- });
107
-
108
- const addr = server.address();
109
- const port = typeof addr === "object" && addr ? addr.port : null;
110
- if (!port) {
111
- server.close();
112
- throw new Error("Failed to bind local callback server");
113
- }
114
-
115
- const callbackUrl = `http://127.0.0.1:${port}${callbackPath}`;
116
-
117
- const timer = setTimeout(() => {
118
- if (resolved) return;
119
- resolved = true;
120
- rejectResult(new Error("Authentication timed out"));
121
- server.close();
122
- }, timeoutMs);
123
-
124
- async function waitForCallback() {
125
- try {
126
- return await resultPromise;
127
- } finally {
128
- clearTimeout(timer);
129
- server.close();
130
- }
131
- }
132
-
133
- return { callbackUrl, waitForCallback };
134
- }
135
-
136
6
  function detectDefaultBrowser() {
137
7
  try {
138
8
  const raw = cp.execFileSync("defaults", [
@@ -285,23 +155,7 @@ end if
285
155
  spawnBrowserCommand(cmd, args, { url, platform, stderr });
286
156
  }
287
157
 
288
- function resolvePostAuthRedirect({ dashboardUrl, authUrl }) {
289
- try {
290
- if (dashboardUrl) {
291
- const target = new URL("/", dashboardUrl);
292
- if (target.protocol === "http:" || target.protocol === "https:") {
293
- return target.toString();
294
- }
295
- return null;
296
- }
297
- } catch (_e) {
298
- return null;
299
- }
300
- return null;
301
- }
302
-
303
158
  module.exports = {
304
- beginBrowserAuth,
305
159
  openInBrowser,
306
160
  buildBrowserOpenErrorMessage,
307
161
  spawnBrowserCommand,