@ipv9/tokentracker-cli 0.39.37 → 0.39.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +41 -9
  2. package/dashboard/dist/assets/{Card-DKmo-v8a.js → Card-BbRYRVXf.js} +1 -1
  3. package/dashboard/dist/assets/DashboardPage-DYKHv0m3.js +60 -0
  4. package/dashboard/dist/assets/{FadeIn-Cf30rPTW.js → FadeIn-CjAacJ5t.js} +1 -1
  5. package/dashboard/dist/assets/{IpCheckPage-D3CL8XP0.js → IpCheckPage-BGALK8Vy.js} +1 -1
  6. package/dashboard/dist/assets/{LimitsPage-BqST7DxD.js → LimitsPage-LkZGPYvg.js} +2 -2
  7. package/dashboard/dist/assets/{LocalOnlyNotice-CqHzPooT.js → LocalOnlyNotice-Dxo2vFMQ.js} +1 -1
  8. package/dashboard/dist/assets/PopoverPopup-DP9fwdtl.js +1 -0
  9. package/dashboard/dist/assets/Select-DPE53NmM.js +1 -0
  10. package/dashboard/dist/assets/SelectItemText-D69v77is.js +12 -0
  11. package/dashboard/dist/assets/SettingsPage-Dxzqpi02.js +1 -0
  12. package/dashboard/dist/assets/SkillsPage-Bo7e39AB.js +1 -0
  13. package/dashboard/dist/assets/WidgetsPage-HHcIr_Eq.js +1 -0
  14. package/dashboard/dist/assets/WrappedPage-C-_jLXXU.js +1 -0
  15. package/dashboard/dist/assets/{arrow-up-right-BJzUcPig.js → arrow-up-right-BJvJHUGj.js} +1 -1
  16. package/dashboard/dist/assets/{download-CsXweDLj.js → download-CnzY_EgN.js} +1 -1
  17. package/dashboard/dist/assets/format-1w-q5n2f.js +1 -0
  18. package/dashboard/dist/assets/main-1kxAVM3m.css +1 -0
  19. package/dashboard/dist/assets/main-3gBEI6Jl.js +694 -0
  20. package/dashboard/dist/assets/mock-data-CJOMGcL9.js +1 -0
  21. package/dashboard/dist/assets/use-limits-display-prefs-D1nT9-mr.js +1 -0
  22. package/dashboard/dist/assets/use-native-settings-Dh6NzqjY.js +1 -0
  23. package/dashboard/dist/assets/use-usage-limits-BXR6lzGf.js +1 -0
  24. package/dashboard/dist/assets/useCurrency-Hhs5jbWw.js +1 -0
  25. package/dashboard/dist/index.html +16 -28
  26. package/dashboard/dist/llms.txt +0 -2
  27. package/dashboard/dist/sitemap.xml +0 -12
  28. package/package.json +7 -4
  29. package/src/cli.js +4 -13
  30. package/src/commands/doctor.js +1 -3
  31. package/src/commands/init.js +63 -53
  32. package/src/commands/serve.js +2 -50
  33. package/src/commands/status.js +1 -55
  34. package/src/commands/sync.js +134 -227
  35. package/src/lib/{browser-auth.js → browser-open.js} +3 -149
  36. package/src/lib/claude-categorizer.js +204 -32
  37. package/src/lib/codex-context-breakdown.js +83 -7
  38. package/src/lib/codex-rollout-parser.js +52 -10
  39. package/src/lib/diagnostics.js +1 -49
  40. package/src/lib/doctor.js +12 -87
  41. package/src/lib/file-identity.js +69 -0
  42. package/src/lib/init-flow.js +6 -3
  43. package/src/lib/local-api.js +6 -364
  44. package/src/lib/pricing/curated-overrides.json +1 -1
  45. package/src/lib/pricing/seed-snapshot.json +1 -1
  46. package/src/lib/rollout.js +392 -144
  47. package/src/lib/runtime-config.js +0 -32
  48. package/src/lib/sqlite-reader.js +83 -8
  49. package/dashboard/dist/assets/ActivityHeatmap-DPZ_Aj6E.js +0 -42
  50. package/dashboard/dist/assets/DashboardPage-C9Q5HCDh.js +0 -19
  51. package/dashboard/dist/assets/DevicePage-DhQFtROo.js +0 -1
  52. package/dashboard/dist/assets/DialogTitle-DduJCLiL.js +0 -1
  53. package/dashboard/dist/assets/InsforgeUserHeaderControls-CDqYVWBQ.js +0 -1
  54. package/dashboard/dist/assets/LandingPage-9345o0vr.js +0 -4356
  55. package/dashboard/dist/assets/LeaderboardAvatar-DNORVh3n.js +0 -1
  56. package/dashboard/dist/assets/LeaderboardPage-DBsdCCzY.js +0 -6
  57. package/dashboard/dist/assets/LeaderboardProfileModal-BuYdBVh_.js +0 -72
  58. package/dashboard/dist/assets/LeaderboardProfilePage-D4Fe1LOk.js +0 -1
  59. package/dashboard/dist/assets/LoginPage-BmcybmeK.js +0 -1
  60. package/dashboard/dist/assets/PopoverPopup-CDL7kqy5.js +0 -1
  61. package/dashboard/dist/assets/Select-BkzhG41t.js +0 -1
  62. package/dashboard/dist/assets/SelectItemText-BmRzs9ud.js +0 -1
  63. package/dashboard/dist/assets/SettingsPage-gXQl_o6s.js +0 -1
  64. package/dashboard/dist/assets/SkillsPage-BRXYEsKm.js +0 -1
  65. package/dashboard/dist/assets/WidgetsPage-CLHLz2UW.js +0 -1
  66. package/dashboard/dist/assets/WrappedPage-p4ZFwQjO.js +0 -1
  67. package/dashboard/dist/assets/index-BeoRn2gJ.js +0 -2
  68. package/dashboard/dist/assets/info-CuLr7ZIa.js +0 -1
  69. package/dashboard/dist/assets/main-CPGI-smk.js +0 -1004
  70. package/dashboard/dist/assets/main-DAcSUW2t.css +0 -1
  71. package/dashboard/dist/assets/use-limits-display-prefs-DOB09__q.js +0 -1
  72. package/dashboard/dist/assets/use-native-settings-JSylS2xS.js +0 -1
  73. package/dashboard/dist/assets/use-spring-qjUEvMQX.js +0 -1
  74. package/dashboard/dist/assets/use-usage-limits-B0bzPxNR.js +0 -1
  75. package/dashboard/dist/assets/useCurrency-BRorShq7.js +0 -1
  76. package/dashboard/dist/assets/useOpenInteractionType-DPP-6ZsL.js +0 -12
  77. package/dashboard/dist/share.html +0 -140
  78. package/src/commands/device-login.js +0 -161
  79. package/src/lib/upload-throttle.js +0 -148
@@ -2,7 +2,6 @@ const http = require("node:http");
2
2
  const os = require("node:os");
3
3
  const path = require("node:path");
4
4
  const fssync = require("node:fs");
5
- const cp = require("node:child_process");
6
5
 
7
6
  const { resolveTrackerPaths } = require("../lib/tracker-paths");
8
7
  const { createLocalApiHandler, resolveQueuePath } = require("../lib/local-api");
@@ -12,7 +11,7 @@ const {
12
11
  } = require("../lib/local-data-preflight");
13
12
  const { ensurePricingLoaded } = require("../lib/pricing");
14
13
  const { serveStaticFile } = require("../lib/static-server");
15
- const { openInBrowser } = require("../lib/browser-auth");
14
+ const { openInBrowser } = require("../lib/browser-open");
16
15
 
17
16
  const DEFAULT_PORT = 7680;
18
17
  const DEFAULT_MAX_PORT_ATTEMPTS = 20;
@@ -20,7 +19,7 @@ const NPM_PACKAGE_NAME = "@ipv9/tokentracker-cli";
20
19
  const LOCAL_BIND_HOST = "127.0.0.1";
21
20
 
22
21
  function buildPortInUseHint(port) {
23
- return `Port ${port} is still in use after cleanup. Try: npx ${NPM_PACKAGE_NAME} serve --port ${port + 1}\n`;
22
+ return `Port ${port} is unavailable. Try: npx ${NPM_PACKAGE_NAME} serve --port ${port + 1}\n`;
24
23
  }
25
24
 
26
25
  function isPortUnavailableError(error) {
@@ -155,7 +154,6 @@ async function cmdServe(argv) {
155
154
  try {
156
155
  port = await listenOnAvailablePort(server, opts.port, {
157
156
  allowFallback: !opts.portExplicit,
158
- ensurePortFreeFn: opts.portExplicit ? ensurePortFree : null,
159
157
  onRetry: (failedPort) => {
160
158
  process.stdout.write(`Port ${failedPort} unavailable, trying ${failedPort + 1}...\n`);
161
159
  },
@@ -208,47 +206,6 @@ async function cmdServe(argv) {
208
206
  await new Promise(() => {});
209
207
  }
210
208
 
211
- function findPidOnPort(port) {
212
- try {
213
- const out = cp.execFileSync("lsof", ["-ti", `tcp:${port}`], { encoding: "utf8", timeout: 5000 });
214
- const pids = out.trim().split(/\s+/).map(Number).filter((n) => Number.isFinite(n) && n > 0);
215
- return pids;
216
- } catch (_e) {
217
- return [];
218
- }
219
- }
220
-
221
- async function ensurePortFree(port) {
222
- const pids = findPidOnPort(port);
223
- if (pids.length === 0) return;
224
-
225
- // Don't kill ourselves
226
- const self = process.pid;
227
- const targets = pids.filter((p) => p !== self);
228
- if (targets.length === 0) return;
229
-
230
- process.stdout.write(`Stopping previous server on port ${port} (pid ${targets.join(", ")})...\n`);
231
- for (const pid of targets) {
232
- try {
233
- process.kill(pid, "SIGTERM");
234
- } catch (_e) {}
235
- }
236
-
237
- // Wait briefly for port to free up
238
- for (let i = 0; i < 10; i++) {
239
- await new Promise((r) => setTimeout(r, 300));
240
- if (findPidOnPort(port).length === 0) return;
241
- }
242
-
243
- // Force kill if still alive
244
- for (const pid of targets) {
245
- try {
246
- process.kill(pid, "SIGKILL");
247
- } catch (_e) {}
248
- }
249
- await new Promise((r) => setTimeout(r, 500));
250
- }
251
-
252
209
  function listenOnce(server, port, host) {
253
210
  return new Promise((resolve, reject) => {
254
211
  let settled = false;
@@ -283,7 +240,6 @@ async function listenOnAvailablePort(
283
240
  host = LOCAL_BIND_HOST,
284
241
  allowFallback = false,
285
242
  maxAttempts = DEFAULT_MAX_PORT_ATTEMPTS,
286
- ensurePortFreeFn = null,
287
243
  onRetry = null,
288
244
  } = {},
289
245
  ) {
@@ -292,10 +248,6 @@ async function listenOnAvailablePort(
292
248
  let lastError = null;
293
249
 
294
250
  for (let i = 0; i < attempts && port < 65536; i++, port++) {
295
- if (ensurePortFreeFn) {
296
- await ensurePortFreeFn(port);
297
- }
298
-
299
251
  try {
300
252
  await listenOnce(server, port, host);
301
253
  return port;
@@ -25,9 +25,6 @@ const {
25
25
  describeCopilotOtelStatus,
26
26
  readCopilotOauthToken,
27
27
  } = require("../lib/usage-limits");
28
- const {
29
- normalizeState: normalizeUploadState,
30
- } = require("../lib/upload-throttle");
31
28
  const { collectTrackerDiagnostics } = require("../lib/diagnostics");
32
29
  const { detectPassiveProviders, isPassiveModeActive } = require("../lib/passive-mode");
33
30
  const { probeOpenclawHookState } = require("../lib/openclaw-hook");
@@ -68,15 +65,11 @@ async function cmdStatus(argv = []) {
68
65
 
69
66
  const home = os.homedir();
70
67
  const { trackerDir, binDir } = await resolveTrackerPaths({ home });
71
- const configPath = path.join(trackerDir, "config.json");
72
68
  const queuePath = path.join(trackerDir, "queue.jsonl");
73
- const queueStatePath = path.join(trackerDir, "queue.state.json");
74
69
  const cursorsPath = path.join(trackerDir, "cursors.json");
75
70
  const notifySignalPath = path.join(trackerDir, "notify.signal");
76
71
  const openclawSignalPath = path.join(trackerDir, "openclaw.signal");
77
72
  const throttlePath = path.join(trackerDir, "sync.throttle");
78
- const uploadThrottlePath = path.join(trackerDir, "upload.throttle.json");
79
- const autoRetryPath = path.join(trackerDir, "auto.retry.json");
80
73
  const codexHome = process.env.CODEX_HOME || path.join(home, ".codex");
81
74
  const codexConfigPath = path.join(codexHome, "config.toml");
82
75
  const codeHome = process.env.CODE_HOME || path.join(home, ".code");
@@ -99,16 +92,9 @@ async function cmdStatus(argv = []) {
99
92
  const codebuddyHookCommand = buildHookCommand(notifyPath, "codebuddy");
100
93
  const geminiHookCommand = buildGeminiHookCommand(notifyPath);
101
94
 
102
- const config = await readJson(configPath);
103
95
  const cursors = await readJson(cursorsPath);
104
- const queueState = (await readJson(queueStatePath)) || { offset: 0 };
105
- const uploadThrottle = normalizeUploadState(
106
- await readJson(uploadThrottlePath),
107
- );
108
- const autoRetry = await readJson(autoRetryPath);
109
96
 
110
97
  const queueSize = await safeStatSize(queuePath);
111
- const pendingBytes = Math.max(0, queueSize - (queueState.offset || 0));
112
98
 
113
99
  const lastNotify = (await safeReadText(notifySignalPath))?.trim() || null;
114
100
  const lastOpenclawSync =
@@ -148,23 +134,6 @@ async function cmdStatus(argv = []) {
148
134
  env: process.env,
149
135
  });
150
136
 
151
- const lastUpload = uploadThrottle.lastSuccessMs
152
- ? parseEpochMsToIso(uploadThrottle.lastSuccessMs)
153
- : typeof queueState.updatedAt === "string"
154
- ? queueState.updatedAt
155
- : null;
156
- const nextUpload = parseEpochMsToIso(uploadThrottle.nextAllowedAtMs || null);
157
- const backoffUntil = parseEpochMsToIso(uploadThrottle.backoffUntilMs || null);
158
- const lastUploadError = uploadThrottle.lastError
159
- ? `${uploadThrottle.lastErrorAt || "unknown"} ${uploadThrottle.lastError}`
160
- : null;
161
- const autoRetryAt = parseEpochMsToIso(autoRetry?.retryAtMs || null);
162
- const autoRetryLine = autoRetryAt
163
- ? `- Auto retry after: ${autoRetryAt} (${autoRetry?.reason || "scheduled"}, pending ${Number(
164
- autoRetry?.pendingBytes || 0,
165
- )} bytes)`
166
- : null;
167
-
168
137
  const subscriptions = await collectLocalSubscriptions({
169
138
  home,
170
139
  env: process.env,
@@ -277,22 +246,13 @@ async function cmdStatus(argv = []) {
277
246
  if (opts.json || opts.light) {
278
247
  const summary = {
279
248
  generated_at: new Date().toISOString(),
280
- base_url: config?.baseUrl || null,
281
- device_token_set: Boolean(config?.deviceToken),
282
249
  queue: {
283
- pending_bytes: pendingBytes,
284
250
  size_bytes: queueSize,
285
- offset: queueState.offset || 0,
286
251
  },
287
252
  last_parse: cursors?.updatedAt || null,
288
253
  last_notify: lastNotify || null,
289
254
  last_openclaw_sync: lastOpenclawSync || null,
290
255
  last_notify_spawn: lastNotifySpawn || null,
291
- last_upload: lastUpload || null,
292
- next_upload_after: nextUpload || null,
293
- backoff_until: backoffUntil || null,
294
- last_upload_error: lastUploadError || null,
295
- auto_retry: autoRetry || null,
296
256
  hooks: {
297
257
  codex_notify: notifyConfigured,
298
258
  every_code_notify: everyCodeConfigured,
@@ -371,18 +331,11 @@ async function cmdStatus(argv = []) {
371
331
  process.stdout.write(
372
332
  [
373
333
  "Status:",
374
- `- Base URL: ${config?.baseUrl || "unset"}`,
375
- `- Device token: ${config?.deviceToken ? "set" : "unset"}`,
376
- `- Queue: ${pendingBytes} bytes pending`,
334
+ `- Queue: ${queueSize} bytes`,
377
335
  `- Last parse: ${cursors?.updatedAt || "never"}`,
378
336
  `- Last notify: ${lastNotify || "never"}`,
379
337
  `- Last OpenClaw-triggered sync: ${lastOpenclawSync || "never"}`,
380
338
  `- Last notify-triggered sync: ${lastNotifySpawn || "never"}`,
381
- `- Last upload: ${lastUpload || "never"}`,
382
- `- Next upload after: ${nextUpload || "never"}`,
383
- `- Backoff until: ${backoffUntil || "never"}`,
384
- lastUploadError ? `- Last upload error: ${lastUploadError}` : null,
385
- autoRetryLine,
386
339
  `- Codex notify: ${notifyConfigured ? JSON.stringify(codexNotify) : "unset"}`,
387
340
  `- Every Code notify: ${everyCodeConfigured ? JSON.stringify(everyCodeNotify) : "unset"}`,
388
341
  `- Claude hooks: ${claudeHookConfigured ? "set" : "unset"}`,
@@ -538,16 +491,9 @@ function renderLightTable(summary) {
538
491
  const rows = [];
539
492
  const push = (k, v) => rows.push([k, v == null || v === "" ? "—" : String(v)]);
540
493
 
541
- push("Base URL", summary.base_url);
542
- push("Device token", summary.device_token_set ? "set" : "unset");
543
- push("Queue pending (bytes)", summary.queue.pending_bytes);
544
494
  push("Queue size (bytes)", summary.queue.size_bytes);
545
495
  push("Last parse", summary.last_parse);
546
496
  push("Last notify", summary.last_notify);
547
- push("Last upload", summary.last_upload);
548
- push("Next upload after", summary.next_upload_after);
549
- push("Backoff until", summary.backoff_until);
550
- if (summary.last_upload_error) push("Last upload error", summary.last_upload_error);
551
497
 
552
498
  for (const [name, state] of Object.entries(summary.hooks || {})) {
553
499
  push(`Hook · ${name}`, state ? "set" : "unset");
@@ -62,13 +62,6 @@ const {
62
62
  } = require("../lib/rollout");
63
63
  const { computeClaudeGroundTruthBuckets } = require("../lib/claude-categorizer");
64
64
  const { createProgress, renderBar, formatNumber, formatBytes } = require("../lib/progress");
65
- const {
66
- normalizeState: normalizeUploadState,
67
- decideAutoUpload,
68
- recordUploadFailure,
69
- recordUploadSuccess,
70
- parseRetryAfterMs,
71
- } = require("../lib/upload-throttle");
72
65
  const {
73
66
  isCursorInstalled,
74
67
  extractCursorSessionToken,
@@ -77,10 +70,16 @@ const {
77
70
  } = require("../lib/cursor-config");
78
71
  const { purgeProjectUsage } = require("../lib/project-usage-purge");
79
72
  const { resolveTrackerPaths } = require("../lib/tracker-paths");
80
- const { resolveRuntimeConfig } = require("../lib/runtime-config");
81
73
 
82
74
  const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
83
75
  const ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY = "rolloutCumulativeDeltaReparse_2026_05";
76
+ // #75: rebuild Codex/every-code aggregates that were accumulated with the
77
+ // pre-fix parser, which counted replayed subagent-fork parent history as child
78
+ // consumption (up to ~40x Cache Read inflation on the local fork corpus). The
79
+ // rolloutCumulativeDeltaReparse migration above already rebuilt these once, but
80
+ // with the still-buggy parser, so a fresh key is required to rebuild again now
81
+ // that the parser is fixed.
82
+ const CODEX_FORK_OVERCOUNT_MIGRATION_KEY = "codexForkOvercountRebuild_2026_07";
84
83
  const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
85
84
  const GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY = "grokAppendOnlyRepair_2026_05_v4";
86
85
  const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
@@ -146,14 +145,11 @@ async function cmdSync(argv) {
146
145
  const queueStatePath = path.join(trackerDir, "queue.state.json");
147
146
  const projectQueuePath = path.join(trackerDir, "project.queue.jsonl");
148
147
  const projectQueueStatePath = path.join(trackerDir, "project.queue.state.json");
149
- const uploadThrottlePath = path.join(trackerDir, "upload.throttle.json");
150
148
  const grokSignalPath = path.join(trackerDir, "grok-last-session.json");
151
149
  const legacyGrokSignalPath = path.join(trackerDir, "tracker", "grok-last-session.json");
152
150
 
153
151
  const config = await readJson(configPath);
154
152
  const cursors = (await readJson(cursorsPath)) || { version: 1, files: {}, updatedAt: null };
155
- const uploadThrottle = normalizeUploadState(await readJson(uploadThrottlePath));
156
- let uploadThrottleState = uploadThrottle;
157
153
  let grokHookSignal = null;
158
154
  let grokHookSignalPath = null;
159
155
  for (const candidate of [grokSignalPath, legacyGrokSignalPath]) {
@@ -199,6 +195,7 @@ async function cmdSync(argv) {
199
195
  }
200
196
 
201
197
  await migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rolloutFiles });
198
+ await migrateCodexForkOvercountRebuild({ cursors, queuePath, projectQueuePath, rolloutFiles });
202
199
 
203
200
  const openclawFiles = openclawSignal?.sessionFile
204
201
  ? [{ path: openclawSignal.sessionFile, source: "openclaw" }]
@@ -928,69 +925,7 @@ async function cmdSync(argv) {
928
925
 
929
926
  progress?.stop();
930
927
 
931
- const runtime = resolveRuntimeConfig({ config: config || {}, env: process.env });
932
-
933
- let uploadResult = { inserted: 0, skipped: 0 };
934
- let uploadAttempted = false;
935
-
936
- if (runtime.deviceToken && runtime.baseUrl) {
937
- uploadAttempted = true;
938
- try {
939
- uploadResult = await drainQueueToCloud({
940
- baseUrl: runtime.baseUrl,
941
- deviceToken: runtime.deviceToken,
942
- queuePath,
943
- queueStatePath,
944
- maxBatches: opts.drain ? 100 : 5,
945
- batchSize: 200,
946
- });
947
- // Record success so the exponential backoff step resets — otherwise
948
- // a single past failure keeps us pessimistically throttled forever.
949
- uploadThrottleState = recordUploadSuccess({
950
- nowMs: Date.now(),
951
- state: uploadThrottleState,
952
- });
953
- await writeJson(uploadThrottlePath, uploadThrottleState);
954
- } catch (e) {
955
- // Persist a backoff on 429 / 5xx so the next auto-sync waits instead
956
- // of retrying immediately and making the rate-limit worse. The
957
- // throttle module already parses Retry-After when we surface it on
958
- // the error object (drainQueueToCloud stamps err.status + err.retryAfterMs).
959
- uploadThrottleState = recordUploadFailure({
960
- nowMs: Date.now(),
961
- state: uploadThrottleState,
962
- error: e,
963
- });
964
- await writeJson(uploadThrottlePath, uploadThrottleState);
965
- if (!opts.auto) {
966
- process.stderr.write(`Upload error: ${e?.message || e}\n`);
967
- }
968
- }
969
- }
970
928
 
971
- const afterState = (await readJson(queueStatePath)) || { offset: 0 };
972
- const queueSize = await safeStatSize(queuePath);
973
- const projectAfterState = (await readJson(projectQueueStatePath)) || { offset: 0 };
974
- const projectQueueSize = await safeStatSize(projectQueuePath);
975
- const pendingBytes =
976
- Math.max(0, queueSize - Number(afterState.offset || 0)) +
977
- Math.max(0, projectQueueSize - Number(projectAfterState.offset || 0));
978
-
979
- if (pendingBytes <= 0) {
980
- await clearAutoRetry(trackerDir);
981
- } else if (opts.auto && uploadAttempted) {
982
- const retryAtMs = Number(uploadThrottleState?.nextAllowedAtMs || 0);
983
- if (retryAtMs > Date.now()) {
984
- await scheduleAutoRetry({
985
- trackerDir,
986
- retryAtMs,
987
- reason: "backlog",
988
- pendingBytes,
989
- source: "auto-backlog",
990
- autoRetryNoSpawn: runtime.autoRetryNoSpawn,
991
- });
992
- }
993
- }
994
929
 
995
930
  const totalParsed =
996
931
  parseResult.filesProcessed +
@@ -1045,9 +980,6 @@ async function cmdSync(argv) {
1045
980
  const summary = {
1046
981
  totalParsed,
1047
982
  totalBuckets,
1048
- upload: uploadResult,
1049
- uploadAttempted,
1050
- pendingBytes,
1051
983
  };
1052
984
 
1053
985
  if (!opts.auto) {
@@ -1056,12 +988,6 @@ async function cmdSync(argv) {
1056
988
  "Sync finished:",
1057
989
  `- Parsed files: ${totalParsed}`,
1058
990
  `- New 30-min buckets queued: ${totalBuckets}`,
1059
- runtime.deviceToken
1060
- ? `- Uploaded: ${uploadResult.inserted} inserted, ${uploadResult.skipped} skipped`
1061
- : "- Uploaded: skipped (no device token)",
1062
- runtime.deviceToken && pendingBytes > 0 && !opts.drain
1063
- ? `- Remaining: ${formatBytes(pendingBytes)} pending (run sync again, or use --drain)`
1064
- : null,
1065
991
  "",
1066
992
  ]
1067
993
  .filter(Boolean)
@@ -1102,10 +1028,12 @@ module.exports = {
1102
1028
  cmdSync,
1103
1029
  migrateCursorUnknownBuckets,
1104
1030
  migrateRolloutCumulativeDeltaBuckets,
1031
+ migrateCodexForkOvercountRebuild,
1105
1032
  reincludeClaudeMemObserverFiles,
1106
1033
  repairGrokQueueFromSessionSnapshots,
1107
1034
  CURSOR_UNKNOWN_MIGRATION_KEY,
1108
1035
  ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY,
1036
+ CODEX_FORK_OVERCOUNT_MIGRATION_KEY,
1109
1037
  CLAUDE_MEM_OBSERVER_REINCLUDE_KEY,
1110
1038
  GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY,
1111
1039
  };
@@ -1276,102 +1204,10 @@ function deriveAutoSkipReason({ decision, state }) {
1276
1204
  return "throttled";
1277
1205
  }
1278
1206
 
1279
- async function scheduleAutoRetry({
1280
- trackerDir,
1281
- retryAtMs,
1282
- reason,
1283
- pendingBytes,
1284
- source,
1285
- autoRetryNoSpawn,
1286
- }) {
1287
- const retryMs = coerceRetryMs(retryAtMs);
1288
- if (!retryMs) return { scheduled: false, retryAtMs: 0 };
1289
-
1290
- const retryPath = path.join(trackerDir, AUTO_RETRY_FILENAME);
1291
- const nowMs = Date.now();
1292
- const existing = await readJson(retryPath);
1293
- const existingMs = coerceRetryMs(existing?.retryAtMs);
1294
- if (existingMs && existingMs >= retryMs - 1000) {
1295
- return { scheduled: false, retryAtMs: existingMs };
1296
- }
1297
-
1298
- const payload = {
1299
- version: 1,
1300
- retryAtMs: retryMs,
1301
- retryAt: new Date(retryMs).toISOString(),
1302
- reason: typeof reason === "string" && reason.length > 0 ? reason : "throttled",
1303
- pendingBytes: Math.max(0, Number(pendingBytes || 0)),
1304
- scheduledAt: new Date(nowMs).toISOString(),
1305
- source: typeof source === "string" ? source : "auto",
1306
- };
1307
-
1308
- await writeJson(retryPath, payload);
1309
-
1310
- const delayMs = Math.min(AUTO_RETRY_MAX_DELAY_MS, Math.max(0, retryMs - nowMs));
1311
- if (delayMs <= 0) return { scheduled: false, retryAtMs: retryMs };
1312
- if (autoRetryNoSpawn) {
1313
- return { scheduled: false, retryAtMs: retryMs };
1314
- }
1315
-
1316
- spawnAutoRetryProcess({
1317
- retryPath,
1318
- trackerBinPath: path.join(trackerDir, "app", "bin", "tracker.js"),
1319
- fallbackPkg: "@ipv9/tokentracker-cli",
1320
- delayMs,
1321
- });
1322
- return { scheduled: true, retryAtMs: retryMs };
1323
- }
1324
1207
 
1325
- async function clearAutoRetry(trackerDir) {
1326
- const retryPath = path.join(trackerDir, AUTO_RETRY_FILENAME);
1327
- await fs.unlink(retryPath).catch(() => {});
1328
- }
1329
1208
 
1330
- function spawnAutoRetryProcess({ retryPath, trackerBinPath, fallbackPkg, delayMs }) {
1331
- const script = buildAutoRetryScript({ retryPath, trackerBinPath, fallbackPkg, delayMs });
1332
- try {
1333
- const child = cp.spawn(process.execPath, ["-e", script], {
1334
- detached: true,
1335
- stdio: "ignore",
1336
- env: process.env,
1337
- });
1338
- child.unref();
1339
- } catch (_e) {}
1340
- }
1341
1209
 
1342
- function buildAutoRetryScript({ retryPath, trackerBinPath, fallbackPkg, delayMs }) {
1343
- return (
1344
- `'use strict';\n` +
1345
- `const fs = require('node:fs');\n` +
1346
- `const cp = require('node:child_process');\n` +
1347
- `const retryPath = ${JSON.stringify(retryPath)};\n` +
1348
- `const trackerBinPath = ${JSON.stringify(trackerBinPath)};\n` +
1349
- `const fallbackPkg = ${JSON.stringify(fallbackPkg)};\n` +
1350
- `const delayMs = ${Math.max(0, Math.floor(delayMs || 0))};\n` +
1351
- `setTimeout(() => {\n` +
1352
- ` let retryAtMs = 0;\n` +
1353
- ` try {\n` +
1354
- ` const raw = fs.readFileSync(retryPath, 'utf8');\n` +
1355
- ` retryAtMs = Number(JSON.parse(raw).retryAtMs || 0);\n` +
1356
- ` } catch (_) {}\n` +
1357
- ` if (!retryAtMs || Date.now() + 1000 < retryAtMs) process.exit(0);\n` +
1358
- ` const argv = ['sync', '--auto', '--from-retry'];\n` +
1359
- ` const cmd = fs.existsSync(trackerBinPath)\n` +
1360
- ` ? [process.execPath, trackerBinPath, ...argv]\n` +
1361
- ` : ['npx', '--yes', fallbackPkg, ...argv];\n` +
1362
- ` try {\n` +
1363
- ` const child = cp.spawn(cmd[0], cmd.slice(1), { detached: true, stdio: 'ignore', env: process.env });\n` +
1364
- ` child.unref();\n` +
1365
- ` } catch (_) {}\n` +
1366
- `}, delayMs);\n`
1367
- );
1368
- }
1369
1210
 
1370
- function coerceRetryMs(v) {
1371
- const n = Number(v);
1372
- if (!Number.isFinite(n) || n <= 0) return 0;
1373
- return Math.floor(n);
1374
- }
1375
1211
 
1376
1212
  async function writeOpenclawSignal(trackerDir) {
1377
1213
  const openclawSignalPath = path.join(trackerDir, "openclaw.signal");
@@ -1382,63 +1218,10 @@ async function writeOpenclawSignal(trackerDir) {
1382
1218
  }
1383
1219
  }
1384
1220
 
1385
- const AUTO_RETRY_FILENAME = "auto.retry.json";
1386
1221
  const AUTO_RETRY_MAX_DELAY_MS = 2 * 60 * 60 * 1000;
1387
1222
 
1388
- const INGEST_SLUG = "tokentracker-ingest";
1389
1223
  const MAX_INGEST_BUCKETS = 500;
1390
1224
 
1391
- async function drainQueueToCloud({ baseUrl, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200 }) {
1392
- const state = (await readJson(queueStatePath)) || { offset: 0 };
1393
- let offset = Number(state.offset || 0);
1394
- let inserted = 0;
1395
- let skipped = 0;
1396
-
1397
- const queueSize = await safeStatSize(queuePath);
1398
- const limit = Math.min(Math.max(1, Math.floor(Number(batchSize || 200))), MAX_INGEST_BUCKETS);
1399
-
1400
- for (let batch = 0; batch < maxBatches; batch++) {
1401
- if (offset >= queueSize) break;
1402
- const result = await readQueueBatch(queuePath, offset, limit);
1403
- if (result.buckets.length === 0) break;
1404
-
1405
- const root = baseUrl.replace(/\/$/, "");
1406
- const anonKey = process.env.TOKENTRACKER_INSFORGE_ANON_KEY || "";
1407
- const headers = {
1408
- "Content-Type": "application/json",
1409
- Accept: "application/json",
1410
- Authorization: `Bearer ${deviceToken}`,
1411
- };
1412
- if (anonKey) headers.apikey = anonKey;
1413
- const res = await fetch(`${root}/functions/${INGEST_SLUG}`, {
1414
- method: "POST",
1415
- headers,
1416
- body: JSON.stringify({ hourly: result.buckets }),
1417
- });
1418
-
1419
- const rawText = await res.text().catch(() => "");
1420
- let data = {};
1421
- try { data = JSON.parse(rawText); } catch { data = {}; }
1422
- if (!res.ok) {
1423
- const err = new Error(`HTTP ${res.status}: ${rawText.substring(0, 500)}`);
1424
- err.status = res.status;
1425
- const retryAfter = res.headers?.get?.("Retry-After") ?? null;
1426
- const retryAfterMs = parseRetryAfterMs(retryAfter);
1427
- if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
1428
- throw err;
1429
- }
1430
-
1431
- inserted += Number(data?.inserted || 0);
1432
- skipped += Number(data?.skipped || 0);
1433
-
1434
- offset = result.nextOffset;
1435
- state.offset = offset;
1436
- state.updatedAt = new Date().toISOString();
1437
- await writeJson(queueStatePath, state);
1438
- }
1439
-
1440
- return { inserted, skipped };
1441
- }
1442
1225
 
1443
1226
  async function readQueueBatch(queuePath, startOffset, maxBuckets) {
1444
1227
  const st = await fs.stat(queuePath).catch(() => null);
@@ -1922,6 +1705,130 @@ async function migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rollou
1922
1705
  cursors.migrations[ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY] = new Date().toISOString();
1923
1706
  }
1924
1707
 
1708
+ // One-time rebuild for issue #75. The prior parser counted a Codex subagent
1709
+ // fork's replayed parent token_count history as the child's consumption. After
1710
+ // the parser fix (PR #77) that only corrects go-forward parsing — already-
1711
+ // ingested files sit at offset = EOF and are never re-read — so their inflated
1712
+ // contribution stays baked into cursors.hourly / project aggregates and the
1713
+ // dashboard until this migration retracts it and forces a clean re-parse.
1714
+ //
1715
+ // Choreography mirrors migrateRolloutCumulativeDeltaBuckets (codex-scoped, so
1716
+ // no other provider is touched): (1) drop the per-file cursors so the same-run
1717
+ // sync re-parses from offset 0 with the fixed accounting; (2) delete the
1718
+ // inflated hourly buckets and append zero rows to queue.jsonl — the dashboard
1719
+ // keep-lasts per source|model|hour_start (local-api.js readQueueData), so the
1720
+ // re-parse's corrected row supersedes the zero, or the zero stands for a file
1721
+ // no longer on disk; (3) do the same for project aggregates, which are on by
1722
+ // default (sync passes projectQueuePath) and would otherwise double-count on
1723
+ // re-parse — readProjectQueueData also keep-lasts per project_key|source|
1724
+ // hour_start. Run-once via the migrations sentinel (the issue asks for a
1725
+ // one-time rebuild, not a repeatable command).
1726
+ //
1727
+ // Project aggregates are handled conservatively: the accumulator buckets are
1728
+ // reset (so files still on disk rebuild clean and their corrected row wins via
1729
+ // keep-last), but no zero rows are written. A project bucket whose file was
1730
+ // deleted or whose repo moved keeps its prior queue row rather than being
1731
+ // zeroed — we never destroy project history we can no longer recompute. Main-
1732
+ // dashboard (source) totals are always rebuilt regardless of project
1733
+ // resolution. See step (3).
1734
+ async function migrateCodexForkOvercountRebuild({
1735
+ cursors,
1736
+ queuePath,
1737
+ projectQueuePath,
1738
+ rolloutFiles,
1739
+ }) {
1740
+ if (!cursors || typeof cursors !== "object") return;
1741
+ cursors.migrations = cursors.migrations || {};
1742
+ if (cursors.migrations[CODEX_FORK_OVERCOUNT_MIGRATION_KEY]) return;
1743
+
1744
+ const rolloutPaths = new Set();
1745
+ for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
1746
+ const filePath = typeof entry === "string" ? entry : entry?.path;
1747
+ const source = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
1748
+ if (filePath && (source === "codex" || source === "every-code")) {
1749
+ rolloutPaths.add(filePath);
1750
+ }
1751
+ }
1752
+
1753
+ // (1) Drop per-file cursors -> clean re-parse from offset 0 with the fix.
1754
+ if (cursors.files && typeof cursors.files === "object") {
1755
+ for (const filePath of rolloutPaths) delete cursors.files[filePath];
1756
+ }
1757
+
1758
+ // (2) Retract + delete inflated hourly codex/every-code buckets.
1759
+ const hourlyRetractions = [];
1760
+ const buckets = cursors.hourly?.buckets;
1761
+ if (buckets && typeof buckets === "object") {
1762
+ for (const key of Object.keys(buckets)) {
1763
+ const [source, model, ...hourParts] = key.split("|");
1764
+ if (source !== "codex" && source !== "every-code") continue;
1765
+ hourlyRetractions.push(
1766
+ JSON.stringify({
1767
+ source,
1768
+ model: model || "unknown",
1769
+ hour_start: hourParts.join("|"),
1770
+ input_tokens: 0,
1771
+ cached_input_tokens: 0,
1772
+ cache_creation_input_tokens: 0,
1773
+ output_tokens: 0,
1774
+ reasoning_output_tokens: 0,
1775
+ total_tokens: 0,
1776
+ billable_total_tokens: 0,
1777
+ conversation_count: 0,
1778
+ }),
1779
+ );
1780
+ delete buckets[key];
1781
+ }
1782
+ }
1783
+ const groupQueued = cursors.hourly?.groupQueued;
1784
+ if (groupQueued && typeof groupQueued === "object") {
1785
+ for (const key of Object.keys(groupQueued)) {
1786
+ if (key.startsWith("codex|") || key.startsWith("every-code|")) {
1787
+ delete groupQueued[key];
1788
+ }
1789
+ }
1790
+ }
1791
+ if (hourlyRetractions.length > 0) {
1792
+ await ensureDir(path.dirname(queuePath));
1793
+ await fs.appendFile(queuePath, hourlyRetractions.join("\n") + "\n");
1794
+ }
1795
+
1796
+ // (3) Reset the inflated project codex/every-code accumulator buckets, but
1797
+ // do NOT append zero rows to project.queue.jsonl. Deleting the in-memory
1798
+ // bucket is what prevents the double-count: the same-run re-parse of a file
1799
+ // still on disk starts that project bucket fresh from zero and appends a
1800
+ // corrected row, which the keep-last project reader (local-api.js
1801
+ // readProjectQueueData, per project_key|source|hour_start) picks over the old
1802
+ // inflated row. A bucket whose file has since been deleted or whose repo
1803
+ // moved is NOT re-parsed; leaving its prior queue row in place keeps that
1804
+ // (possibly fork-inflated) historical value visible rather than zeroing data
1805
+ // we can no longer recompute. Unlike the hourly path, project aggregates were
1806
+ // never retracted by the earlier delta migration, so a zero here would be a
1807
+ // brand-new, unrecoverable loss for orphaned files — hence the conservative
1808
+ // reset-accumulator-only approach.
1809
+ let projectReset = 0;
1810
+ const projectBuckets = cursors.projectHourly?.buckets;
1811
+ if (projectQueuePath && projectBuckets && typeof projectBuckets === "object") {
1812
+ for (const key of Object.keys(projectBuckets)) {
1813
+ const source = projectBuckets[key]?.source;
1814
+ if (source !== "codex" && source !== "every-code") continue;
1815
+ delete projectBuckets[key];
1816
+ projectReset += 1;
1817
+ }
1818
+ }
1819
+
1820
+ if (process.env.TOKENTRACKER_DEBUG) {
1821
+ process.stderr.write(
1822
+ `[codex-fork] #75 rebuild: reset ${rolloutPaths.size} codex/every-code file ` +
1823
+ `cursors, retracted ${hourlyRetractions.length} hourly bucket(s), reset ` +
1824
+ `${projectReset} project accumulator bucket(s) (no project rows zeroed); ` +
1825
+ `this sync re-parses files still on disk with the corrected accounting\n`,
1826
+ );
1827
+ }
1828
+
1829
+ cursors.migrations[CODEX_FORK_OVERCOUNT_MIGRATION_KEY] = new Date().toISOString();
1830
+ }
1831
+
1925
1832
  // One-time repair migration: rebuild source=claude rows in queue.jsonl from
1926
1833
  // the actual jsonl files using ccusage's algorithm (msgId+reqId global
1927
1834
  // dedup). Earlier `reincludeClaudeMemObserverFiles` versions (v1/v2/v3) each