@ipv9/tokentracker-cli 0.39.38 → 0.39.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -11
- package/dashboard/dist/assets/{Card-DD4s0V6T.js → Card-JS4gjUZf.js} +1 -1
- package/dashboard/dist/assets/DashboardPage-BoXhzJb1.js +60 -0
- package/dashboard/dist/assets/{FadeIn-CUSBK0jm.js → FadeIn-DvLfvwys.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-PHF4lPax.js → IpCheckPage-Dcfc0Uts.js} +1 -1
- package/dashboard/dist/assets/LimitsPage-Vz-GVdi9.js +2 -0
- package/dashboard/dist/assets/{LocalOnlyNotice-DCmtS_j5.js → LocalOnlyNotice-8xF5couh.js} +1 -1
- package/dashboard/dist/assets/PopoverPopup-7HrUzeRx.js +1 -0
- package/dashboard/dist/assets/Select-9XDsaO_C.js +1 -0
- package/dashboard/dist/assets/SelectItemText-DE5Fs9p6.js +12 -0
- package/dashboard/dist/assets/SettingsPage-CMnfAptE.js +1 -0
- package/dashboard/dist/assets/SkillsPage-CbvLHN0L.js +1 -0
- package/dashboard/dist/assets/WidgetsPage-CeGQlyt7.js +1 -0
- package/dashboard/dist/assets/WrappedPage-Bux8qLUC.js +1 -0
- package/dashboard/dist/assets/{arrow-up-right-CjaBQnwI.js → arrow-up-right-B7rCoZdW.js} +1 -1
- package/dashboard/dist/assets/{download-DLBqkSwp.js → download-CtTYpbcx.js} +1 -1
- package/dashboard/dist/assets/format-BuQKK6kQ.js +1 -0
- package/dashboard/dist/assets/limitDisplay-WwuTdS_a.js +1 -0
- package/dashboard/dist/assets/main-Ck4SUH4m.js +696 -0
- package/dashboard/dist/assets/main-ZrWkoMlr.css +1 -0
- package/dashboard/dist/assets/mock-data-Bvbe8tYK.js +1 -0
- package/dashboard/dist/assets/use-limits-display-prefs-BKWYPgeA.js +1 -0
- package/dashboard/dist/assets/use-native-settings-D7Nj1dXR.js +1 -0
- package/dashboard/dist/assets/useCurrency-BX3pvyhh.js +1 -0
- package/dashboard/dist/index.html +15 -27
- package/dashboard/dist/llms.txt +0 -2
- package/dashboard/dist/sitemap.xml +0 -12
- package/package.json +7 -4
- package/src/cli.js +4 -13
- package/src/commands/doctor.js +1 -3
- package/src/commands/init.js +63 -53
- package/src/commands/serve.js +2 -50
- package/src/commands/status.js +1 -55
- package/src/commands/sync.js +134 -227
- package/src/lib/{browser-auth.js → browser-open.js} +3 -149
- package/src/lib/claude-categorizer.js +204 -32
- package/src/lib/codex-context-breakdown.js +83 -7
- package/src/lib/codex-rollout-parser.js +52 -10
- package/src/lib/diagnostics.js +1 -49
- package/src/lib/doctor.js +12 -87
- package/src/lib/file-identity.js +69 -0
- package/src/lib/init-flow.js +6 -3
- package/src/lib/local-api.js +6 -364
- package/src/lib/pricing/curated-overrides.json +1 -1
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +392 -144
- package/src/lib/runtime-config.js +0 -32
- package/src/lib/sqlite-reader.js +83 -8
- package/dashboard/dist/assets/ActivityHeatmap-BrehqUIb.js +0 -42
- package/dashboard/dist/assets/DashboardPage-CyUShBRJ.js +0 -19
- package/dashboard/dist/assets/DevicePage-BA74Su1l.js +0 -1
- package/dashboard/dist/assets/DialogTitle-DoKsz5wi.js +0 -1
- package/dashboard/dist/assets/InsforgeUserHeaderControls-Bk_U2dHr.js +0 -1
- package/dashboard/dist/assets/LandingPage-Bbeu84Iu.js +0 -4356
- package/dashboard/dist/assets/LeaderboardAvatar-ByqeB6bM.js +0 -1
- package/dashboard/dist/assets/LeaderboardPage-BeWXF00S.js +0 -6
- package/dashboard/dist/assets/LeaderboardProfileModal-C4yvbZd_.js +0 -72
- package/dashboard/dist/assets/LeaderboardProfilePage-D3cbSh5o.js +0 -1
- package/dashboard/dist/assets/LimitsPage-dS9C9Vdt.js +0 -2
- package/dashboard/dist/assets/LoginPage-Ch3UQoWj.js +0 -1
- package/dashboard/dist/assets/PopoverPopup-DjG0gjlA.js +0 -1
- package/dashboard/dist/assets/Select-HzIjWJpE.js +0 -1
- package/dashboard/dist/assets/SelectItemText-f0tCwtvZ.js +0 -1
- package/dashboard/dist/assets/SettingsPage-DJOZAlZy.js +0 -1
- package/dashboard/dist/assets/SkillsPage-BQcyJIdx.js +0 -1
- package/dashboard/dist/assets/WidgetsPage-B_vxuV09.js +0 -1
- package/dashboard/dist/assets/WrappedPage-CZN9Y3Vz.js +0 -1
- package/dashboard/dist/assets/index-BeoRn2gJ.js +0 -2
- package/dashboard/dist/assets/info-BAIxlHIB.js +0 -1
- package/dashboard/dist/assets/main-DAcSUW2t.css +0 -1
- package/dashboard/dist/assets/main-pi07CfeC.js +0 -1004
- package/dashboard/dist/assets/use-limits-display-prefs-CMfASZsh.js +0 -1
- package/dashboard/dist/assets/use-native-settings-CfPik-Ao.js +0 -1
- package/dashboard/dist/assets/use-spring-wsmJQRqb.js +0 -1
- package/dashboard/dist/assets/use-usage-limits-Bt0Dp6kx.js +0 -1
- package/dashboard/dist/assets/useCurrency-CsaAMc3j.js +0 -1
- package/dashboard/dist/assets/useOpenInteractionType-CPzi5Z8M.js +0 -12
- package/dashboard/dist/share.html +0 -140
- package/src/commands/device-login.js +0 -161
- package/src/lib/upload-throttle.js +0 -148
package/src/commands/sync.js
CHANGED
|
@@ -62,13 +62,6 @@ const {
|
|
|
62
62
|
} = require("../lib/rollout");
|
|
63
63
|
const { computeClaudeGroundTruthBuckets } = require("../lib/claude-categorizer");
|
|
64
64
|
const { createProgress, renderBar, formatNumber, formatBytes } = require("../lib/progress");
|
|
65
|
-
const {
|
|
66
|
-
normalizeState: normalizeUploadState,
|
|
67
|
-
decideAutoUpload,
|
|
68
|
-
recordUploadFailure,
|
|
69
|
-
recordUploadSuccess,
|
|
70
|
-
parseRetryAfterMs,
|
|
71
|
-
} = require("../lib/upload-throttle");
|
|
72
65
|
const {
|
|
73
66
|
isCursorInstalled,
|
|
74
67
|
extractCursorSessionToken,
|
|
@@ -77,10 +70,16 @@ const {
|
|
|
77
70
|
} = require("../lib/cursor-config");
|
|
78
71
|
const { purgeProjectUsage } = require("../lib/project-usage-purge");
|
|
79
72
|
const { resolveTrackerPaths } = require("../lib/tracker-paths");
|
|
80
|
-
const { resolveRuntimeConfig } = require("../lib/runtime-config");
|
|
81
73
|
|
|
82
74
|
const CURSOR_UNKNOWN_MIGRATION_KEY = "cursorUnknownPurge_2026_04";
|
|
83
75
|
const ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY = "rolloutCumulativeDeltaReparse_2026_05";
|
|
76
|
+
// #75: rebuild Codex/every-code aggregates that were accumulated with the
|
|
77
|
+
// pre-fix parser, which counted replayed subagent-fork parent history as child
|
|
78
|
+
// consumption (up to ~40x Cache Read inflation on the local fork corpus). The
|
|
79
|
+
// rolloutCumulativeDeltaReparse migration above already rebuilt these once, but
|
|
80
|
+
// with the still-buggy parser, so a fresh key is required to rebuild again now
|
|
81
|
+
// that the parser is fixed.
|
|
82
|
+
const CODEX_FORK_OVERCOUNT_MIGRATION_KEY = "codexForkOvercountRebuild_2026_07";
|
|
84
83
|
const CLAUDE_MEM_OBSERVER_REINCLUDE_KEY = "claudeMemObserverReinclude_2026_05_v3";
|
|
85
84
|
const GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY = "grokAppendOnlyRepair_2026_05_v4";
|
|
86
85
|
const CLAUDE_MEM_OBSERVER_PATH_SEGMENT = "--claude-mem-observer-sessions";
|
|
@@ -146,14 +145,11 @@ async function cmdSync(argv) {
|
|
|
146
145
|
const queueStatePath = path.join(trackerDir, "queue.state.json");
|
|
147
146
|
const projectQueuePath = path.join(trackerDir, "project.queue.jsonl");
|
|
148
147
|
const projectQueueStatePath = path.join(trackerDir, "project.queue.state.json");
|
|
149
|
-
const uploadThrottlePath = path.join(trackerDir, "upload.throttle.json");
|
|
150
148
|
const grokSignalPath = path.join(trackerDir, "grok-last-session.json");
|
|
151
149
|
const legacyGrokSignalPath = path.join(trackerDir, "tracker", "grok-last-session.json");
|
|
152
150
|
|
|
153
151
|
const config = await readJson(configPath);
|
|
154
152
|
const cursors = (await readJson(cursorsPath)) || { version: 1, files: {}, updatedAt: null };
|
|
155
|
-
const uploadThrottle = normalizeUploadState(await readJson(uploadThrottlePath));
|
|
156
|
-
let uploadThrottleState = uploadThrottle;
|
|
157
153
|
let grokHookSignal = null;
|
|
158
154
|
let grokHookSignalPath = null;
|
|
159
155
|
for (const candidate of [grokSignalPath, legacyGrokSignalPath]) {
|
|
@@ -199,6 +195,7 @@ async function cmdSync(argv) {
|
|
|
199
195
|
}
|
|
200
196
|
|
|
201
197
|
await migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rolloutFiles });
|
|
198
|
+
await migrateCodexForkOvercountRebuild({ cursors, queuePath, projectQueuePath, rolloutFiles });
|
|
202
199
|
|
|
203
200
|
const openclawFiles = openclawSignal?.sessionFile
|
|
204
201
|
? [{ path: openclawSignal.sessionFile, source: "openclaw" }]
|
|
@@ -928,69 +925,7 @@ async function cmdSync(argv) {
|
|
|
928
925
|
|
|
929
926
|
progress?.stop();
|
|
930
927
|
|
|
931
|
-
const runtime = resolveRuntimeConfig({ config: config || {}, env: process.env });
|
|
932
|
-
|
|
933
|
-
let uploadResult = { inserted: 0, skipped: 0 };
|
|
934
|
-
let uploadAttempted = false;
|
|
935
|
-
|
|
936
|
-
if (runtime.deviceToken && runtime.baseUrl) {
|
|
937
|
-
uploadAttempted = true;
|
|
938
|
-
try {
|
|
939
|
-
uploadResult = await drainQueueToCloud({
|
|
940
|
-
baseUrl: runtime.baseUrl,
|
|
941
|
-
deviceToken: runtime.deviceToken,
|
|
942
|
-
queuePath,
|
|
943
|
-
queueStatePath,
|
|
944
|
-
maxBatches: opts.drain ? 100 : 5,
|
|
945
|
-
batchSize: 200,
|
|
946
|
-
});
|
|
947
|
-
// Record success so the exponential backoff step resets — otherwise
|
|
948
|
-
// a single past failure keeps us pessimistically throttled forever.
|
|
949
|
-
uploadThrottleState = recordUploadSuccess({
|
|
950
|
-
nowMs: Date.now(),
|
|
951
|
-
state: uploadThrottleState,
|
|
952
|
-
});
|
|
953
|
-
await writeJson(uploadThrottlePath, uploadThrottleState);
|
|
954
|
-
} catch (e) {
|
|
955
|
-
// Persist a backoff on 429 / 5xx so the next auto-sync waits instead
|
|
956
|
-
// of retrying immediately and making the rate-limit worse. The
|
|
957
|
-
// throttle module already parses Retry-After when we surface it on
|
|
958
|
-
// the error object (drainQueueToCloud stamps err.status + err.retryAfterMs).
|
|
959
|
-
uploadThrottleState = recordUploadFailure({
|
|
960
|
-
nowMs: Date.now(),
|
|
961
|
-
state: uploadThrottleState,
|
|
962
|
-
error: e,
|
|
963
|
-
});
|
|
964
|
-
await writeJson(uploadThrottlePath, uploadThrottleState);
|
|
965
|
-
if (!opts.auto) {
|
|
966
|
-
process.stderr.write(`Upload error: ${e?.message || e}\n`);
|
|
967
|
-
}
|
|
968
|
-
}
|
|
969
|
-
}
|
|
970
928
|
|
|
971
|
-
const afterState = (await readJson(queueStatePath)) || { offset: 0 };
|
|
972
|
-
const queueSize = await safeStatSize(queuePath);
|
|
973
|
-
const projectAfterState = (await readJson(projectQueueStatePath)) || { offset: 0 };
|
|
974
|
-
const projectQueueSize = await safeStatSize(projectQueuePath);
|
|
975
|
-
const pendingBytes =
|
|
976
|
-
Math.max(0, queueSize - Number(afterState.offset || 0)) +
|
|
977
|
-
Math.max(0, projectQueueSize - Number(projectAfterState.offset || 0));
|
|
978
|
-
|
|
979
|
-
if (pendingBytes <= 0) {
|
|
980
|
-
await clearAutoRetry(trackerDir);
|
|
981
|
-
} else if (opts.auto && uploadAttempted) {
|
|
982
|
-
const retryAtMs = Number(uploadThrottleState?.nextAllowedAtMs || 0);
|
|
983
|
-
if (retryAtMs > Date.now()) {
|
|
984
|
-
await scheduleAutoRetry({
|
|
985
|
-
trackerDir,
|
|
986
|
-
retryAtMs,
|
|
987
|
-
reason: "backlog",
|
|
988
|
-
pendingBytes,
|
|
989
|
-
source: "auto-backlog",
|
|
990
|
-
autoRetryNoSpawn: runtime.autoRetryNoSpawn,
|
|
991
|
-
});
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
929
|
|
|
995
930
|
const totalParsed =
|
|
996
931
|
parseResult.filesProcessed +
|
|
@@ -1045,9 +980,6 @@ async function cmdSync(argv) {
|
|
|
1045
980
|
const summary = {
|
|
1046
981
|
totalParsed,
|
|
1047
982
|
totalBuckets,
|
|
1048
|
-
upload: uploadResult,
|
|
1049
|
-
uploadAttempted,
|
|
1050
|
-
pendingBytes,
|
|
1051
983
|
};
|
|
1052
984
|
|
|
1053
985
|
if (!opts.auto) {
|
|
@@ -1056,12 +988,6 @@ async function cmdSync(argv) {
|
|
|
1056
988
|
"Sync finished:",
|
|
1057
989
|
`- Parsed files: ${totalParsed}`,
|
|
1058
990
|
`- New 30-min buckets queued: ${totalBuckets}`,
|
|
1059
|
-
runtime.deviceToken
|
|
1060
|
-
? `- Uploaded: ${uploadResult.inserted} inserted, ${uploadResult.skipped} skipped`
|
|
1061
|
-
: "- Uploaded: skipped (no device token)",
|
|
1062
|
-
runtime.deviceToken && pendingBytes > 0 && !opts.drain
|
|
1063
|
-
? `- Remaining: ${formatBytes(pendingBytes)} pending (run sync again, or use --drain)`
|
|
1064
|
-
: null,
|
|
1065
991
|
"",
|
|
1066
992
|
]
|
|
1067
993
|
.filter(Boolean)
|
|
@@ -1102,10 +1028,12 @@ module.exports = {
|
|
|
1102
1028
|
cmdSync,
|
|
1103
1029
|
migrateCursorUnknownBuckets,
|
|
1104
1030
|
migrateRolloutCumulativeDeltaBuckets,
|
|
1031
|
+
migrateCodexForkOvercountRebuild,
|
|
1105
1032
|
reincludeClaudeMemObserverFiles,
|
|
1106
1033
|
repairGrokQueueFromSessionSnapshots,
|
|
1107
1034
|
CURSOR_UNKNOWN_MIGRATION_KEY,
|
|
1108
1035
|
ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY,
|
|
1036
|
+
CODEX_FORK_OVERCOUNT_MIGRATION_KEY,
|
|
1109
1037
|
CLAUDE_MEM_OBSERVER_REINCLUDE_KEY,
|
|
1110
1038
|
GROK_APPEND_ONLY_REPAIR_MIGRATION_KEY,
|
|
1111
1039
|
};
|
|
@@ -1276,102 +1204,10 @@ function deriveAutoSkipReason({ decision, state }) {
|
|
|
1276
1204
|
return "throttled";
|
|
1277
1205
|
}
|
|
1278
1206
|
|
|
1279
|
-
async function scheduleAutoRetry({
|
|
1280
|
-
trackerDir,
|
|
1281
|
-
retryAtMs,
|
|
1282
|
-
reason,
|
|
1283
|
-
pendingBytes,
|
|
1284
|
-
source,
|
|
1285
|
-
autoRetryNoSpawn,
|
|
1286
|
-
}) {
|
|
1287
|
-
const retryMs = coerceRetryMs(retryAtMs);
|
|
1288
|
-
if (!retryMs) return { scheduled: false, retryAtMs: 0 };
|
|
1289
|
-
|
|
1290
|
-
const retryPath = path.join(trackerDir, AUTO_RETRY_FILENAME);
|
|
1291
|
-
const nowMs = Date.now();
|
|
1292
|
-
const existing = await readJson(retryPath);
|
|
1293
|
-
const existingMs = coerceRetryMs(existing?.retryAtMs);
|
|
1294
|
-
if (existingMs && existingMs >= retryMs - 1000) {
|
|
1295
|
-
return { scheduled: false, retryAtMs: existingMs };
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
|
-
const payload = {
|
|
1299
|
-
version: 1,
|
|
1300
|
-
retryAtMs: retryMs,
|
|
1301
|
-
retryAt: new Date(retryMs).toISOString(),
|
|
1302
|
-
reason: typeof reason === "string" && reason.length > 0 ? reason : "throttled",
|
|
1303
|
-
pendingBytes: Math.max(0, Number(pendingBytes || 0)),
|
|
1304
|
-
scheduledAt: new Date(nowMs).toISOString(),
|
|
1305
|
-
source: typeof source === "string" ? source : "auto",
|
|
1306
|
-
};
|
|
1307
|
-
|
|
1308
|
-
await writeJson(retryPath, payload);
|
|
1309
|
-
|
|
1310
|
-
const delayMs = Math.min(AUTO_RETRY_MAX_DELAY_MS, Math.max(0, retryMs - nowMs));
|
|
1311
|
-
if (delayMs <= 0) return { scheduled: false, retryAtMs: retryMs };
|
|
1312
|
-
if (autoRetryNoSpawn) {
|
|
1313
|
-
return { scheduled: false, retryAtMs: retryMs };
|
|
1314
|
-
}
|
|
1315
|
-
|
|
1316
|
-
spawnAutoRetryProcess({
|
|
1317
|
-
retryPath,
|
|
1318
|
-
trackerBinPath: path.join(trackerDir, "app", "bin", "tracker.js"),
|
|
1319
|
-
fallbackPkg: "@ipv9/tokentracker-cli",
|
|
1320
|
-
delayMs,
|
|
1321
|
-
});
|
|
1322
|
-
return { scheduled: true, retryAtMs: retryMs };
|
|
1323
|
-
}
|
|
1324
1207
|
|
|
1325
|
-
async function clearAutoRetry(trackerDir) {
|
|
1326
|
-
const retryPath = path.join(trackerDir, AUTO_RETRY_FILENAME);
|
|
1327
|
-
await fs.unlink(retryPath).catch(() => {});
|
|
1328
|
-
}
|
|
1329
1208
|
|
|
1330
|
-
function spawnAutoRetryProcess({ retryPath, trackerBinPath, fallbackPkg, delayMs }) {
|
|
1331
|
-
const script = buildAutoRetryScript({ retryPath, trackerBinPath, fallbackPkg, delayMs });
|
|
1332
|
-
try {
|
|
1333
|
-
const child = cp.spawn(process.execPath, ["-e", script], {
|
|
1334
|
-
detached: true,
|
|
1335
|
-
stdio: "ignore",
|
|
1336
|
-
env: process.env,
|
|
1337
|
-
});
|
|
1338
|
-
child.unref();
|
|
1339
|
-
} catch (_e) {}
|
|
1340
|
-
}
|
|
1341
1209
|
|
|
1342
|
-
function buildAutoRetryScript({ retryPath, trackerBinPath, fallbackPkg, delayMs }) {
|
|
1343
|
-
return (
|
|
1344
|
-
`'use strict';\n` +
|
|
1345
|
-
`const fs = require('node:fs');\n` +
|
|
1346
|
-
`const cp = require('node:child_process');\n` +
|
|
1347
|
-
`const retryPath = ${JSON.stringify(retryPath)};\n` +
|
|
1348
|
-
`const trackerBinPath = ${JSON.stringify(trackerBinPath)};\n` +
|
|
1349
|
-
`const fallbackPkg = ${JSON.stringify(fallbackPkg)};\n` +
|
|
1350
|
-
`const delayMs = ${Math.max(0, Math.floor(delayMs || 0))};\n` +
|
|
1351
|
-
`setTimeout(() => {\n` +
|
|
1352
|
-
` let retryAtMs = 0;\n` +
|
|
1353
|
-
` try {\n` +
|
|
1354
|
-
` const raw = fs.readFileSync(retryPath, 'utf8');\n` +
|
|
1355
|
-
` retryAtMs = Number(JSON.parse(raw).retryAtMs || 0);\n` +
|
|
1356
|
-
` } catch (_) {}\n` +
|
|
1357
|
-
` if (!retryAtMs || Date.now() + 1000 < retryAtMs) process.exit(0);\n` +
|
|
1358
|
-
` const argv = ['sync', '--auto', '--from-retry'];\n` +
|
|
1359
|
-
` const cmd = fs.existsSync(trackerBinPath)\n` +
|
|
1360
|
-
` ? [process.execPath, trackerBinPath, ...argv]\n` +
|
|
1361
|
-
` : ['npx', '--yes', fallbackPkg, ...argv];\n` +
|
|
1362
|
-
` try {\n` +
|
|
1363
|
-
` const child = cp.spawn(cmd[0], cmd.slice(1), { detached: true, stdio: 'ignore', env: process.env });\n` +
|
|
1364
|
-
` child.unref();\n` +
|
|
1365
|
-
` } catch (_) {}\n` +
|
|
1366
|
-
`}, delayMs);\n`
|
|
1367
|
-
);
|
|
1368
|
-
}
|
|
1369
1210
|
|
|
1370
|
-
function coerceRetryMs(v) {
|
|
1371
|
-
const n = Number(v);
|
|
1372
|
-
if (!Number.isFinite(n) || n <= 0) return 0;
|
|
1373
|
-
return Math.floor(n);
|
|
1374
|
-
}
|
|
1375
1211
|
|
|
1376
1212
|
async function writeOpenclawSignal(trackerDir) {
|
|
1377
1213
|
const openclawSignalPath = path.join(trackerDir, "openclaw.signal");
|
|
@@ -1382,63 +1218,10 @@ async function writeOpenclawSignal(trackerDir) {
|
|
|
1382
1218
|
}
|
|
1383
1219
|
}
|
|
1384
1220
|
|
|
1385
|
-
const AUTO_RETRY_FILENAME = "auto.retry.json";
|
|
1386
1221
|
const AUTO_RETRY_MAX_DELAY_MS = 2 * 60 * 60 * 1000;
|
|
1387
1222
|
|
|
1388
|
-
const INGEST_SLUG = "tokentracker-ingest";
|
|
1389
1223
|
const MAX_INGEST_BUCKETS = 500;
|
|
1390
1224
|
|
|
1391
|
-
async function drainQueueToCloud({ baseUrl, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200 }) {
|
|
1392
|
-
const state = (await readJson(queueStatePath)) || { offset: 0 };
|
|
1393
|
-
let offset = Number(state.offset || 0);
|
|
1394
|
-
let inserted = 0;
|
|
1395
|
-
let skipped = 0;
|
|
1396
|
-
|
|
1397
|
-
const queueSize = await safeStatSize(queuePath);
|
|
1398
|
-
const limit = Math.min(Math.max(1, Math.floor(Number(batchSize || 200))), MAX_INGEST_BUCKETS);
|
|
1399
|
-
|
|
1400
|
-
for (let batch = 0; batch < maxBatches; batch++) {
|
|
1401
|
-
if (offset >= queueSize) break;
|
|
1402
|
-
const result = await readQueueBatch(queuePath, offset, limit);
|
|
1403
|
-
if (result.buckets.length === 0) break;
|
|
1404
|
-
|
|
1405
|
-
const root = baseUrl.replace(/\/$/, "");
|
|
1406
|
-
const anonKey = process.env.TOKENTRACKER_INSFORGE_ANON_KEY || "";
|
|
1407
|
-
const headers = {
|
|
1408
|
-
"Content-Type": "application/json",
|
|
1409
|
-
Accept: "application/json",
|
|
1410
|
-
Authorization: `Bearer ${deviceToken}`,
|
|
1411
|
-
};
|
|
1412
|
-
if (anonKey) headers.apikey = anonKey;
|
|
1413
|
-
const res = await fetch(`${root}/functions/${INGEST_SLUG}`, {
|
|
1414
|
-
method: "POST",
|
|
1415
|
-
headers,
|
|
1416
|
-
body: JSON.stringify({ hourly: result.buckets }),
|
|
1417
|
-
});
|
|
1418
|
-
|
|
1419
|
-
const rawText = await res.text().catch(() => "");
|
|
1420
|
-
let data = {};
|
|
1421
|
-
try { data = JSON.parse(rawText); } catch { data = {}; }
|
|
1422
|
-
if (!res.ok) {
|
|
1423
|
-
const err = new Error(`HTTP ${res.status}: ${rawText.substring(0, 500)}`);
|
|
1424
|
-
err.status = res.status;
|
|
1425
|
-
const retryAfter = res.headers?.get?.("Retry-After") ?? null;
|
|
1426
|
-
const retryAfterMs = parseRetryAfterMs(retryAfter);
|
|
1427
|
-
if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
|
|
1428
|
-
throw err;
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
inserted += Number(data?.inserted || 0);
|
|
1432
|
-
skipped += Number(data?.skipped || 0);
|
|
1433
|
-
|
|
1434
|
-
offset = result.nextOffset;
|
|
1435
|
-
state.offset = offset;
|
|
1436
|
-
state.updatedAt = new Date().toISOString();
|
|
1437
|
-
await writeJson(queueStatePath, state);
|
|
1438
|
-
}
|
|
1439
|
-
|
|
1440
|
-
return { inserted, skipped };
|
|
1441
|
-
}
|
|
1442
1225
|
|
|
1443
1226
|
async function readQueueBatch(queuePath, startOffset, maxBuckets) {
|
|
1444
1227
|
const st = await fs.stat(queuePath).catch(() => null);
|
|
@@ -1922,6 +1705,130 @@ async function migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rollou
|
|
|
1922
1705
|
cursors.migrations[ROLLOUT_CUMULATIVE_DELTA_MIGRATION_KEY] = new Date().toISOString();
|
|
1923
1706
|
}
|
|
1924
1707
|
|
|
1708
|
+
// One-time rebuild for issue #75. The prior parser counted a Codex subagent
|
|
1709
|
+
// fork's replayed parent token_count history as the child's consumption. After
|
|
1710
|
+
// the parser fix (PR #77) that only corrects go-forward parsing — already-
|
|
1711
|
+
// ingested files sit at offset = EOF and are never re-read — so their inflated
|
|
1712
|
+
// contribution stays baked into cursors.hourly / project aggregates and the
|
|
1713
|
+
// dashboard until this migration retracts it and forces a clean re-parse.
|
|
1714
|
+
//
|
|
1715
|
+
// Choreography mirrors migrateRolloutCumulativeDeltaBuckets (codex-scoped, so
|
|
1716
|
+
// no other provider is touched): (1) drop the per-file cursors so the same-run
|
|
1717
|
+
// sync re-parses from offset 0 with the fixed accounting; (2) delete the
|
|
1718
|
+
// inflated hourly buckets and append zero rows to queue.jsonl — the dashboard
|
|
1719
|
+
// keep-lasts per source|model|hour_start (local-api.js readQueueData), so the
|
|
1720
|
+
// re-parse's corrected row supersedes the zero, or the zero stands for a file
|
|
1721
|
+
// no longer on disk; (3) do the same for project aggregates, which are on by
|
|
1722
|
+
// default (sync passes projectQueuePath) and would otherwise double-count on
|
|
1723
|
+
// re-parse — readProjectQueueData also keep-lasts per project_key|source|
|
|
1724
|
+
// hour_start. Run-once via the migrations sentinel (the issue asks for a
|
|
1725
|
+
// one-time rebuild, not a repeatable command).
|
|
1726
|
+
//
|
|
1727
|
+
// Project aggregates are handled conservatively: the accumulator buckets are
|
|
1728
|
+
// reset (so files still on disk rebuild clean and their corrected row wins via
|
|
1729
|
+
// keep-last), but no zero rows are written. A project bucket whose file was
|
|
1730
|
+
// deleted or whose repo moved keeps its prior queue row rather than being
|
|
1731
|
+
// zeroed — we never destroy project history we can no longer recompute. Main-
|
|
1732
|
+
// dashboard (source) totals are always rebuilt regardless of project
|
|
1733
|
+
// resolution. See step (3).
|
|
1734
|
+
async function migrateCodexForkOvercountRebuild({
|
|
1735
|
+
cursors,
|
|
1736
|
+
queuePath,
|
|
1737
|
+
projectQueuePath,
|
|
1738
|
+
rolloutFiles,
|
|
1739
|
+
}) {
|
|
1740
|
+
if (!cursors || typeof cursors !== "object") return;
|
|
1741
|
+
cursors.migrations = cursors.migrations || {};
|
|
1742
|
+
if (cursors.migrations[CODEX_FORK_OVERCOUNT_MIGRATION_KEY]) return;
|
|
1743
|
+
|
|
1744
|
+
const rolloutPaths = new Set();
|
|
1745
|
+
for (const entry of Array.isArray(rolloutFiles) ? rolloutFiles : []) {
|
|
1746
|
+
const filePath = typeof entry === "string" ? entry : entry?.path;
|
|
1747
|
+
const source = typeof entry === "string" ? "codex" : String(entry?.source || "codex");
|
|
1748
|
+
if (filePath && (source === "codex" || source === "every-code")) {
|
|
1749
|
+
rolloutPaths.add(filePath);
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
// (1) Drop per-file cursors -> clean re-parse from offset 0 with the fix.
|
|
1754
|
+
if (cursors.files && typeof cursors.files === "object") {
|
|
1755
|
+
for (const filePath of rolloutPaths) delete cursors.files[filePath];
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
// (2) Retract + delete inflated hourly codex/every-code buckets.
|
|
1759
|
+
const hourlyRetractions = [];
|
|
1760
|
+
const buckets = cursors.hourly?.buckets;
|
|
1761
|
+
if (buckets && typeof buckets === "object") {
|
|
1762
|
+
for (const key of Object.keys(buckets)) {
|
|
1763
|
+
const [source, model, ...hourParts] = key.split("|");
|
|
1764
|
+
if (source !== "codex" && source !== "every-code") continue;
|
|
1765
|
+
hourlyRetractions.push(
|
|
1766
|
+
JSON.stringify({
|
|
1767
|
+
source,
|
|
1768
|
+
model: model || "unknown",
|
|
1769
|
+
hour_start: hourParts.join("|"),
|
|
1770
|
+
input_tokens: 0,
|
|
1771
|
+
cached_input_tokens: 0,
|
|
1772
|
+
cache_creation_input_tokens: 0,
|
|
1773
|
+
output_tokens: 0,
|
|
1774
|
+
reasoning_output_tokens: 0,
|
|
1775
|
+
total_tokens: 0,
|
|
1776
|
+
billable_total_tokens: 0,
|
|
1777
|
+
conversation_count: 0,
|
|
1778
|
+
}),
|
|
1779
|
+
);
|
|
1780
|
+
delete buckets[key];
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
const groupQueued = cursors.hourly?.groupQueued;
|
|
1784
|
+
if (groupQueued && typeof groupQueued === "object") {
|
|
1785
|
+
for (const key of Object.keys(groupQueued)) {
|
|
1786
|
+
if (key.startsWith("codex|") || key.startsWith("every-code|")) {
|
|
1787
|
+
delete groupQueued[key];
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
if (hourlyRetractions.length > 0) {
|
|
1792
|
+
await ensureDir(path.dirname(queuePath));
|
|
1793
|
+
await fs.appendFile(queuePath, hourlyRetractions.join("\n") + "\n");
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
// (3) Reset the inflated project codex/every-code accumulator buckets, but
|
|
1797
|
+
// do NOT append zero rows to project.queue.jsonl. Deleting the in-memory
|
|
1798
|
+
// bucket is what prevents the double-count: the same-run re-parse of a file
|
|
1799
|
+
// still on disk starts that project bucket fresh from zero and appends a
|
|
1800
|
+
// corrected row, which the keep-last project reader (local-api.js
|
|
1801
|
+
// readProjectQueueData, per project_key|source|hour_start) picks over the old
|
|
1802
|
+
// inflated row. A bucket whose file has since been deleted or whose repo
|
|
1803
|
+
// moved is NOT re-parsed; leaving its prior queue row in place keeps that
|
|
1804
|
+
// (possibly fork-inflated) historical value visible rather than zeroing data
|
|
1805
|
+
// we can no longer recompute. Unlike the hourly path, project aggregates were
|
|
1806
|
+
// never retracted by the earlier delta migration, so a zero here would be a
|
|
1807
|
+
// brand-new, unrecoverable loss for orphaned files — hence the conservative
|
|
1808
|
+
// reset-accumulator-only approach.
|
|
1809
|
+
let projectReset = 0;
|
|
1810
|
+
const projectBuckets = cursors.projectHourly?.buckets;
|
|
1811
|
+
if (projectQueuePath && projectBuckets && typeof projectBuckets === "object") {
|
|
1812
|
+
for (const key of Object.keys(projectBuckets)) {
|
|
1813
|
+
const source = projectBuckets[key]?.source;
|
|
1814
|
+
if (source !== "codex" && source !== "every-code") continue;
|
|
1815
|
+
delete projectBuckets[key];
|
|
1816
|
+
projectReset += 1;
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
if (process.env.TOKENTRACKER_DEBUG) {
|
|
1821
|
+
process.stderr.write(
|
|
1822
|
+
`[codex-fork] #75 rebuild: reset ${rolloutPaths.size} codex/every-code file ` +
|
|
1823
|
+
`cursors, retracted ${hourlyRetractions.length} hourly bucket(s), reset ` +
|
|
1824
|
+
`${projectReset} project accumulator bucket(s) (no project rows zeroed); ` +
|
|
1825
|
+
`this sync re-parses files still on disk with the corrected accounting\n`,
|
|
1826
|
+
);
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
cursors.migrations[CODEX_FORK_OVERCOUNT_MIGRATION_KEY] = new Date().toISOString();
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1925
1832
|
// One-time repair migration: rebuild source=claude rows in queue.jsonl from
|
|
1926
1833
|
// the actual jsonl files using ccusage's algorithm (msgId+reqId global
|
|
1927
1834
|
// dedup). Earlier `reincludeClaudeMemObserverFiles` versions (v1/v2/v3) each
|
|
@@ -1,138 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
// Opens a URL in the user's browser. This module used to also run the cloud
|
|
2
|
+
// sign-in flow (a local callback server plus a redirect dance); that went away
|
|
3
|
+
// with the cloud, and the file was renamed so its name stops implying auth.
|
|
3
4
|
const cp = require("node:child_process");
|
|
4
5
|
|
|
5
|
-
const { DEFAULT_BASE_URL } = require("./runtime-config");
|
|
6
|
-
|
|
7
|
-
async function beginBrowserAuth({ baseUrl, dashboardUrl, timeoutMs, open }) {
|
|
8
|
-
const nonce = crypto.randomBytes(16).toString("hex");
|
|
9
|
-
const callbackPath = `/tokentracker/callback/${nonce}`;
|
|
10
|
-
const authUrl = dashboardUrl ? new URL("/", dashboardUrl) : new URL("/auth/sign-up", baseUrl);
|
|
11
|
-
const postAuthRedirect = resolvePostAuthRedirect({ dashboardUrl, authUrl });
|
|
12
|
-
const { callbackUrl, waitForCallback } = await startLocalCallbackServer({
|
|
13
|
-
callbackPath,
|
|
14
|
-
timeoutMs,
|
|
15
|
-
redirectUrl: postAuthRedirect,
|
|
16
|
-
});
|
|
17
|
-
authUrl.searchParams.set("redirect", callbackUrl);
|
|
18
|
-
if (dashboardUrl && baseUrl && baseUrl !== DEFAULT_BASE_URL) {
|
|
19
|
-
authUrl.searchParams.set("base_url", baseUrl);
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
if (open !== false) openInBrowser(authUrl.toString());
|
|
23
|
-
|
|
24
|
-
return { authUrl: authUrl.toString(), waitForCallback };
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
async function startLocalCallbackServer({ callbackPath, timeoutMs, redirectUrl }) {
|
|
28
|
-
let resolved = false;
|
|
29
|
-
let resolveResult;
|
|
30
|
-
let rejectResult;
|
|
31
|
-
|
|
32
|
-
const resultPromise = new Promise((resolve, reject) => {
|
|
33
|
-
resolveResult = resolve;
|
|
34
|
-
rejectResult = reject;
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
const server = http.createServer((req, res) => {
|
|
38
|
-
if (resolved) {
|
|
39
|
-
res.writeHead(409, { "Content-Type": "text/plain; charset=utf-8" });
|
|
40
|
-
res.end("Already authenticated.\n");
|
|
41
|
-
return;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
const method = req.method || "GET";
|
|
45
|
-
if (method !== "GET") {
|
|
46
|
-
res.writeHead(405, { "Content-Type": "text/plain; charset=utf-8" });
|
|
47
|
-
res.end("Method not allowed.\n");
|
|
48
|
-
return;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
const url = new URL(req.url || "/", "http://127.0.0.1");
|
|
52
|
-
if (url.pathname !== callbackPath) {
|
|
53
|
-
res.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" });
|
|
54
|
-
res.end("Not found.\n");
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
const accessToken = url.searchParams.get("access_token") || "";
|
|
59
|
-
if (!accessToken) {
|
|
60
|
-
res.writeHead(400, { "Content-Type": "text/plain; charset=utf-8" });
|
|
61
|
-
res.end("Missing access_token.\n");
|
|
62
|
-
return;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
resolved = true;
|
|
66
|
-
if (redirectUrl) {
|
|
67
|
-
res.writeHead(302, {
|
|
68
|
-
Location: redirectUrl,
|
|
69
|
-
"Content-Type": "text/html; charset=utf-8",
|
|
70
|
-
});
|
|
71
|
-
res.end(
|
|
72
|
-
[
|
|
73
|
-
"<!doctype html>",
|
|
74
|
-
'<html><head><meta charset="utf-8"><title>TokenTracker</title></head>',
|
|
75
|
-
"<body>",
|
|
76
|
-
"<h2>Login succeeded</h2>",
|
|
77
|
-
`<p>Redirecting to <a href="${redirectUrl}">dashboard</a>...</p>`,
|
|
78
|
-
"</body></html>",
|
|
79
|
-
].join(""),
|
|
80
|
-
);
|
|
81
|
-
} else {
|
|
82
|
-
res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
|
|
83
|
-
res.end(
|
|
84
|
-
[
|
|
85
|
-
"<!doctype html>",
|
|
86
|
-
'<html><head><meta charset="utf-8"><title>TokenTracker</title></head>',
|
|
87
|
-
"<body>",
|
|
88
|
-
"<h2>Login succeeded</h2>",
|
|
89
|
-
"<p>You can close this tab and return to the CLI.</p>",
|
|
90
|
-
"</body></html>",
|
|
91
|
-
].join(""),
|
|
92
|
-
);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
resolveResult({
|
|
96
|
-
accessToken,
|
|
97
|
-
userId: url.searchParams.get("user_id") || null,
|
|
98
|
-
email: url.searchParams.get("email") || null,
|
|
99
|
-
name: url.searchParams.get("name") || null,
|
|
100
|
-
});
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
await new Promise((resolve, reject) => {
|
|
104
|
-
server.once("error", reject);
|
|
105
|
-
server.listen(0, "127.0.0.1", resolve);
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
const addr = server.address();
|
|
109
|
-
const port = typeof addr === "object" && addr ? addr.port : null;
|
|
110
|
-
if (!port) {
|
|
111
|
-
server.close();
|
|
112
|
-
throw new Error("Failed to bind local callback server");
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
const callbackUrl = `http://127.0.0.1:${port}${callbackPath}`;
|
|
116
|
-
|
|
117
|
-
const timer = setTimeout(() => {
|
|
118
|
-
if (resolved) return;
|
|
119
|
-
resolved = true;
|
|
120
|
-
rejectResult(new Error("Authentication timed out"));
|
|
121
|
-
server.close();
|
|
122
|
-
}, timeoutMs);
|
|
123
|
-
|
|
124
|
-
async function waitForCallback() {
|
|
125
|
-
try {
|
|
126
|
-
return await resultPromise;
|
|
127
|
-
} finally {
|
|
128
|
-
clearTimeout(timer);
|
|
129
|
-
server.close();
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
return { callbackUrl, waitForCallback };
|
|
134
|
-
}
|
|
135
|
-
|
|
136
6
|
function detectDefaultBrowser() {
|
|
137
7
|
try {
|
|
138
8
|
const raw = cp.execFileSync("defaults", [
|
|
@@ -285,23 +155,7 @@ end if
|
|
|
285
155
|
spawnBrowserCommand(cmd, args, { url, platform, stderr });
|
|
286
156
|
}
|
|
287
157
|
|
|
288
|
-
function resolvePostAuthRedirect({ dashboardUrl, authUrl }) {
|
|
289
|
-
try {
|
|
290
|
-
if (dashboardUrl) {
|
|
291
|
-
const target = new URL("/", dashboardUrl);
|
|
292
|
-
if (target.protocol === "http:" || target.protocol === "https:") {
|
|
293
|
-
return target.toString();
|
|
294
|
-
}
|
|
295
|
-
return null;
|
|
296
|
-
}
|
|
297
|
-
} catch (_e) {
|
|
298
|
-
return null;
|
|
299
|
-
}
|
|
300
|
-
return null;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
158
|
module.exports = {
|
|
304
|
-
beginBrowserAuth,
|
|
305
159
|
openInBrowser,
|
|
306
160
|
buildBrowserOpenErrorMessage,
|
|
307
161
|
spawnBrowserCommand,
|