@token-dashboard/codex-usage-uploader 0.1.0 → 0.1.2

package/README.md

@@ -18,14 +18,6 @@ npx @token-dashboard/codex-usage-uploader init
 
 安装过程中会自动读取 Codex 登录态（`~/.codex/auth.json`）获取身份信息。如果读取不到邮箱，会在终端交互式提示你手动填写。
 
-如需跳过交互提示，可以直接指定邮箱：
-
-```bash
-npx @token-dashboard/codex-usage-uploader init \
-  --email you@company.com \
-  --yes
-```
-
 安装完成后，`codex-usage-uploader` 命令会被写入 `~/bin/`，后续可直接使用。
 
 ## 命令一览
@@ -82,16 +74,16 @@ codex-usage-uploader uninstall
 
 ## 常用选项
 
-| 选项                       | 说明                            |
-| -------------------------- | ------------------------------- |
-| `--backend-url <url>`      | Dashboard 后端地址（默认 `http://101.126.66.51:8086`） |
-| `--email <email>`          | 绑定邮箱                        |
-| `--employee-name <name>`   | 绑定姓名                        |
-| `--employee-id <id>`       | 绑定工号                        |
-| `--interval <seconds>`     | 扫描间隔秒数（默认 30）         |
-| `--yes`                    | 跳过所有交互确认                |
-| `--lines <n>`              | `logs` 命令输出行数（默认 100） |
-| `-h, --help`               | 显示帮助                        |
+| 选项                     | 说明                                                   |
+| ------------------------ | ------------------------------------------------------ |
+| `--backend-url <url>`    | Dashboard 后端地址（默认 `http://101.126.66.51:8086`） |
+| `--email <email>`        | 绑定邮箱                                               |
+| `--employee-name <name>` | 绑定姓名                                               |
+| `--employee-id <id>`     | 绑定工号                                               |
+| `--interval <seconds>`   | 扫描间隔秒数（默认 30）                                |
+| `--yes`                  | 跳过所有交互确认                                       |
+| `--lines <n>`            | `logs` 命令输出行数（默认 100）                        |
+| `-h, --help`             | 显示帮助                                               |
 
 ## 工作原理
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@token-dashboard/codex-usage-uploader",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Codex 用量上报 CLI",
   "type": "module",
   "bin": {
@@ -12,6 +12,7 @@
     "README.md"
   ],
   "scripts": {
+    "dev": "node bin/codex-usage-uploader.js init --backend-url http://localhost:8086",
     "test": "node --no-warnings --test tests/*.test.mjs"
   },
   "engines": {

package/src/cli.js

@@ -1,7 +1,8 @@
 import fs from 'node:fs';
+import path from 'node:path';
 import { CodexUsageUploader } from './collector.js';
 import { identityIsBound, promptConfirm } from './auth.js';
-import { CLI_NAME, DEFAULT_CONFIG_FILE, PRODUCT_NAME } from './constants.js';
+import { CLI_NAME, DEFAULT_BACKEND_URL, DEFAULT_CONFIG_FILE, PRODUCT_NAME } from './constants.js';
 import { findPackageRoot, installCurrentPackage } from './install.js';
 import { LaunchdServiceManager } from './launchd.js';
 import { formatStatusOutput, mergeRuntimeConfig } from './runtime-config.js';
@@ -256,44 +257,21 @@ function formatDuration(durationMs) {
 function printCatchUpProgress(event) {
   switch (event.phase) {
     case 'start':
-      console.log(
-        `[catch-up] start files=${event.totalFiles} queued_batches=${
-          event.bufferingBatchCount +
-          event.pendingBatchCount +
-          event.retryingBatchCount
-        } queued_events=${event.queuedEvents}`,
-      );
+      console.log(`[scan] start files=${event.totalFiles}`);
       return;
     case 'file':
       console.log(
-        `[catch-up] file ${event.filesProcessed}/${event.totalFiles} current=${event.file} events=${event.eventsParsed} queued_batches=${
-          event.bufferingBatchCount +
-          event.pendingBatchCount +
-          event.retryingBatchCount
-        } queued_events=${event.queuedEvents}`,
-      );
-      return;
-    case 'upload':
-      console.log(
-        `[catch-up] uploaded_batches=${event.batchesUploaded} last_batch=${event.batchKey} remaining_batches=${
-          event.bufferingBatchCount +
-          event.pendingBatchCount +
-          event.retryingBatchCount
-        } remaining_events=${event.queuedEvents}`,
+        `[scan] file ${event.filesProcessed}/${event.totalFiles} current=${event.file} events=${event.eventsParsed}`,
       );
       return;
     case 'done':
       console.log(
-        `[catch-up] complete files=${event.filesProcessed}/${event.totalFiles} events=${event.eventsParsed} uploaded_batches=${event.batchesUploaded} remaining_batches=${event.remainingQueuedBatches} remaining_events=${event.remainingQueuedEvents} duration=${formatDuration(event.durationMs)}`,
+        `[scan] complete files=${event.filesProcessed}/${event.totalFiles} events=${event.eventsParsed} pending_batches=${event.pendingBatches} duration=${formatDuration(event.durationMs)}`,
       );
       return;
     case 'error':
       console.error(
-        `[catch-up] failed stage=${event.stage} message=${event.message} remaining_batches=${
-          event.bufferingBatchCount +
-          event.pendingBatchCount +
-          event.retryingBatchCount
-        } remaining_events=${event.queuedEvents}`,
+        `[scan] failed stage=${event.stage} message=${event.message}`,
       );
       return;
   }
@@ -307,8 +285,26 @@ function wrapCatchUpFailure(error) {
   );
 }
 
+function printPathHint(runtime) {
+  const linkDir = path.dirname(runtime.homeBinLink);
+  const dirs = (process.env.PATH || '').split(path.delimiter);
+  if (dirs.includes(linkDir)) return;
+  console.log('');
+  console.log(
+    `Note: ${linkDir} is not in your PATH. To use \`${CLI_NAME}\` directly, run:`,
+  );
+  console.log(`  export PATH="${linkDir}:$PATH"`);
+  console.log(
+    'You can add this line to ~/.zshrc or ~/.bashrc to make it permanent.',
+  );
+}
+
 async function runInit(options) {
-  let runtime = mergeRuntimeConfig(options.configFile, runtimeOverrides(options));
+  const overrides = runtimeOverrides(options);
+  if (!overrides.backendUrl) {
+    overrides.backendUrl = DEFAULT_BACKEND_URL;
+  }
+  let runtime = mergeRuntimeConfig(options.configFile, overrides);
   if (!runtime.backendUrl) {
     throw new Error(
       '--backend-url is required for init unless already configured in config.json',
@@ -332,10 +328,11 @@ async function runInit(options) {
     console.log(`Install root: ${runtime.installRoot}`);
     console.log(`Config file: ${runtime.configFile}`);
     printIdentity(identity);
-    console.log('Starting foreground historical catch-up.');
+    console.log('Scanning local Codex sessions...');
 
+    let scanResult;
     try {
-      await uploader.runForegroundCatchUp({
+      scanResult = await uploader.runForegroundCatchUp({
         onProgress: printCatchUpProgress,
       });
     } catch (error) {
@@ -344,7 +341,13 @@ async function runInit(options) {
 
     manager.start();
     console.log(`${PRODUCT_NAME} initialized and started.`);
+    if (scanResult.pendingBatches > 0) {
+      console.log(
+        `Background service is uploading ${scanResult.pendingBatches} batch(es) with ${scanResult.pendingEvents} event(s).`,
+      );
+    }
     console.log(`Use \`${CLI_NAME} status\` to check the local service.`);
+    printPathHint(runtime);
   } finally {
     uploader.close();
   }

package/src/collector.js

@@ -465,7 +465,7 @@ export class CodexUsageUploader {
     const response = await fetch(`${this.backendUrl}${apiPath}`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: stableStringify(payload),
+      body: JSON.stringify(payload),
     });
     if (!response.ok) {
       const text = await response.text();
@@ -489,7 +489,7 @@ export class CodexUsageUploader {
     this.stateDb.setCheckpoint('last_register_at', String(nowTs()));
   }
 
-  async flushPendingBatches({ failFast = false, onBatchUploaded } = {}) {
+  async flushPendingBatches({ failFast = false, concurrency = 5, onBatchUploaded } = {}) {
     if (!this.backendUrl) {
       return { uploadedBatches: 0, failedBatches: 0, lastError: null };
     }
@@ -509,42 +509,70 @@ export class CodexUsageUploader {
     let uploadedBatches = 0;
     let failedBatches = 0;
     let lastError = null;
+    const collectorBody = this.collectorRequestBody();
 
-    for (const row of this.stateDb.iterDuePendingBatches()) {
+    const rows = this.stateDb.iterDuePendingBatches();
+
+    const uploadOne = async (row) => {
       const payload = this.sanitizeUploadPayload(JSON.parse(row.payload_json));
       const requestBody = {
         idempotencyKey: row.batch_key,
-        collector: this.collectorRequestBody(),
+        collector: collectorBody,
         payloadSizeBytes: Number(row.payload_bytes),
         sessions: payload.sessions ?? [],
         turns: payload.turns ?? [],
         events: payload.events ?? [],
       };
+      await this.postJson('/codex-usage/upload', requestBody);
+    };
+
+    const pending = new Set();
+    let rowIndex = 0;
+
+    const enqueue = () => {
+      while (pending.size < concurrency && rowIndex < rows.length) {
+        const row = rows[rowIndex++];
+        const task = uploadOne(row)
+          .then(() => {
+            this.stateDb.markBatchUploaded(row.id);
+            uploadedBatches += 1;
+            onBatchUploaded?.({ row, uploadedBatches });
+          })
+          .catch((error) => {
+            this.stateDb.markBatchFailed(
+              row.id,
+              Number(row.attempt_count) + 1,
+              error instanceof Error ? error.message : String(error),
+            );
+            failedBatches += 1;
+            lastError = error;
+            if (failFast) {
+              throw error;
+            }
+          })
+          .finally(() => {
+            pending.delete(task);
+          });
+        pending.add(task);
+      }
+    };
+
+    enqueue();
+    while (pending.size > 0) {
       try {
-        await this.postJson('/codex-usage/upload', requestBody);
-        this.stateDb.markBatchUploaded(row.id);
-        uploadedBatches += 1;
-        onBatchUploaded?.({
-          row,
-          uploadedBatches,
-          queueStats: this.getQueueStats(),
-        });
-      } catch (error) {
-        this.stateDb.markBatchFailed(
-          row.id,
-          Number(row.attempt_count) + 1,
-          error instanceof Error ? error.message : String(error),
-        );
-        failedBatches += 1;
-        lastError = error;
-        if (failFast) {
-          throw new Error(
-            `Upload batch ${row.batch_key} failed: ${
-              error instanceof Error ? error.message : String(error)
-            }`,
-          );
-        }
+        await Promise.race(pending);
+      } catch {
+        if (failFast) break;
       }
+      enqueue();
+    }
+
+    if (failFast && lastError) {
+      throw new Error(
+        `Upload batch failed: ${
+          lastError instanceof Error ? lastError.message : String(lastError)
+        }`,
+      );
     }
 
     return { uploadedBatches, failedBatches, lastError };
@@ -578,15 +606,12 @@ export class CodexUsageUploader {
       filesProcessed: 0,
       eventsParsed: 0,
       batchesQueued: 0,
-      batchesUploaded: 0,
-      ...this.getQueueStats(),
     });
 
     let scanResult;
     try {
       scanResult = await this.scanSnapshotEntries(backfillSnapshot, {
         onFileProcessed: ({ entry, filesProcessed, totals }) => {
-          const queueStats = this.getQueueStats();
           onProgress?.({
             phase: 'file',
             file: entry.progressPath ?? entry.relpath,
@@ -594,8 +619,6 @@ export class CodexUsageUploader {
             filesProcessed,
             eventsParsed: totals.events,
             batchesQueued: totals.batchesQueued,
-            batchesUploaded: 0,
-            ...queueStats,
           });
         },
       });
@@ -604,44 +627,13 @@ export class CodexUsageUploader {
         phase: 'error',
         stage: 'scan',
         message: error instanceof Error ? error.message : String(error),
-        ...this.getQueueStats(),
       });
       throw error;
     }
 
     scanResult.batchesQueued += this.stateDb.sealStaleBatches(true);
-    this.stateDb.markAllPendingDue();
 
-    let flushResult;
-    try {
-      flushResult = await this.flushPendingBatches({
-        failFast: true,
-        onBatchUploaded: ({ row, uploadedBatches, queueStats }) => {
-          onProgress?.({
-            phase: 'upload',
-            batchKey: row.batch_key,
-            totalFiles,
-            filesProcessed: totalFiles,
-            eventsParsed: scanResult.events,
-            batchesQueued: scanResult.batchesQueued,
-            batchesUploaded: uploadedBatches,
-            ...queueStats,
-          });
-        },
-      });
-    } catch (error) {
-      onProgress?.({
-        phase: 'error',
-        stage: 'upload',
-        message: error instanceof Error ? error.message : String(error),
-        totalFiles,
-        filesProcessed: totalFiles,
-        eventsParsed: scanResult.events,
-        batchesQueued: scanResult.batchesQueued,
-        ...this.getQueueStats(),
-      });
-      throw error;
-    }
+    await this.ensureRemoteRegistration();
 
     const queueStats = this.getQueueStats();
     const result = {
@@ -651,14 +643,10 @@ export class CodexUsageUploader {
       sessionsParsed: scanResult.sessions,
       turnsParsed: scanResult.turns,
       batchesQueued: scanResult.batchesQueued,
-      batchesUploaded: flushResult.uploadedBatches,
-      remainingQueuedBatches:
-        queueStats.bufferingBatchCount +
-        queueStats.pendingBatchCount +
-        queueStats.retryingBatchCount,
-      remainingQueuedEvents: queueStats.queuedEvents,
+      pendingBatches:
+        queueStats.pendingBatchCount + queueStats.retryingBatchCount,
+      pendingEvents: queueStats.queuedEvents,
       durationMs: Date.now() - startedAt,
-      ...queueStats,
     };
 
     onProgress?.({