@token-dashboard/codex-usage-uploader 0.1.1 → 0.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@token-dashboard/codex-usage-uploader",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Codex 用量上报 CLI",
   "type": "module",
   "bin": {
@@ -12,6 +12,7 @@
     "README.md"
   ],
   "scripts": {
+    "dev": "node bin/codex-usage-uploader.js init --backend-url http://localhost:8086",
     "test": "node --no-warnings --test tests/*.test.mjs"
   },
   "engines": {

package/src/cli.js

@@ -257,44 +257,21 @@ function formatDuration(durationMs) {
 function printCatchUpProgress(event) {
   switch (event.phase) {
     case 'start':
-      console.log(
-        `[catch-up] start files=${event.totalFiles} queued_batches=${
-          event.bufferingBatchCount +
-          event.pendingBatchCount +
-          event.retryingBatchCount
-        } queued_events=${event.queuedEvents}`,
-      );
+      console.log(`[scan] start files=${event.totalFiles}`);
       return;
     case 'file':
       console.log(
-        `[catch-up] file ${event.filesProcessed}/${event.totalFiles} current=${event.file} events=${event.eventsParsed} queued_batches=${
-          event.bufferingBatchCount +
-          event.pendingBatchCount +
-          event.retryingBatchCount
-        } queued_events=${event.queuedEvents}`,
-      );
-      return;
-    case 'upload':
-      console.log(
-        `[catch-up] uploaded_batches=${event.batchesUploaded} last_batch=${event.batchKey} remaining_batches=${
-          event.bufferingBatchCount +
-          event.pendingBatchCount +
-          event.retryingBatchCount
-        } remaining_events=${event.queuedEvents}`,
+        `[scan] file ${event.filesProcessed}/${event.totalFiles} current=${event.file} events=${event.eventsParsed}`,
       );
       return;
     case 'done':
       console.log(
-        `[catch-up] complete files=${event.filesProcessed}/${event.totalFiles} events=${event.eventsParsed} uploaded_batches=${event.batchesUploaded} remaining_batches=${event.remainingQueuedBatches} remaining_events=${event.remainingQueuedEvents} duration=${formatDuration(event.durationMs)}`,
+        `[scan] complete files=${event.filesProcessed}/${event.totalFiles} events=${event.eventsParsed} pending_batches=${event.pendingBatches} duration=${formatDuration(event.durationMs)}`,
       );
       return;
     case 'error':
       console.error(
-        `[catch-up] failed stage=${event.stage} message=${event.message} remaining_batches=${
-          event.bufferingBatchCount +
-          event.pendingBatchCount +
-          event.retryingBatchCount
-        } remaining_events=${event.queuedEvents}`,
+        `[scan] failed stage=${event.stage} message=${event.message}`,
       );
       return;
   }
@@ -351,10 +328,11 @@ async function runInit(options) {
     console.log(`Install root: ${runtime.installRoot}`);
     console.log(`Config file: ${runtime.configFile}`);
     printIdentity(identity);
-    console.log('Starting foreground historical catch-up.');
+    console.log('Scanning local Codex sessions...');
 
+    let scanResult;
     try {
-      await uploader.runForegroundCatchUp({
+      scanResult = await uploader.runForegroundCatchUp({
         onProgress: printCatchUpProgress,
       });
     } catch (error) {
@@ -363,6 +341,11 @@ async function runInit(options) {
 
     manager.start();
     console.log(`${PRODUCT_NAME} initialized and started.`);
+    if (scanResult.pendingBatches > 0) {
+      console.log(
+        `Background service is uploading ${scanResult.pendingBatches} batch(es) with ${scanResult.pendingEvents} event(s).`,
+      );
+    }
     console.log(`Use \`${CLI_NAME} status\` to check the local service.`);
     printPathHint(runtime);
   } finally {

package/src/collector.js

@@ -465,7 +465,7 @@ export class CodexUsageUploader {
     const response = await fetch(`${this.backendUrl}${apiPath}`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: stableStringify(payload),
+      body: JSON.stringify(payload),
     });
     if (!response.ok) {
       const text = await response.text();
@@ -489,7 +489,7 @@ export class CodexUsageUploader {
     this.stateDb.setCheckpoint('last_register_at', String(nowTs()));
   }
 
-  async flushPendingBatches({ failFast = false, onBatchUploaded } = {}) {
+  async flushPendingBatches({ failFast = false, concurrency = 5, onBatchUploaded } = {}) {
     if (!this.backendUrl) {
       return { uploadedBatches: 0, failedBatches: 0, lastError: null };
     }
@@ -509,42 +509,70 @@ export class CodexUsageUploader {
     let uploadedBatches = 0;
     let failedBatches = 0;
     let lastError = null;
+    const collectorBody = this.collectorRequestBody();
 
-    for (const row of this.stateDb.iterDuePendingBatches()) {
+    const rows = this.stateDb.iterDuePendingBatches();
+
+    const uploadOne = async (row) => {
       const payload = this.sanitizeUploadPayload(JSON.parse(row.payload_json));
       const requestBody = {
         idempotencyKey: row.batch_key,
-        collector: this.collectorRequestBody(),
+        collector: collectorBody,
         payloadSizeBytes: Number(row.payload_bytes),
         sessions: payload.sessions ?? [],
         turns: payload.turns ?? [],
         events: payload.events ?? [],
       };
+      await this.postJson('/codex-usage/upload', requestBody);
+    };
+
+    const pending = new Set();
+    let rowIndex = 0;
+
+    const enqueue = () => {
+      while (pending.size < concurrency && rowIndex < rows.length) {
+        const row = rows[rowIndex++];
+        const task = uploadOne(row)
+          .then(() => {
+            this.stateDb.markBatchUploaded(row.id);
+            uploadedBatches += 1;
+            onBatchUploaded?.({ row, uploadedBatches });
+          })
+          .catch((error) => {
+            this.stateDb.markBatchFailed(
+              row.id,
+              Number(row.attempt_count) + 1,
+              error instanceof Error ? error.message : String(error),
+            );
+            failedBatches += 1;
+            lastError = error;
+            if (failFast) {
+              throw error;
+            }
+          })
+          .finally(() => {
+            pending.delete(task);
+          });
+        pending.add(task);
+      }
+    };
+
+    enqueue();
+    while (pending.size > 0) {
       try {
-        await this.postJson('/codex-usage/upload', requestBody);
-        this.stateDb.markBatchUploaded(row.id);
-        uploadedBatches += 1;
-        onBatchUploaded?.({
-          row,
-          uploadedBatches,
-          queueStats: this.getQueueStats(),
-        });
-      } catch (error) {
-        this.stateDb.markBatchFailed(
-          row.id,
-          Number(row.attempt_count) + 1,
-          error instanceof Error ? error.message : String(error),
-        );
-        failedBatches += 1;
-        lastError = error;
-        if (failFast) {
-          throw new Error(
-            `Upload batch ${row.batch_key} failed: ${
-              error instanceof Error ? error.message : String(error)
-            }`,
-          );
-        }
+        await Promise.race(pending);
+      } catch {
+        if (failFast) break;
       }
+      enqueue();
+    }
+
+    if (failFast && lastError) {
+      throw new Error(
+        `Upload batch failed: ${
+          lastError instanceof Error ? lastError.message : String(lastError)
+        }`,
+      );
     }
 
     return { uploadedBatches, failedBatches, lastError };
@@ -578,15 +606,12 @@ export class CodexUsageUploader {
       filesProcessed: 0,
       eventsParsed: 0,
       batchesQueued: 0,
-      batchesUploaded: 0,
-      ...this.getQueueStats(),
     });
 
     let scanResult;
     try {
       scanResult = await this.scanSnapshotEntries(backfillSnapshot, {
         onFileProcessed: ({ entry, filesProcessed, totals }) => {
-          const queueStats = this.getQueueStats();
           onProgress?.({
             phase: 'file',
             file: entry.progressPath ?? entry.relpath,
@@ -594,8 +619,6 @@ export class CodexUsageUploader {
             filesProcessed,
             eventsParsed: totals.events,
             batchesQueued: totals.batchesQueued,
-            batchesUploaded: 0,
-            ...queueStats,
           });
         },
       });
@@ -604,44 +627,13 @@ export class CodexUsageUploader {
         phase: 'error',
         stage: 'scan',
         message: error instanceof Error ? error.message : String(error),
-        ...this.getQueueStats(),
       });
       throw error;
     }
 
     scanResult.batchesQueued += this.stateDb.sealStaleBatches(true);
-    this.stateDb.markAllPendingDue();
 
-    let flushResult;
-    try {
-      flushResult = await this.flushPendingBatches({
-        failFast: true,
-        onBatchUploaded: ({ row, uploadedBatches, queueStats }) => {
-          onProgress?.({
-            phase: 'upload',
-            batchKey: row.batch_key,
-            totalFiles,
-            filesProcessed: totalFiles,
-            eventsParsed: scanResult.events,
-            batchesQueued: scanResult.batchesQueued,
-            batchesUploaded: uploadedBatches,
-            ...queueStats,
-          });
-        },
-      });
-    } catch (error) {
-      onProgress?.({
-        phase: 'error',
-        stage: 'upload',
-        message: error instanceof Error ? error.message : String(error),
-        totalFiles,
-        filesProcessed: totalFiles,
-        eventsParsed: scanResult.events,
-        batchesQueued: scanResult.batchesQueued,
-        ...this.getQueueStats(),
-      });
-      throw error;
-    }
+    await this.ensureRemoteRegistration();
 
     const queueStats = this.getQueueStats();
     const result = {
@@ -651,14 +643,10 @@ export class CodexUsageUploader {
       sessionsParsed: scanResult.sessions,
       turnsParsed: scanResult.turns,
       batchesQueued: scanResult.batchesQueued,
-      batchesUploaded: flushResult.uploadedBatches,
-      remainingQueuedBatches:
-        queueStats.bufferingBatchCount +
-        queueStats.pendingBatchCount +
-        queueStats.retryingBatchCount,
-      remainingQueuedEvents: queueStats.queuedEvents,
+      pendingBatches:
+        queueStats.pendingBatchCount + queueStats.retryingBatchCount,
+      pendingEvents: queueStats.queuedEvents,
       durationMs: Date.now() - startedAt,
-      ...queueStats,
     };
 
     onProgress?.({