@poco-ai/tokenarena 0.1.6 → 0.2.0

package/dist/index.js

@@ -1645,6 +1645,150 @@ function toProjectIdentity(input2) {
   };
 }
 
+// src/domain/upload-manifest.ts
+import { createHash as createHash2 } from "crypto";
+var MANIFEST_VERSION = 1;
+function shortHash(value) {
+  return createHash2("sha256").update(value).digest("hex").slice(0, 16);
+}
+function normalizeApiUrl(apiUrl) {
+  return apiUrl.replace(/\/+$/, "");
+}
+function buildUploadManifestScope(input2) {
+  return {
+    apiKeyHash: shortHash(input2.apiKey),
+    apiUrl: normalizeApiUrl(input2.apiUrl),
+    deviceId: input2.deviceId,
+    projectHashSaltHash: shortHash(input2.settings.projectHashSalt),
+    projectMode: input2.settings.projectMode
+  };
+}
+function describeUploadManifestScopeChanges(previous, current) {
+  const changes = [];
+  if (previous.apiUrl !== current.apiUrl || previous.apiKeyHash !== current.apiKeyHash) {
+    changes.push("server_or_api_key");
+  }
+  if (previous.deviceId !== current.deviceId) {
+    changes.push("device_id");
+  }
+  if (previous.projectMode !== current.projectMode || previous.projectHashSaltHash !== current.projectHashSaltHash) {
+    changes.push("project_identity");
+  }
+  return changes;
+}
+function getUploadBucketManifestKey(bucket) {
+  return [
+    bucket.deviceId,
+    bucket.source,
+    bucket.model,
+    bucket.projectKey,
+    bucket.bucketStart
+  ].join("|");
+}
+function getUploadSessionManifestKey(session) {
+  return [session.deviceId, session.source, session.sessionHash].join("|");
+}
+function getUploadBucketContentHash(bucket) {
+  return shortHash(
+    JSON.stringify({
+      cachedTokens: bucket.cachedTokens,
+      hostname: bucket.hostname,
+      inputTokens: bucket.inputTokens,
+      outputTokens: bucket.outputTokens,
+      projectLabel: bucket.projectLabel,
+      reasoningTokens: bucket.reasoningTokens,
+      totalTokens: bucket.totalTokens
+    })
+  );
+}
+function getUploadSessionContentHash(session) {
+  return shortHash(
+    JSON.stringify({
+      activeSeconds: session.activeSeconds,
+      cachedTokens: session.cachedTokens,
+      durationSeconds: session.durationSeconds,
+      firstMessageAt: session.firstMessageAt,
+      hostname: session.hostname,
+      inputTokens: session.inputTokens,
+      lastMessageAt: session.lastMessageAt,
+      messageCount: session.messageCount,
+      modelUsages: session.modelUsages,
+      outputTokens: session.outputTokens,
+      primaryModel: session.primaryModel,
+      projectKey: session.projectKey,
+      projectLabel: session.projectLabel,
+      reasoningTokens: session.reasoningTokens,
+      totalTokens: session.totalTokens,
+      userMessageCount: session.userMessageCount
+    })
+  );
+}
+function buildRecordHashes(items, getKey, getHash) {
+  const hashes = {};
+  for (const item of items) {
+    hashes[getKey(item)] = getHash(item);
+  }
+  return hashes;
+}
+function createUploadManifest(input2) {
+  return {
+    buckets: buildRecordHashes(
+      input2.buckets,
+      getUploadBucketManifestKey,
+      getUploadBucketContentHash
+    ),
+    scope: input2.scope,
+    sessions: buildRecordHashes(
+      input2.sessions,
+      getUploadSessionManifestKey,
+      getUploadSessionContentHash
+    ),
+    updatedAt: input2.updatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+    version: MANIFEST_VERSION
+  };
+}
+function countRemovedRecords(previous, current) {
+  let removed = 0;
+  for (const key of Object.keys(previous)) {
+    if (!(key in current)) {
+      removed++;
+    }
+  }
+  return removed;
+}
+function diffUploadManifest(input2) {
+  const nextManifest = createUploadManifest({
+    buckets: input2.buckets,
+    scope: input2.scope,
+    sessions: input2.sessions,
+    updatedAt: input2.updatedAt
+  });
+  const scopeChangedReasons = input2.previous ? describeUploadManifestScopeChanges(input2.previous.scope, input2.scope) : [];
+  const previousBuckets = input2.previous && scopeChangedReasons.length === 0 ? input2.previous.buckets : {};
+  const previousSessions = input2.previous && scopeChangedReasons.length === 0 ? input2.previous.sessions : {};
+  const bucketsToUpload = input2.buckets.filter((bucket) => {
+    const key = getUploadBucketManifestKey(bucket);
+    return previousBuckets[key] !== nextManifest.buckets[key];
+  });
+  const sessionsToUpload = input2.sessions.filter((session) => {
+    const key = getUploadSessionManifestKey(session);
+    return previousSessions[key] !== nextManifest.sessions[key];
+  });
+  return {
+    bucketsToUpload,
+    nextManifest,
+    removedBuckets: countRemovedRecords(previousBuckets, nextManifest.buckets),
+    removedSessions: countRemovedRecords(
+      previousSessions,
+      nextManifest.sessions
+    ),
+    scopeChangedReasons,
+    sessionsToUpload,
+    unchangedBuckets: input2.buckets.length - bucketsToUpload.length,
+    unchangedSessions: input2.sessions.length - sessionsToUpload.length
+  };
+}
+
 // src/infrastructure/api/client.ts
 import http from "http";
 import https from "https";
@@ -1898,6 +2042,9 @@ function getSyncLockPath() {
 function getSyncStatePath() {
   return join10(getStateDir(), "status.json");
 }
+function getUploadManifestPath() {
+  return join10(getStateDir(), "upload-manifest.json");
+}
 function ensureAppDirs() {
   mkdirSync2(getRuntimeDirPath(), { recursive: true });
   mkdirSync2(getStateDir(), { recursive: true });
@@ -2054,6 +2201,43 @@ function markSyncFailed(source, error, status) {
   });
 }
 
+// src/infrastructure/runtime/upload-manifest.ts
+import { existsSync as existsSync11, readFileSync as readFileSync9, writeFileSync as writeFileSync4 } from "fs";
+function isRecordOfStrings(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  return Object.values(value).every((entry) => typeof entry === "string");
+}
+function isUploadManifest(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  const manifest = value;
+  return manifest.version === 1 && !!manifest.scope && typeof manifest.scope === "object" && typeof manifest.scope.apiUrl === "string" && typeof manifest.scope.apiKeyHash === "string" && typeof manifest.scope.deviceId === "string" && typeof manifest.scope.projectMode === "string" && typeof manifest.scope.projectHashSaltHash === "string" && typeof manifest.updatedAt === "string" && isRecordOfStrings(manifest.buckets) && isRecordOfStrings(manifest.sessions);
+}
+function loadUploadManifest() {
+  const path = getUploadManifestPath();
+  if (!existsSync11(path)) {
+    return null;
+  }
+  try {
+    const parsed = JSON.parse(readFileSync9(path, "utf-8"));
+    return isUploadManifest(parsed) ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+function saveUploadManifest(manifest) {
+  ensureAppDirs();
+  writeFileSync4(
+    getUploadManifestPath(),
+    `${JSON.stringify(manifest, null, 2)}
+`,
+    "utf-8"
+  );
+}
+
 // src/services/parser-service.ts
 async function runAllParsers() {
   const allBuckets = [];
@@ -2105,6 +2289,27 @@ function writeUploadProgress(sent, total, batchNum, totalBatches) {
     `\r  Uploading ${progressBar} ${String(pct).padStart(3, " ")}% \xB7 ${formatBytes(sent)}/${formatBytes(total)}${batchLabel}\x1B[K`
   );
 }
+function formatScopeChangeReason(reason) {
+  switch (reason) {
+    case "server_or_api_key":
+      return "server/API key";
+    case "device_id":
+      return "device ID";
+    case "project_identity":
+      return "project identity settings";
+  }
+}
+function persistUploadManifest(manifest, quiet) {
+  try {
+    saveUploadManifest(manifest);
+  } catch (error) {
+    if (!quiet) {
+      logger.warn(
+        `Uploaded data, but failed to update the local sync manifest: ${error.message}`
+      );
+    }
+  }
+}
 function toDeviceMetadata(config) {
   return {
     deviceId: getOrCreateDeviceId(config),
@@ -2250,22 +2455,72 @@ async function runSync(config, opts = {}) {
       );
     }
     const device = toDeviceMetadata(config);
+    const manifestScope = buildUploadManifestScope({
+      apiKey: config.apiKey,
+      apiUrl,
+      deviceId: device.deviceId,
+      settings
+    });
     const uploadBuckets = toUploadBuckets(allBuckets, settings, device);
     const uploadSessions = toUploadSessions(allSessions, settings, device);
-    const bucketBatches = Math.ceil(uploadBuckets.length / BATCH_SIZE);
+    const uploadDiff = diffUploadManifest({
+      buckets: uploadBuckets,
+      previous: loadUploadManifest(),
+      scope: manifestScope,
+      sessions: uploadSessions
+    });
+    const changedBuckets = uploadDiff.bucketsToUpload;
+    const changedSessions = uploadDiff.sessionsToUpload;
+    if (!quiet && uploadDiff.scopeChangedReasons.length > 0) {
+      logger.warn(
+        `Upload scope changed (${uploadDiff.scopeChangedReasons.map(formatScopeChangeReason).join(", ")}). TokenArena will upload the current snapshot again, but existing remote records from the previous scope will not be deleted automatically.`
+      );
+    }
+    if (!quiet && (uploadDiff.removedBuckets > 0 || uploadDiff.removedSessions > 0)) {
+      const parts = [];
+      if (uploadDiff.removedBuckets > 0) {
+        parts.push(`${uploadDiff.removedBuckets} buckets`);
+      }
+      if (uploadDiff.removedSessions > 0) {
+        parts.push(`${uploadDiff.removedSessions} sessions`);
+      }
+      logger.warn(
+        `Detected ${parts.join(" + ")} that were present in the previous local snapshot but are missing now. Remote deletions are not supported yet, so renamed projects or removed local logs may leave stale data online.`
+      );
+    }
+    if (changedBuckets.length === 0 && changedSessions.length === 0) {
+      if (!quiet) {
+        const skippedParts = [];
+        if (uploadDiff.unchangedBuckets > 0) {
+          skippedParts.push(`${uploadDiff.unchangedBuckets} unchanged buckets`);
+        }
+        if (uploadDiff.unchangedSessions > 0) {
+          skippedParts.push(
+            `${uploadDiff.unchangedSessions} unchanged sessions`
+          );
+        }
+        logger.info(
+          skippedParts.length > 0 ? `No new or updated usage data to upload. Skipped ${skippedParts.join(" + ")}.` : "No new or updated usage data to upload."
+        );
+      }
+      persistUploadManifest(uploadDiff.nextManifest, quiet);
+      markSyncSucceeded(source, { buckets: 0, sessions: 0 });
+      return { buckets: 0, sessions: 0 };
+    }
+    const bucketBatches = Math.ceil(changedBuckets.length / BATCH_SIZE);
     const sessionBatches = Math.ceil(
-      uploadSessions.length / SESSION_BATCH_SIZE
+      changedSessions.length / SESSION_BATCH_SIZE
     );
     const totalBatches = Math.max(bucketBatches, sessionBatches, 1);
     const batchPayloadSizes = Array.from(
       { length: totalBatches },
       (_, batchIdx) => getIngestPayloadSize(
         device,
-        uploadBuckets.slice(
+        changedBuckets.slice(
           batchIdx * BATCH_SIZE,
           (batchIdx + 1) * BATCH_SIZE
         ),
-        uploadSessions.slice(
+        changedSessions.slice(
           batchIdx * SESSION_BATCH_SIZE,
           (batchIdx + 1) * SESSION_BATCH_SIZE
         )
@@ -2278,22 +2533,29 @@ async function runSync(config, opts = {}) {
     let uploadedBytesBeforeBatch = 0;
     if (!quiet) {
       const parts = [];
-      if (uploadBuckets.length > 0) {
-        parts.push(`${uploadBuckets.length} buckets`);
+      if (changedBuckets.length > 0) {
+        parts.push(`${changedBuckets.length} buckets`);
+      }
+      if (changedSessions.length > 0) {
+        parts.push(`${changedSessions.length} sessions`);
+      }
+      const skippedParts = [];
+      if (uploadDiff.unchangedBuckets > 0) {
+        skippedParts.push(`${uploadDiff.unchangedBuckets} unchanged buckets`);
       }
-      if (uploadSessions.length > 0) {
-        parts.push(`${uploadSessions.length} sessions`);
+      if (uploadDiff.unchangedSessions > 0) {
+        skippedParts.push(`${uploadDiff.unchangedSessions} unchanged sessions`);
       }
       logger.info(
-        `Uploading ${parts.join(" + ")} (${totalBatches} batch${totalBatches > 1 ? "es" : ""})...`
+        `Uploading ${parts.join(" + ")} (${totalBatches} batch${totalBatches > 1 ? "es" : ""}${skippedParts.length > 0 ? `, skipped ${skippedParts.join(" + ")}` : ""})...`
       );
     }
     for (let batchIdx = 0; batchIdx < totalBatches; batchIdx++) {
-      const batch = uploadBuckets.slice(
+      const batch = changedBuckets.slice(
         batchIdx * BATCH_SIZE,
         (batchIdx + 1) * BATCH_SIZE
       );
-      const batchSessions = uploadSessions.slice(
+      const batchSessions = changedSessions.slice(
         batchIdx * SESSION_BATCH_SIZE,
         (batchIdx + 1) * SESSION_BATCH_SIZE
       );
@@ -2315,7 +2577,7 @@ async function runSync(config, opts = {}) {
       totalSessionsSynced += result.sessions ?? batchSessions.length;
       uploadedBytesBeforeBatch += batchPayloadSizes[batchIdx] ?? 0;
     }
-    if (!quiet && (totalBatches > 1 || uploadBuckets.length > 0)) {
+    if (!quiet && (totalBatches > 1 || changedBuckets.length > 0)) {
       writeUploadProgress(
         totalPayloadBytes,
         totalPayloadBytes,
@@ -2333,6 +2595,7 @@ async function runSync(config, opts = {}) {
       logger.info(`
 View your dashboard at: ${apiUrl}/usage`);
     }
+    persistUploadManifest(uploadDiff.nextManifest, quiet);
     markSyncSucceeded(source, {
       buckets: totalIngested,
       sessions: totalSessionsSynced
@@ -2383,7 +2646,7 @@ View your dashboard at: ${apiUrl}/usage`);
 
 // src/commands/init.ts
 import { execFileSync as execFileSync2, spawn } from "child_process";
-import { existsSync as existsSync11 } from "fs";
+import { existsSync as existsSync12 } from "fs";
 import { appendFile, mkdir, readFile } from "fs/promises";
 import { homedir as homedir8, platform } from "os";
 import { dirname as dirname2, join as join11, posix, win32 } from "path";
@@ -2460,7 +2723,7 @@ function resolveShellAliasSetup(options = {}) {
   const currentPlatform = options.currentPlatform ?? platform();
   const env = options.env ?? process.env;
   const homeDir = options.homeDir ?? homedir8();
-  const pathExists = options.exists ?? existsSync11;
+  const pathExists = options.exists ?? existsSync12;
   const shellFromEnv = env.SHELL ? basenameLikeShell(env.SHELL).toLowerCase() : "";
   const shellName = shellFromEnv || (currentPlatform === "win32" ? "powershell" : "");
   const aliasName = "ta";
@@ -2633,7 +2896,7 @@ async function setupShellAlias() {
   try {
     await mkdir(dirname2(setup.configFile), { recursive: true });
     let existingContent = "";
-    if (existsSync11(setup.configFile)) {
+    if (existsSync12(setup.configFile)) {
       existingContent = await readFile(setup.configFile, "utf-8");
     }
     const normalizedContent = existingContent.toLowerCase();
@@ -2826,7 +3089,7 @@ async function runSyncCommand(opts = {}) {
 }
 
 // src/commands/uninstall.ts
-import { existsSync as existsSync12, readFileSync as readFileSync9, rmSync as rmSync2, writeFileSync as writeFileSync4 } from "fs";
+import { existsSync as existsSync13, readFileSync as readFileSync10, rmSync as rmSync2, writeFileSync as writeFileSync5 } from "fs";
 import { homedir as homedir9, platform as platform2 } from "os";
 function removeShellAlias() {
   const shell = process.env.SHELL;
@@ -2839,7 +3102,7 @@ function removeShellAlias() {
       configFile = `${homedir9()}/.zshrc`;
       break;
     case "bash":
-      if (platform2() === "darwin" && existsSync12(`${homedir9()}/.bash_profile`)) {
+      if (platform2() === "darwin" && existsSync13(`${homedir9()}/.bash_profile`)) {
         configFile = `${homedir9()}/.bash_profile`;
       } else {
         configFile = `${homedir9()}/.bashrc`;
@@ -2851,9 +3114,9 @@ function removeShellAlias() {
     default:
       return;
   }
-  if (!existsSync12(configFile)) return;
+  if (!existsSync13(configFile)) return;
   try {
-    let content = readFileSync9(configFile, "utf-8");
+    let content = readFileSync10(configFile, "utf-8");
     const aliasPatterns = [
       // zsh / bash format: alias ta="tokenarena"
       new RegExp(
@@ -2875,7 +3138,7 @@ function removeShellAlias() {
         content = next;
       }
     }
-    writeFileSync4(configFile, content, "utf-8");
+    writeFileSync5(configFile, content, "utf-8");
     logger.info(`Removed shell alias from ${configFile}`);
   } catch (err) {
     logger.warn(
@@ -2886,7 +3149,7 @@ function removeShellAlias() {
 async function runUninstall() {
   const configPath = getConfigPath();
   const configDir = getConfigDir();
-  if (!existsSync12(configPath)) {
+  if (!existsSync13(configPath)) {
     logger.info(formatHeader("\u5378\u8F7D TokenArena"));
     logger.info(formatBullet("\u672A\u53D1\u73B0\u672C\u5730\u914D\u7F6E\uFF0C\u65E0\u9700\u5378\u8F7D\u3002"));
     return;
@@ -2915,7 +3178,7 @@ async function runUninstall() {
   deleteConfig();
   logger.info(formatSection("\u6267\u884C\u7ED3\u679C"));
   logger.info(formatBullet("\u5DF2\u5220\u9664\u914D\u7F6E\u6587\u4EF6\u3002", "success"));
-  if (existsSync12(configDir)) {
+  if (existsSync13(configDir)) {
     try {
       rmSync2(configDir, { recursive: false, force: true });
       logger.info(formatBullet("\u5DF2\u5220\u9664\u914D\u7F6E\u76EE\u5F55\u3002", "success"));
@@ -2923,12 +3186,12 @@ async function runUninstall() {
     }
   }
   const stateDir = getStateDir();
-  if (existsSync12(stateDir)) {
+  if (existsSync13(stateDir)) {
     rmSync2(stateDir, { recursive: true, force: true });
     logger.info(formatBullet("\u5DF2\u5220\u9664\u72B6\u6001\u6570\u636E\u3002", "success"));
   }
   const runtimeDir = getRuntimeDirPath();
-  if (existsSync12(runtimeDir)) {
+  if (existsSync13(runtimeDir)) {
     rmSync2(runtimeDir, { recursive: true, force: true });
     logger.info(formatBullet("\u5DF2\u5220\u9664\u8FD0\u884C\u65F6\u6570\u636E\u3002", "success"));
   }
@@ -3052,7 +3315,7 @@ async function runHome(program) {
 }
 
 // src/infrastructure/runtime/cli-version.ts
-import { readFileSync as readFileSync10 } from "fs";
+import { readFileSync as readFileSync11 } from "fs";
 import { dirname as dirname3, join as join12 } from "path";
 import { fileURLToPath } from "url";
 var FALLBACK_VERSION = "0.0.0";
@@ -3067,7 +3330,7 @@ function getCliVersion(metaUrl = import.meta.url) {
     "package.json"
   );
   try {
-    const packageJson = JSON.parse(readFileSync10(packageJsonPath, "utf-8"));
+    const packageJson = JSON.parse(readFileSync11(packageJsonPath, "utf-8"));
     cachedVersion = typeof packageJson.version === "string" ? packageJson.version : FALLBACK_VERSION;
   } catch {
     cachedVersion = FALLBACK_VERSION;
@@ -3117,7 +3380,7 @@ function createCli() {
 }
 
 // src/infrastructure/runtime/main-module.ts
-import { existsSync as existsSync13, realpathSync } from "fs";
+import { existsSync as existsSync14, realpathSync } from "fs";
 import { resolve } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 function isMainModule(argvEntry = process.argv[1], metaUrl = import.meta.url) {
@@ -3128,7 +3391,7 @@ function isMainModule(argvEntry = process.argv[1], metaUrl = import.meta.url) {
   try {
     return realpathSync(argvEntry) === realpathSync(currentModulePath);
   } catch {
-    if (!existsSync13(argvEntry)) {
+    if (!existsSync14(argvEntry)) {
       return false;
     }
     return resolve(argvEntry) === resolve(currentModulePath);