@yishiguji/tokenarena 0.16.5 → 0.17.0

package/dist/index.js

@@ -4494,57 +4494,314 @@ var RooCodeParser = class {
 registerParser(new RooCodeParser());
 
 // src/parsers/snow.ts
-import { existsSync as existsSync22 } from "fs";
+import { existsSync as existsSync22, readdirSync as readdirSync12, statSync as statSync6 } from "fs";
 import { homedir as homedir22 } from "os";
-import { join as join23 } from "path";
-var DEFAULT_DATA_DIR6 = join23(homedir22(), ".snow", "usage");
+import { basename as basename10, join as join23 } from "path";
+var DEFAULT_SNOW_DIR = join23(homedir22(), ".snow");
+var USAGE_DIR_NAME = "usage";
+var SESSIONS_DIR_NAME = "sessions";
+var NON_TRANSCRIPT_DIRS = /* @__PURE__ */ new Set(["subagent", "compressed"]);
+var SESSION_MATCH_WINDOW_MS = 5 * 60 * 1e3;
+var DAMAGED_TRANSCRIPT_SETTLE_MS = 60 * 1e3;
+var MIN_TIMESTAMP_MS = Date.UTC(2020, 0, 1);
+var MAX_TIMESTAMP_MS = Date.UTC(2080, 0, 1);
 function toNonNegativeNumber2(value) {
+  if (typeof value !== "number" && typeof value !== "string") return 0;
   const numberValue = Number(value);
   return Number.isFinite(numberValue) && numberValue >= 0 ? numberValue : 0;
 }
+function toTime(value) {
+  const time = typeof value === "string" ? Date.parse(value) : typeof value === "number" ? value : Number.NaN;
+  return time >= MIN_TIMESTAMP_MS && time < MAX_TIMESTAMP_MS ? time : null;
+}
+function isRecord(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function findSourceFiles(dir, extension, results = []) {
+  let entries;
+  try {
+    entries = readdirSync12(dir, { withFileTypes: true });
+  } catch (error) {
+    const code = error.code;
+    if (code === "ENOENT" || code === "ENOTDIR") return results;
+    throw error;
+  }
+  for (const entry of entries) {
+    const fullPath = join23(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (extension === ".json" && NON_TRANSCRIPT_DIRS.has(entry.name)) {
+        continue;
+      }
+      findSourceFiles(fullPath, extension, results);
+    } else if (entry.name.endsWith(extension)) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+function resolveProject(session) {
+  if (typeof session.projectPath === "string" && session.projectPath) {
+    const segments = session.projectPath.split(/[\\/]+/).filter(Boolean);
+    return segments.at(-1) || "unknown";
+  }
+  return "unknown";
+}
+function readTranscript(filePath) {
+  const content = readFileSafe(filePath);
+  if (content === null) return { status: "damaged" };
+  let parsed;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return { status: "damaged" };
+  }
+  if (!isRecord(parsed) || !Array.isArray(parsed.messages)) {
+    return { status: "foreign" };
+  }
+  const session = parsed;
+  const compressedFrom = typeof session.compressedFrom === "string" && session.compressedFrom ? session.compressedFrom : void 0;
+  const copiedUntil = compressedFrom !== void 0 || typeof session.branchedFrom === "string" ? Math.max(
+    toTime(session.compressedAt) ?? Number.NEGATIVE_INFINITY,
+    toTime(session.createdAt) ?? Number.NEGATIVE_INFINITY
+  ) : Number.NEGATIVE_INFINITY;
+  const messages = [];
+  let lastMessageAt = 0;
+  for (const message of parsed.messages) {
+    if (!isRecord(message)) continue;
+    if (message.role !== "user" && message.role !== "assistant") continue;
+    const time = toTime(message.timestamp);
+    if (time === null) continue;
+    if (time > lastMessageAt) lastMessageAt = time;
+    if (time <= copiedUntil) continue;
+    messages.push({ role: message.role, time });
+  }
+  return {
+    status: "ok",
+    transcript: {
+      filePath,
+      sessionId: typeof session.id === "string" && session.id ? session.id : basename10(filePath, ".json"),
+      project: resolveProject(session),
+      updatedAt: toTime(session.updatedAt) ?? lastMessageAt,
+      ...compressedFrom !== void 0 ? { compressedFrom } : {},
+      messages
+    }
+  };
+}
+function isNewerCopy(candidate, current) {
+  if (candidate.updatedAt !== current.updatedAt) {
+    return candidate.updatedAt > current.updatedAt;
+  }
+  if (candidate.messages.length !== current.messages.length) {
+    return candidate.messages.length > current.messages.length;
+  }
+  return candidate.filePath < current.filePath;
+}
+function resolveConversationId(transcript, transcripts) {
+  let current = transcript;
+  const visited = /* @__PURE__ */ new Set([current.sessionId]);
+  while (current.compressedFrom && !visited.has(current.compressedFrom)) {
+    visited.add(current.compressedFrom);
+    const source = transcripts.get(current.compressedFrom);
+    if (!source) return current.compressedFrom;
+    current = source;
+  }
+  return current.sessionId;
+}
+function mayStillBeWriting(filePath, now) {
+  try {
+    return now - statSync6(filePath).mtimeMs < DAMAGED_TRANSCRIPT_SETTLE_MS;
+  } catch {
+    return true;
+  }
+}
+function distanceToNearest(sortedTimes, time) {
+  let low = 0;
+  let high = sortedTimes.length;
+  while (low < high) {
+    const middle = low + high >>> 1;
+    if (sortedTimes[middle] < time) low = middle + 1;
+    else high = middle;
+  }
+  let distance = Number.POSITIVE_INFINITY;
+  if (low < sortedTimes.length) distance = sortedTimes[low] - time;
+  if (low > 0) distance = Math.min(distance, time - sortedTimes[low - 1]);
+  return distance;
+}
+function attributeUsage(entries, windows) {
+  if (windows.length === 0) return;
+  const byStart = [...windows].sort((left, right) => left.start - right.start);
+  const byTime = entries.map((entry) => ({ entry, time: entry.timestamp.getTime() })).sort((left, right) => left.time - right.time);
+  const open = [];
+  let nextWindow = 0;
+  for (const { entry, time } of byTime) {
+    while (nextWindow < byStart.length && byStart[nextWindow].start <= time) {
+      open.push(byStart[nextWindow]);
+      nextWindow++;
+    }
+    let kept = 0;
+    let best;
+    let bestDistance = Number.POSITIVE_INFINITY;
+    for (const window of open) {
+      if (window.end < time) continue;
+      open[kept++] = window;
+      const distance = distanceToNearest(window.assistantTimes, time);
+      if (best === void 0 || distance < bestDistance || distance === bestDistance && window.sessionId < best.sessionId) {
+        best = window;
+        bestDistance = distance;
+      }
+    }
+    open.length = kept;
+    if (best) entry.sessionId = best.sessionId;
+  }
+}
 var SnowParser = class {
-  constructor(dataDir = DEFAULT_DATA_DIR6) {
-    this.dataDir = dataDir;
-    this.tool = { id: "snow", name: "Snow CLI", dataDir };
+  constructor(snowDir = DEFAULT_SNOW_DIR) {
+    this.snowDir = snowDir;
+    this.tool = {
+      id: "snow",
+      name: "Snow CLI",
+      dataDir: join23(snowDir, USAGE_DIR_NAME)
+    };
   }
-  dataDir;
+  snowDir;
   tool;
   async parse() {
+    const usage = this.parseUsage();
+    const sessions = this.parseSessions();
+    attributeUsage(usage.entries, sessions.windows);
+    return {
+      buckets: aggregateToBuckets(usage.entries),
+      sessions: extractSessions(sessions.events, usage.entries),
+      ...usage.incomplete ? { incomplete: true } : {}
+    };
+  }
+  listSourceFiles() {
+    return [
+      ...findSourceFiles(join23(this.snowDir, USAGE_DIR_NAME), ".jsonl"),
+      ...findSourceFiles(join23(this.snowDir, SESSIONS_DIR_NAME), ".json")
+    ];
+  }
+  /**
+   * Build session timing events plus the windows used to attribute usage
+   * records. Message timestamps drive turn boundaries, so active time, session
+   * duration, and message counts all come from real transcripts.
+   *
+   * Transcripts only feed sessions, never buckets, so a damaged one is skipped
+   * rather than holding back the whole source: Snow cannot open it either and
+   * will not rewrite it, so deferring would block Snow uploads for good.
+   */
+  parseSessions() {
+    const transcripts = /* @__PURE__ */ new Map();
+    const damaged = [];
+    const now = Date.now();
+    for (const filePath of findSourceFiles(
+      join23(this.snowDir, SESSIONS_DIR_NAME),
+      ".json"
+    )) {
+      const read = readTranscript(filePath);
+      if (read.status === "damaged") {
+        if (!mayStillBeWriting(filePath, now)) damaged.push(filePath);
+        continue;
+      }
+      if (read.status === "foreign") continue;
+      const current = transcripts.get(read.transcript.sessionId);
+      if (!current || isNewerCopy(read.transcript, current)) {
+        transcripts.set(read.transcript.sessionId, read.transcript);
+      }
+    }
+    if (damaged.length > 0) {
+      const more = damaged.length > 1 ? ` and ${damaged.length - 1} more` : "";
+      logger.warn(
+        `Skipped damaged Snow session file ${damaged[0]}${more}. Token usage is still counted.`
+      );
+    }
+    const events = [];
+    const windows = [];
+    for (const transcript of transcripts.values()) {
+      if (transcript.messages.length === 0) continue;
+      const sessionId = resolveConversationId(transcript, transcripts);
+      const assistantTimes = [];
+      let start = Number.POSITIVE_INFINITY;
+      let end = Number.NEGATIVE_INFINITY;
+      for (const message of transcript.messages) {
+        events.push({
+          sessionId,
+          source: "snow",
+          project: transcript.project,
+          timestamp: new Date(message.time),
+          role: message.role
+        });
+        if (message.role === "assistant") assistantTimes.push(message.time);
+        if (message.time < start) start = message.time;
+        if (message.time > end) end = message.time;
+      }
+      windows.push({
+        sessionId,
+        assistantTimes: assistantTimes.sort((left, right) => left - right),
+        start: start - SESSION_MATCH_WINDOW_MS,
+        end: end + SESSION_MATCH_WINDOW_MS
+      });
+    }
+    return { events, windows };
+  }
+  /**
+   * Parse usage JSONL. Session attribution happens afterwards, because it needs
+   * the transcripts.
+   *
+   * The bucket project stays `unknown` on purpose; see the note on
+   * `SESSION_MATCH_WINDOW_MS`.
+   */
+  parseUsage() {
     const entries = [];
-    for (const filePath of findJsonlFiles(this.dataDir)) {
+    let incomplete = false;
+    for (const filePath of findSourceFiles(
+      join23(this.snowDir, USAGE_DIR_NAME),
+      ".jsonl"
+    )) {
       const content = readFileSafe(filePath);
-      if (!content) continue;
+      if (content === null) {
+        if (existsSync22(filePath)) incomplete = true;
+        continue;
+      }
       for (const line of content.split("\n")) {
         if (!line.trim()) continue;
+        let record;
         try {
-          const record = JSON.parse(line);
-          if (typeof record.timestamp !== "string") continue;
-          const timestamp = new Date(record.timestamp);
-          if (Number.isNaN(timestamp.getTime())) continue;
-          const inputTokens = toNonNegativeNumber2(record.inputTokens);
-          const outputTokens = toNonNegativeNumber2(record.outputTokens);
-          const cachedTokens = toNonNegativeNumber2(record.cacheReadInputTokens);
-          const reasoningTokens = toNonNegativeNumber2(record.reasoningTokens);
-          if (inputTokens + outputTokens + cachedTokens + reasoningTokens === 0)
-            continue;
-          entries.push({
-            source: "snow",
-            model: typeof record.model === "string" && record.model ? record.model : "unknown",
-            project: "unknown",
-            timestamp,
-            inputTokens,
-            outputTokens,
-            reasoningTokens,
-            cachedTokens
-          });
+          const parsed = JSON.parse(line);
+          if (!isRecord(parsed)) continue;
+          record = parsed;
         } catch {
+          continue;
         }
+        const time = toTime(record.timestamp);
+        if (time === null) continue;
+        const reportedInputTokens = toNonNegativeNumber2(record.inputTokens);
+        const outputTokens = toNonNegativeNumber2(record.outputTokens);
+        const cachedTokens = toNonNegativeNumber2(record.cacheReadInputTokens);
+        const cacheCreationTokens = toNonNegativeNumber2(
+          record.cacheCreationInputTokens
+        );
+        const reasoningTokens = toNonNegativeNumber2(record.reasoningTokens);
+        const inputTokens = record.cacheCreationInputTokens === void 0 && cachedTokens <= reportedInputTokens ? reportedInputTokens - cachedTokens : reportedInputTokens;
+        if (inputTokens + outputTokens + cachedTokens + cacheCreationTokens + reasoningTokens === 0)
+          continue;
+        entries.push({
+          source: "snow",
+          model: typeof record.model === "string" && record.model ? record.model : "unknown",
+          project: "unknown",
+          timestamp: new Date(time),
+          inputTokens,
+          outputTokens,
+          reasoningTokens,
+          cachedTokens,
+          cacheCreationTokens
+        });
       }
     }
-    return { buckets: aggregateToBuckets(entries), sessions: [] };
+    return { entries, incomplete };
   }
   isInstalled() {
-    return existsSync22(this.dataDir);
+    return existsSync22(join23(this.snowDir, USAGE_DIR_NAME));
   }
 };
 registerParser(new SnowParser());
@@ -5174,9 +5431,9 @@ var ZCodeParser = class {
 registerParser(new ZCodeParser());
 
 // src/parsers/qodercli.ts
-import { existsSync as existsSync25, readdirSync as readdirSync12 } from "fs";
+import { existsSync as existsSync25, readdirSync as readdirSync13 } from "fs";
 import { homedir as homedir25 } from "os";
-import { basename as basename10, join as join26 } from "path";
+import { basename as basename11, join as join26 } from "path";
 var TOOL_ID16 = "qodercli";
 var TOOL_NAME15 = "Qoder CLI";
 var DEFAULT_PROJECTS_DIR = join26(homedir25(), ".qoder", "projects");
@@ -5241,7 +5498,7 @@ function projectFromEncodedSlug(slug) {
 }
 function projectFromCwd(cwd) {
   if (!cwd) return "unknown";
-  const leaf = basename10(cwd.replace(/[\\/]+$/, ""));
+  const leaf = basename11(cwd.replace(/[\\/]+$/, ""));
   return leaf || "unknown";
 }
 function totalCreditsUsed(quota) {
@@ -5344,7 +5601,7 @@ var QoderCliParser = class {
   parseProjectSessions(sessionEvents) {
     if (!existsSync25(this.projectsDir)) return;
     for (const filePath of findJsonlFiles(this.projectsDir)) {
-      if (basename10(filePath).startsWith("verified-")) continue;
+      if (basename11(filePath).startsWith("verified-")) continue;
       const content = readFileSafe(filePath);
       if (!content) continue;
       if (classifyQoderEntrypoint(content) !== CLI_ENTRYPOINT) continue;
@@ -5382,7 +5639,7 @@ var QoderCliParser = class {
     if (!existsSync25(this.runsDir)) return [];
     let runDirs;
     try {
-      runDirs = readdirSync12(this.runsDir, { withFileTypes: true }).filter(
+      runDirs = readdirSync13(this.runsDir, { withFileTypes: true }).filter(
         (d) => d.isDirectory()
       );
     } catch {
@@ -5416,12 +5673,12 @@ var QoderCliParser = class {
 registerParser(new QoderCliParser());
 
 // src/parsers/grok-build.ts
-import { existsSync as existsSync26, readdirSync as readdirSync13 } from "fs";
+import { existsSync as existsSync26, readdirSync as readdirSync14 } from "fs";
 import { homedir as homedir26 } from "os";
-import { basename as basename11, join as join27 } from "path";
+import { basename as basename12, join as join27 } from "path";
 var TOOL_ID17 = "grok-build";
 var TOOL_NAME16 = "Grok Build";
-var DEFAULT_DATA_DIR7 = join27(homedir26(), ".grok", "sessions");
+var DEFAULT_DATA_DIR6 = join27(homedir26(), ".grok", "sessions");
 var DEFAULT_ARCHIVE_DIR2 = join27(homedir26(), ".grok", "archived_sessions");
 function createToolDefinition12(dataDir) {
   return {
@@ -5437,10 +5694,10 @@ function toSafeNumber12(value) {
 function projectFromEncodedCwd(encoded) {
   try {
     const decoded = decodeURIComponent(encoded);
-    const leaf = basename11(decoded.replace(/[\\/]+$/, ""));
+    const leaf = basename12(decoded.replace(/[\\/]+$/, ""));
     return leaf || "unknown";
   } catch {
-    const leaf = basename11(encoded);
+    const leaf = basename12(encoded);
     return leaf || "unknown";
   }
 }
@@ -5494,7 +5751,7 @@ function findSessionDirs(dataDir) {
   if (!existsSync26(dataDir)) return results;
   let projectDirs;
   try {
-    projectDirs = readdirSync13(dataDir, { withFileTypes: true });
+    projectDirs = readdirSync14(dataDir, { withFileTypes: true });
   } catch {
     return results;
   }
@@ -5505,7 +5762,7 @@ function findSessionDirs(dataDir) {
     const project = projectFromEncodedCwd(projectEntry.name);
     let sessionEntries;
     try {
-      sessionEntries = readdirSync13(projectDir, { withFileTypes: true });
+      sessionEntries = readdirSync14(projectDir, { withFileTypes: true });
     } catch {
       continue;
     }
@@ -5534,7 +5791,7 @@ function readFallbackModel(sessionDir) {
   }
 }
 var GrokBuildParser = class {
-  constructor(dataDir = DEFAULT_DATA_DIR7, archiveDir = dataDir === DEFAULT_DATA_DIR7 ? DEFAULT_ARCHIVE_DIR2 : void 0) {
+  constructor(dataDir = DEFAULT_DATA_DIR6, archiveDir = dataDir === DEFAULT_DATA_DIR6 ? DEFAULT_ARCHIVE_DIR2 : void 0) {
     this.dataDir = dataDir;
     this.archiveDir = archiveDir;
     this.tool = createToolDefinition12(dataDir);
@@ -5635,7 +5892,7 @@ registerParser(new GrokBuildParser());
 // src/parsers/atomcode.ts
 import { existsSync as existsSync27 } from "fs";
 import { homedir as homedir27 } from "os";
-import { basename as basename12, join as join28 } from "path";
+import { basename as basename13, join as join28 } from "path";
 var TOOL_ID18 = "atomcode";
 var TOOL_NAME17 = "AtomCode";
 var DEFAULT_SESSIONS_DIR6 = join28(homedir27(), ".atomcode", "sessions");
@@ -5690,7 +5947,7 @@ function getMetaModel(meta) {
 }
 function getMetaProject(meta) {
   if (typeof meta?.working_dir === "string" && meta.working_dir) {
-    return basename12(meta.working_dir) || "unknown";
+    return basename13(meta.working_dir) || "unknown";
   }
   return "unknown";
 }
@@ -5781,9 +6038,9 @@ var AtomCodeParser = class {
 registerParser(new AtomCodeParser());
 
 // src/parsers/dsh.ts
-import { existsSync as existsSync28, readdirSync as readdirSync14, readFileSync as readFileSync9 } from "fs";
+import { existsSync as existsSync28, readdirSync as readdirSync15, readFileSync as readFileSync9 } from "fs";
 import { homedir as homedir28 } from "os";
-import { basename as basename13, join as join29 } from "path";
+import { basename as basename14, join as join29 } from "path";
 import * as zlib from "zlib";
 var TOOL_ID19 = "dsh";
 var TOOL_NAME18 = "DeepSeek Harness";
@@ -5883,7 +6140,7 @@ function findSessionLogs(dir) {
   const results = [];
   if (!existsSync28(dir)) return results;
   try {
-    for (const entry of readdirSync14(dir, { withFileTypes: true })) {
+    for (const entry of readdirSync15(dir, { withFileTypes: true })) {
       const fullPath = join29(dir, entry.name);
       if (entry.isDirectory()) {
         results.push(...findSessionLogs(fullPath));
@@ -5933,7 +6190,7 @@ var DshParser = class {
         const relativeParts = filePath.slice(sessionsDir.length + 1).split(/[\\/]/);
         const header = rows[0]?.type === "session" ? rows[0] : void 0;
         const sessionId = (header?.id ?? decodeSegment(relativeParts[1] ?? "")) || "unknown";
-        const project = typeof header?.cwd === "string" && header.cwd ? basename13(header.cwd) || "unknown" : projectFromDirName(relativeParts[0] ?? "");
+        const project = typeof header?.cwd === "string" && header.cwd ? basename14(header.cwd) || "unknown" : projectFromDirName(relativeParts[0] ?? "");
         let currentModel = "unknown";
         for (const row of rows) {
           if (row.type === "request/context") {
@@ -7311,7 +7568,7 @@ function saveUploadManifest(manifest) {
 
 // src/services/parse-cache.ts
 import { createHash as createHash7 } from "crypto";
-import { mkdirSync as mkdirSync3, readFileSync as readFileSync14, statSync as statSync6, writeFileSync as writeFileSync5 } from "fs";
+import { mkdirSync as mkdirSync3, readFileSync as readFileSync14, statSync as statSync7, writeFileSync as writeFileSync5 } from "fs";
 import { hostname as hostname6 } from "os";
 import { join as join35 } from "path";
 
@@ -7367,7 +7624,7 @@ function computeScanFingerprint(files, now = Date.now()) {
   for (const file of [...files].sort()) {
     let stats;
     try {
-      stats = statSync6(file);
+      stats = statSync7(file);
     } catch {
       return null;
     }
@@ -7575,12 +7832,12 @@ function createProjectIdentityResolver(settings) {
     return identity;
   };
 }
-function toUploadBuckets(buckets, settings, device, resolveProject = createProjectIdentityResolver(
+function toUploadBuckets(buckets, settings, device, resolveProject2 = createProjectIdentityResolver(
   settings
 )) {
   const aggregated = /* @__PURE__ */ new Map();
   for (const bucket of buckets) {
-    const project = resolveProject(bucket.project);
+    const project = resolveProject2(bucket.project);
     const key = [
       bucket.source,
       bucket.model,
@@ -7616,11 +7873,11 @@ function toUploadBuckets(buckets, settings, device, resolveProject = createProje
   }
   return Array.from(aggregated.values());
 }
-function toUploadSessions(sessions, settings, device, resolveProject = createProjectIdentityResolver(
+function toUploadSessions(sessions, settings, device, resolveProject2 = createProjectIdentityResolver(
   settings
 )) {
   return sessions.map((session) => {
-    const project = resolveProject(session.project);
+    const project = resolveProject2(session.project);
     return {
       source: session.source,
       projectKey: project.projectKey,
@@ -7799,18 +8056,18 @@ async function runSync(config, opts = {}) {
       deviceId: device.deviceId,
       settings
     });
-    const resolveProject = createProjectIdentityResolver(settings);
+    const resolveProject2 = createProjectIdentityResolver(settings);
     const uploadBuckets = toUploadBuckets(
       allBuckets,
       settings,
       device,
-      resolveProject
+      resolveProject2
     );
     const uploadSessions = toUploadSessions(
       allSessions,
       settings,
       device,
-      resolveProject
+      resolveProject2
     );
     const previousManifest = loadUploadManifest();
     const uploadDiff = diffUploadManifest({