@yishiguji/tokenarena 0.18.1 → 0.18.3

package/dist/index.js

@@ -6297,12 +6297,13 @@ registerParser(new AtomCodeParser());
 // src/parsers/dsh.ts
 import { existsSync as existsSync28, readdirSync as readdirSync15, readFileSync as readFileSync9 } from "fs";
 import { homedir as homedir29 } from "os";
-import { basename as basename14, join as join30 } from "path";
+import { basename as basename14, dirname as dirname7, join as join30 } from "path";
 import * as zlib from "zlib";
 var TOOL_ID20 = "dsh";
 var TOOL_NAME18 = "DeepSeek Harness";
 var DEFAULT_SESSIONS_DIR7 = join30(homedir29(), ".dsh", "sessions");
-var LOG_BASENAME = "session";
+var SESSION_LOG_PATTERN = /^session(?:\.v(\d+))?\.jsonl(?:\.zstd)?$/;
+var LEGACY_GENERATION = 0;
 function getDshSessionsDirs(env = process.env) {
   const dirs = [
     env.TOKEN_ARENA_DSH_DIR,
@@ -6311,10 +6312,18 @@ function getDshSessionsDirs(env = process.env) {
   ].filter((value) => Boolean(value));
   return Array.from(new Set(dirs));
 }
+function getSessionLogGeneration(fileName) {
+  const match = SESSION_LOG_PATTERN.exec(fileName);
+  if (!match) return null;
+  return match[1] === void 0 ? LEGACY_GENERATION : Number(match[1]);
+}
 function toNonNegativeNumber4(value) {
   const numberValue = Number(value);
   return Number.isFinite(numberValue) && numberValue >= 0 ? numberValue : 0;
 }
+function toModelName(value) {
+  return typeof value === "string" && value ? value : null;
+}
 function scanZstdFrames(buffer) {
   const ZSTD_MAGIC = 4247762216;
   const frames = [];
@@ -6370,45 +6379,85 @@ function scanZstdFrames(buffer) {
   return frames;
 }
 function decompressZstdLog(buffer) {
-  if (typeof zlib.zstdDecompressSync !== "function") return null;
+  if (typeof zlib.zstdDecompressSync !== "function") {
+    return { text: null, damaged: false };
+  }
+  const frames = scanZstdFrames(buffer);
+  if (frames.length === 0) return { text: null, damaged: false };
   const chunks = [];
-  for (const [start, end] of scanZstdFrames(buffer)) {
+  let damaged = false;
+  for (const [start, end] of frames) {
     try {
       chunks.push(zlib.zstdDecompressSync(buffer.subarray(start, end)));
     } catch {
+      damaged = true;
     }
   }
-  if (chunks.length === 0) return null;
-  return Buffer.concat(chunks).toString("utf-8");
+  if (chunks.length === 0) return { text: null, damaged };
+  return { text: Buffer.concat(chunks).toString("utf-8"), damaged };
 }
-function readSessionLog(filePath) {
+function readSessionLog(log2) {
   let buffer;
   try {
-    buffer = readFileSync9(filePath);
+    buffer = readFileSync9(log2.filePath);
   } catch {
-    return null;
+    return { text: null, damaged: false };
   }
-  if (filePath.endsWith(".zstd")) {
+  if (log2.compressed) {
     return decompressZstdLog(buffer);
   }
-  return buffer.toString("utf-8");
+  return { text: buffer.toString("utf-8"), damaged: false };
 }
-function findSessionLogs(dir) {
+function findSessionLogs(dir, rootDir = dir) {
   const results = [];
   if (!existsSync28(dir)) return results;
   try {
     for (const entry of readdirSync15(dir, { withFileTypes: true })) {
       const fullPath = join30(dir, entry.name);
       if (entry.isDirectory()) {
-        results.push(...findSessionLogs(fullPath));
-      } else if (entry.name === `${LOG_BASENAME}.jsonl` || entry.name === `${LOG_BASENAME}.jsonl.zstd`) {
-        results.push(fullPath);
+        results.push(...findSessionLogs(fullPath, rootDir));
+        continue;
       }
+      const generation = getSessionLogGeneration(entry.name);
+      if (generation === null) continue;
+      results.push({
+        filePath: fullPath,
+        generation,
+        compressed: entry.name.endsWith(".zstd"),
+        rootDir
+      });
     }
   } catch {
   }
   return results;
 }
+function isNewerLog(candidate, current) {
+  if (candidate.generation !== current.generation) {
+    return candidate.generation > current.generation;
+  }
+  if (candidate.compressed !== current.compressed) return candidate.compressed;
+  return candidate.filePath < current.filePath;
+}
+function selectCurrentSessionLogs(logs) {
+  const byDirectory = /* @__PURE__ */ new Map();
+  for (const log2 of logs) {
+    const directory = dirname7(log2.filePath);
+    const current = byDirectory.get(directory);
+    if (!current || isNewerLog(log2, current)) {
+      byDirectory.set(directory, log2);
+    }
+  }
+  return Array.from(byDirectory.values()).sort(
+    (left, right) => left.filePath < right.filePath ? -1 : left.filePath > right.filePath ? 1 : 0
+  );
+}
+function currentSessionLogs(roots) {
+  const logs = [];
+  for (const dir of roots) {
+    logs.push(...findSessionLogs(dir));
+  }
+  return selectCurrentSessionLogs(logs);
+}
 function decodeSegment(segment) {
   return segment.replace(
     /~([0-9A-F]{4})/g,
@@ -6438,101 +6487,121 @@ var DshParser = class {
     const entries = [];
     const sessionEvents = [];
     const seenEntryKeys = /* @__PURE__ */ new Set();
-    for (const sessionsDir of this.sessionsDirs) {
-      for (const filePath of findSessionLogs(sessionsDir)) {
-        const content = readSessionLog(filePath);
-        if (!content) continue;
-        const rows = parseJsonl(content);
-        if (rows.length === 0) continue;
-        const relativeParts = filePath.slice(sessionsDir.length + 1).split(/[\\/]/);
-        const header = rows[0]?.type === "session" ? rows[0] : void 0;
-        const sessionId = (header?.id ?? decodeSegment(relativeParts[1] ?? "")) || "unknown";
-        const project = typeof header?.cwd === "string" && header.cwd ? basename14(header.cwd) || "unknown" : projectFromDirName(relativeParts[0] ?? "");
-        let currentModel = "unknown";
-        for (const row of rows) {
-          if (row.type === "request/context") {
-            const model2 = row.data?.model;
-            if (typeof model2 === "string" && model2) currentModel = model2;
-            continue;
-          }
-          if (row.type === "request/header") {
-            const model2 = row.data?.header?.config?.model;
-            if (typeof model2 === "string" && model2) currentModel = model2;
-            continue;
-          }
-          const timestamp = typeof row.time === "number" && Number.isFinite(row.time) ? new Date(row.time) : null;
-          if (row.type === "user/message") {
-            const sourceKind = row.data?.source?.kind ?? row.data?.message?.source?.kind;
-            if (sourceKind !== "user") continue;
-            if (timestamp) {
-              sessionEvents.push({
-                sessionId,
-                source: TOOL_ID20,
-                project,
-                timestamp,
-                role: "user"
-              });
-            }
-            continue;
-          }
-          const isCompaction = row.type === "compaction/summary";
-          if (row.type !== "assistant/message" && !isCompaction) continue;
-          if (timestamp && !isCompaction) {
+    let incomplete = false;
+    const logs = currentSessionLogs(this.sessionsDirs);
+    for (const log2 of logs) {
+      const { text: content, damaged } = readSessionLog(log2);
+      if (content === null) {
+        if (existsSync28(log2.filePath)) incomplete = true;
+        continue;
+      }
+      if (damaged) {
+        logger.warn(
+          `dsh session log ${log2.filePath} has frames that could not be decompressed.`
+        );
+        incomplete = true;
+        continue;
+      }
+      const rows = parseJsonl(content);
+      if (rows.length === 0) continue;
+      const relativeParts = log2.filePath.slice(log2.rootDir.length + 1).split(/[\\/]/);
+      const header = rows[0]?.type === "session" ? rows[0] : void 0;
+      const sessionId = (header?.id ?? decodeSegment(relativeParts[1] ?? "")) || "unknown";
+      const project = typeof header?.cwd === "string" && header.cwd ? basename14(header.cwd) || "unknown" : projectFromDirName(relativeParts[0] ?? "");
+      let currentModel = "unknown";
+      for (const row of rows) {
+        if (row.type === "request/context") {
+          const model2 = row.data?.model;
+          if (typeof model2 === "string" && model2) currentModel = model2;
+          continue;
+        }
+        if (row.type === "request/header") {
+          const model2 = row.data?.header?.config?.model;
+          if (typeof model2 === "string" && model2) currentModel = model2;
+          continue;
+        }
+        const timestamp = typeof row.time === "number" && Number.isFinite(row.time) ? new Date(row.time) : null;
+        if (row.type === "user/message") {
+          const sourceKind = row.data?.source?.kind ?? row.data?.message?.source?.kind;
+          if (sourceKind !== "user") continue;
+          if (timestamp) {
             sessionEvents.push({
               sessionId,
               source: TOOL_ID20,
               project,
               timestamp,
-              role: "assistant"
+              role: "user"
             });
           }
-          const usage = row.data?.usage;
-          if (!usage || timestamp === null) continue;
-          const model = (isCompaction && typeof row.data?.model === "string" ? row.data.model : "") || currentModel;
-          const inputTokens = toNonNegativeNumber4(usage.inputTokens);
-          const cachedTokens = toNonNegativeNumber4(usage.cacheReadTokens);
-          const cacheCreationTokens = toNonNegativeNumber4(
-            usage.cacheWriteTokens
-          );
-          const reasoningTokens = toNonNegativeNumber4(usage.reasoningTokens);
-          const outputTokens = Math.max(
-            0,
-            toNonNegativeNumber4(usage.outputTokens) - reasoningTokens
-          );
-          if (inputTokens + outputTokens + cachedTokens + cacheCreationTokens + reasoningTokens === 0)
-            continue;
-          const entryKey = [
-            sessionId,
-            timestamp.toISOString(),
-            model,
-            inputTokens,
-            outputTokens,
-            cachedTokens,
-            cacheCreationTokens,
-            reasoningTokens
-          ].join("|");
-          if (seenEntryKeys.has(entryKey)) continue;
-          seenEntryKeys.add(entryKey);
-          entries.push({
+          continue;
+        }
+        const isCompaction = row.type === "compaction/summary";
+        if (row.type !== "assistant/message" && !isCompaction) continue;
+        if (timestamp && !isCompaction) {
+          sessionEvents.push({
             sessionId,
             source: TOOL_ID20,
-            model,
             project,
             timestamp,
-            inputTokens,
-            outputTokens,
-            reasoningTokens,
-            cachedTokens,
-            cacheCreationTokens
+            role: "assistant"
           });
         }
+        const usage = row.data?.usage;
+        if (!usage || timestamp === null) continue;
+        const declaredModel = toModelName(row.data?.model) ?? toModelName(row.data?.message?.source?.model);
+        if (declaredModel) currentModel = declaredModel;
+        const model = declaredModel ?? currentModel;
+        const inputTokens = toNonNegativeNumber4(usage.inputTokens);
+        const cachedTokens = toNonNegativeNumber4(usage.cacheReadTokens);
+        const cacheCreationTokens = toNonNegativeNumber4(usage.cacheWriteTokens);
+        const reasoningTokens = toNonNegativeNumber4(usage.reasoningTokens);
+        const outputTokens = Math.max(
+          0,
+          toNonNegativeNumber4(usage.outputTokens) - reasoningTokens
+        );
+        if (inputTokens + outputTokens + cachedTokens + cacheCreationTokens + reasoningTokens === 0)
+          continue;
+        const entryKey = [
+          sessionId,
+          timestamp.toISOString(),
+          model,
+          inputTokens,
+          outputTokens,
+          cachedTokens,
+          cacheCreationTokens,
+          reasoningTokens
+        ].join("|");
+        if (seenEntryKeys.has(entryKey)) continue;
+        seenEntryKeys.add(entryKey);
+        entries.push({
+          sessionId,
+          source: TOOL_ID20,
+          model,
+          project,
+          timestamp,
+          inputTokens,
+          outputTokens,
+          reasoningTokens,
+          cachedTokens,
+          cacheCreationTokens
+        });
       }
     }
     return {
       buckets: aggregateToBuckets(entries),
-      sessions: extractSessions(sessionEvents, entries)
+      sessions: extractSessions(sessionEvents, entries),
+      ...incomplete ? { incomplete: true } : {}
     };
   }
+  /**
+   * Exhaustive: every file `parse()` reads, generation selection included.
+   *
+   * A log that is discovered but no longer selected must not appear here either,
+   * or an upgraded session would look changed while contributing nothing.
+   */
+  listSourceFiles() {
+    return currentSessionLogs(this.sessionsDirs).map((log2) => log2.filePath);
+  }
   isInstalled() {
     return this.sessionsDirs.some((dir) => existsSync28(dir));
   }
@@ -6542,7 +6611,7 @@ registerParser(new DshParser());
 // src/parsers/cherry-studio.ts
 import { existsSync as existsSync29 } from "fs";
 import { homedir as homedir30 } from "os";
-import { dirname as dirname7, join as join31, resolve as resolve3 } from "path";
+import { dirname as dirname8, join as join31, resolve as resolve3 } from "path";
 var TOOL_ID21 = "cherry-studio";
 var TOOL_NAME19 = "Cherry Studio";
 var DB_RELATIVE = join31("Data", "cherrystudio.sqlite");
@@ -6597,7 +6666,7 @@ function createToolDefinition13(dbPath) {
   return {
     id: TOOL_ID21,
     name: TOOL_NAME19,
-    dataDir: dbPath ? dirname7(dbPath) : join31(getDefaultUserDataDir(), "Data")
+    dataDir: dbPath ? dirname8(dbPath) : join31(getDefaultUserDataDir(), "Data")
   };
 }
 function toSafeCount(value) {
@@ -7830,7 +7899,7 @@ import { hostname as hostname6 } from "os";
 import { join as join36 } from "path";
 
 // src/infrastructure/runtime/cli-version.ts
-import { dirname as dirname8, join as join35 } from "path";
+import { dirname as dirname9, join as join35 } from "path";
 import { fileURLToPath } from "url";
 var FALLBACK_VERSION = "0.0.0";
 var versionCache = /* @__PURE__ */ new Map();
@@ -7846,7 +7915,7 @@ function readPackageVersion(startDir) {
         return void 0;
       }
     }
-    const parent = dirname8(dir);
+    const parent = dirname9(dir);
     if (parent === dir) {
       return void 0;
     }
@@ -7854,7 +7923,7 @@ function readPackageVersion(startDir) {
   }
 }
 function getCliVersion(metaUrl = import.meta.url) {
-  const startDir = dirname8(fileURLToPath(metaUrl));
+  const startDir = dirname9(fileURLToPath(metaUrl));
   const cached = versionCache.get(startDir);
   if (cached !== void 0) {
     return cached;
@@ -8547,7 +8616,7 @@ import { execFileSync as execFileSync8, spawn } from "child_process";
 import { existsSync as existsSync36 } from "fs";
 import { appendFile, mkdir, readFile } from "fs/promises";
 import { homedir as homedir34, platform as platform5 } from "os";
-import { dirname as dirname9, join as join37, posix as posix3, win32 } from "path";
+import { dirname as dirname10, join as join37, posix as posix3, win32 } from "path";
 
 // src/infrastructure/service/index.ts
 import { platform as platform4 } from "os";
@@ -9521,7 +9590,7 @@ async function setupShellAlias() {
     return;
   }
   try {
-    await mkdir(dirname9(setup.configFile), { recursive: true });
+    await mkdir(dirname10(setup.configFile), { recursive: true });
     let existingContent = "";
     if (existsSync36(setup.configFile)) {
       existingContent = await readFile(setup.configFile, "utf-8");