aicp-tracker 1.3.2 → 1.3.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aicp-tracker",
-  "version": "1.3.2",
+  "version": "1.3.4",
   "description": "AI Code Pulse — Claude Code usage tracker for JIRA cost attribution",
   "main": "src/daemon.js",
   "bin": {

package/src/log-parser.js

@@ -12,6 +12,17 @@ function extractTasksFromBranch(branch) {
 }
 
 function sumUsage(entries) {
+  // Claude Code writes multiple JSONL lines per message.id (streaming snapshots).
+  // All snapshots for the same message.id have identical token counts, so we count
+  // only the LAST occurrence of each message.id to avoid double-counting.
+  // The last occurrence also has the most complete content (all tool calls present).
+  const lastIdxByMsgId = new Map();
+  for (let i = 0; i < entries.length; i++) {
+    const msgId = entries[i].message?.id;
+    if (msgId && entries[i].message?.usage) lastIdxByMsgId.set(msgId, i);
+  }
+  const countIdxSet = new Set(lastIdxByMsgId.values());
+
   let input = 0, cc = 0, cr = 0, out = 0;
   let ephUser = 0, ephAsst = 0, ephTool = 0;
   let webSearch = 0, webFetch = 0;
@@ -19,17 +30,21 @@ function sumUsage(entries) {
   const models = new Set();
   const uuids = [];
 
-  for (const e of entries) {
-    const u = e.message?.usage || {};
+  for (let i = 0; i < entries.length; i++) {
+    const e = entries[i];
+    const u = e.message?.usage;
+    if (!u) continue;
+    // Skip all but the last snapshot for each message.id
+    const msgId = e.message?.id;
+    if (msgId && !countIdxSet.has(i)) continue;
+
     input    += u.input_tokens                     || 0;
     cc       += u.cache_creation_input_tokens      || 0;
     cr       += u.cache_read_input_tokens          || 0;
     out      += u.output_tokens                    || 0;
-    // Ephemeral tokens may be at top level (older format) or in cache_creation object (newer format)
     ephUser  += u.ephemeral_user_input_tokens      || u.cache_creation?.ephemeral_user_input_tokens || 0;
     ephAsst  += u.ephemeral_assistant_input_tokens || u.cache_creation?.ephemeral_assistant_input_tokens || 0;
     ephTool  += u.ephemeral_tool_input_tokens      || u.cache_creation?.ephemeral_tool_input_tokens || 0;
-    // Also sum ephemeral cache creation tokens (5m and 1h variants)
     ephUser  += u.cache_creation?.ephemeral_5m_input_tokens || 0;
     ephUser  += u.cache_creation?.ephemeral_1h_input_tokens || 0;
     if (!serviceTier && u.service_tier) serviceTier = u.service_tier;
@@ -43,7 +58,6 @@ function sumUsage(entries) {
         if (block.name === 'web_fetch'  || block.name === 'WebFetch')  webFetch++;
       }
     }
-    // Also check server_tool_use object for web requests
     const srvTool = u.server_tool_use;
     if (srvTool) {
       webSearch += srvTool.web_search_requests || 0;
@@ -55,6 +69,59 @@ function sumUsage(entries) {
   return { input, cc, cr, out, ephUser, ephAsst, ephTool, webSearch, webFetch, serviceTier, speed, models, uuids };
 }
 
+// Merge two usage objects (both have models as Set, uuids as array)
+function mergeUsages(a, b) {
+  return {
+    input:       a.input + b.input,
+    cc:          a.cc + b.cc,
+    cr:          a.cr + b.cr,
+    out:         a.out + b.out,
+    ephUser:     a.ephUser + b.ephUser,
+    ephAsst:     a.ephAsst + b.ephAsst,
+    ephTool:     a.ephTool + b.ephTool,
+    webSearch:   a.webSearch + b.webSearch,
+    webFetch:    a.webFetch + b.webFetch,
+    serviceTier: a.serviceTier || b.serviceTier,
+    speed:       a.speed || b.speed,
+    models:      new Set([...a.models, ...b.models]),
+    uuids:       [...a.uuids, ...b.uuids],
+  };
+}
+
+// Convert usage (with Set) to JSON-serializable form for state storage
+function usageToPlain(u) {
+  return {
+    input: u.input, cc: u.cc, cr: u.cr, out: u.out,
+    ephUser: u.ephUser, ephAsst: u.ephAsst, ephTool: u.ephTool,
+    webSearch: u.webSearch, webFetch: u.webFetch,
+    serviceTier: u.serviceTier, speed: u.speed,
+    models: [...u.models],
+    uuids: u.uuids,
+  };
+}
+
+// Restore usage from plain state (convert models array back to Set)
+function usageFromPlain(p) {
+  return {
+    input: p.input || 0, cc: p.cc || 0, cr: p.cr || 0, out: p.out || 0,
+    ephUser: p.ephUser || 0, ephAsst: p.ephAsst || 0, ephTool: p.ephTool || 0,
+    webSearch: p.webSearch || 0, webFetch: p.webFetch || 0,
+    serviceTier: p.serviceTier || null, speed: p.speed || null,
+    models: new Set(p.models || []),
+    uuids: p.uuids || [],
+  };
+}
+
+// Extract the minimal ref fields needed by makeRecord from any JSONL entry
+function refFromEntry(entry) {
+  return {
+    sessionId:  entry.sessionId  || null,
+    timestamp:  entry.timestamp  || null,
+    gitBranch:  entry.gitBranch  || null,
+    parentUuid: entry.parentUuid || null,
+  };
+}
+
 function makeRecord(ref, rootUuid, usage, filePaths) {
   const { input, cc, cr, out, ephUser, ephAsst, ephTool, webSearch, webFetch, serviceTier, speed, models, uuids } = usage;
   return {
@@ -107,13 +174,69 @@ function extractAllFilePaths(entry) {
   return paths;
 }
 
+function extractGroupFilePaths(allEntries) {
+  const paths = [];
+  for (const entry of allEntries) {
+    for (const fp of extractAllFilePaths(entry)) paths.push(fp);
+  }
+  return [...new Set(paths)];
+}
+
+// Emit a closed group with per-file splitting (used for groups fully contained in one tick)
+function emitGroup(ref, baseUuid, turns, allEntries) {
+  const allFiles = new Set();
+  for (const entry of allEntries) {
+    for (const fp of extractAllFilePaths(entry)) allFiles.add(fp);
+  }
+
+  if (!allFiles.size) {
+    return [makeRecord(ref, baseUuid, sumUsage(turns), [])];
+  }
+
+  const turnFiles      = turns.map(extractFilePath);
+  const fileGroups     = new Map();
+  const firstAnchorIdx = turnFiles.findIndex(f => f !== null);
+  let lastFile         = turnFiles[firstAnchorIdx];
+
+  for (let i = 0; i < turns.length; i++) {
+    const file = turnFiles[i] !== null ? turnFiles[i]
+      : (i < firstAnchorIdx ? turnFiles[firstAnchorIdx] : lastFile);
+    if (!fileGroups.has(file)) fileGroups.set(file, []);
+    fileGroups.get(file).push(turns[i]);
+    if (turnFiles[i] !== null) lastFile = turnFiles[i];
+  }
+
+  const records = [];
+  let idx = 0;
+  for (const [fp, fileTurns] of fileGroups) {
+    const uuid = idx === 0 ? baseUuid : `${baseUuid}:${idx}`;
+    const filesForRecord = [...new Set([fp, ...fileTurns.flatMap(extractAllFilePaths)])];
+    records.push(makeRecord(ref, uuid, sumUsage(fileTurns), filesForRecord));
+    idx++;
+  }
+  return records;
+}
+
 function parseNewLines(filePath) {
   const carryOverPromptId = state.getLastPromptId(filePath);
+  const pendingGroup      = state.getPendingGroup(filePath);
 
   const offset = state.getOffset(filePath);
   let stat;
   try { stat = fs.statSync(filePath); } catch { return []; }
-  if (stat.size <= offset) return [];
+
+  // No new bytes — flush any pending group now (the session is done or paused long enough)
+  if (stat.size <= offset) {
+    if (pendingGroup) {
+      const usage = usageFromPlain(pendingGroup.usage);
+      if (usage.uuids.length > 0) {
+        state.clearPendingGroup(filePath);
+        return [makeRecord(pendingGroup.ref, pendingGroup.promptId, usage, pendingGroup.filePaths || [])];
+      }
+      state.clearPendingGroup(filePath);
+    }
+    return [];
+  }
 
   const fd  = fs.openSync(filePath, 'r');
   const buf = Buffer.alloc(stat.size - offset);
@@ -121,13 +244,10 @@ function parseNewLines(filePath) {
   fs.closeSync(fd);
   state.setOffset(filePath, offset + read);
 
-  // Parse all entries. Only deduplicate by message.id for entries that carry
-  // usage data — those are the only ones where duplicate lines are truly
-  // identical and would cause token double-counting.  Entries without usage
-  // (pure tool-call lines, tool-result lines, etc.) are all kept so that no
-  // file-path information is accidentally discarded.
+  // Keep ALL entries — deduplication by message.id happens inside sumUsage.
+  // We must retain every line here because later snapshots of the same message.id
+  // contain the Edit/Write tool calls that earlier snapshots lack.
   const entries = [];
-  const seenUsageMsgIds = new Set();
 
   for (const raw of buf.slice(0, read).toString('utf8').split('\n')) {
     const line = raw.trim();
@@ -135,34 +255,19 @@ function parseNewLines(filePath) {
     let entry;
     try { entry = JSON.parse(line); } catch { continue; }
     if (!entry.uuid) continue;
-
-    const msgId = entry.message?.id;
-    if (msgId && entry.message?.usage) {
-      if (seenUsageMsgIds.has(msgId)) continue;
-      seenUsageMsgIds.add(msgId);
-    }
     entries.push(entry);
   }
 
   // ── Sequential grouping ───────────────────────────────────────────────────
   // Any entry that carries a promptId opens (or re-opens) a group.
-  // Every subsequent entry — regardless of type or role — belongs to that
-  // group until the next promptId appears.  This is the approach the user
-  // requested: "start from the record that has a promptId and go through all
-  // records down, counting them in, even if they don't have a promptId, until
-  // you find a new promptId."
-  //
-  // carryOverPromptId: the last promptId seen in a previous batch so that
-  // assistant entries at the very start of this batch (before any new
-  // promptId) are still collected rather than dropped.
-
-  let currentPromptId    = carryOverPromptId;
-  let latestPromptId     = carryOverPromptId;
-  let carryOverSeenInBatch = false;
+  // Every subsequent entry belongs to that group until the next promptId appears.
+
+  let currentPromptId = carryOverPromptId;
+  let latestPromptId  = carryOverPromptId;
 
   const groupOrder      = [];
-  const groupAllEntries = new Map(); // promptId → every non-prompt entry in the turn
-  const groupRef        = new Map(); // promptId → the entry that opened the group
+  const groupAllEntries = new Map();
+  const groupRef        = new Map();
 
   function ensureGroup(pid, refEntry) {
     if (!groupAllEntries.has(pid)) {
@@ -174,16 +279,13 @@ function parseNewLines(filePath) {
 
   for (const entry of entries) {
     if (entry.promptId) {
-      // This entry opens a new group (could be any role/type)
-      if (entry.promptId === carryOverPromptId) carryOverSeenInBatch = true;
       currentPromptId = entry.promptId;
       latestPromptId  = entry.promptId;
-      ensureGroup(currentPromptId, entry);
-    } else if (currentPromptId) {
-      // No promptId → belongs to whatever group is currently open
-      ensureGroup(currentPromptId, entry);
-      groupAllEntries.get(currentPromptId).push(entry);
+    } else if (!currentPromptId) {
+      continue;
     }
+    ensureGroup(currentPromptId, entry);
+    groupAllEntries.get(currentPromptId).push(entry);
   }
 
   if (latestPromptId) state.setLastPromptId(filePath, latestPromptId);
@@ -191,54 +293,56 @@ function parseNewLines(filePath) {
   // ── Emit records ──────────────────────────────────────────────────────────
   const records = [];
 
-  for (const promptId of groupOrder) {
-    const allEntries = groupAllEntries.get(promptId);
-    // Only entries that actually carry usage data contribute to token counts
-    const turns = allEntries.filter(e => e.message?.usage);
-    if (!turns.length) continue;
-
-    const ref = groupRef.get(promptId);
-
-    // Suffix :carry when this is a pure continuation from a previous batch so
-    // both records land in the DB rather than one being deduplicated away.
-    const isCarryOver = promptId === carryOverPromptId && !carryOverSeenInBatch;
-    const baseUuid    = isCarryOver ? `${promptId}:carry` : promptId;
-
-    // Collect file paths from EVERY entry in the group (tool-use lines,
-    // tool-result lines, etc.) not just the ones that carry usage.
-    const allFiles = new Set();
-    for (const entry of allEntries) {
-      for (const fp of extractAllFilePaths(entry)) {
-        allFiles.add(fp);
-      }
-    }
-
-    if (!allFiles.size) {
-      records.push(makeRecord(ref, baseUuid, sumUsage(turns), []));
-      continue;
+  // If a pending group exists but its promptId doesn't appear in this batch at all,
+  // it has been superseded by new prompts — emit it now before processing new groups.
+  if (pendingGroup && groupOrder.length > 0 && !groupOrder.includes(pendingGroup.promptId)) {
+    const usage = usageFromPlain(pendingGroup.usage);
+    if (usage.uuids.length > 0) {
+      records.push(makeRecord(pendingGroup.ref, pendingGroup.promptId, usage, pendingGroup.filePaths || []));
     }
+    state.clearPendingGroup(filePath);
+  }
 
-    // Attribute orphan turns (no file anchor) to the nearest anchor:
-    // leading orphans → first anchor; trailing orphans → last anchor seen.
-    const turnFiles      = turns.map(extractFilePath);
-    const fileGroups     = new Map();
-    const firstAnchorIdx = turnFiles.findIndex(f => f !== null);
-    let lastFile         = turnFiles[firstAnchorIdx];
-
-    for (let i = 0; i < turns.length; i++) {
-      const file = turnFiles[i] !== null ? turnFiles[i]
-        : (i < firstAnchorIdx ? turnFiles[firstAnchorIdx] : lastFile);
-      if (!fileGroups.has(file)) fileGroups.set(file, []);
-      fileGroups.get(file).push(turns[i]);
-      if (turnFiles[i] !== null) lastFile = turnFiles[i];
+  for (let gi = 0; gi < groupOrder.length; gi++) {
+    const promptId   = groupOrder[gi];
+    const isLast     = gi === groupOrder.length - 1;
+    const allEntries = groupAllEntries.get(promptId);
+    const turns      = allEntries.filter(e => e.message?.usage);
+    const hasPending = pendingGroup && pendingGroup.promptId === promptId;
+
+    let effectiveUsage, effectiveFilePaths, effectiveRef;
+
+    if (hasPending) {
+      // Merge accumulated pending data with new entries from this tick
+      const pendingUsage = usageFromPlain(pendingGroup.usage);
+      effectiveUsage     = turns.length > 0 ? mergeUsages(pendingUsage, sumUsage(turns)) : pendingUsage;
+      effectiveFilePaths = [...new Set([...(pendingGroup.filePaths || []), ...extractGroupFilePaths(allEntries)])];
+      effectiveRef       = pendingGroup.ref;
+    } else {
+      if (!turns.length) continue;
+      effectiveUsage     = sumUsage(turns);
+      effectiveFilePaths = extractGroupFilePaths(allEntries);
+      effectiveRef       = refFromEntry(groupRef.get(promptId));
     }
 
-    let idx = 0;
-    for (const [fp, fileTurns] of fileGroups) {
-      const uuid = idx === 0 ? baseUuid : `${baseUuid}:${idx}`;
-      const filesForRecord = [...new Set([fp, ...fileTurns.flatMap(extractAllFilePaths)])];
-      records.push(makeRecord(ref, uuid, sumUsage(fileTurns), filesForRecord));
-      idx++;
+    if (isLast) {
+      // Group may still grow in the next tick — save as pending, don't emit yet
+      state.setPendingGroup(filePath, {
+        promptId,
+        ref:       effectiveRef,
+        usage:     usageToPlain(effectiveUsage),
+        filePaths: effectiveFilePaths,
+      });
+    } else {
+      // Group is closed (a subsequent promptId appeared) — emit now
+      if (hasPending) state.clearPendingGroup(filePath);
+      if (hasPending) {
+        // Cross-tick group: emit as one record (tokens already merged, no per-file split)
+        records.push(makeRecord(effectiveRef, promptId, effectiveUsage, effectiveFilePaths));
+      } else {
+        // Single-tick group: use per-file splitting as before
+        records.push(...emitGroup(effectiveRef, promptId, turns, allEntries));
+      }
     }
   }
 

package/src/state.js

@@ -44,4 +44,25 @@ function setLastPromptId(filePath, promptId) {
   save();
 }
 
-module.exports = { getOffset, setOffset, getLastPromptId, setLastPromptId };
+// Pending group: the last open group that has not yet been closed by a subsequent promptId.
+// Stored until the group is definitively closed (different promptId appears, or file stops growing).
+function getPendingGroup(filePath) {
+  return load().pendingGroups?.[filePath] || null;
+}
+
+function setPendingGroup(filePath, data) {
+  const s = load();
+  if (!s.pendingGroups) s.pendingGroups = {};
+  s.pendingGroups[filePath] = data;
+  save();
+}
+
+function clearPendingGroup(filePath) {
+  const s = load();
+  if (s.pendingGroups?.[filePath]) {
+    delete s.pendingGroups[filePath];
+    save();
+  }
+}
+
+module.exports = { getOffset, setOffset, getLastPromptId, setLastPromptId, getPendingGroup, setPendingGroup, clearPendingGroup };