@assuredloop/orca-bot-kit 0.3.0 → 0.5.0

package/src/usage.js

@@ -41,6 +41,21 @@ import { harnessOf } from './launch.js';
 /** The kinds a call's tokens are reported in, the same on either harness. */
 const KINDS = ['input', 'output', 'cache_read', 'cache_write', 'reasoning'];
 
+/**
+ * What could not be counted, by why. A figure that quietly leaves something out
+ * reads as complete, so what is left out is said, and the totals cover only what
+ * was counted: nothing missing is guessed at, nor taken as zero, nor placed in a
+ * window it has no time to be placed in.
+ */
+const GAPS = ['unreadable_transcripts', 'broken_lines', 'records_without_numbers', 'records_without_time'];
+
+const noGaps = () => Object.fromEntries(GAPS.map((gap) => [gap, 0]));
+
+const addGaps = (into, more) => {
+  for (const gap of GAPS) into[gap] += more[gap];
+  return into;
+};
+
 /**
  * What each bot's sessions have used. `since` counts the calls made at or after
  * that moment rather than the conversations begun after it: a conversation that
@@ -99,20 +114,28 @@ function forBot(bots, name, onlySession, window) {
 
   const sessions = bot.sessions
     .filter((session) => onlySession === undefined || session.name === onlySession)
-    .map((session) => ({
-      name: session.name,
-      conversations: idsIn(book.sessions[session.name] ?? {})
-        .filter((id) => onRecord.has(id))
-        .map((id) => counted(onRecord.get(id), window))
-        .filter((one) => one !== undefined),
-    }));
-
-  const unclaimed = [...onRecord.values()]
-    .filter((one) => !claimed.has(one.id))
-    .map((one) => counted(one, window))
-    .filter((one) => one !== undefined);
-
-  return { bot: name, home, sessions, unclaimed };
+    .map((session) => {
+      const { conversations, gaps } = all(
+        idsIn(book.sessions[session.name] ?? {}).filter((id) => onRecord.has(id)).map((id) => onRecord.get(id)),
+        window,
+      );
+      return { name: session.name, conversations, not_counted: gaps };
+    });
+
+  const unclaimed = all([...onRecord.values()].filter((one) => !claimed.has(one.id)), window);
+
+  return { bot: name, home, sessions, unclaimed: unclaimed.conversations, unclaimed_not_counted: unclaimed.gaps };
+}
+
+/** Some conversations counted, and what all of them together could not count. */
+function all(transcripts, window) {
+  const gaps = noGaps();
+  const conversations = transcripts.flatMap((one) => {
+    const { row, gaps: its } = counted(one, window);
+    addGaps(gaps, its);
+    return row === undefined ? [] : [row];
+  });
+  return { conversations, gaps };
 }
 
 /** The conversations a book entry names: the one it is in, then the ones before. */
@@ -122,9 +145,10 @@ const idsIn = (entry) => [
 ];
 
 /**
- * One conversation, counted. A conversation that spent nothing in the window is
- * left out rather than reported as a row of zeroes: it is not part of what has
- * happened since, and a page of zeroes is harder to read than a shorter page.
+ * One conversation, counted, and what in it could not be. A conversation that
+ * spent nothing in the window has no row rather than a row of zeroes: it is not
+ * part of what has happened since, and a page of zeroes is harder to read than a
+ * shorter page. What it could not count is still said.
  */
 function counted(one, window) {
   const read = one.harness === 'claude' ? fromClaude : fromCodex;
@@ -139,12 +163,16 @@ function counted(one, window) {
     byModel: new Map(),
     first: undefined,
     last: undefined,
+    gaps: noGaps(),
   };
 
-  read(lines(one.file), window, tally);
-  if (tally.calls === 0 && KINDS.every((kind) => tally.tokens[kind] === 0)) return undefined;
+  const { entries, unreadable, broken } = transcript(one.file);
+  tally.gaps.unreadable_transcripts = unreadable;
+  tally.gaps.broken_lines = broken;
+  read(entries, window, tally);
+  if (tally.calls === 0 && KINDS.every((kind) => tally.tokens[kind] === 0)) return { gaps: tally.gaps };
 
-  return {
+  return { gaps: tally.gaps, row: {
     id: one.id,
     calls: tally.calls,
     tokens: tally.tokens,
@@ -154,7 +182,7 @@ function counted(one, window) {
     compactions: tally.compactions,
     first: new Date(tally.first).toISOString(),
     last: new Date(tally.last).toISOString(),
-  };
+  } };
 }
 
 /**
@@ -175,6 +203,13 @@ function counted(one, window) {
  * start. A transcript only grows, so one run's end is the next run's start and
  * the runs add up to the whole, with a call still being written at a boundary
  * charged once, part to each side (#169).
+ *
+ * A record with no time, or with a figure missing, is left out whole and said to
+ * be; the call it belongs to is counted from its other records, if it has any.
+ * But when the record last written before the window's start is one of those,
+ * what the call had grown to by then is not known, and what it grew by inside
+ * the window cannot be told apart from what it grew by before. So the call is
+ * not counted in that window, and is said to be.
  */
 function fromClaude(entries, window, tally) {
   const byCall = new Map();
@@ -182,25 +217,34 @@ function fromClaude(entries, window, tally) {
   for (const entry of entries) {
     const when = Date.parse(entry.timestamp ?? '');
     if (entry.type === 'system' && entry.subtype === 'compact_boundary') {
-      if (inside(when, window)) tally.compactions += 1;
+      if (Number.isNaN(when)) tally.gaps.records_without_time += 1;
+      else if (inside(when, window)) tally.compactions += 1;
       continue;
     }
     if (entry.type !== 'assistant') continue;
 
     const usage = entry.message?.usage;
     if (usage === undefined || usage === null) continue;
+    if (Number.isNaN(when)) {
+      tally.gaps.records_without_time += 1;
+      continue;
+    }
+    const broken = !complete(usage, CLAUDE_FIELDS);
+    if (broken && inside(when, window)) tally.gaps.records_without_numbers += 1;
     const call = `${entry.requestId}\u0000${entry.message?.id}`;
     if (!byCall.has(call)) byCall.set(call, []);
-    byCall.get(call).push({ entry, when });
+    byCall.get(call).push({ entry, when, broken });
   }
 
   for (const records of byCall.values()) {
     const made = records[0].when;
-    // A record with no time of its own says nothing about being outside, as in
-    // `inside`: it counts as written before the end and not before the start.
-    const atEnd = records.findLast(({ when }) => Number.isNaN(when) || when < window.to);
+    const atEnd = records.findLast(({ when, broken }) => !broken && when < window.to);
     if (atEnd === undefined) continue;
-    const atStart = records.findLast(({ when }) => !Number.isNaN(when) && when < window.from);
+    const atStart = records.findLast(({ when }) => when < window.from);
+    if (atStart?.broken) {
+      if (inside(atEnd.when, window)) tally.gaps.records_without_numbers += 1;
+      continue;
+    }
 
     const now = figures(atEnd.entry);
     const was = atStart === undefined ? undefined : figures(atStart.entry);
@@ -212,6 +256,9 @@ function fromClaude(entries, window, tally) {
   }
 }
 
+/** The figures Claude Code writes on every call, all of which a record needs to be counted. */
+const CLAUDE_FIELDS = ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'];
+
 /** What one Claude Code record says its call used so far. */
 function figures(entry) {
   const usage = entry.message.usage;
@@ -252,11 +299,23 @@ const CODEX_FIELDS = [
  *
  * The running total is followed through events outside the window as well, since
  * what a call added can only be measured against the event before it.
+ *
+ * An event whose running total has a figure missing is left out and said to be,
+ * and after it the running total is not known, so neither is what the next
+ * complete event added. Measured against the event before the broken one, it
+ * would take in what the broken one used, at its own time and perhaps in
+ * another window; taken by its own per-call figure, it may be the broken one
+ * written down again. Telling those apart is guessing, so it is not counted
+ * either, and is said to be; the running total follows on from it. A call whose
+ * figure is needed and has something missing is not counted, and is said to
+ * be; nothing missing is taken as zero.
  */
 function fromCodex(entries, window, tally) {
   let model;
   let effort;
   let running;
+  // Whether a running total with something missing came since the last complete one.
+  let broken = false;
 
   for (const entry of entries) {
     const when = Date.parse(entry.timestamp ?? '');
@@ -269,48 +328,73 @@ function fromCodex(entries, window, tally) {
       continue;
     }
     if (entry.type === 'compacted') {
-      if (inside(when, window)) tally.compactions += 1;
+      if (Number.isNaN(when)) tally.gaps.records_without_time += 1;
+      else if (inside(when, window)) tally.compactions += 1;
       continue;
     }
     if (entry.type !== 'event_msg' || entry.payload?.type !== 'token_count') continue;
 
+    // No `info` is a note about rate limits, not a record of usage.
     const info = entry.payload?.info;
     if (info === undefined || info === null) continue;
 
-    const used = spent(running, info);
-    if (info.total_token_usage !== undefined && info.total_token_usage !== null) {
-      running = info.total_token_usage;
+    const total = info.total_token_usage ?? undefined;
+    let used;
+    if (total === undefined) {
+      // Nothing to measure against: the per-call figure is all there is. After a
+      // broken running total it may be the broken one written again, so it is
+      // not counted either until a complete running total says where things are.
+      used = broken ? null : perCall(info);
+    } else if (!complete(total, CODEX_READ)) {
+      used = null;
+      broken = true;
+    } else {
+      used = broken ? null : spent(running, total, info);
+      running = total;
+      broken = false;
     }
+
     // A repeat is not a call, whether or not it is inside the window.
     if (used === undefined) continue;
+    if (Number.isNaN(when)) {
+      tally.gaps.records_without_time += 1;
+      continue;
+    }
     if (!inside(when, window)) continue;
+    if (used === null) {
+      tally.gaps.records_without_numbers += 1;
+      continue;
+    }
 
     count(tally, used, model, effort, when);
   }
 }
 
+/** The fields Codex itself writes; it has no cache writes to report. */
+const CODEX_READ = CODEX_FIELDS.filter((field) => field !== 'cache_write_input_tokens').concat('total_tokens');
+
 /**
- * What one Codex event says its call used, measured against the event before it.
- * `undefined` where the event is the one before written down again.
+ * What one Codex event says its call used, measured against the running total
+ * before it: `undefined` where the event is the one before written down again,
+ * and `null` where what it used cannot be known.
  */
-function spent(running, info) {
-  const total = info.total_token_usage;
-  if (total === undefined || total === null) {
-    // Nothing to measure against: the per-call figure is all there is.
-    return info.last_token_usage === undefined ? undefined : kindsOf(info.last_token_usage);
-  }
+function spent(running, total, info) {
   if (running === undefined) return kindsOf(total);
 
-  const moved = number(total.total_tokens) - number(running.total_tokens);
+  const moved = total.total_tokens - running.total_tokens;
   if (moved === 0) return undefined;
   // A fall is a new window, and the running total is counting again from there.
-  if (moved < 0) return kindsOf(info.last_token_usage ?? total);
+  if (moved < 0) return perCall(info);
 
   return kindsOf(Object.fromEntries(
     CODEX_FIELDS.map((field) => [field, number(total[field]) - number(running[field])]),
   ));
 }
 
+/** A Codex event's own per-call figure, or `null` when something in it is missing. */
+const perCall = (info) =>
+  complete(info.last_token_usage, CODEX_READ) ? kindsOf(info.last_token_usage) : null;
+
 /**
  * One Codex figure in the kinds this command reports. The cached tokens sit
  * inside `input_tokens` here and outside it on Claude Code, so they come out of
@@ -351,24 +435,32 @@ function count(tally, used, model, effort, when, calls = 1) {
 }
 
 /** The lines of a transcript that are readable JSON; the rest say nothing. */
-function lines(file) {
+export const lines = (file) => transcript(file).entries;
+
+/**
+ * The lines of a transcript that are JSON records, and how many were not. A
+ * transcript that cannot be read is said to be, rather than a run that stops.
+ */
+function transcript(file) {
   let text;
   try {
     text = readFileSync(file, 'utf8');
   } catch {
-    // A transcript that cannot be read is a conversation the kit knows nothing
-    // about, which is the honest answer rather than a run that stops.
-    return [];
+    return { entries: [], unreadable: 1, broken: 0 };
   }
-  return text.split('\n').flatMap((line) => {
+  let broken = 0;
+  const entries = text.split('\n').flatMap((line) => {
     if (line.trim() === '') return [];
     try {
       const entry = JSON.parse(line);
-      return entry !== null && typeof entry === 'object' ? [entry] : [];
+      if (entry !== null && typeof entry === 'object' && !Array.isArray(entry)) return [entry];
     } catch {
-      return [];
+      // Counted below with any other line that is not a record.
     }
+    broken += 1;
+    return [];
   });
+  return { entries, unreadable: 0, broken };
 }
 
 /** A bot home as the file system knows it, or as it was given when it is not there. */
@@ -382,20 +474,23 @@ function realHome(home) {
 
 const number = (value) => (typeof value === 'number' && Number.isFinite(value) ? value : 0);
 
+/** Whether a figure has every field it should, each a number. */
+const complete = (figure, fields) =>
+  figure !== undefined && figure !== null
+  && fields.every((field) => typeof figure[field] === 'number' && Number.isFinite(figure[field]));
+
 const add = (set, value) => {
   if (typeof value === 'string' && value !== '') set.add(value);
 };
 
 /**
  * Whether a moment is in the window: at or after its start, before its end. A
- * record with no time of its own is counted, as it always has been: nothing
- * says it is outside.
+ * record with no time of its own is in no window; it is left out and said to be.
  */
-const inside = (when, { from, to }) => Number.isNaN(when) || (when >= from && when < to);
+const inside = (when, { from, to }) => when >= from && when < to;
 
 /** When the first and the last counted call of this conversation were. */
 function mark(tally, when) {
-  if (Number.isNaN(when)) return;
   if (tally.first === undefined || when < tally.first) tally.first = when;
   if (tally.last === undefined || when > tally.last) tally.last = when;
 }