@ymys4ever/opencode-throughput-v2 4.0.0 → 4.0.2

package/dist/index.js

@@ -0,0 +1,698 @@
+// src/core/stats.ts
+function aggregateAt(entries, now) {
+  const groups = new Map;
+  for (const entry of entries) {
+    let g = groups.get(entry.model);
+    if (!g) {
+      g = {
+        count: 0,
+        ttft: newMetricFold(),
+        tps: newMetricFold(),
+        latency: newMetricFold(),
+        totalInputTokens: 0,
+        totalOutputTokens: 0,
+        totalReasoningTokens: 0,
+        totalCacheReadTokens: 0,
+        totalCacheWriteTokens: 0,
+        totalCost: 0
+      };
+      groups.set(entry.model, g);
+    }
+    g.count++;
+    if (entry.ttft_ms != null)
+      foldMetric(g.ttft, entry.ttft_ms);
+    if (entry.tps != null)
+      foldMetric(g.tps, entry.tps);
+    if (entry.latency_ms != null)
+      foldMetric(g.latency, entry.latency_ms);
+    g.totalInputTokens += entry.inputTokens;
+    g.totalOutputTokens += entry.outputTokens;
+    g.totalReasoningTokens += entry.reasoningTokens;
+    g.totalCacheReadTokens += entry.cacheReadTokens;
+    g.totalCacheWriteTokens += entry.cacheWriteTokens;
+    g.totalCost += entry.cost;
+  }
+  const result = new Map;
+  for (const [model, g] of groups) {
+    result.set(model, {
+      model,
+      count: g.count,
+      avgTTFT: metricAvg(g.ttft),
+      minTTFT: metricMin(g.ttft),
+      maxTTFT: metricMax(g.ttft),
+      avgTPS: metricAvg(g.tps),
+      maxTPS: metricMax(g.tps),
+      avgLatency: metricAvg(g.latency),
+      minLatency: metricMin(g.latency),
+      maxLatency: metricMax(g.latency),
+      totalInputTokens: g.totalInputTokens,
+      totalOutputTokens: g.totalOutputTokens,
+      totalReasoningTokens: g.totalReasoningTokens,
+      totalCacheReadTokens: g.totalCacheReadTokens,
+      totalCacheWriteTokens: g.totalCacheWriteTokens,
+      totalCost: g.totalCost,
+      lastUpdated: now
+    });
+  }
+  return result;
+}
+function newMetricFold() {
+  return { sum: 0, count: 0, min: Infinity, max: -Infinity };
+}
+function foldMetric(m, v) {
+  m.sum += v;
+  m.count++;
+  if (v < m.min)
+    m.min = v;
+  if (v > m.max)
+    m.max = v;
+}
+function metricAvg(m) {
+  return m.count > 0 ? m.sum / m.count : null;
+}
+function metricMin(m) {
+  return m.count > 0 ? m.min : null;
+}
+function metricMax(m) {
+  return m.count > 0 ? m.max : null;
+}
+
+// src/plugins/throughput.ts
+import { Plugin } from "@opencode/plugin";
+
+// src/core/step-tracker.ts
+function getModelKey(providerID, modelID) {
+  return `${providerID}/${modelID}`;
+}
+var ZERO_TOKENS = {
+  input: 0,
+  output: 0,
+  reasoning: 0,
+  cache: { read: 0, write: 0 }
+};
+function buildLogEntry(state, eventAt, terminal, recordedAt) {
+  const tokens = terminal.tokens ?? ZERO_TOKENS;
+  const latencyMs = eventAt - state.started;
+  const ttftMs = state.firstOutput != null ? state.firstOutput - state.started : null;
+  const genEnd = state.streamEnded ?? eventAt;
+  const genTimeMs = state.firstOutput != null ? genEnd - state.firstOutput : eventAt - state.started;
+  const tps = tokens.output > 0 && genTimeMs > 0 ? tokens.output / genTimeMs * 1000 : null;
+  return {
+    ts: new Date(recordedAt).toISOString(),
+    model: state.modelKey,
+    providerID: state.providerID,
+    modelID: state.modelID,
+    sessionID: terminal.sessionID,
+    messageID: terminal.assistantMessageID,
+    ttft_ms: ttftMs,
+    tps,
+    latency_ms: latencyMs,
+    inputTokens: tokens.input,
+    outputTokens: tokens.output,
+    reasoningTokens: tokens.reasoning,
+    cacheReadTokens: tokens.cache.read,
+    cacheWriteTokens: tokens.cache.write,
+    cost: terminal.cost ?? 0,
+    finish: terminal.finish ?? "error"
+  };
+}
+function buildAbortedEntry(state, assistantMessageID, now) {
+  return buildLogEntry(state, now, { sessionID: state.sessionID, assistantMessageID, finish: "aborted" }, now);
+}
+
+class StepTracker {
+  active = new Map;
+  parked = new Map;
+  activeTtlMs;
+  constructor(options = {}) {
+    const ttl = options.activeTtlMs ?? 600000;
+    this.activeTtlMs = ttl > 0 ? ttl : 0;
+  }
+  start(data) {
+    this.active.set(data.assistantMessageID, {
+      sessionID: data.sessionID,
+      started: data.started,
+      providerID: data.model.providerID,
+      modelID: data.model.id,
+      modelKey: getModelKey(data.model.providerID, data.model.id),
+      firstOutput: null,
+      streamEnded: null
+    });
+  }
+  markFirstOutput(assistantMessageID, created) {
+    const state = this.lookup(assistantMessageID);
+    if (state && state.firstOutput == null)
+      state.firstOutput = created;
+  }
+  markStreamed(assistantMessageID, created) {
+    const state = this.lookup(assistantMessageID);
+    if (state && state.streamEnded == null)
+      state.streamEnded = created;
+  }
+  settle(assistantMessageID, eventAt, terminal, recordedAt) {
+    const activeState = this.active.get(assistantMessageID);
+    if (activeState) {
+      this.active.delete(assistantMessageID);
+      return buildLogEntry(activeState, eventAt, terminal, recordedAt);
+    }
+    const parked = this.parked.get(assistantMessageID);
+    if (parked) {
+      this.parked.delete(assistantMessageID);
+      return buildLogEntry(parked.state, eventAt, terminal, recordedAt);
+    }
+    return null;
+  }
+  parkSession(sessionID, now, ttlMs) {
+    const deadline = now + ttlMs;
+    for (const [id, state] of this.active) {
+      if (state.sessionID === sessionID) {
+        this.parked.set(id, { state, deadline });
+        this.active.delete(id);
+      }
+    }
+  }
+  collectExpired(now) {
+    const expired = [];
+    for (const [id, parked] of this.parked) {
+      if (parked.deadline <= now) {
+        expired.push({ assistantMessageID: id, state: parked.state });
+        this.parked.delete(id);
+      }
+    }
+    if (this.activeTtlMs > 0) {
+      for (const [id, state] of this.active) {
+        if (now - state.started >= this.activeTtlMs) {
+          expired.push({ assistantMessageID: id, state });
+          this.active.delete(id);
+        }
+      }
+    }
+    return expired;
+  }
+  nextDeadline() {
+    let next = null;
+    const consider = (deadline) => {
+      if (next == null || deadline < next)
+        next = deadline;
+    };
+    for (const { deadline } of this.parked.values())
+      consider(deadline);
+    if (this.activeTtlMs > 0) {
+      for (const { started } of this.active.values())
+        consider(started + this.activeTtlMs);
+    }
+    return next;
+  }
+  get size() {
+    return this.active.size + this.parked.size;
+  }
+  lookup(assistantMessageID) {
+    return this.active.get(assistantMessageID) ?? this.parked.get(assistantMessageID)?.state;
+  }
+}
+
+// src/core/events.ts
+var SUBSCRIBED_EVENT_TYPES = [
+  "session.step.started",
+  "session.text.started",
+  "session.reasoning.started",
+  "session.step.streamed",
+  "session.step.ended",
+  "session.step.failed",
+  "session.idle"
+];
+function normalizeEvent(raw) {
+  if (typeof raw === "object" && raw != null && "type" in raw) {
+    const type = raw.type;
+    if (typeof type === "string" && SUBSCRIBED_EVENT_TYPES.includes(type)) {
+      return { ok: true, event: raw };
+    }
+  }
+  return { ok: false, reason: "unknown" };
+}
+function applyThroughputEvent(tracker, ev, deps) {
+  switch (ev.type) {
+    case "session.step.started":
+      tracker.start(ev.data);
+      return null;
+    case "session.text.started":
+    case "session.reasoning.started":
+      tracker.markFirstOutput(ev.data.assistantMessageID, ev.created);
+      return null;
+    case "session.step.streamed":
+      tracker.markStreamed(ev.data.assistantMessageID, ev.created);
+      return null;
+    case "session.step.ended":
+    case "session.step.failed":
+      return tracker.settle(ev.data.assistantMessageID, ev.created, ev.data, deps.recordedAt);
+    case "session.idle":
+      tracker.parkSession(ev.data.sessionID, ev.created, deps.abortTtlMs);
+      return null;
+  }
+  return null;
+}
+
+// src/runtime/pipeline.ts
+function makeSafeOnError(onError) {
+  return (e) => {
+    try {
+      onError(e);
+    } catch (reporterError) {
+      console.error(reporterError);
+    }
+  };
+}
+function createPipeline(deps) {
+  const timers = deps.timers ?? {
+    set: (cb, ms) => setTimeout(cb, ms),
+    clear: (h) => clearTimeout(h)
+  };
+  const safeOnError = makeSafeOnError(deps.onError);
+  const { tracker, now, abortTtlMs, onEntry } = deps;
+  let flushTimer = null;
+  let disposed = false;
+  function drainExpired() {
+    for (const { assistantMessageID, state } of tracker.collectExpired(now())) {
+      try {
+        onEntry(buildAbortedEntry(state, assistantMessageID, now()));
+      } catch (e) {
+        safeOnError(e);
+      }
+    }
+  }
+  function rearm() {
+    if (flushTimer != null)
+      timers.clear(flushTimer);
+    flushTimer = null;
+    if (disposed)
+      return;
+    const next = tracker.nextDeadline();
+    if (next == null)
+      return;
+    flushTimer = timers.set(() => {
+      flushTimer = null;
+      if (disposed)
+        return;
+      try {
+        drainExpired();
+      } catch (e) {
+        safeOnError(e);
+      } finally {
+        rearm();
+      }
+    }, Math.max(0, next - now()));
+  }
+  function dispatch(raw) {
+    if (disposed)
+      return;
+    let owned;
+    try {
+      owned = deps.owns(raw);
+    } catch (e) {
+      safeOnError(e);
+      return;
+    }
+    if (!owned)
+      return;
+    const n = normalizeEvent(raw);
+    if (!n.ok && n.reason === "malformed") {
+      safeOnError(new Error("throughput: malformed event rejected"));
+      return;
+    }
+    try {
+      if (n.ok) {
+        const recordedAt = now();
+        const entry = applyThroughputEvent(tracker, n.event, { recordedAt, abortTtlMs });
+        if (entry)
+          onEntry(entry);
+      }
+    } catch (e) {
+      safeOnError(e);
+    } finally {
+      drainExpired();
+      rearm();
+    }
+  }
+  function dispose() {
+    if (disposed)
+      return;
+    disposed = true;
+    if (flushTimer != null) {
+      timers.clear(flushTimer);
+      flushTimer = null;
+    }
+  }
+  return { dispatch, dispose };
+}
+
+// src/core/ownership.ts
+function makeOwns(selfDir) {
+  return (ev) => {
+    const dir = ev.location?.directory;
+    return dir == null || selfDir == null || dir === selfDir;
+  };
+}
+
+// src/core/log-codec.ts
+function isString(v) {
+  return typeof v === "string";
+}
+function isFiniteNumber(v) {
+  return typeof v === "number" && Number.isFinite(v);
+}
+function decodeLogLine(line) {
+  let raw;
+  try {
+    raw = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  if (typeof raw !== "object" || raw == null)
+    return null;
+  const o = raw;
+  if (!isString(o.model) || !isString(o.providerID) || !isString(o.modelID) || !isString(o.sessionID) || !isString(o.messageID)) {
+    return null;
+  }
+  const metrics = [o.ttft_ms, o.tps, o.latency_ms];
+  const amounts = [
+    o.inputTokens,
+    o.outputTokens,
+    o.reasoningTokens,
+    o.cacheReadTokens,
+    o.cacheWriteTokens,
+    o.cost
+  ];
+  const validMetric = (v) => v == null || isFiniteNumber(v);
+  if (!metrics.every(validMetric) || !amounts.every(validMetric))
+    return null;
+  const metric = (v) => isFiniteNumber(v) ? v : null;
+  const amount = (v) => isFiniteNumber(v) ? v : 0;
+  return {
+    ts: isString(o.ts) ? o.ts : "",
+    model: o.model,
+    providerID: o.providerID,
+    modelID: o.modelID,
+    sessionID: o.sessionID,
+    messageID: o.messageID,
+    ttft_ms: metric(o.ttft_ms),
+    tps: metric(o.tps),
+    latency_ms: metric(o.latency_ms),
+    inputTokens: amount(o.inputTokens),
+    outputTokens: amount(o.outputTokens),
+    reasoningTokens: amount(o.reasoningTokens),
+    cacheReadTokens: amount(o.cacheReadTokens),
+    cacheWriteTokens: amount(o.cacheWriteTokens),
+    cost: amount(o.cost),
+    finish: isString(o.finish) ? o.finish : "error"
+  };
+}
+function decodeLogContent(content) {
+  const entries = [];
+  let skipped = 0;
+  for (const line of content.split(`
+`)) {
+    if (!line.trim())
+      continue;
+    const entry = decodeLogLine(line);
+    if (entry)
+      entries.push(entry);
+    else
+      skipped++;
+  }
+  return { entries, skipped };
+}
+function encodeLogEntry(entry) {
+  return JSON.stringify(entry) + `
+`;
+}
+
+// src/io/log-store.ts
+import fs from "fs";
+import os from "os";
+import path from "path";
+function createLogStore(opts) {
+  const file = opts?.file ?? path.join(os.homedir(), ".opencode", "throughput.jsonl");
+  const onError = opts?.onError ?? ((e) => console.warn(e));
+  let dirReady = false;
+  const ensureDir = () => {
+    if (dirReady)
+      return;
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    dirReady = true;
+  };
+  return {
+    append(entry) {
+      try {
+        ensureDir();
+        fs.appendFileSync(file, encodeLogEntry(entry), "utf-8");
+      } catch (e) {
+        onError(e);
+      }
+    },
+    read() {
+      let content;
+      try {
+        content = fs.readFileSync(file, "utf-8");
+      } catch (e) {
+        if (e.code === "ENOENT")
+          return { entries: [], skipped: 0 };
+        throw e;
+      }
+      return decodeLogContent(content);
+    }
+  };
+}
+
+// src/format/entry.ts
+function formatMs(ms) {
+  if (ms == null)
+    return "N/A";
+  if (ms >= 60000)
+    return (ms / 60000).toFixed(1) + "m";
+  if (ms >= 1000)
+    return (ms / 1000).toFixed(1) + "s";
+  return ms.toFixed(0) + "ms";
+}
+function formatTokenCount(n) {
+  if (n >= 1e6)
+    return (n / 1e6).toFixed(2) + "M";
+  if (n >= 1000)
+    return (n / 1000).toFixed(1) + "k";
+  return n.toFixed(n >= 100 ? 0 : 1);
+}
+function formatNum(n) {
+  if (n == null)
+    return "N/A";
+  return formatTokenCount(n);
+}
+function truncate(s, max) {
+  return s.length > max ? s.slice(0, max - 1) + "…" : s;
+}
+function buildLogMsg(entry) {
+  const parts = [
+    truncate(entry.modelID, 28),
+    formatMs(entry.ttft_ms) + " TTFT",
+    formatNum(entry.tps) + " tok/s",
+    formatMs(entry.latency_ms),
+    "↑" + formatNum(entry.inputTokens) + " ↓" + formatNum(entry.outputTokens)
+  ];
+  return parts.join(" | ");
+}
+
+// src/format/perf.ts
+function formatPerfLine(entry) {
+  const finishSuffix = entry.finish && entry.finish !== "stop" ? " | " + entry.finish : "";
+  return `${entry.ts} | ${entry.modelID} | TTFT ${formatMs(entry.ttft_ms)}` + ` | TPS ${formatNum(entry.tps)} tok/s | Latency ${formatMs(entry.latency_ms)}` + ` | ↑${formatNum(entry.inputTokens)} ↓${formatNum(entry.outputTokens)}` + ` ↓r${formatNum(entry.reasoningTokens)} | Cost $${entry.cost.toFixed(4)}` + `${finishSuffix}
+`;
+}
+
+// src/io/perf-file.ts
+import fs2 from "fs";
+import path2 from "path";
+function createPerfFileWriter(directory, onError) {
+  const file = path2.join(directory, ".opencode", "throughput.md");
+  const handleError = onError ?? (() => {});
+  let dirReady = false;
+  return (entry) => {
+    try {
+      if (!dirReady) {
+        fs2.mkdirSync(path2.dirname(file), { recursive: true });
+        dirReady = true;
+      }
+      fs2.appendFileSync(file, formatPerfLine(entry), "utf-8");
+    } catch (e) {
+      handleError(e);
+    }
+  };
+}
+
+// src/benchmark.ts
+async function executeBenchmark(argsRaw, deps) {
+  const args = argsRaw ?? {};
+  const store = deps?.logStore ?? createLogStore();
+  let entries;
+  let skipped = 0;
+  try {
+    const read = store.read();
+    entries = read.entries;
+    skipped = read.skipped;
+  } catch (e) {
+    return {
+      content: `Failed to read log file: ${e instanceof Error ? e.message : String(e)}`
+    };
+  }
+  if (entries.length === 0) {
+    return {
+      content: "No benchmark data yet. The throughput plugin needs to process at least one LLM response before stats are available."
+    };
+  }
+  const keyword = args.model?.toLowerCase();
+  const filtered = keyword ? entries.filter((l) => l.model.toLowerCase().includes(keyword) || l.modelID.toLowerCase().includes(keyword)) : entries;
+  if (keyword && filtered.length === 0) {
+    return { content: `No entries match filter: ${args.model}` };
+  }
+  const last = args.last !== undefined && Number.isInteger(args.last) && args.last >= 1 ? args.last : undefined;
+  const sliced = last ? filtered.slice(-last) : filtered;
+  const byModel = new Map;
+  for (const entry of sliced) {
+    const arr = byModel.get(entry.model) ?? [];
+    arr.push(entry);
+    byModel.set(entry.model, arr);
+  }
+  const statsByModel = aggregateAt(sliced, Date.now());
+  const lines = [];
+  lines.push(`## Model Performance Benchmark`);
+  lines.push(`Total requests: ${sliced.length}`);
+  if (args.model)
+    lines.push(`Filter: ${args.model}`);
+  lines.push(``);
+  for (const [model, modelEntries] of byModel) {
+    const s = statsByModel.get(model);
+    if (!s)
+      continue;
+    lines.push(`### ${model}`);
+    lines.push(`  Requests: ${modelEntries.length}`);
+    lines.push(``);
+    lines.push(`  **Latency:**`);
+    lines.push(`    TTFT:  avg ${formatMs(s.avgTTFT)} | min ${formatMs(s.minTTFT)} | max ${formatMs(s.maxTTFT)}`);
+    lines.push(`    Total: avg ${formatMs(s.avgLatency)} | min ${formatMs(s.minLatency)} | max ${formatMs(s.maxLatency)}`);
+    lines.push(``);
+    lines.push(`  **Throughput:**`);
+    lines.push(`    TPS:  avg ${formatNum(s.avgTPS)} | max ${formatNum(s.maxTPS)}`);
+    lines.push(``);
+    lines.push(`  **Tokens:**`);
+    lines.push(`    Input:     ${formatNum(s.totalInputTokens)}`);
+    lines.push(`    Output:    ${formatNum(s.totalOutputTokens)}`);
+    lines.push(`    Reasoning: ${formatNum(s.totalReasoningTokens)}`);
+    lines.push(`    Cache R/W: ${formatNum(s.totalCacheReadTokens)} / ${formatNum(s.totalCacheWriteTokens)}`);
+    lines.push(``);
+    lines.push(`  **Cost:** $${s.totalCost.toFixed(4)}`);
+    if (modelEntries.length <= 5) {
+      lines.push(``);
+      lines.push(`  Recent entries:`);
+      for (const e of modelEntries) {
+        lines.push(`    ${e.ts} | ${formatMs(e.ttft_ms)} TTFT | ${formatNum(e.tps)} tok/s | ${formatMs(e.latency_ms)} | cost $${e.cost.toFixed(4)}`);
+      }
+    }
+    lines.push(``);
+  }
+  if (skipped > 0) {
+    lines.push(`(${skipped} corrupt lines ignored)`);
+  }
+  return { content: lines.join(`
+`) };
+}
+
+// src/plugins/throughput.ts
+function createThroughputPlugin(options = {}) {
+  return Plugin.define({
+    id: "opencode-throughput",
+    async setup(ctx) {
+      const onError = options.onError ?? ((e) => console.error(e));
+      const tracker = new StepTracker({ activeTtlMs: options.activeTtlMs });
+      const logStore = options.logStore ?? createLogStore({ onError });
+      const writePerf = createPerfFileWriter(ctx.location.directory, onError);
+      const owns = makeOwns(ctx.location.directory);
+      const now = options.now ?? Date.now;
+      const abortTtlMs = options.abortTtlMs ?? 5000;
+      await ctx.tool.transform((editor) => {
+        editor.add({
+          name: "benchmark",
+          description: "Query model performance benchmarks including latency, throughput (TTFT, TPS), token usage, and cost. Shows stats aggregated by model.",
+          input: {
+            type: "object",
+            properties: {
+              model: {
+                type: "string",
+                description: "Filter by model name (partial match, e.g. 'claude' or 'gpt'). Omit for all models."
+              },
+              last: {
+                type: "integer",
+                minimum: 1,
+                description: "Only show the last N entries (default: show all)"
+              }
+            },
+            additionalProperties: false
+          },
+          async execute(args) {
+            return await executeBenchmark(args, { logStore });
+          }
+        });
+      });
+      const controller = new AbortController;
+      function record(entry) {
+        try {
+          logStore.append(entry);
+        } catch (e) {
+          onError(e);
+        }
+        try {
+          writePerf(entry);
+        } catch (e) {
+          onError(e);
+        }
+      }
+      const pipeline = createPipeline({
+        tracker,
+        owns,
+        now,
+        abortTtlMs,
+        onEntry: record,
+        onError,
+        timers: {
+          set: (cb, ms) => setTimeout(cb, ms),
+          clear: (h) => clearTimeout(h)
+        }
+      });
+      const task = (async () => {
+        for await (const ev of ctx.event.subscribe({ signal: controller.signal })) {
+          pipeline.dispatch(ev);
+        }
+      })();
+      const reportSubscriptionDeath = makeSafeOnError(onError);
+      task.catch((e) => {
+        reportSubscriptionDeath(new Error(`throughput: event subscription died: ${e instanceof Error ? e.message : String(e)}`, {
+          cause: e
+        }));
+      });
+      return () => {
+        pipeline.dispose();
+        controller.abort();
+      };
+    }
+  });
+}
+var ThroughputPlugin = createThroughputPlugin();
+var throughput_default = ThroughputPlugin;
+
+// src/index.ts
+function aggregate(entries) {
+  return aggregateAt(entries, Date.now());
+}
+export {
+  throughput_default as default,
+  createThroughputPlugin,
+  createLogStore,
+  aggregate,
+  ThroughputPlugin
+};