@ericsanchezok/synergy-workbench 0.0.0-stage → 3.1.1

package/src/stats/rollup.ts

@@ -0,0 +1,522 @@
+import type { OperationDigest } from "./types"
+import { ModelLimit } from "@ericsanchezok/synergy-util/model-limit"
+import { RolloutAccounting } from "@ericsanchezok/synergy-harness/session/rollout/accounting"
+import type {
+  AgentStats,
+  AgentUsage,
+  ChannelStats,
+  ChannelUsage,
+  CodeChangeStats,
+  DailyBucket,
+  HourlyBucket,
+  ModelStats,
+  ModelUsage,
+  OverviewStats,
+  SessionDigest,
+  SessionLifecycleStats,
+  StatsSnapshot,
+  TimeSeriesStats,
+  TokenBreakdown,
+  TokenCostStats,
+  ToolStats,
+  ToolUsage,
+} from "./types"
+
+export namespace Rollup {
+  // -----------------------------------------------------------------------
+  // Token helpers
+  // -----------------------------------------------------------------------
+
+  const ZERO_TOKENS: TokenBreakdown = {
+    input: 0,
+    output: 0,
+    reasoning: 0,
+    cache: { read: 0, write: 0 },
+  }
+
+  function addTokens(a: TokenBreakdown, b: TokenBreakdown): TokenBreakdown {
+    return {
+      input: a.input + b.input,
+      output: a.output + b.output,
+      reasoning: a.reasoning + b.reasoning,
+      cache: {
+        read: a.cache.read + b.cache.read,
+        write: a.cache.write + b.cache.write,
+      },
+    }
+  }
+
+  function totalTokens(t: TokenBreakdown): number {
+    return ModelLimit.totalTokens(t)
+  }
+
+  function dayKey(timestamp: number): string {
+    const d = new Date(timestamp)
+    return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`
+  }
+
+  function hourOfDay(timestamp: number): number {
+    return new Date(timestamp).getHours()
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 1 — Overview
+  // -----------------------------------------------------------------------
+
+  function computeOverview(digests: SessionDigest[]): OverviewStats {
+    const archived = digests.filter((d) => d.archived !== undefined)
+    const daySet = new Set<string>()
+    for (const d of digests) daySet.add(dayKey(d.created))
+    const scopeSet = new Set<string>()
+    for (const d of digests) scopeSet.add(d.scopeID)
+
+    let longestStreak = 0
+    let currentStreak = 0
+    if (daySet.size > 0) {
+      const sortedDays = [...daySet].sort()
+      let prev = 0
+      for (const day of sortedDays) {
+        const ts = new Date(day).getTime()
+        if (prev > 0 && ts - prev <= 86_400_000) {
+          currentStreak++
+        } else {
+          currentStreak = 1
+        }
+        longestStreak = Math.max(longestStreak, currentStreak)
+        prev = ts
+      }
+    }
+
+    // Check if current streak extends to today
+    const today = dayKey(Date.now())
+    const sortedDays = [...daySet].sort()
+    if (sortedDays.length > 0 && sortedDays[sortedDays.length - 1] !== today) {
+      // Current streak already ended
+    }
+
+    return {
+      totalSessions: digests.length,
+      activeSessions: digests.length - archived.length,
+      archivedSessions: archived.length,
+      totalMessages: digests.reduce((s, d) => s + d.messages, 0),
+      totalTurns: digests.reduce((s, d) => s + d.turns, 0),
+      totalDays: daySet.size,
+      longestStreak,
+      currentStreak,
+      projectCount: scopeSet.size,
+    }
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 2 — Tokens & Cost
+  // -----------------------------------------------------------------------
+
+  function computeTokenCost(digests: Array<SessionDigest | OperationDigest>): TokenCostStats {
+    const tokens = digests.reduce((acc, d) => addTokens(acc, d.tokens), { ...ZERO_TOKENS })
+    const cost = digests.reduce((s, d) => s + d.cost, 0)
+    const turns = digests.reduce((s, d) => s + d.turns, 0)
+    const days = new Set(digests.map((d) => dayKey(d.created))).size || 1
+    const cacheTotal = tokens.input + tokens.cache.read + tokens.cache.write
+    const total = totalTokens(tokens)
+
+    return {
+      tokens,
+      cost,
+      accounting: RolloutAccounting.merge(digests.flatMap((d) => (d.accounting ? [d.accounting] : []))),
+      cacheHitRate: cacheTotal > 0 ? tokens.cache.read / cacheTotal : 0,
+      avgCostPerTurn: turns > 0 ? cost / turns : 0,
+      avgTokensPerTurn: turns > 0 ? total / turns : 0,
+      dailyCost: cost / days,
+      dailyTokens: total / days,
+    }
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 3 — By Model
+  // -----------------------------------------------------------------------
+
+  function computeModels(digests: Array<SessionDigest | OperationDigest>): ModelStats {
+    const map = new Map<string, ModelUsage>()
+    for (const d of digests) {
+      for (const [key, usage] of Object.entries(d.modelUsage)) {
+        const existing = map.get(key)
+        if (existing) {
+          const previousCalls = existing.accounting
+            ? existing.accounting.calls + existing.accounting.legacy.messages
+            : existing.messages
+          const incomingCalls = usage.accounting
+            ? usage.accounting.calls + usage.accounting.legacy.messages
+            : usage.messages
+          const responseMs = existing.avgResponseMs * previousCalls + usage.totalResponseMs
+          existing.messages += usage.messages
+          existing.turns += usage.messages
+          existing.tokens = addTokens(existing.tokens, usage.tokens)
+          existing.cost += usage.cost
+          existing.accounting = RolloutAccounting.merge([
+            existing.accounting ?? RolloutAccounting.empty(),
+            usage.accounting ?? RolloutAccounting.empty(),
+          ])
+          existing.avgResponseMs = previousCalls + incomingCalls > 0 ? responseMs / (previousCalls + incomingCalls) : 0
+        } else {
+          const [providerID, modelID] = key.split("/")
+          map.set(key, {
+            providerID: providerID ?? "",
+            modelID: modelID ?? "",
+            messages: usage.messages,
+            turns: usage.messages,
+            tokens: { ...usage.tokens, cache: { ...usage.tokens.cache } },
+            cost: usage.cost,
+            accounting: usage.accounting,
+            avgResponseMs:
+              usage.totalResponseMs /
+              Math.max(
+                1,
+                usage.accounting ? usage.accounting.calls + usage.accounting.legacy.messages : usage.messages,
+              ),
+          })
+        }
+      }
+    }
+    const models = [...map.values()].sort((a, b) => b.messages - a.messages)
+    return { models }
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 4 — By Agent
+  // -----------------------------------------------------------------------
+
+  function computeAgents(digests: Array<SessionDigest | OperationDigest>): AgentStats {
+    const map = new Map<string, AgentUsage>()
+    const sessionAgents = new Map<string, Set<string>>()
+    let totalSubagentCalls = 0
+
+    for (const d of digests) {
+      if ("parentID" in d && d.parentID) totalSubagentCalls++
+
+      for (const [agent, usage] of Object.entries(d.agentUsage)) {
+        const existing = map.get(agent)
+        if (existing) {
+          existing.messages += usage.messages
+          existing.tokens = addTokens(existing.tokens, usage.tokens)
+          existing.cost += usage.cost
+          existing.accounting = RolloutAccounting.merge([
+            existing.accounting ?? RolloutAccounting.empty(),
+            usage.accounting ?? RolloutAccounting.empty(),
+          ])
+        } else {
+          map.set(agent, {
+            agent,
+            messages: usage.messages,
+            sessions: 0,
+            tokens: { ...usage.tokens, cache: { ...usage.tokens.cache } },
+            cost: usage.cost,
+            accounting: usage.accounting,
+            subagentInvocations: 0,
+          })
+        }
+
+        if ("sessionID" in d) {
+          if (!sessionAgents.has(d.sessionID)) sessionAgents.set(d.sessionID, new Set())
+          sessionAgents.get(d.sessionID)!.add(agent)
+        }
+      }
+    }
+
+    // Count sessions per agent
+    for (const agents of sessionAgents.values()) {
+      for (const agent of agents) {
+        const entry = map.get(agent)
+        if (entry) entry.sessions++
+      }
+    }
+
+    // Subagent invocations: for each session that IS a subagent (has parentID),
+    // credit its agent
+    for (const d of digests) {
+      if ("parentID" in d && d.parentID) {
+        // The agent of this subagent session
+        const topAgent = Object.keys(d.agentUsage)[0]
+        if (topAgent) {
+          const entry = map.get(topAgent)
+          if (entry) entry.subagentInvocations++
+        }
+      }
+    }
+
+    const agents = [...map.values()].sort((a, b) => b.messages - a.messages)
+    return { agents, totalSubagentCalls }
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 5 — By Tool
+  // -----------------------------------------------------------------------
+
+  function computeTools(digests: SessionDigest[]): ToolStats {
+    const map = new Map<string, ToolUsage>()
+    const timings = new Map<string, { duration: number; samples: number }>()
+    for (const d of digests) {
+      for (const [tool, usage] of Object.entries(d.toolUsage)) {
+        const timing = timings.get(tool) ?? { duration: 0, samples: 0 }
+        timing.duration += usage.totalDurationMs
+        timing.samples += usage.timedSamples ?? usage.calls
+        timings.set(tool, timing)
+        const existing = map.get(tool)
+        if (existing) {
+          existing.calls += usage.calls
+          existing.successes += usage.successes
+          existing.errors += usage.errors
+          existing.avgDurationMs = timing.samples > 0 ? timing.duration / timing.samples : 0
+        } else {
+          map.set(tool, {
+            tool,
+            calls: usage.calls,
+            successes: usage.successes,
+            errors: usage.errors,
+            avgDurationMs: timing.samples > 0 ? timing.duration / timing.samples : 0,
+          })
+        }
+      }
+    }
+    const tools = [...map.values()].sort((a, b) => b.calls - a.calls)
+    return { tools }
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 6 — Code Changes
+  // -----------------------------------------------------------------------
+
+  function computeCodeChanges(digests: SessionDigest[]): CodeChangeStats {
+    const totalAdditions = digests.reduce((s, d) => s + d.additions, 0)
+    const totalDeletions = digests.reduce((s, d) => s + d.deletions, 0)
+    const days = new Set(digests.map((d) => dayKey(d.created))).size || 1
+
+    return {
+      totalAdditions,
+      totalDeletions,
+      totalFiles: digests.reduce((s, d) => s + d.files, 0),
+      netLines: totalAdditions - totalDeletions,
+      dailyAdditions: totalAdditions / days,
+      dailyDeletions: totalDeletions / days,
+    }
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 7 — Session Lifecycle
+  // -----------------------------------------------------------------------
+
+  function computeLifecycle(digests: SessionDigest[]): SessionLifecycleStats {
+    const pinnedCount = digests.filter((d) => d.pinned).length
+    const turnsPerSession = digests.map((d) => d.turns).sort((a, b) => a - b)
+    const avgTurns =
+      turnsPerSession.length > 0 ? turnsPerSession.reduce((s, t) => s + t, 0) / turnsPerSession.length : 0
+    const mid = Math.floor(turnsPerSession.length / 2)
+    const medianTurns =
+      turnsPerSession.length === 0
+        ? 0
+        : turnsPerSession.length % 2 === 0
+          ? (turnsPerSession[mid - 1] + turnsPerSession[mid]) / 2
+          : turnsPerSession[mid]
+
+    const compactionCount = digests.reduce((s, d) => s + d.compactionCount, 0)
+    const retryCount = digests.reduce((s, d) => s + d.retryCount, 0)
+    const errorCount = digests.reduce((s, d) => s + d.errorCount, 0)
+    const totalMessages = digests.reduce((s, d) => s + d.messages, 0)
+
+    let short = 0
+    let medium = 0
+    let long = 0
+    for (const d of digests) {
+      if (d.durationMs < 5 * 60_000) short++
+      else if (d.durationMs < 30 * 60_000) medium++
+      else long++
+    }
+
+    return {
+      pinnedCount,
+      avgTurnsPerSession: avgTurns,
+      medianTurnsPerSession: medianTurns,
+      compactionCount,
+      retryCount,
+      errorCount,
+      errorRate: totalMessages > 0 ? errorCount / totalMessages : 0,
+      durationBuckets: { short, medium, long },
+    }
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 8 — By Channel
+  // -----------------------------------------------------------------------
+
+  function computeChannels(digests: SessionDigest[]): ChannelStats {
+    const map = new Map<string, ChannelUsage>()
+    let interactiveSessions = 0
+    let unattendedSessions = 0
+
+    for (const d of digests) {
+      const channel = d.endpoint?.type ?? d.endpoint?.kind ?? "web"
+      const existing = map.get(channel)
+      if (existing) {
+        existing.sessions++
+        existing.messages += d.messages
+      } else {
+        map.set(channel, { channel, sessions: 1, messages: d.messages })
+      }
+
+      if (d.interaction?.mode === "unattended") unattendedSessions++
+      else interactiveSessions++
+    }
+
+    const channels = [...map.values()].sort((a, b) => b.sessions - a.sessions)
+    return { channels, interactiveSessions, unattendedSessions }
+  }
+
+  // -----------------------------------------------------------------------
+  // Dimension 9 — Time Series
+  // -----------------------------------------------------------------------
+
+  function computeTimeSeries(digests: SessionDigest[]): TimeSeriesStats {
+    const dayMap = new Map<string, DailyBucket>()
+    const hourMap = new Map<string, number>()
+    const hourlyActivity = new Array(24).fill(0) as number[]
+
+    for (const d of digests) {
+      const day = dayKey(d.created)
+      const existing = dayMap.get(day)
+      if (existing) {
+        existing.sessions++
+        existing.turns += d.turns
+        existing.tokens = addTokens(existing.tokens, d.tokens)
+        existing.cost += d.cost
+        existing.accounting = RolloutAccounting.merge([
+          existing.accounting ?? RolloutAccounting.empty(),
+          d.accounting ?? RolloutAccounting.empty(),
+        ])
+        existing.additions += d.additions
+        existing.deletions += d.deletions
+        existing.files += d.files
+        const totalToolCalls = Object.values(d.toolUsage).reduce((s, u) => s + u.calls, 0)
+        existing.toolCalls += totalToolCalls
+        existing.errors += d.errorCount
+      } else {
+        const totalToolCalls = Object.values(d.toolUsage).reduce((s, u) => s + u.calls, 0)
+        dayMap.set(day, {
+          day,
+          sessions: 1,
+          turns: d.turns,
+          tokens: { ...d.tokens, cache: { ...d.tokens.cache } },
+          cost: d.cost,
+          accounting: d.accounting,
+          additions: d.additions,
+          deletions: d.deletions,
+          files: d.files,
+          toolCalls: totalToolCalls,
+          errors: d.errorCount,
+        })
+      }
+
+      for (const [hour, turns] of Object.entries(d.hourlyTurns ?? {})) {
+        hourMap.set(hour, (hourMap.get(hour) ?? 0) + turns)
+        const hourNumber = Number(hour.slice(-2))
+        hourlyActivity[hourNumber] += turns
+      }
+    }
+
+    const days = [...dayMap.values()].sort((a, b) => a.day.localeCompare(b.day))
+    const hours: HourlyBucket[] = [...hourMap.entries()]
+      .sort((a, b) => a[0].localeCompare(b[0]))
+      .map(([hour, turns]) => ({ hour, turns }))
+    return { days, hours, hourlyActivity }
+  }
+
+  // -----------------------------------------------------------------------
+  // Full snapshot
+  // -----------------------------------------------------------------------
+
+  export function snapshot(
+    digests: SessionDigest[],
+    watermark: number,
+    operations: OperationDigest[] = [],
+  ): StatsSnapshot {
+    const usage = [...digests, ...operations]
+    const timeSeries = computeTimeSeries(digests)
+    const days = new Map(timeSeries.days.map((day) => [day.day, day]))
+    for (const operation of operations) {
+      const day = dayKey(operation.created)
+      days.set(
+        day,
+        mergeDailyBucket(days.get(day), {
+          day,
+          sessions: 0,
+          turns: 0,
+          tokens: operation.tokens,
+          cost: operation.cost,
+          accounting: operation.accounting,
+          additions: 0,
+          deletions: 0,
+          files: 0,
+          toolCalls: 0,
+          errors: 0,
+        }),
+      )
+    }
+    timeSeries.days = [...days.values()].sort((a, b) => a.day.localeCompare(b.day))
+    return {
+      overview: computeOverview(digests),
+      tokenCost: computeTokenCost(usage),
+      models: computeModels(usage),
+      agents: computeAgents(usage),
+      tools: computeTools(digests),
+      codeChanges: computeCodeChanges(digests),
+      lifecycle: computeLifecycle(digests),
+      channels: computeChannels(digests),
+      timeSeries,
+      computedAt: Date.now(),
+      watermark,
+    }
+  }
+
+  // -----------------------------------------------------------------------
+  // Incremental daily bucket merge
+  // -----------------------------------------------------------------------
+
+  export function mergeDailyBucket(existing: DailyBucket | undefined, incoming: DailyBucket): DailyBucket {
+    if (!existing) return incoming
+    return {
+      day: incoming.day,
+      sessions: existing.sessions + incoming.sessions,
+      turns: existing.turns + incoming.turns,
+      tokens: addTokens(existing.tokens, incoming.tokens),
+      cost: existing.cost + incoming.cost,
+      accounting: RolloutAccounting.merge([
+        existing.accounting ?? RolloutAccounting.empty(),
+        incoming.accounting ?? RolloutAccounting.empty(),
+      ]),
+      additions: existing.additions + incoming.additions,
+      deletions: existing.deletions + incoming.deletions,
+      files: existing.files + incoming.files,
+      toolCalls: existing.toolCalls + incoming.toolCalls,
+      errors: existing.errors + incoming.errors,
+    }
+  }
+
+  // -----------------------------------------------------------------------
+  // Sub-digest from a single session (for daily bucket)
+  // -----------------------------------------------------------------------
+
+  export function sessionToDailyBucket(d: SessionDigest): DailyBucket {
+    const totalToolCalls = Object.values(d.toolUsage).reduce((s, u) => s + u.calls, 0)
+    return {
+      day: dayKey(d.created),
+      sessions: 1,
+      turns: d.turns,
+      tokens: { ...d.tokens, cache: { ...d.tokens.cache } },
+      cost: d.cost,
+      ...(d.accounting ? { accounting: d.accounting } : {}),
+      additions: d.additions,
+      deletions: d.deletions,
+      files: d.files,
+      toolCalls: totalToolCalls,
+      errors: d.errorCount,
+    }
+  }
+}

package/src/stats/routes/stats.ts

@@ -0,0 +1,107 @@
+import { Hono } from "hono"
+import { streamSSE } from "hono/streaming"
+import { describeRoute, validator, resolver } from "hono-openapi"
+import z from "zod"
+import { errors } from "@ericsanchezok/synergy-server/server/error"
+import { Engine } from ".."
+import { ProgressEvent, StatsSnapshot } from "../types"
+import { UsageRoute } from "./usage"
+
+const StatsQuery = z.object({
+  recompute: z
+    .enum(["true", "false"])
+    .optional()
+    .default("false")
+    .meta({ description: "Set to 'true' to force a full recompute from scratch" }),
+})
+
+export const StatsRoute = () =>
+  new Hono()
+    .route("/usage", UsageRoute())
+    .get(
+      "/",
+      describeRoute({
+        summary: "Get stats snapshot",
+        description:
+          "Read the last computed stats snapshot without scanning history; null means no snapshot exists. Use the progress stream to refresh, or ?recompute=true to force a full recompute.",
+        operationId: "global.stats.get",
+        responses: {
+          200: {
+            description: "Stats snapshot",
+            content: {
+              "application/json": {
+                schema: resolver(StatsSnapshot.meta({ ref: "StatsSnapshot" }).nullable()),
+              },
+            },
+          },
+          ...errors(400),
+        },
+      }),
+      validator("query", StatsQuery),
+      async (c) => {
+        const { recompute } = c.req.valid("query")
+        try {
+          const snapshot = recompute === "true" ? await Engine.recompute() : await Engine.get()
+          return c.json(snapshot)
+        } catch (err: any) {
+          return c.json({ message: err?.message ?? String(err) }, 400)
+        }
+      },
+    )
+    .get(
+      "/progress",
+      describeRoute({
+        summary: "Stream stats recompute progress",
+        description:
+          "Refresh changed statistics and stream progress updates until the final snapshot is ready. Concurrent refreshes share one computation.",
+        operationId: "global.stats.progress",
+        responses: {
+          200: {
+            description: "Server-sent events containing progress and final snapshot payloads",
+            content: {
+              "text/event-stream": {
+                schema: resolver(
+                  z.object({
+                    type: z.enum(["progress", "done", "error"]),
+                    progress: ProgressEvent.optional(),
+                    snapshot: StatsSnapshot.optional(),
+                    message: z.string().optional(),
+                  }),
+                ),
+              },
+            },
+          },
+        },
+      }),
+      async (c) => {
+        c.header("X-Accel-Buffering", "no")
+        c.header("Cache-Control", "no-cache, no-transform")
+        return streamSSE(c, async (stream) => {
+          try {
+            const snapshot = await Engine.update(async (event) => {
+              await stream.writeSSE({
+                data: JSON.stringify({
+                  type: "progress",
+                  progress: event,
+                }),
+              })
+            })
+            await stream.writeSSE({
+              data: JSON.stringify({
+                type: "done",
+                snapshot,
+              }),
+            })
+          } catch (err: any) {
+            await stream.writeSSE({
+              data: JSON.stringify({
+                type: "error",
+                message: err?.message ?? String(err),
+              }),
+            })
+          } finally {
+            stream.close()
+          }
+        })
+      },
+    )