@saadnvd1/agent-os 0.3.555 → 0.3.557

package/lib/experiments/meter.ts

@@ -0,0 +1,327 @@
+/**
+ * What experiments cost and whether they earn it. Every model call an
+ * experiment makes goes through experimentRun, which records it here and in
+ * the usage ledger; non-model work (scripts, suggestions shown, accepted,
+ * dismissed) is counted with countEvent.
+ */
+
+import { db } from "../db";
+import {
+  runClaude,
+  type ClaudeRun,
+  type ClaudeRunner,
+} from "../orchestrator/claude-cli";
+import type { RunUsage } from "../usage/check-runs";
+import { recordLooseCall } from "../usage/calls";
+import { dayKey } from "../usage/aggregate";
+import { experimentOn } from ".";
+import { getExperiment, type ExperimentId } from "./registry";
+
+export type ExperimentEvent =
+  | "model"
+  | "script"
+  | "shown"
+  | "accepted"
+  | "dismissed"
+  | "applied"
+  | "error";
+
+export interface MeterContext {
+  trigger: string;
+  sessionId?: string | null;
+  workspaceId?: string | null;
+  detail?: string | null;
+}
+
+const clip = (s: string | null | undefined, n = 120) =>
+  s ? s.slice(0, n) : null;
+
+function insert(
+  id: ExperimentId,
+  event: ExperimentEvent,
+  ctx: MeterContext,
+  u?: RunUsage,
+  ok = true,
+  at = Date.now()
+): number {
+  const info = db
+    .prepare(
+      `INSERT INTO experiment_usage (experiment_id, event, trigger, session_id, workspace_id, model, provider,
+         input_tokens, cache_write_tokens, cache_read_tokens, output_tokens, cost_usd, duration_ms, ok, detail, at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+    )
+    .run(
+      id,
+      event,
+      ctx.trigger,
+      ctx.sessionId ?? null,
+      ctx.workspaceId ?? null,
+      u?.model ?? null,
+      u ? (u.provider ?? "anthropic") : null,
+      u?.inputTokens ?? 0,
+      u?.cacheWriteTokens ?? 0,
+      u?.cacheReadTokens ?? 0,
+      u?.outputTokens ?? 0,
+      u?.costUsd ?? 0,
+      u?.durationMs ?? null,
+      ok ? 1 : 0,
+      clip(ctx.detail),
+      at
+    );
+  return Number(info.lastInsertRowid);
+}
+
+function recordModel(id: ExperimentId, ctx: MeterContext, u: RunUsage) {
+  try {
+    const rowId = insert(id, "model", ctx, u);
+    recordLooseCall({
+      callKey: `experiment:${rowId}`,
+      at: Date.now(),
+      sessionId: ctx.sessionId ?? null,
+      taskId: null,
+      workspaceId: ctx.workspaceId ?? null,
+      projectId: null,
+      sessionName: getExperiment(id).name,
+      role: `experiment:${id}`,
+      kind: "helper",
+      agentType: null,
+      model: u.model,
+      provider: u.provider ?? "anthropic",
+      input: u.inputTokens,
+      cacheRead: u.cacheReadTokens,
+      cacheWrite5m: u.cacheWriteTokens,
+      cacheWrite1h: 0,
+      output: u.outputTokens,
+      source: "helper",
+    });
+  } catch (e) {
+    console.error(`[experiments] recording ${id}:`, e);
+  }
+}
+
+/**
+ * The only way an experiment calls a model. Returns null, having started
+ * nothing, when the experiment is off. A failed run is recorded too (its
+ * usage, or an error row when it reported none) and then thrown.
+ */
+export async function experimentRun(
+  id: ExperimentId,
+  ctx: MeterContext,
+  run: ClaudeRun,
+  runner: ClaudeRunner = runClaude
+): Promise<unknown | null> {
+  if (!experimentOn(id)) return null;
+  let usage: RunUsage | null = null;
+  try {
+    const out = await runner({
+      ...run,
+      onUsage: (u) => {
+        usage = u;
+        run.onUsage?.(u);
+      },
+    });
+    if (usage) recordModel(id, ctx, usage);
+    return out;
+  } catch (error) {
+    try {
+      if (usage) recordModel(id, ctx, usage);
+      insert(
+        id,
+        "error",
+        {
+          ...ctx,
+          detail: error instanceof Error ? error.message : String(error),
+        },
+        undefined,
+        false
+      );
+    } catch (e) {
+      console.error(`[experiments] recording ${id}:`, e);
+    }
+    throw error;
+  }
+}
+
+/**
+ * Non-model work. Does nothing when the experiment is off, unless `force`:
+ * a person deciding on a card that is already on screen is still counted.
+ */
+export function countEvent(
+  id: ExperimentId,
+  event: Exclude<ExperimentEvent, "model">,
+  ctx: MeterContext,
+  force = false
+): void {
+  if (!force && !experimentOn(id)) return;
+  try {
+    insert(id, event, ctx);
+  } catch (e) {
+    console.error(`[experiments] counting ${id}:`, e);
+  }
+}
+
+// Whether this session already has this event, so a hook runs once.
+export function hasEvent(
+  id: ExperimentId,
+  sessionId: string,
+  event: ExperimentEvent,
+  trigger?: string
+): boolean {
+  return !!db
+    .prepare(
+      `SELECT 1 FROM experiment_usage WHERE experiment_id = ? AND session_id = ? AND event = ?
+         AND (? IS NULL OR trigger = ?) LIMIT 1`
+    )
+    .get(id, sessionId, event, trigger ?? null, trigger ?? null);
+}
+
+export interface WindowTotals {
+  calls: number;
+  tokens: number;
+  costUsd: number;
+  scripts: number;
+  shown: number;
+  accepted: number;
+  dismissed: number;
+  // accepted / (accepted + dismissed); null with no decisions yet.
+  acceptRate: number | null;
+}
+
+export interface UsageSummary {
+  today: WindowTotals;
+  d7: WindowTotals;
+  d30: WindowTotals;
+}
+
+const DAY = 86_400_000;
+
+export function startOfDay(now: number): number {
+  const d = new Date(now);
+  d.setHours(0, 0, 0, 0);
+  return d.getTime();
+}
+
+function windowTotals(id: ExperimentId, since: number): WindowTotals {
+  const r = db
+    .prepare(
+      `SELECT
+         coalesce(sum(event = 'model'), 0) AS calls,
+         coalesce(sum(input_tokens + cache_write_tokens + cache_read_tokens + output_tokens), 0) AS tokens,
+         coalesce(sum(cost_usd), 0) AS costUsd,
+         coalesce(sum(event = 'script'), 0) AS scripts,
+         coalesce(sum(event = 'shown'), 0) AS shown,
+         coalesce(sum(event = 'accepted'), 0) AS accepted,
+         coalesce(sum(event = 'dismissed'), 0) AS dismissed
+       FROM experiment_usage WHERE experiment_id = ? AND at >= ?`
+    )
+    .get(id, since) as Omit<WindowTotals, "acceptRate">;
+  const decided = r.accepted + r.dismissed;
+  return { ...r, acceptRate: decided ? r.accepted / decided : null };
+}
+
+export function usageSummary(id: ExperimentId, now = Date.now()): UsageSummary {
+  const today = startOfDay(now);
+  return {
+    today: windowTotals(id, today),
+    d7: windowTotals(id, today - 6 * DAY),
+    d30: windowTotals(id, today - 29 * DAY),
+  };
+}
+
+export interface DayRow {
+  day: string;
+  calls: number;
+  tokens: number;
+  costUsd: number;
+  accepted: number;
+  dismissed: number;
+}
+
+export interface Breakdown {
+  key: string;
+  calls: number;
+  tokens: number;
+  costUsd: number;
+}
+
+export interface RecentEvent {
+  at: number;
+  event: string;
+  trigger: string;
+  model: string | null;
+  tokens: number;
+  costUsd: number;
+  ok: boolean;
+  detail: string | null;
+}
+
+export interface UsageDetail {
+  days: DayRow[];
+  byTrigger: Breakdown[];
+  byModel: Breakdown[];
+  recent: RecentEvent[];
+}
+
+export function usageDetail(id: ExperimentId, now = Date.now()): UsageDetail {
+  const since = startOfDay(now) - 29 * DAY;
+  const rows = db
+    .prepare(
+      `SELECT at, event, trigger, model, cost_usd AS costUsd,
+         input_tokens + cache_write_tokens + cache_read_tokens + output_tokens AS tokens
+       FROM experiment_usage WHERE experiment_id = ? AND at >= ?`
+    )
+    .all(id, since) as {
+    at: number;
+    event: string;
+    trigger: string;
+    model: string | null;
+    costUsd: number;
+    tokens: number;
+  }[];
+  const days = new Map<string, DayRow>();
+  for (let i = 29; i >= 0; i--) {
+    const day = dayKey(startOfDay(now) - i * DAY);
+    days.set(day, {
+      day,
+      calls: 0,
+      tokens: 0,
+      costUsd: 0,
+      accepted: 0,
+      dismissed: 0,
+    });
+  }
+  const trig = new Map<string, Breakdown>();
+  const mod = new Map<string, Breakdown>();
+  const add = (m: Map<string, Breakdown>, key: string, r: (typeof rows)[0]) => {
+    const b = m.get(key) ?? { key, calls: 0, tokens: 0, costUsd: 0 };
+    b.calls += r.event === "model" ? 1 : 0;
+    b.tokens += r.tokens;
+    b.costUsd += r.costUsd;
+    m.set(key, b);
+  };
+  for (const r of rows) {
+    const d = days.get(dayKey(r.at));
+    if (d) {
+      d.calls += r.event === "model" ? 1 : 0;
+      d.tokens += r.tokens;
+      d.costUsd += r.costUsd;
+      d.accepted += r.event === "accepted" ? 1 : 0;
+      d.dismissed += r.event === "dismissed" ? 1 : 0;
+    }
+    add(trig, r.trigger, r);
+    if (r.event === "model") add(mod, r.model ?? "unrecorded", r);
+  }
+  const recent = db
+    .prepare(
+      `SELECT at, event, trigger, model, input_tokens + cache_write_tokens + cache_read_tokens + output_tokens AS tokens,
+         cost_usd AS costUsd, ok, detail
+       FROM experiment_usage WHERE experiment_id = ? ORDER BY at DESC, id DESC LIMIT 50`
+    )
+    .all(id) as (Omit<RecentEvent, "ok"> & { ok: number })[];
+  return {
+    days: [...days.values()],
+    byTrigger: [...trig.values()],
+    byModel: [...mod.values()],
+    recent: recent.map((r) => ({ ...r, ok: !!r.ok })),
+  };
+}

package/lib/experiments/registry.ts

@@ -0,0 +1,37 @@
+// Every experiment AgentOS knows about. All are off until someone turns them
+// on. To add one: an entry here, `experimentOn(id)` where it does its work,
+// and `experimentRun(id, ...)` for any model call it makes.
+
+export type ExperimentStatus = "experimental" | "beta";
+
+export interface Experiment {
+  // Kebab-case and stable: it names the setting and keys the usage rows.
+  id: string;
+  name: string;
+  description: string;
+  // Who owns it: "sessions", "tasks", "orchestrator"...
+  area: string;
+  status: ExperimentStatus;
+  // It shows suggestions, so an accept rate means something for it.
+  suggests?: boolean;
+}
+
+export const EXPERIMENTS = [
+  {
+    id: "learning-cards",
+    name: "Learning cards",
+    description:
+      "When a session ends, suggest saving it as a recipe or keeping a lesson, in the one place it belongs.",
+    area: "sessions",
+    status: "experimental",
+    suggests: true,
+  },
+] as const satisfies readonly Experiment[];
+
+export type ExperimentId = (typeof EXPERIMENTS)[number]["id"];
+
+export const isExperimentId = (v: unknown): v is ExperimentId =>
+  EXPERIMENTS.some((e) => e.id === v);
+
+export const getExperiment = (id: ExperimentId): Experiment =>
+  EXPERIMENTS.find((e) => e.id === id)!;

package/lib/experiments/view.ts

@@ -0,0 +1,19 @@
+import { experimentOn, experimentsLocked } from ".";
+import { EXPERIMENTS, type Experiment, type ExperimentId } from "./registry";
+import { usageSummary, type UsageSummary } from "./meter";
+
+export interface ExperimentView extends Experiment {
+  on: boolean;
+  usage: UsageSummary;
+}
+
+export const experimentView = (id: ExperimentId): ExperimentView => ({
+  ...EXPERIMENTS.find((e) => e.id === id)!,
+  on: experimentOn(id),
+  usage: usageSummary(id),
+});
+
+export const experimentsView = () => ({
+  locked: experimentsLocked(),
+  experiments: EXPERIMENTS.map((e) => experimentView(e.id)),
+});

package/lib/recipes/settle.ts

@@ -11,6 +11,7 @@ import { blockedReason } from "../tasks/state";
 import { runClaude, type ClaudeRunner } from "../orchestrator/claude-cli";
 import { releasePieces } from "../plans/release";
 import { queueEvent } from "../orchestrator/events";
+import { sessionEnded } from "../experiments/hooks";
 import { untrusted } from "../orchestrator/untrusted";
 import {
   cleanOutputs,
@@ -172,8 +173,10 @@ function finish(
       run.id
     ).changes;
   idleSince.delete(run.id);
-  if (changed)
+  if (changed) {
     tell(run, status, summary, result, session?.pr_number ?? run.pr_number);
+    if (run.session_id) void sessionEnded(run.session_id, "recipe-settled");
+  }
   return changed === 1;
 }
 

package/lib/settings-store.ts

@@ -0,0 +1,22 @@
+import { getDb } from "@/lib/db";
+
+// The settings table: a key and a string, shared by every feature.
+export function getSetting(key: string): string | null {
+  const row = getDb()
+    .prepare(`SELECT value FROM settings WHERE key = ?`)
+    .get(key) as { value: string } | undefined;
+  return row?.value ?? null;
+}
+
+export function putSetting(key: string, value: string | null): void {
+  if (value === null) {
+    getDb().prepare(`DELETE FROM settings WHERE key = ?`).run(key);
+    return;
+  }
+  getDb()
+    .prepare(
+      `INSERT INTO settings (key, value) VALUES (?, ?)
+       ON CONFLICT(key) DO UPDATE SET value = excluded.value`
+    )
+    .run(key, value);
+}

package/lib/settings-url.ts

@@ -16,6 +16,7 @@ export const SETTINGS_SECTIONS = [
   "audits",
   "setup",
   "updates",
+  "experiments",
 ] as const;
 
 export type SettingsSection = (typeof SETTINGS_SECTIONS)[number];

package/lib/tasks/finish.ts

@@ -26,6 +26,7 @@ import { canSignOff } from "./state";
 import { forgetPR, getTaskSession, prFor, projectPathFor } from "./session";
 import { dropSessionDatabase } from "../project-config/database";
 import { captureMergedTask } from "../session-logs";
+import { sessionEnded } from "../experiments/hooks";
 
 // Never deleted, whatever a setting says.
 export const PROTECTED_BRANCHES = new Set(["main", "master", "HEAD"]);
@@ -163,6 +164,7 @@ export async function signOffTask(
   pushMerged(id);
   // Reads the plan file now: cleanup below removes the worktree.
   void captureMergedTask(id, { head: opts.head ?? pr!.head ?? null });
+  void sessionEnded(id, "merged");
   const head = opts.head ?? pr!.head;
   resolveMergedTaskAsks(
     id,

package/lib/tasks/session.ts

@@ -16,6 +16,7 @@ import { findPRStrict } from "./gh";
 import { currentBranch, git } from "../git-exec";
 import { lookupPR } from "./pr-poll";
 import { captureMergedTask } from "../session-logs";
+import { sessionEnded } from "../experiments/hooks";
 import { getOrchestrator } from "../orchestrator/home";
 import {
   pushCiFailed,
@@ -211,6 +212,7 @@ export async function prFor(
     if (pr.state === "MERGED") {
       if (session.pr_status !== "merged") pushMerged(session.id);
       void captureMergedTask(session.id, { head: pr.head });
+      void sessionEnded(session.id, "merged");
     }
   }
   // Only an answer just read from GitHub: a cached one may predate a

package/lib/updates/settings.ts

@@ -1,4 +1,4 @@
-import { getDb } from "@/lib/db";
+import { getSetting, putSetting } from "../settings-store";
 
 // Whether this server shows the update banner, and the checks behind it.
 // AGENTOS_UPDATE_BANNER=off pins it off, like the network settings.
@@ -17,22 +17,4 @@ export function bannerEnabled(env: Env = process.env): boolean {
 export const setBannerEnabled = (on: boolean) =>
   putSetting(KEY, on ? "1" : "0");
 
-export function getSetting(key: string): string | null {
-  const row = getDb()
-    .prepare(`SELECT value FROM settings WHERE key = ?`)
-    .get(key) as { value: string } | undefined;
-  return row?.value ?? null;
-}
-
-export function putSetting(key: string, value: string | null): void {
-  if (value === null) {
-    getDb().prepare(`DELETE FROM settings WHERE key = ?`).run(key);
-    return;
-  }
-  getDb()
-    .prepare(
-      `INSERT INTO settings (key, value) VALUES (?, ?)
-       ON CONFLICT(key) DO UPDATE SET value = excluded.value`
-    )
-    .run(key, value);
-}
+export { getSetting, putSetting } from "../settings-store";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saadnvd1/agent-os",
-  "version": "0.3.555",
+  "version": "0.3.557",
   "description": "Self-hosted web UI for managing Claude Code sessions",
   "bin": {
     "agent-os": "scripts/agent-os"