@saadnvd1/agent-os 0.3.556 → 0.3.558

package/components/Settings/ExperimentsSection.test.ts

@@ -0,0 +1,89 @@
+import "@/components/Chat/composer/test-dom";
+import { act, createElement } from "react";
+import { createRoot, type Root } from "react-dom/client";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+for (const key of ["Event", "CustomEvent", "FocusEvent", "HTMLButtonElement"])
+  Object.defineProperty(globalThis, key, {
+    value: (window as unknown as Record<string, unknown>)[key],
+    configurable: true,
+    writable: true,
+  });
+(
+  globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }
+).IS_REACT_ACT_ENVIRONMENT = true;
+
+const w = (calls: number) => ({
+  calls,
+  tokens: calls * 1000,
+  costUsd: calls * 0.01,
+  scripts: 0,
+  shown: 0,
+  accepted: 1,
+  dismissed: 1,
+  acceptRate: 0.5,
+});
+const state = vi.hoisted(() => ({ mutate: vi.fn() }));
+vi.mock("@/data/experiments", () => ({
+  useExperimentsQuery: () => ({
+    isPending: false,
+    isError: false,
+    data: {
+      locked: false,
+      experiments: [
+        {
+          id: "learning-cards",
+          name: "Learning cards",
+          description: "Suggest things when a session ends.",
+          area: "sessions",
+          status: "experimental",
+          suggests: true,
+          on: false,
+          usage: { today: w(1), d7: w(2), d30: w(3) },
+        },
+      ],
+    },
+  }),
+  useSetExperiment: () => ({ mutate: state.mutate }),
+  useExperimentDetailQuery: () => ({ isPending: true }),
+}));
+
+const { ExperimentsSection } = await import("./ExperimentsSection");
+
+let root: Root;
+let host: HTMLDivElement;
+beforeEach(() => {
+  state.mutate.mockReset();
+  host = document.createElement("div");
+  document.body.appendChild(host);
+  root = createRoot(host);
+});
+afterEach(() => {
+  act(() => root.unmount());
+  host.remove();
+});
+
+describe("ExperimentsSection", () => {
+  it("lists each experiment, off, with its usage", () => {
+    act(() => root.render(createElement(ExperimentsSection)));
+    expect(host.textContent).toContain("Learning cards");
+    expect(host.textContent).toContain("Suggest things when a session ends.");
+    expect(host.textContent).toContain("50% (1/2)");
+    expect(
+      host.querySelector('[role="switch"]')?.getAttribute("aria-checked")
+    ).toBe("false");
+  });
+
+  it("turns one on from its toggle", () => {
+    act(() => root.render(createElement(ExperimentsSection)));
+    act(() => {
+      host
+        .querySelector('[role="switch"]')
+        ?.dispatchEvent(new window.MouseEvent("click", { bubbles: true }));
+    });
+    expect(state.mutate).toHaveBeenCalledWith(
+      { id: "learning-cards", on: true },
+      expect.any(Object)
+    );
+  });
+});

package/components/Settings/ExperimentsSection.tsx

@@ -0,0 +1,225 @@
+"use client";
+
+import { useState } from "react";
+import { toast } from "sonner";
+import { ArrowLeft } from "lucide-react";
+import { Button } from "@/components/ui/button";
+import { SettingToggle } from "@/components/Settings/SettingToggle";
+import { formatTokens } from "@/lib/chat/context";
+import {
+  useExperimentDetailQuery,
+  useExperimentsQuery,
+  useSetExperiment,
+  type ExperimentView,
+} from "@/data/experiments";
+
+const usd = (n: number) => `$${n.toFixed(2)}`;
+const pct = (n: number | null) =>
+  n === null ? "-" : `${Math.round(n * 100)}%`;
+
+const WINDOWS = [
+  ["today", "Today"],
+  ["d7", "7 days"],
+  ["d30", "30 days"],
+] as const;
+
+function Usage({ e }: { e: ExperimentView }) {
+  return (
+    <table className="w-full text-xs tabular-nums">
+      <thead className="text-muted-foreground">
+        <tr>
+          <th className="py-1 text-left font-normal" />
+          <th className="py-1 text-right font-normal">Calls</th>
+          <th className="py-1 text-right font-normal">Tokens</th>
+          <th className="py-1 text-right font-normal">Cost</th>
+          {e.suggests && (
+            <th className="py-1 text-right font-normal">Accepted</th>
+          )}
+        </tr>
+      </thead>
+      <tbody>
+        {WINDOWS.map(([key, label]) => {
+          const w = e.usage[key];
+          return (
+            <tr key={key} className="border-border border-t">
+              <td className="py-1">{label}</td>
+              <td className="py-1 text-right">{w.calls}</td>
+              <td className="py-1 text-right">{formatTokens(w.tokens)}</td>
+              <td className="py-1 text-right">{usd(w.costUsd)}</td>
+              {e.suggests && (
+                <td className="py-1 text-right">
+                  {pct(w.acceptRate)}
+                  {w.accepted + w.dismissed > 0 &&
+                    ` (${w.accepted}/${w.accepted + w.dismissed})`}
+                </td>
+              )}
+            </tr>
+          );
+        })}
+      </tbody>
+    </table>
+  );
+}
+
+function Detail({ id, onBack }: { id: string; onBack: () => void }) {
+  const { data, isPending, isError, error } = useExperimentDetailQuery(id);
+  return (
+    <div className="space-y-4">
+      <Button
+        variant="ghost"
+        size="sm"
+        onClick={onBack}
+        className="min-h-11 md:min-h-0"
+      >
+        <ArrowLeft className="h-4 w-4" /> Experiments
+      </Button>
+      {isPending && (
+        <div className="bg-muted/40 h-24 animate-pulse rounded-lg" />
+      )}
+      {isError && <p className="text-destructive text-sm">{error.message}</p>}
+      {data && (
+        <>
+          <div>
+            <h3 className="text-sm font-medium">{data.name}</h3>
+            <p className="text-muted-foreground text-xs">{data.description}</p>
+          </div>
+          <Usage e={data} />
+          <Breakdowns title="By trigger" rows={data.detail.byTrigger} />
+          <Breakdowns title="By model" rows={data.detail.byModel} />
+          <section>
+            <h4 className="text-muted-foreground mb-1 text-xs">Last 30 days</h4>
+            <table className="w-full text-xs tabular-nums">
+              <tbody>
+                {data.detail.days
+                  .filter((d) => d.calls || d.accepted || d.dismissed)
+                  .reverse()
+                  .map((d) => (
+                    <tr key={d.day} className="border-border border-t">
+                      <td className="py-1">{d.day}</td>
+                      <td className="py-1 text-right">{d.calls} calls</td>
+                      <td className="py-1 text-right">
+                        {formatTokens(d.tokens)}
+                      </td>
+                      <td className="py-1 text-right">{usd(d.costUsd)}</td>
+                      <td className="py-1 text-right">
+                        {d.accepted}/{d.accepted + d.dismissed}
+                      </td>
+                    </tr>
+                  ))}
+                <tr className="border-border border-t font-medium">
+                  <td className="py-1">Total</td>
+                  <td className="py-1 text-right">
+                    {data.detail.days.reduce((t, d) => t + d.calls, 0)} calls
+                  </td>
+                  <td className="py-1 text-right">
+                    {formatTokens(
+                      data.detail.days.reduce((t, d) => t + d.tokens, 0)
+                    )}
+                  </td>
+                  <td className="py-1 text-right">
+                    {usd(data.detail.days.reduce((t, d) => t + d.costUsd, 0))}
+                  </td>
+                  <td />
+                </tr>
+              </tbody>
+            </table>
+          </section>
+          {data.detail.recent.length > 0 && (
+            <section>
+              <h4 className="text-muted-foreground mb-1 text-xs">Recent</h4>
+              <ul className="space-y-1 text-xs">
+                {data.detail.recent.map((r, i) => (
+                  <li key={i} className="flex justify-between gap-3">
+                    <span className="min-w-0 truncate">
+                      {new Date(r.at).toLocaleString()} · {r.event} ·{" "}
+                      {r.trigger}
+                      {r.detail ? ` · ${r.detail}` : ""}
+                    </span>
+                    <span className="text-muted-foreground tabular-nums">
+                      {r.tokens ? formatTokens(r.tokens) : ""}
+                    </span>
+                  </li>
+                ))}
+              </ul>
+            </section>
+          )}
+        </>
+      )}
+    </div>
+  );
+}
+
+function Breakdowns({
+  title,
+  rows,
+}: {
+  title: string;
+  rows: { key: string; calls: number; tokens: number; costUsd: number }[];
+}) {
+  if (!rows.length) return null;
+  return (
+    <section>
+      <h4 className="text-muted-foreground mb-1 text-xs">{title}</h4>
+      <table className="w-full text-xs tabular-nums">
+        <tbody>
+          {rows.map((r) => (
+            <tr key={r.key} className="border-border border-t">
+              <td className="py-1">{r.key}</td>
+              <td className="py-1 text-right">{r.calls} calls</td>
+              <td className="py-1 text-right">{formatTokens(r.tokens)}</td>
+              <td className="py-1 text-right">{usd(r.costUsd)}</td>
+            </tr>
+          ))}
+        </tbody>
+      </table>
+    </section>
+  );
+}
+
+export function ExperimentsSection() {
+  const { data, isPending, isError, error } = useExperimentsQuery();
+  const set = useSetExperiment();
+  const [open, setOpen] = useState<string | null>(null);
+
+  if (open) return <Detail id={open} onBack={() => setOpen(null)} />;
+  if (isPending)
+    return <div className="bg-muted/40 h-16 animate-pulse rounded-lg" />;
+  if (isError)
+    return <p className="text-destructive text-sm">{error.message}</p>;
+
+  return (
+    <div className="space-y-5">
+      <p className="text-muted-foreground text-sm">
+        Features still being tried. Each is off until you turn it on, does no
+        work while off, and shows what it costs here.
+      </p>
+      {data.experiments.map((e) => (
+        <div key={e.id} className="space-y-2">
+          <SettingToggle
+            title={`${e.name} · ${e.status === "beta" ? "Beta" : "Experimental"}`}
+            detail={e.description}
+            checked={e.on}
+            locked={data.locked}
+            onChange={(on) =>
+              set.mutate(
+                { id: e.id, on },
+                { onError: (err) => toast.error(err.message) }
+              )
+            }
+          />
+          <div className="px-3">
+            <Usage e={e} />
+            <Button
+              variant="link"
+              size="sm"
+              className="min-h-11 px-0 md:min-h-0"
+              onClick={() => setOpen(e.id)}
+            >
+              Details
+            </Button>
+          </div>
+        </div>
+      ))}
+    </div>
+  );
+}

package/components/Settings/SettingsDialog.test.ts

@@ -47,6 +47,9 @@ vi.mock("@/components/Audits/AuditsPanel", () => ({
 vi.mock("@/components/Updates/UpdatesSection", () => ({
   UpdatesSection: stub("updates"),
 }));
+vi.mock("./ExperimentsSection", () => ({
+  ExperimentsSection: stub("experiments"),
+}));
 vi.mock("./NotificationsSection", () => ({
   NotificationsSection: stub("notifications"),
 }));

package/components/Settings/SettingsDialog.tsx

@@ -14,6 +14,7 @@ import {
   LayoutGrid,
   ListChecks,
   ShieldOff,
+  FlaskConical,
   RefreshCw,
   Smartphone,
   X,
@@ -33,6 +34,7 @@ import { GlobalMergeSection } from "@/components/Merge/GlobalMergeSection";
 import { ModelsSection } from "@/components/Models/ModelsSection";
 import { SchedulesPanel } from "@/components/Schedules";
 import { UpdatesSection } from "@/components/Updates/UpdatesSection";
+import { ExperimentsSection } from "./ExperimentsSection";
 import { WorkspaceSettings } from "@/components/Workspaces";
 import { useSettingsUrl } from "@/hooks/useSettingsUrl";
 import { SETTINGS_SECTIONS, type SettingsSection } from "@/lib/settings-url";
@@ -106,6 +108,11 @@ export const SECTION_INFO: Record<
     hint: "Version, check for updates, the update banner",
     icon: RefreshCw,
   },
+  experiments: {
+    label: "Experiments",
+    hint: "Try features that aren't finished, and see what they cost",
+    icon: FlaskConical,
+  },
 };
 
 function SectionBody({
@@ -138,6 +145,8 @@ function SectionBody({
       return <SetupSection />;
     case "updates":
       return <UpdatesSection />;
+    case "experiments":
+      return <ExperimentsSection />;
   }
 }
 

package/components/Settings/commands.ts

@@ -3,6 +3,7 @@ import {
   ClipboardList,
   Clock,
   Cpu,
+  FlaskConical,
   GitMerge,
   LayoutGrid,
   RefreshCw,
@@ -111,5 +112,13 @@ export function settingsCommands(workspaceId: string | null): PaletteCommand[] {
       icon: RefreshCw,
       run: () => settingsUiActions.open("updates"),
     },
+    {
+      id: "app.experiments",
+      title: "Experiments",
+      group,
+      keywords: ["experiment", "labs", "beta", "feature", "try"],
+      icon: FlaskConical,
+      run: () => settingsUiActions.open("experiments"),
+    },
   ];
 }

package/components/Settings/entry-points.test.ts

@@ -28,6 +28,7 @@ describe("⌘K entry points", () => {
     ["app.schedules", "schedules"],
     ["app.audits", "audits"],
     ["app.updates", "updates"],
+    ["app.experiments", "experiments"],
   ] as const)("%s opens Settings at %s", (id, section) => {
     run(id);
     expect(settingsUi.open).toBe(true);

package/data/experiments/index.ts

@@ -0,0 +1,67 @@
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import type { ExperimentView } from "@/lib/experiments/view";
+import type { UsageDetail } from "@/lib/experiments/meter";
+import type { ExperimentId } from "@/lib/experiments/registry";
+
+export type { ExperimentView, UsageDetail };
+
+export interface ExperimentsList {
+  locked: boolean;
+  experiments: ExperimentView[];
+}
+
+export interface ExperimentDetail extends ExperimentView {
+  locked: boolean;
+  detail: UsageDetail;
+}
+
+export const experimentsKeys = {
+  all: ["experiments"] as const,
+  detail: (id: string) => ["experiments", id] as const,
+};
+
+async function json<T>(res: Response): Promise<T> {
+  const data = await res.json();
+  if (!res.ok) throw new Error(data.error || "Request failed");
+  return data;
+}
+
+export function useExperimentsQuery() {
+  return useQuery({
+    queryKey: experimentsKeys.all,
+    queryFn: async () => json<ExperimentsList>(await fetch("/api/experiments")),
+    staleTime: 10_000,
+  });
+}
+
+export function useExperimentDetailQuery(id: string | null) {
+  return useQuery({
+    queryKey: experimentsKeys.detail(id ?? ""),
+    queryFn: async () =>
+      json<ExperimentDetail>(await fetch(`/api/experiments/${id}`)),
+    enabled: !!id,
+    staleTime: 10_000,
+  });
+}
+
+// The client's one check: false until the list has loaded.
+export function useExperiment(id: ExperimentId): boolean {
+  const { data } = useExperimentsQuery();
+  return !!data?.experiments.find((e) => e.id === id)?.on;
+}
+
+export function useSetExperiment() {
+  const queryClient = useQueryClient();
+  return useMutation({
+    mutationFn: async ({ id, on }: { id: string; on: boolean }) =>
+      json<ExperimentView>(
+        await fetch(`/api/experiments/${id}`, {
+          method: "PUT",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ on }),
+        })
+      ),
+    onSuccess: () =>
+      queryClient.invalidateQueries({ queryKey: experimentsKeys.all }),
+  });
+}

package/data/learning-cards/index.ts

@@ -0,0 +1,79 @@
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { toast } from "sonner";
+import type { Card } from "@/lib/learning-cards/types";
+import { useExperiment } from "../experiments";
+
+export type { Card };
+
+export interface CardEdits {
+  lesson?: string;
+  brief?: string;
+  taskPrompt?: string;
+}
+
+export const learningCardsKeys = {
+  session: (sessionId: string) => ["learning-cards", sessionId] as const,
+};
+
+async function json<T>(res: Response): Promise<T> {
+  const data = await res.json();
+  if (!res.ok) throw new Error(data.error || "Request failed");
+  return data;
+}
+
+// Pending cards of a session; nothing is asked while the experiment is off.
+export function useLearningCards(sessionId: string) {
+  const enabled = useExperiment("learning-cards");
+  return useQuery({
+    queryKey: learningCardsKeys.session(sessionId),
+    queryFn: async () =>
+      (
+        await json<{ cards: Card[] }>(
+          await fetch(
+            `/api/sessions/${encodeURIComponent(sessionId)}/learning-cards`
+          )
+        )
+      ).cards,
+    enabled,
+    staleTime: 10_000,
+    refetchInterval: 30_000,
+  });
+}
+
+function useDecide(sessionId: string) {
+  const queryClient = useQueryClient();
+  return useMutation({
+    mutationFn: async (
+      v: { id: number; action: "apply" | "dismiss" } & CardEdits
+    ) =>
+      json<{ card: Card }>(
+        await fetch(`/api/learning-cards/${encodeURIComponent(String(v.id))}`, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify(v),
+        })
+      ),
+    onSuccess: (_d, v) => {
+      queryClient.invalidateQueries({
+        queryKey: learningCardsKeys.session(sessionId),
+      });
+      // A saved recipe or a new rule shows up in their own lists.
+      if (v.action === "apply") {
+        queryClient.invalidateQueries({ queryKey: ["recipes"] });
+        queryClient.invalidateQueries({ queryKey: ["orchestrator"] });
+      }
+    },
+    onError: (error) =>
+      toast.error(error instanceof Error ? error.message : "Couldn't do that"),
+  });
+}
+
+export const useApplyCard = (sessionId: string) => {
+  const m = useDecide(sessionId);
+  return {
+    ...m,
+    apply: (id: number, edits: CardEdits = {}) =>
+      m.mutate({ id, action: "apply", ...edits }),
+    dismiss: (id: number) => m.mutate({ id, action: "dismiss" }),
+  };
+};

package/lib/chat/store.ts

@@ -180,6 +180,22 @@ export function itemsOfKind<K extends ChatItem["kind"]>(
   ).map((r) => JSON.parse(r.data));
 }
 
+// Several kinds in the order they happened.
+export function itemsOfKinds<K extends ChatItem["kind"]>(
+  sessionId: string,
+  kinds: K[]
+): Extract<ChatItem, { kind: K }>[] {
+  flushItems();
+  return (
+    db
+      .prepare(
+        `SELECT data FROM chat_items WHERE session_id = ?
+         AND kind IN (${kinds.map(() => "?").join(",")}) ORDER BY seq`
+      )
+      .all(sessionId, ...kinds) as { data: string }[]
+  ).map((r) => JSON.parse(r.data));
+}
+
 // A chat's background tasks still running, or ended after `since` (ms): what
 // the composer lists, read without parsing the rest of a long conversation.
 export function recentTasks(

package/lib/db/migrations.ts

@@ -2028,6 +2028,65 @@ const migrations: Migration[] = [
       `);
     },
   },
+  {
+    id: 85,
+    name: "experiment_usage",
+    up: (db) => {
+      // What each experiment did: model calls with their cost, and the
+      // non-model work (scripts run, suggestions shown and decided).
+      db.exec(`
+        CREATE TABLE IF NOT EXISTS experiment_usage (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          experiment_id TEXT NOT NULL,
+          event TEXT NOT NULL,
+          trigger TEXT NOT NULL,
+          session_id TEXT,
+          workspace_id TEXT,
+          model TEXT,
+          provider TEXT,
+          input_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_write_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+          output_tokens INTEGER NOT NULL DEFAULT 0,
+          cost_usd REAL NOT NULL DEFAULT 0,
+          duration_ms INTEGER,
+          ok INTEGER NOT NULL DEFAULT 1,
+          detail TEXT,
+          at INTEGER NOT NULL
+        );
+        CREATE INDEX IF NOT EXISTS idx_experiment_usage_exp_at
+          ON experiment_usage(experiment_id, at);
+        CREATE INDEX IF NOT EXISTS idx_experiment_usage_session
+          ON experiment_usage(session_id, experiment_id, event);
+      `);
+    },
+  },
+  {
+    id: 86,
+    name: "learning_cards",
+    up: (db) => {
+      // End-of-session suggestions (save as recipe, keep a lesson). An
+      // accepted lesson row is the record of where it was applied.
+      db.exec(`
+        CREATE TABLE IF NOT EXISTS learning_cards (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id TEXT NOT NULL,
+          workspace_id TEXT,
+          kind TEXT NOT NULL,
+          status TEXT NOT NULL DEFAULT 'pending',
+          payload TEXT NOT NULL,
+          destination TEXT,
+          applied_ref TEXT,
+          shown_at INTEGER,
+          decided_at INTEGER,
+          created_at INTEGER NOT NULL,
+          UNIQUE (session_id, kind)
+        );
+        CREATE INDEX IF NOT EXISTS idx_learning_cards_dest
+          ON learning_cards(destination, decided_at);
+      `);
+    },
+  },
 ];
 
 // `upTo`: stop after this id (tests that start from an older database).

package/lib/done/index.ts

@@ -32,6 +32,7 @@ import { doneOnPeer, peerSessionLink } from "../hosts/peer-actions";
 import { hostLink } from "../hosts/remote-api";
 import { doneHeldTask } from "../tasks/held-remote";
 import { captureMergedTask } from "../session-logs";
+import { sessionEnded } from "../experiments/hooks";
 import { pushMerged } from "../notify/push/events";
 
 export interface DoneOptions {
@@ -112,6 +113,7 @@ function mark(s: Session, status: "merged" | "done", pr: TaskPR | null) {
       : `UPDATE sessions SET task_status = 'done' WHERE id = ?`
   ).run(s.id);
   syncTaskCardInBackground(s, status, pr);
+  void sessionEnded(s.id, status);
   if (status === "merged") {
     pushMerged(s.id);
     void captureMergedTask(s.id, { head: pr?.head });

package/lib/experiments/hooks.ts

@@ -0,0 +1,39 @@
+import { experimentOn } from ".";
+import { EXPERIMENTS, type ExperimentId } from "./registry";
+
+export type EndHow = "done" | "merged" | "recipe-settled";
+export type EndHook = (sessionId: string, how: EndHow) => Promise<void> | void;
+
+// What each experiment does when a session ends. An experiment adds itself
+// here; one that is off is never called.
+// Loaded on first use so the places that end sessions don't pull in the
+// experiment's code. A hook does its own once-only check in the database: this
+// process forgets what it has seen on a restart.
+export const END_HOOKS: Partial<Record<ExperimentId, EndHook>> = {
+  "learning-cards": async (id, how) =>
+    (await import("../learning-cards/analyze")).analyzeSession(id, how),
+};
+
+// A merge is seen again on every poll; each session is handed on once. A
+// recipe run settling is its own event: its session has usually ended already.
+const seen = new Set<string>();
+export const resetEndHooks = () => seen.clear();
+
+// Called where sessions end. Never makes the caller wait or fail.
+export function sessionEnded(sessionId: string, how: EndHow): Promise<void> {
+  const key = how === "recipe-settled" ? `${sessionId}:settled` : sessionId;
+  if (seen.has(key)) return Promise.resolve();
+  seen.add(key);
+  return (async () => {
+    await new Promise<void>((r) => setImmediate(r));
+    for (const { id } of EXPERIMENTS) {
+      const hook = END_HOOKS[id];
+      if (!hook || !experimentOn(id)) continue;
+      try {
+        await hook(sessionId, how);
+      } catch (error) {
+        console.error(`[experiments] ${id} on ${sessionId}:`, error);
+      }
+    }
+  })();
+}