debutly 0.0.0-stage → 0.2.0

package/src/store/pg-run-store.ts

@@ -0,0 +1,163 @@
+import { and, asc, eq, inArray } from "drizzle-orm";
+import type { CustomerUnit, Finding, Persona, Session, Step } from "../types";
+import type { Db } from "./db";
+import type { NewRun, RunStore, RunUsage, SideEffect, StoredRun } from "./run-store";
+import * as t from "./schema";
+
+const membersOf = (u: CustomerUnit): Persona[] => (u.kind === "individual" ? [u.person] : [u.owner, ...u.employees]);
+
+export class PgRunStore implements RunStore {
+  constructor(private readonly db: Db) {}
+
+  async createRun(r: NewRun): Promise<string> {
+    const [row] = await this.db
+      .insert(t.runs)
+      .values({
+        app: r.app ?? null,
+        trigger: r.trigger ?? null,
+        ciUrl: r.ciUrl ?? null,
+        targetUrl: r.targetUrl,
+        goal: r.goal,
+        icp: r.icp,
+        tier: r.tier,
+        unitKind: r.unitKind,
+        model: r.model,
+        estimateLowUsd: r.estimate?.low,
+        estimateHighUsd: r.estimate?.high,
+        status: "running",
+        startedAt: new Date(),
+      })
+      .returning({ id: t.runs.id });
+    return row.id;
+  }
+
+  async recordUnits(runId: string, units: CustomerUnit[]): Promise<void> {
+    if (!units.length) return;
+    await this.db.insert(t.customerUnits).values(units.map((u) => ({ runId, id: u.id, kind: u.kind, business: u.kind === "business" ? u.business : null })));
+    const people = units.flatMap((u) => membersOf(u).map((p) => ({ unitId: u.id, p })));
+    await this.db.insert(t.agents).values(
+      people.map(({ unitId, p }) => ({ runId, id: p.id, unitId, name: p.name, role: p.role ?? null, inclusion: p.inclusion ?? null, persona: p })),
+    );
+  }
+
+  async startSession(runId: string, agentId: string, goal: string): Promise<void> {
+    await this.db.insert(t.sessions).values({ runId, agentId, goal }).onConflictDoNothing();
+  }
+
+  async appendStep(runId: string, agentId: string, s: Step): Promise<void> {
+    await this.db
+      .insert(t.steps)
+      .values({
+        runId,
+        agentId,
+        index: s.index,
+        url: s.observation.url,
+        title: s.observation.title,
+        action: s.action,
+        reasoning: s.reasoning,
+        observation: s.observation,
+        signals: s.signals,
+        error: s.error ?? null,
+        blocked: s.blocked ?? null,
+        screenshotUrl: s.screenshotPath ?? null,
+      })
+      .onConflictDoNothing();
+  }
+
+  async finishSession(runId: string, s: Session): Promise<void> {
+    const values = {
+      outcome: s.outcome,
+      outcomeNote: s.outcomeNote,
+      inputTokens: s.usage?.inputTokens ?? null,
+      outputTokens: s.usage?.outputTokens ?? null,
+      costUsd: s.usage?.costUsd ?? null,
+      finishedAt: new Date(),
+    };
+    await this.db
+      .insert(t.sessions)
+      .values({ runId, agentId: s.agentId, goal: s.goal, ...values })
+      .onConflictDoUpdate({ target: [t.sessions.runId, t.sessions.agentId], set: values });
+  }
+
+  async finishRun(
+    runId: string,
+    r: { status: "finished" | "failed" | "cancelled"; findings: Finding[]; reportMd: string; usage: RunUsage; sideEffects: SideEffect[] },
+  ): Promise<void> {
+    if (r.findings.length) {
+      await this.db.insert(t.findings).values(
+        r.findings.map((f) => ({
+          runId,
+          signature: f.signature,
+          kind: f.kind,
+          severity: f.severity,
+          confidence: f.confidence,
+          title: f.title,
+          detail: f.detail,
+          pagePath: f.pagePath,
+          hitBy: f.hitBy,
+          visitors: f.visitors,
+          evidence: f.evidence,
+          sandboxCause: f.sandboxCause ?? null,
+        })),
+      );
+    }
+    if (r.sideEffects.length) await this.db.insert(t.sideEffects).values(r.sideEffects.map((e) => ({ runId, ...e })));
+    await this.db
+      .update(t.runs)
+      .set({
+        status: r.status,
+        reportMd: r.reportMd,
+        inputTokens: r.usage.inputTokens,
+        outputTokens: r.usage.outputTokens,
+        costUsd: r.usage.costUsd ?? null,
+        finishedAt: new Date(),
+      })
+      .where(eq(t.runs.id, runId));
+  }
+
+  async requestCancel(runId: string): Promise<void> {
+    await this.db
+      .update(t.runs)
+      .set({ status: "cancelling" })
+      .where(and(eq(t.runs.id, runId), inArray(t.runs.status, ["queued", "running"])));
+  }
+
+  async isCancelled(runId: string): Promise<boolean> {
+    const [row] = await this.db.select({ status: t.runs.status }).from(t.runs).where(eq(t.runs.id, runId));
+    return row?.status === "cancelling" || row?.status === "cancelled";
+  }
+
+  async getRun(runId: string): Promise<StoredRun | null> {
+    const [run] = await this.db.select().from(t.runs).where(eq(t.runs.id, runId));
+    if (!run) return null;
+    const [units, agents, sessions, steps, findings, sideEffects] = await Promise.all([
+      this.db.select().from(t.customerUnits).where(eq(t.customerUnits.runId, runId)).orderBy(asc(t.customerUnits.id)),
+      this.db.select().from(t.agents).where(eq(t.agents.runId, runId)).orderBy(asc(t.agents.id)),
+      this.db.select().from(t.sessions).where(eq(t.sessions.runId, runId)).orderBy(asc(t.sessions.agentId)),
+      this.db.select().from(t.steps).where(eq(t.steps.runId, runId)).orderBy(asc(t.steps.agentId), asc(t.steps.index)),
+      this.db.select().from(t.findings).where(eq(t.findings.runId, runId)),
+      this.db.select().from(t.sideEffects).where(eq(t.sideEffects.runId, runId)).orderBy(asc(t.sideEffects.id)),
+    ]);
+    return {
+      ...run,
+      units,
+      agents,
+      sessions,
+      steps,
+      findings: findings.map((f) => ({
+        signature: f.signature,
+        kind: f.kind as Finding["kind"],
+        severity: f.severity as Finding["severity"],
+        confidence: f.confidence as Finding["confidence"],
+        title: f.title,
+        detail: f.detail,
+        pagePath: f.pagePath,
+        hitBy: f.hitBy,
+        visitors: f.visitors,
+        evidence: f.evidence as Finding["evidence"],
+        ...(f.sandboxCause ? { sandboxCause: f.sandboxCause } : {}),
+      })),
+      sideEffects: sideEffects.map(({ outcome, rule, channel, host, url, recipient, summary, at }) => ({ outcome, rule, channel, host, url, recipient, summary, at })),
+    };
+  }
+}

package/src/store/predictions.ts

@@ -0,0 +1,70 @@
+import { and, desc, eq, isNotNull, isNull } from "drizzle-orm";
+import type { Db } from "./db";
+import * as t from "./schema";
+
+export type PredictionMetric = "winner" | "top_objection" | "intent_share";
+
+export interface NewPrediction {
+  metric: PredictionMetric;
+  claim: string;
+  predicted: Record<string, unknown>;
+}
+
+export interface StoredPrediction extends NewPrediction {
+  id: string;
+  runId: string;
+  actual: unknown;
+  correct: boolean | null;
+  createdAt: Date;
+  checkedAt: Date | null;
+}
+
+/** Whether a real outcome matches the prediction; null where we can't score it yet. */
+function isCorrect(metric: string, predicted: Record<string, unknown>, actual: string): boolean | null {
+  if (metric === "winner") return String(predicted.winner) === actual.trim();
+  if (metric === "top_objection") return String(predicted.category) === actual.trim();
+  // Shares (e.g. intent) are not conversion rates; they are scored once calibration exists.
+  return null;
+}
+
+/** Calibration loop: verdicts as predictions, checked against real outcomes. */
+export class PgPredictionStore {
+  constructor(private readonly db: Db) {}
+
+  async add(runId: string, items: NewPrediction[]): Promise<string[]> {
+    if (!items.length) return [];
+    const rows = await this.db
+      .insert(t.predictions)
+      .values(items.map((p) => ({ runId, metric: p.metric, claim: p.claim, predicted: p.predicted })))
+      .returning({ id: t.predictions.id });
+    return rows.map((r) => r.id);
+  }
+
+  async list(o: { unresolved?: boolean } = {}): Promise<StoredPrediction[]> {
+    const rows = await this.db
+      .select()
+      .from(t.predictions)
+      .where(o.unresolved ? isNull(t.predictions.checkedAt) : undefined)
+      .orderBy(desc(t.predictions.createdAt));
+    return rows.map((r) => ({ ...r, metric: r.metric as PredictionMetric, predicted: r.predicted as Record<string, unknown> }));
+  }
+
+  /** Records what really happened (e.g. "B" won the real A/B test, or "price" was the top objection in sales calls). */
+  async record(id: string, actual: string): Promise<StoredPrediction> {
+    const [row] = await this.db.select().from(t.predictions).where(eq(t.predictions.id, id));
+    if (!row) throw new Error(`No prediction ${id}`);
+    const correct = isCorrect(row.metric, row.predicted as Record<string, unknown>, actual);
+    const [updated] = await this.db
+      .update(t.predictions)
+      .set({ actual, correct, checkedAt: new Date() })
+      .where(eq(t.predictions.id, id))
+      .returning();
+    return { ...updated, metric: updated.metric as PredictionMetric, predicted: updated.predicted as Record<string, unknown> };
+  }
+
+  async accuracy(): Promise<{ scored: number; correct: number; rate: number | null }> {
+    const scored = await this.db.select({ correct: t.predictions.correct }).from(t.predictions).where(and(isNotNull(t.predictions.correct)));
+    const correct = scored.filter((r) => r.correct).length;
+    return { scored: scored.length, correct, rate: scored.length ? correct / scored.length : null };
+  }
+}

package/src/store/run-store.ts

@@ -0,0 +1,78 @@
+import type { CustomerUnit, Finding, Session, Step } from "../types";
+
+export interface NewRun {
+  /** Which app is under test, e.g. "acme". */
+  app?: string;
+  /** nightly | pr | manual | local */
+  trigger?: string;
+  /** The CI run behind this one, e.g. its GitHub Actions page. */
+  ciUrl?: string;
+  targetUrl: string;
+  goal: string;
+  icp?: string;
+  tier?: string;
+  unitKind?: string;
+  model?: string;
+  estimate?: { low: number; high: number } | null;
+}
+
+export interface RunUsage {
+  inputTokens: number;
+  outputTokens: number;
+  costUsd?: number;
+}
+
+/** Something the app under test tried to send or reach, from its sandbox outbox. */
+export interface SideEffect {
+  outcome: string;
+  rule: string | null;
+  channel: string;
+  host: string;
+  url: string;
+  recipient: string | null;
+  summary: string | null;
+  at: Date;
+}
+
+export interface StoredRun {
+  id: string;
+  app: string | null;
+  trigger: string | null;
+  ciUrl: string | null;
+  targetUrl: string;
+  goal: string;
+  tier: string | null;
+  status: string;
+  model: string | null;
+  estimateLowUsd: number | null;
+  estimateHighUsd: number | null;
+  costUsd: number | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
+  reportMd: string | null;
+  createdAt: Date;
+  finishedAt: Date | null;
+  units: { id: string; kind: string; business: unknown }[];
+  agents: { id: string; unitId: string; name: string; role: string | null; inclusion: string | null }[];
+  sessions: { agentId: string; goal: string; outcome: string | null; outcomeNote: string | null; inputTokens: number | null; outputTokens: number | null; costUsd: number | null }[];
+  steps: { agentId: string; index: number; url: string; action: unknown; reasoning: string | null; error: string | null; blocked: string | null; screenshotUrl: string | null }[];
+  findings: (Finding & { sandboxCause?: string[] })[];
+  sideEffects: SideEffect[];
+}
+
+/** Records a run while it happens, so it can be watched live, compared and audited. */
+export interface RunStore {
+  createRun(r: NewRun): Promise<string>;
+  recordUnits(runId: string, units: CustomerUnit[]): Promise<void>;
+  startSession(runId: string, agentId: string, goal: string): Promise<void>;
+  appendStep(runId: string, agentId: string, step: Step): Promise<void>;
+  finishSession(runId: string, session: Session): Promise<void>;
+  finishRun(
+    runId: string,
+    r: { status: "finished" | "failed" | "cancelled"; findings: Finding[]; reportMd: string; usage: RunUsage; sideEffects: SideEffect[] },
+  ): Promise<void>;
+  /** The run kill switch: agents stop at their next step. */
+  requestCancel(runId: string): Promise<void>;
+  isCancelled(runId: string): Promise<boolean>;
+  getRun(runId: string): Promise<StoredRun | null>;
+}

package/src/store/schema.ts

@@ -0,0 +1,193 @@
+// Postgres schema for stored runs. Organizations, apps as records and
+// multi-tenancy arrive later; runs name their app, and the dashboard reads all of this.
+import { sql } from "drizzle-orm";
+import { boolean, customType, doublePrecision, index, integer, jsonb, pgTable, primaryKey, text, timestamp, uuid } from "drizzle-orm/pg-core";
+
+export const runs = pgTable(
+  "runs",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    /** Which app was tested (e.g. "acme"), so the dashboard can group runs by it. */
+    app: text("app"),
+    /** nightly | pr | manual | local: what started the run. */
+    trigger: text("trigger"),
+    /** The CI run behind this one (e.g. its GitHub Actions page), when there is one. */
+    ciUrl: text("ci_url"),
+    targetUrl: text("target_url").notNull(),
+    goal: text("goal").notNull(),
+    icp: text("icp"),
+    tier: text("tier"),
+    unitKind: text("unit_kind"),
+    /** queued | running | cancelling | cancelled | finished | failed */
+    status: text("status").notNull().default("queued"),
+    model: text("model"),
+    estimateLowUsd: doublePrecision("estimate_low_usd"),
+    estimateHighUsd: doublePrecision("estimate_high_usd"),
+    costUsd: doublePrecision("cost_usd"),
+    inputTokens: integer("input_tokens"),
+    outputTokens: integer("output_tokens"),
+    reportMd: text("report_md"),
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+    startedAt: timestamp("started_at", { withTimezone: true }),
+    finishedAt: timestamp("finished_at", { withTimezone: true }),
+  },
+  (t) => [index("runs_created_at_idx").on(t.createdAt.desc()), index("runs_app_created_at_idx").on(t.app, t.createdAt.desc())],
+);
+
+// postgres-js returns a Buffer, PGlite a Uint8Array: hand back a Buffer either way.
+const bytea = customType<{ data: Buffer; driverData: Buffer | Uint8Array }>({
+  dataType: () => "bytea",
+  fromDriver: (value) => (Buffer.isBuffer(value) ? value : Buffer.from(value)),
+});
+
+/**
+ * Step screenshots kept in the database, for a deployment with no object storage (a free
+ * Postgres is enough to run the dashboard). Compact JPEGs, pruned after a few days.
+ */
+export const screenshots = pgTable(
+  "screenshots",
+  {
+    runId: uuid("run_id")
+      .notNull()
+      .references(() => runs.id, { onDelete: "cascade" }),
+    key: text("key").notNull(),
+    contentType: text("content_type").notNull(),
+    data: bytea("data").notNull(),
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  },
+  (t) => [primaryKey({ columns: [t.runId, t.key] }), index("screenshots_created_at_idx").on(t.createdAt)],
+);
+
+export const customerUnits = pgTable(
+  "customer_units",
+  {
+    runId: uuid("run_id")
+      .notNull()
+      .references(() => runs.id, { onDelete: "cascade" }),
+    id: text("id").notNull(),
+    kind: text("kind").notNull(),
+    business: jsonb("business"),
+  },
+  (t) => [primaryKey({ columns: [t.runId, t.id] })],
+);
+
+export const agents = pgTable(
+  "agents",
+  {
+    runId: uuid("run_id")
+      .notNull()
+      .references(() => runs.id, { onDelete: "cascade" }),
+    id: text("id").notNull(),
+    unitId: text("unit_id").notNull(),
+    name: text("name").notNull(),
+    role: text("role"),
+    inclusion: text("inclusion"),
+    persona: jsonb("persona").notNull(),
+  },
+  (t) => [primaryKey({ columns: [t.runId, t.id] })],
+);
+
+export const sessions = pgTable(
+  "sessions",
+  {
+    runId: uuid("run_id")
+      .notNull()
+      .references(() => runs.id, { onDelete: "cascade" }),
+    agentId: text("agent_id").notNull(),
+    goal: text("goal").notNull(),
+    outcome: text("outcome"),
+    outcomeNote: text("outcome_note"),
+    inputTokens: integer("input_tokens"),
+    outputTokens: integer("output_tokens"),
+    costUsd: doublePrecision("cost_usd"),
+    startedAt: timestamp("started_at", { withTimezone: true }).notNull().defaultNow(),
+    finishedAt: timestamp("finished_at", { withTimezone: true }),
+  },
+  (t) => [primaryKey({ columns: [t.runId, t.agentId] })],
+);
+
+export const steps = pgTable(
+  "steps",
+  {
+    runId: uuid("run_id")
+      .notNull()
+      .references(() => runs.id, { onDelete: "cascade" }),
+    agentId: text("agent_id").notNull(),
+    index: integer("index").notNull(),
+    url: text("url").notNull(),
+    title: text("title"),
+    action: jsonb("action").notNull(),
+    reasoning: text("reasoning"),
+    observation: jsonb("observation").notNull(),
+    signals: jsonb("signals").notNull(),
+    error: text("error"),
+    blocked: text("blocked"),
+    screenshotUrl: text("screenshot_url"),
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  },
+  (t) => [primaryKey({ columns: [t.runId, t.agentId, t.index] })],
+);
+
+export const findings = pgTable(
+  "findings",
+  {
+    runId: uuid("run_id")
+      .notNull()
+      .references(() => runs.id, { onDelete: "cascade" }),
+    signature: text("signature").notNull(),
+    kind: text("kind").notNull(),
+    severity: text("severity").notNull(),
+    confidence: text("confidence").notNull(),
+    title: text("title").notNull(),
+    detail: text("detail").notNull(),
+    pagePath: text("page_path").notNull(),
+    hitBy: text("hit_by").array().notNull(),
+    visitors: integer("visitors").notNull(),
+    evidence: jsonb("evidence").notNull(),
+    sandboxCause: text("sandbox_cause").array(),
+  },
+  (t) => [primaryKey({ columns: [t.runId, t.signature] })],
+);
+
+/** What the app under test tried to send or reach, from its sandbox outbox. */
+export const sideEffects = pgTable(
+  "side_effects",
+  {
+    id: integer("id").primaryKey().generatedAlwaysAsIdentity(),
+    runId: uuid("run_id")
+      .notNull()
+      .references(() => runs.id, { onDelete: "cascade" }),
+    outcome: text("outcome").notNull(),
+    rule: text("rule"),
+    channel: text("channel").notNull(),
+    host: text("host").notNull(),
+    url: text("url").notNull(),
+    recipient: text("recipient"),
+    summary: text("summary"),
+    at: timestamp("at", { withTimezone: true }).notNull().default(sql`now()`),
+  },
+  (t) => [index("side_effects_run_idx").on(t.runId)],
+);
+
+/**
+ * Calibration loop: every verdict is a prediction; when the real outcome is known it
+ * is recorded, and accuracy is tracked over time.
+ */
+export const predictions = pgTable(
+  "predictions",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    runId: uuid("run_id")
+      .notNull()
+      .references(() => runs.id, { onDelete: "cascade" }),
+    /** winner | top_objection | intent_share */
+    metric: text("metric").notNull(),
+    claim: text("claim").notNull(),
+    predicted: jsonb("predicted").notNull(),
+    actual: jsonb("actual"),
+    correct: boolean("correct"),
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+    checkedAt: timestamp("checked_at", { withTimezone: true }),
+  },
+  (t) => [index("predictions_run_idx").on(t.runId)],
+);

package/src/store/url.ts

@@ -0,0 +1,15 @@
+/**
+ * A Postgres URL as postgres.js should get it. postgres.js sends query parameters it doesn't
+ * know to the server as settings, and the server rejects libpq-only ones such as
+ * channel_binding, which Neon's connection strings carry. Dropping it keeps the connection on
+ * TLS (sslmode stays), so a connection string can be pasted exactly as the host gives it.
+ */
+export function postgresUrl(url: string): string {
+  const q = url.indexOf("?");
+  if (q < 0) return url;
+  const params = url
+    .slice(q + 1)
+    .split("&")
+    .filter((p) => p && !/^channel_binding=/i.test(p));
+  return params.length ? `${url.slice(0, q)}?${params.join("&")}` : url.slice(0, q);
+}