@jonsoc/console-app 1.1.34

package/src/routes/bench/[id].tsx

@@ -0,0 +1,365 @@
+import { Title } from "@solidjs/meta"
+import { createAsync, query, useParams } from "@solidjs/router"
+import { createSignal, For, Show } from "solid-js"
+import { Database, desc, eq } from "@jonsoc/console-core/drizzle/index.js"
+import { BenchmarkTable } from "@jonsoc/console-core/schema/benchmark.sql.js"
+
+interface TaskSource {
+  repo: string
+  from: string
+  to: string
+}
+
+interface Judge {
+  score: number
+  rationale: string
+  judge: string
+}
+
+interface ScoreDetail {
+  criterion: string
+  weight: number
+  average: number
+  variance?: number
+  judges?: Judge[]
+}
+
+interface RunUsage {
+  input: number
+  output: number
+  cost: number
+}
+
+interface Run {
+  task: string
+  model: string
+  agent: string
+  score: {
+    final: number
+    base: number
+    penalty: number
+  }
+  scoreDetails: ScoreDetail[]
+  usage?: RunUsage
+  duration?: number
+}
+
+interface Prompt {
+  commit: string
+  prompt: string
+}
+
+interface AverageUsage {
+  input: number
+  output: number
+  cost: number
+}
+
+interface Task {
+  averageScore: number
+  averageDuration?: number
+  averageUsage?: AverageUsage
+  model?: string
+  agent?: string
+  summary?: string
+  runs?: Run[]
+  task: {
+    id: string
+    source: TaskSource
+    prompts?: Prompt[]
+  }
+}
+
+interface BenchmarkResult {
+  averageScore: number
+  tasks: Task[]
+}
+
+async function getTaskDetail(benchmarkId: string, taskId: string) {
+  "use server"
+  const rows = await Database.use((tx) =>
+    tx.select().from(BenchmarkTable).where(eq(BenchmarkTable.id, benchmarkId)).limit(1),
+  )
+  if (!rows[0]) return null
+  const parsed = JSON.parse(rows[0].result) as BenchmarkResult
+  const task = parsed.tasks.find((t) => t.task.id === taskId)
+  return task ?? null
+}
+
+const queryTaskDetail = query(getTaskDetail, "benchmark.task.detail")
+
+function formatDuration(ms: number): string {
+  const seconds = Math.floor(ms / 1000)
+  const minutes = Math.floor(seconds / 60)
+  const remainingSeconds = seconds % 60
+  if (minutes > 0) {
+    return `${minutes}m ${remainingSeconds}s`
+  }
+  return `${remainingSeconds}s`
+}
+
+export default function BenchDetail() {
+  const params = useParams()
+  const [benchmarkId, taskId] = (params.id ?? "").split(":")
+  const task = createAsync(() => queryTaskDetail(benchmarkId, taskId))
+
+  return (
+    <main data-page="bench-detail">
+      <Title>Benchmark - {taskId}</Title>
+      <div style={{ padding: "1rem" }}>
+        <Show when={task()} fallback={<p>Task not found</p>}>
+          <div style={{ "margin-bottom": "1rem" }}>
+            <div>
+              <strong>Agent: </strong>
+              {task()?.agent ?? "N/A"}
+            </div>
+            <div>
+              <strong>Model: </strong>
+              {task()?.model ?? "N/A"}
+            </div>
+            <div>
+              <strong>Task: </strong>
+              {task()!.task.id}
+            </div>
+          </div>
+
+          <div style={{ "margin-bottom": "1rem" }}>
+            <div>
+              <strong>Repo: </strong>
+              <a
+                href={`https://github.com/${task()!.task.source.repo}`}
+                target="_blank"
+                rel="noopener noreferrer"
+                style={{ color: "#0066cc" }}
+              >
+                {task()!.task.source.repo}
+              </a>
+            </div>
+            <div>
+              <strong>From: </strong>
+              <a
+                href={`https://github.com/${task()!.task.source.repo}/commit/${task()!.task.source.from}`}
+                target="_blank"
+                rel="noopener noreferrer"
+                style={{ color: "#0066cc" }}
+              >
+                {task()!.task.source.from.slice(0, 7)}
+              </a>
+            </div>
+            <div>
+              <strong>To: </strong>
+              <a
+                href={`https://github.com/${task()!.task.source.repo}/commit/${task()!.task.source.to}`}
+                target="_blank"
+                rel="noopener noreferrer"
+                style={{ color: "#0066cc" }}
+              >
+                {task()!.task.source.to.slice(0, 7)}
+              </a>
+            </div>
+          </div>
+
+          <Show when={task()?.task.prompts && task()!.task.prompts!.length > 0}>
+            <div style={{ "margin-bottom": "1rem" }}>
+              <strong>Prompt:</strong>
+              <For each={task()!.task.prompts}>
+                {(p) => (
+                  <div style={{ "margin-top": "0.5rem" }}>
+                    <div style={{ "font-size": "0.875rem", color: "#666" }}>Commit: {p.commit.slice(0, 7)}</div>
+                    <p style={{ "margin-top": "0.25rem", "white-space": "pre-wrap" }}>{p.prompt}</p>
+                  </div>
+                )}
+              </For>
+            </div>
+          </Show>
+
+          <hr style={{ margin: "1rem 0", border: "none", "border-top": "1px solid #ccc" }} />
+
+          <div style={{ "margin-bottom": "1rem" }}>
+            <div>
+              <strong>Average Duration: </strong>
+              {task()?.averageDuration ? formatDuration(task()!.averageDuration!) : "N/A"}
+            </div>
+            <div>
+              <strong>Average Score: </strong>
+              {task()?.averageScore?.toFixed(3) ?? "N/A"}
+            </div>
+            <div>
+              <strong>Average Cost: </strong>
+              {task()?.averageUsage?.cost ? `$${task()!.averageUsage!.cost.toFixed(4)}` : "N/A"}
+            </div>
+          </div>
+
+          <Show when={task()?.summary}>
+            <div style={{ "margin-bottom": "1rem" }}>
+              <strong>Summary:</strong>
+              <p style={{ "margin-top": "0.5rem", "white-space": "pre-wrap" }}>{task()!.summary}</p>
+            </div>
+          </Show>
+
+          <Show when={task()?.runs && task()!.runs!.length > 0}>
+            <div style={{ "margin-bottom": "1rem" }}>
+              <strong>Runs:</strong>
+              <table style={{ "margin-top": "0.5rem", "border-collapse": "collapse", width: "100%" }}>
+                <thead>
+                  <tr>
+                    <th style={{ border: "1px solid #ccc", padding: "0.5rem", "text-align": "left" }}>Run</th>
+                    <th
+                      style={{
+                        border: "1px solid #ccc",
+                        padding: "0.5rem",
+                        "text-align": "left",
+                        "white-space": "nowrap",
+                      }}
+                    >
+                      Score (Base - Penalty)
+                    </th>
+                    <th style={{ border: "1px solid #ccc", padding: "0.5rem", "text-align": "left" }}>Cost</th>
+                    <th style={{ border: "1px solid #ccc", padding: "0.5rem", "text-align": "left" }}>Duration</th>
+                    <For each={task()!.runs![0]?.scoreDetails}>
+                      {(detail) => (
+                        <th style={{ border: "1px solid #ccc", padding: "0.5rem", "text-align": "left" }}>
+                          {detail.criterion} ({detail.weight})
+                        </th>
+                      )}
+                    </For>
+                  </tr>
+                </thead>
+                <tbody>
+                  <For each={task()!.runs}>
+                    {(run, index) => (
+                      <tr>
+                        <td style={{ border: "1px solid #ccc", padding: "0.5rem" }}>{index() + 1}</td>
+                        <td style={{ border: "1px solid #ccc", padding: "0.5rem", "white-space": "nowrap" }}>
+                          {run.score.final.toFixed(3)} ({run.score.base.toFixed(3)} - {run.score.penalty.toFixed(3)})
+                        </td>
+                        <td style={{ border: "1px solid #ccc", padding: "0.5rem" }}>
+                          {run.usage?.cost ? `$${run.usage.cost.toFixed(4)}` : "N/A"}
+                        </td>
+                        <td style={{ border: "1px solid #ccc", padding: "0.5rem" }}>
+                          {run.duration ? formatDuration(run.duration) : "N/A"}
+                        </td>
+                        <For each={run.scoreDetails}>
+                          {(detail) => (
+                            <td style={{ border: "1px solid #ccc", padding: "0.5rem" }}>
+                              <For each={detail.judges}>
+                                {(judge) => (
+                                  <span
+                                    style={{
+                                      color: judge.score === 1 ? "green" : judge.score === 0 ? "red" : "inherit",
+                                      "margin-right": "0.25rem",
+                                    }}
+                                  >
+                                    {judge.score === 1 ? "✓" : judge.score === 0 ? "✗" : judge.score}
+                                  </span>
+                                )}
+                              </For>
+                            </td>
+                          )}
+                        </For>
+                      </tr>
+                    )}
+                  </For>
+                </tbody>
+              </table>
+              <For each={task()!.runs}>
+                {(run, index) => (
+                  <div style={{ "margin-top": "1rem" }}>
+                    <h3 style={{ margin: "0 0 0.5rem 0" }}>Run {index() + 1}</h3>
+                    <div>
+                      <strong>Score: </strong>
+                      {run.score.final.toFixed(3)} (Base: {run.score.base.toFixed(3)} - Penalty:{" "}
+                      {run.score.penalty.toFixed(3)})
+                    </div>
+                    <For each={run.scoreDetails}>
+                      {(detail) => (
+                        <div style={{ "margin-top": "1rem", "padding-left": "1rem", "border-left": "2px solid #ccc" }}>
+                          <div>
+                            {detail.criterion} (weight: {detail.weight}){" "}
+                            <For each={detail.judges}>
+                              {(judge) => (
+                                <span
+                                  style={{
+                                    color: judge.score === 1 ? "green" : judge.score === 0 ? "red" : "inherit",
+                                    "margin-right": "0.25rem",
+                                  }}
+                                >
+                                  {judge.score === 1 ? "✓" : judge.score === 0 ? "✗" : judge.score}
+                                </span>
+                              )}
+                            </For>
+                          </div>
+                          <Show when={detail.judges && detail.judges.length > 0}>
+                            <For each={detail.judges}>
+                              {(judge) => {
+                                const [expanded, setExpanded] = createSignal(false)
+                                return (
+                                  <div style={{ "margin-top": "0.5rem", "padding-left": "1rem" }}>
+                                    <div
+                                      style={{ "font-size": "0.875rem", cursor: "pointer" }}
+                                      onClick={() => setExpanded(!expanded())}
+                                    >
+                                      <span style={{ "margin-right": "0.5rem" }}>{expanded() ? "▼" : "▶"}</span>
+                                      <span
+                                        style={{
+                                          color: judge.score === 1 ? "green" : judge.score === 0 ? "red" : "inherit",
+                                        }}
+                                      >
+                                        {judge.score === 1 ? "✓" : judge.score === 0 ? "✗" : judge.score}
+                                      </span>{" "}
+                                      {judge.judge}
+                                    </div>
+                                    <Show when={expanded()}>
+                                      <p
+                                        style={{
+                                          margin: "0.25rem 0 0 0",
+                                          "white-space": "pre-wrap",
+                                          "font-size": "0.875rem",
+                                        }}
+                                      >
+                                        {judge.rationale}
+                                      </p>
+                                    </Show>
+                                  </div>
+                                )
+                              }}
+                            </For>
+                          </Show>
+                        </div>
+                      )}
+                    </For>
+                  </div>
+                )}
+              </For>
+            </div>
+          </Show>
+
+          {(() => {
+            const [jsonExpanded, setJsonExpanded] = createSignal(false)
+            return (
+              <div style={{ "margin-top": "1rem" }}>
+                <button
+                  style={{
+                    cursor: "pointer",
+                    padding: "0.75rem 1.5rem",
+                    "font-size": "1rem",
+                    background: "#f0f0f0",
+                    border: "1px solid #ccc",
+                    "border-radius": "4px",
+                  }}
+                  onClick={() => setJsonExpanded(!jsonExpanded())}
+                >
+                  <span style={{ "margin-right": "0.5rem" }}>{jsonExpanded() ? "▼" : "▶"}</span>
+                  Raw JSON
+                </button>
+                <Show when={jsonExpanded()}>
+                  <pre>{JSON.stringify(task(), null, 2)}</pre>
+                </Show>
+              </div>
+            )
+          })()}
+        </Show>
+      </div>
+    </main>
+  )
+}

package/src/routes/bench/index.tsx

@@ -0,0 +1,86 @@
+import { Title } from "@solidjs/meta"
+import { A, createAsync, query } from "@solidjs/router"
+import { createMemo, For, Show } from "solid-js"
+import { Database, desc } from "@jonsoc/console-core/drizzle/index.js"
+import { BenchmarkTable } from "@jonsoc/console-core/schema/benchmark.sql.js"
+
+interface BenchmarkResult {
+  averageScore: number
+  tasks: { averageScore: number; task: { id: string } }[]
+}
+
+async function getBenchmarks() {
+  "use server"
+  const rows = await Database.use((tx) =>
+    tx.select().from(BenchmarkTable).orderBy(desc(BenchmarkTable.timeCreated)).limit(100),
+  )
+  return rows.map((row) => {
+    const parsed = JSON.parse(row.result) as BenchmarkResult
+    const taskScores: Record<string, number> = {}
+    for (const t of parsed.tasks) {
+      taskScores[t.task.id] = t.averageScore
+    }
+    return {
+      id: row.id,
+      agent: row.agent,
+      model: row.model,
+      averageScore: parsed.averageScore,
+      taskScores,
+    }
+  })
+}
+
+const queryBenchmarks = query(getBenchmarks, "benchmarks.list")
+
+export default function Bench() {
+  const benchmarks = createAsync(() => queryBenchmarks())
+
+  const taskIds = createMemo(() => {
+    const ids = new Set<string>()
+    for (const row of benchmarks() ?? []) {
+      for (const id of Object.keys(row.taskScores)) {
+        ids.add(id)
+      }
+    }
+    return [...ids].sort()
+  })
+
+  return (
+    <main data-page="bench" style={{ padding: "2rem" }}>
+      <Title>Benchmark</Title>
+      <h1 style={{ "margin-bottom": "1.5rem" }}>Benchmarks</h1>
+      <table style={{ "border-collapse": "collapse", width: "100%" }}>
+        <thead>
+          <tr>
+            <th style={{ "text-align": "left", padding: "0.75rem" }}>Agent</th>
+            <th style={{ "text-align": "left", padding: "0.75rem" }}>Model</th>
+            <th style={{ "text-align": "left", padding: "0.75rem" }}>Score</th>
+            <For each={taskIds()}>{(id) => <th style={{ "text-align": "left", padding: "0.75rem" }}>{id}</th>}</For>
+          </tr>
+        </thead>
+        <tbody>
+          <For each={benchmarks()}>
+            {(row) => (
+              <tr>
+                <td style={{ padding: "0.75rem" }}>{row.agent}</td>
+                <td style={{ padding: "0.75rem" }}>{row.model}</td>
+                <td style={{ padding: "0.75rem" }}>{row.averageScore.toFixed(3)}</td>
+                <For each={taskIds()}>
+                  {(id) => (
+                    <td style={{ padding: "0.75rem" }}>
+                      <Show when={row.taskScores[id] !== undefined} fallback="">
+                        <A href={`/bench/${row.id}:${id}`} style={{ color: "#0066cc" }}>
+                          {row.taskScores[id]?.toFixed(3)}
+                        </A>
+                      </Show>
+                    </td>
+                  )}
+                </For>
+              </tr>
+            )}
+          </For>
+        </tbody>
+      </table>
+    </main>
+  )
+}

package/src/routes/bench/submission.ts

@@ -0,0 +1,29 @@
+import type { APIEvent } from "@solidjs/start/server"
+import { Database } from "@jonsoc/console-core/drizzle/index.js"
+import { BenchmarkTable } from "@jonsoc/console-core/schema/benchmark.sql.js"
+import { Identifier } from "@jonsoc/console-core/identifier.js"
+
+interface SubmissionBody {
+  model: string
+  agent: string
+  result: string
+}
+
+export async function POST(event: APIEvent) {
+  const body = (await event.request.json()) as SubmissionBody
+
+  if (!body.model || !body.agent || !body.result) {
+    return Response.json({ error: "All fields are required" }, { status: 400 })
+  }
+
+  await Database.use((tx) =>
+    tx.insert(BenchmarkTable).values({
+      id: Identifier.create("benchmark"),
+      model: body.model,
+      agent: body.agent,
+      result: body.result,
+    }),
+  )
+
+  return Response.json({ success: true }, { status: 200 })
+}

package/src/routes/black/common.tsx

@@ -0,0 +1,62 @@
+import { Match, Switch } from "solid-js"
+
+export const plans = [
+  { id: "20", multiplier: null },
+  { id: "100", multiplier: "5x more usage than Black 20" },
+  { id: "200", multiplier: "20x more usage than Black 20" },
+] as const
+
+export type PlanID = (typeof plans)[number]["id"]
+export type Plan = (typeof plans)[number]
+
+export function PlanIcon(props: { plan: string }) {
+  return (
+    <Switch>
+      <Match when={props.plan === "20"}>
+        <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+          <title>Black 20 plan</title>
+          <rect x="0.5" y="0.5" width="23" height="23" stroke="currentColor" />
+        </svg>
+      </Match>
+      <Match when={props.plan === "100"}>
+        <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+          <title>Black 100 plan</title>
+          <rect x="0.5" y="0.5" width="9" height="9" stroke="currentColor" />
+          <rect x="0.5" y="14.5" width="9" height="9" stroke="currentColor" />
+          <rect x="14.5" y="0.5" width="9" height="9" stroke="currentColor" />
+          <rect x="14.5" y="14.5" width="9" height="9" stroke="currentColor" />
+        </svg>
+      </Match>
+      <Match when={props.plan === "200"}>
+        <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+          <title>Black 200 plan</title>
+          <rect x="0.5" y="0.5" width="3" height="3" stroke="currentColor" />
+          <rect x="0.5" y="5.5" width="3" height="3" stroke="currentColor" />
+          <rect x="0.5" y="10.5" width="3" height="3" stroke="currentColor" />
+          <rect x="0.5" y="15.5" width="3" height="3" stroke="currentColor" />
+          <rect x="0.5" y="20.5" width="3" height="3" stroke="currentColor" />
+          <rect x="5.5" y="0.5" width="3" height="3" stroke="currentColor" />
+          <rect x="5.5" y="5.5" width="3" height="3" stroke="currentColor" />
+          <rect x="5.5" y="10.5" width="3" height="3" stroke="currentColor" />
+          <rect x="5.5" y="15.5" width="3" height="3" stroke="currentColor" />
+          <rect x="5.5" y="20.5" width="3" height="3" stroke="currentColor" />
+          <rect x="10.5" y="0.5" width="3" height="3" stroke="currentColor" />
+          <rect x="10.5" y="5.5" width="3" height="3" stroke="currentColor" />
+          <rect x="10.5" y="10.5" width="3" height="3" stroke="currentColor" />
+          <rect x="10.5" y="15.5" width="3" height="3" stroke="currentColor" />
+          <rect x="10.5" y="20.5" width="3" height="3" stroke="currentColor" />
+          <rect x="15.5" y="0.5" width="3" height="3" stroke="currentColor" />
+          <rect x="15.5" y="5.5" width="3" height="3" stroke="currentColor" />
+          <rect x="15.5" y="10.5" width="3" height="3" stroke="currentColor" />
+          <rect x="15.5" y="15.5" width="3" height="3" stroke="currentColor" />
+          <rect x="15.5" y="20.5" width="3" height="3" stroke="currentColor" />
+          <rect x="20.5" y="0.5" width="3" height="3" stroke="currentColor" />
+          <rect x="20.5" y="5.5" width="3" height="3" stroke="currentColor" />
+          <rect x="20.5" y="10.5" width="3" height="3" stroke="currentColor" />
+          <rect x="20.5" y="15.5" width="3" height="3" stroke="currentColor" />
+          <rect x="20.5" y="20.5" width="3" height="3" stroke="currentColor" />
+        </svg>
+      </Match>
+    </Switch>
+  )
+}

package/src/routes/black/index.tsx

@@ -0,0 +1,108 @@
+import { A, useSearchParams } from "@solidjs/router"
+import { Title } from "@solidjs/meta"
+import { createMemo, createSignal, For, Match, onMount, Show, Switch } from "solid-js"
+import { PlanIcon, plans } from "./common"
+
+export default function Black() {
+  const [params] = useSearchParams()
+  const [selected, setSelected] = createSignal<string | null>((params.plan as string) || null)
+  const [mounted, setMounted] = createSignal(false)
+  const selectedPlan = createMemo(() => plans.find((p) => p.id === selected()))
+
+  onMount(() => {
+    requestAnimationFrame(() => setMounted(true))
+  })
+
+  const transition = (action: () => void) => {
+    if (mounted() && "startViewTransition" in document) {
+      ;(document as any).startViewTransition(action)
+      return
+    }
+
+    action()
+  }
+
+  const select = (planId: string) => {
+    if (selected() === planId) {
+      return
+    }
+
+    transition(() => setSelected(planId))
+  }
+
+  const cancel = () => {
+    transition(() => setSelected(null))
+  }
+
+  return (
+    <>
+      <Title>jonsoc</Title>
+      <section data-slot="cta">
+        <Switch>
+          <Match when={!selected()}>
+            <div data-slot="pricing">
+              <For each={plans}>
+                {(plan) => (
+                  <button
+                    type="button"
+                    onClick={() => select(plan.id)}
+                    data-slot="pricing-card"
+                    style={{ "view-transition-name": `card-${plan.id}` }}
+                  >
+                    <div data-slot="icon">
+                      <PlanIcon plan={plan.id} />
+                    </div>
+                    <p data-slot="price">
+                      <span data-slot="amount">${plan.id}</span> <span data-slot="period">per month</span>
+                      <Show when={plan.multiplier}>
+                        <span data-slot="multiplier">{plan.multiplier}</span>
+                      </Show>
+                    </p>
+                  </button>
+                )}
+              </For>
+            </div>
+          </Match>
+          <Match when={selectedPlan()}>
+            {(plan) => (
+              <div data-slot="selected-plan">
+                <div data-slot="selected-card" style={{ "view-transition-name": `card-${plan().id}` }}>
+                  <div data-slot="icon">
+                    <PlanIcon plan={plan().id} />
+                  </div>
+                  <p data-slot="price">
+                    <span data-slot="amount">${plan().id}</span>{" "}
+                    <span data-slot="period">per person billed monthly</span>
+                    <Show when={plan().multiplier}>
+                      <span data-slot="multiplier">{plan().multiplier}</span>
+                    </Show>
+                  </p>
+                  <ul data-slot="terms" style={{ "view-transition-name": `terms-${plan().id}` }}>
+                    <li>Your subscription will not start immediately</li>
+                    <li>You will be added to the waitlist and activated soon</li>
+                    <li>Your card will be only charged when your subscription is activated</li>
+                    <li>Usage limits apply, heavily automated use may reach limits sooner</li>
+                    <li>Subscriptions for individuals, contact Enterprise for teams</li>
+                    <li>Limits may be adjusted and plans may be discontinued in the future</li>
+                    <li>Cancel your subscription at anytime</li>
+                  </ul>
+                  <div data-slot="actions" style={{ "view-transition-name": `actions-${plan().id}` }}>
+                    <button type="button" onClick={() => cancel()} data-slot="cancel">
+                      Cancel
+                    </button>
+                    <a href={`/black/subscribe/${plan().id}`} data-slot="continue">
+                      Continue
+                    </a>
+                  </div>
+                </div>
+              </div>
+            )}
+          </Match>
+        </Switch>
+        <p data-slot="fine-print" style={{ "view-transition-name": "fine-print" }}>
+          Prices shown don't include applicable tax · <A href="/legal/terms-of-service">Terms of Service</A>
+        </p>
+      </section>
+    </>
+  )
+}