@cat-factory/app 0.27.0 → 0.28.0

package/app/components/layout/CommandBar.vue

@@ -181,6 +181,14 @@ const commands = computed<Command[]>(() => {
     keywords: 'local model runner ollama lm studio llamacpp vllm endpoint',
     run: () => ui.openLocalModels(),
   })
+  list.push({
+    id: 'sandbox',
+    label: 'Open Sandbox',
+    group: 'Workspace',
+    icon: 'i-lucide-flask-conical',
+    keywords: 'sandbox prompt model test experiment judge fixture benchmark evaluate',
+    run: () => ui.openSandbox(),
+  })
 
   return list
 })

package/app/components/layout/SideBar.vue

@@ -125,6 +125,19 @@ watch(
         >
           Integrations
         </UButton>
+        <!-- The Sandbox: try prompt versions/models against graded fixtures, off to the
+             side of the board. Opens the on-demand testing window. -->
+        <UButton
+          block
+          color="primary"
+          variant="soft"
+          size="sm"
+          icon="i-lucide-flask-conical"
+          class="justify-start"
+          @click="ui.openSandbox()"
+        >
+          Sandbox
+        </UButton>
       </div>
     </section>
 

package/app/components/sandbox/SandboxPanel.vue

@@ -0,0 +1,542 @@
+<script setup lang="ts">
+// The Sandbox surface: a parallel place to test prompts and models against graded
+// fixtures, without touching the board. Three tabs — Experiments (define a matrix of
+// prompt versions × models × fixtures for one agent kind, run it, read the graded grid),
+// Prompts (clone a shipped baseline into an editable candidate lineage and version it),
+// and Fixtures (the graded inputs each run is scored against). Loaded on demand when the
+// window opens; 503 (the deployment hasn't provisioned the Sandbox DB) shows a notice.
+import { computed, ref, watch } from 'vue'
+import type { SandboxGrade, SandboxPromptVersion, SandboxRun } from '~/types/sandbox'
+
+const ui = useUiStore()
+const store = useSandboxStore()
+const toast = useToast()
+
+const open = computed({
+  get: () => ui.sandboxOpen,
+  set: (v: boolean) => (v ? ui.openSandbox() : ui.closeSandbox()),
+})
+
+const tab = ref<'experiments' | 'prompts' | 'fixtures'>('experiments')
+
+watch(open, (isOpen) => {
+  if (isOpen) void store.load()
+})
+
+// ---- experiment builder ----------------------------------------------------
+const agentKind = ref('requirements-review')
+const name = ref('')
+const selectedPromptIds = ref<string[]>([])
+const selectedModelIds = ref<string[]>([])
+const selectedFixtureIds = ref<string[]>([])
+// The judge model. Empty string = the deployment's routing default (resolved server-side);
+// picking one explicitly is the recourse on a deployment that has no default model wired,
+// where leaving it on default makes every run fail at create time.
+const selectedJudgeModel = ref<string>('')
+
+const judgeModelItems = computed(() => [
+  { label: 'Deployment default', value: '' },
+  ...store.selectableModels.map((m) => ({ label: m.label, value: m.id })),
+])
+
+const kindPrompts = computed(() => store.promptsForKind(agentKind.value))
+const kindFixtures = computed(() => store.fixturesForKind(agentKind.value))
+
+// Reset the builder selections to sensible defaults when the agent kind (or loaded data)
+// changes: every baseline prompt + every fixture for the kind, no models yet.
+watch(
+  [agentKind, () => store.prompts, () => store.fixtures],
+  () => {
+    selectedPromptIds.value = kindPrompts.value
+      .filter((p) => p.origin === 'baseline')
+      .map((p) => p.id)
+    selectedFixtureIds.value = kindFixtures.value.map((f) => f.id)
+  },
+  { immediate: true },
+)
+
+const cellCount = computed(
+  () =>
+    selectedPromptIds.value.length *
+    selectedModelIds.value.length *
+    selectedFixtureIds.value.length,
+)
+
+const canRun = computed(() => cellCount.value > 0 && cellCount.value <= store.maxCells)
+
+function toggle(which: 'prompt' | 'model' | 'fixture', id: string, on: boolean) {
+  const list =
+    which === 'prompt'
+      ? selectedPromptIds
+      : which === 'model'
+        ? selectedModelIds
+        : selectedFixtureIds
+  list.value = on ? [...new Set([...list.value, id])] : list.value.filter((x) => x !== id)
+}
+
+async function createAndRun() {
+  if (!canRun.value) return
+  try {
+    const created = await store.createExperiment({
+      name: name.value.trim() || `${agentKind.value} — sandbox run`,
+      agentKind: agentKind.value,
+      judgeModel: selectedJudgeModel.value || undefined,
+      matrix: {
+        promptVersionIds: selectedPromptIds.value,
+        models: selectedModelIds.value,
+        fixtureIds: selectedFixtureIds.value,
+      },
+    })
+    name.value = ''
+    toast.add({ title: 'Running experiment…', icon: 'i-lucide-flask-conical', color: 'info' })
+    await store.launch(created.id)
+    toast.add({ title: 'Experiment complete', icon: 'i-lucide-check', color: 'success' })
+  } catch (e) {
+    toast.add({
+      title: 'Could not run the experiment',
+      description: e instanceof Error ? e.message : String(e),
+      icon: 'i-lucide-triangle-alert',
+      color: 'error',
+    })
+  }
+}
+
+// ---- results grid ----------------------------------------------------------
+const gradeByRun = computed(() => {
+  const map = new Map<string, SandboxGrade>()
+  for (const g of store.detail?.grades ?? []) map.set(g.runId, g)
+  return map
+})
+const selectedRun = ref<SandboxRun | null>(null)
+
+function scoreColor(score: number): string {
+  if (score >= 4) return 'text-emerald-400'
+  if (score >= 3) return 'text-amber-400'
+  return 'text-rose-400'
+}
+
+// ---- prompt editor ---------------------------------------------------------
+const editing = ref<SandboxPromptVersion | null>(null)
+const editText = ref('')
+const savingPrompt = ref(false)
+
+function edit(prompt: SandboxPromptVersion) {
+  editing.value = prompt
+  editText.value = prompt.systemText
+}
+
+async function saveVersion() {
+  if (!editing.value || !editText.value.trim()) return
+  savingPrompt.value = true
+  try {
+    await store.saveVersion(editing.value.id, editText.value)
+    toast.add({ title: 'Saved a new version', icon: 'i-lucide-check', color: 'success' })
+    editing.value = null
+  } catch (e) {
+    toast.add({
+      title: 'Could not save the version',
+      description: e instanceof Error ? e.message : String(e),
+      icon: 'i-lucide-triangle-alert',
+      color: 'error',
+    })
+  } finally {
+    savingPrompt.value = false
+  }
+}
+
+async function archive(prompt: SandboxPromptVersion) {
+  try {
+    await store.archivePrompt(prompt.id)
+    if (editing.value?.id === prompt.id) editing.value = null
+  } catch (e) {
+    toast.add({
+      title: 'Could not archive',
+      description: e instanceof Error ? e.message : String(e),
+      icon: 'i-lucide-triangle-alert',
+      color: 'error',
+    })
+  }
+}
+
+const fixtureName = (id: string) => store.fixtures.find((f) => f.id === id)?.name ?? id
+</script>
+
+<template>
+  <UModal
+    v-model:open="open"
+    title="Sandbox — prompt & model testing"
+    description="Try prompt versions and models against graded fixtures, scored by a judge model."
+    :ui="{ content: 'max-w-5xl' }"
+  >
+    <template #body>
+      <div v-if="store.loading" class="flex items-center justify-center py-12">
+        <UIcon name="i-lucide-loader-circle" class="h-6 w-6 animate-spin text-slate-400" />
+      </div>
+
+      <div
+        v-else-if="!store.available"
+        class="rounded-lg border border-slate-700 bg-slate-900/50 p-6 text-sm text-slate-300"
+      >
+        <p class="font-medium text-slate-200">The Sandbox isn't enabled for this deployment.</p>
+        <p class="mt-1 text-slate-400">
+          It needs its own database (a dedicated <code>SANDBOX_DB</code> on Cloudflare, or the
+          <code>sandbox</code> Postgres schema on Node). Provision it and reload.
+        </p>
+      </div>
+
+      <div
+        v-else-if="store.error"
+        class="rounded-lg border border-rose-800 bg-rose-950/40 p-6 text-sm text-rose-200"
+      >
+        <p class="font-medium text-rose-100">The Sandbox failed to load.</p>
+        <p class="mt-1 text-rose-300">{{ store.error }}</p>
+        <UButton class="mt-3" size="xs" color="neutral" variant="subtle" @click="store.load()">
+          Retry
+        </UButton>
+      </div>
+
+      <div v-else class="space-y-4">
+        <UTabs
+          v-model="tab"
+          :items="[
+            { label: 'Experiments', value: 'experiments', icon: 'i-lucide-flask-conical' },
+            { label: 'Prompts', value: 'prompts', icon: 'i-lucide-file-text' },
+            { label: 'Fixtures', value: 'fixtures', icon: 'i-lucide-clipboard-list' },
+          ]"
+        />
+
+        <!-- ============================= EXPERIMENTS ============================= -->
+        <div v-if="tab === 'experiments'" class="grid gap-4 lg:grid-cols-2">
+          <!-- builder -->
+          <div class="space-y-3 rounded-lg border border-slate-700 bg-slate-900/40 p-3">
+            <p class="text-[11px] font-semibold uppercase tracking-wide text-slate-400">
+              New experiment
+            </p>
+
+            <UFormField label="Agent">
+              <USelect
+                v-model="agentKind"
+                :items="store.agentKinds.map((k) => ({ label: k.label, value: k.agentKind }))"
+                value-key="value"
+                class="w-full"
+              />
+            </UFormField>
+
+            <div>
+              <span class="mb-1 block text-[10px] uppercase tracking-wide text-slate-500">
+                Prompt versions
+              </span>
+              <div class="max-h-28 space-y-1 overflow-auto pr-1">
+                <label
+                  v-for="p in kindPrompts"
+                  :key="p.id"
+                  class="flex items-center gap-2 text-sm text-slate-300"
+                >
+                  <UCheckbox
+                    :model-value="selectedPromptIds.includes(p.id)"
+                    @update:model-value="
+                      (v: boolean | 'indeterminate') => toggle('prompt', p.id, v === true)
+                    "
+                  />
+                  <span class="truncate">{{ p.name }}</span>
+                  <UBadge
+                    :color="p.origin === 'baseline' ? 'neutral' : 'primary'"
+                    variant="soft"
+                    size="xs"
+                  >
+                    {{ p.origin === 'baseline' ? 'baseline' : `v${p.version}` }}
+                  </UBadge>
+                </label>
+              </div>
+            </div>
+
+            <div>
+              <span class="mb-1 block text-[10px] uppercase tracking-wide text-slate-500">
+                Models
+              </span>
+              <div class="max-h-28 space-y-1 overflow-auto pr-1">
+                <label
+                  v-for="m in store.selectableModels"
+                  :key="m.id"
+                  class="flex items-center gap-2 text-sm text-slate-300"
+                >
+                  <UCheckbox
+                    :model-value="selectedModelIds.includes(m.id)"
+                    @update:model-value="
+                      (v: boolean | 'indeterminate') => toggle('model', m.id, v === true)
+                    "
+                  />
+                  <span class="truncate">{{ m.label }}</span>
+                </label>
+                <p v-if="!store.selectableModels.length" class="text-xs text-slate-500">
+                  No selectable models — configure a provider key or enable Cloudflare AI.
+                </p>
+              </div>
+            </div>
+
+            <div>
+              <span class="mb-1 block text-[10px] uppercase tracking-wide text-slate-500">
+                Fixtures
+              </span>
+              <div class="max-h-28 space-y-1 overflow-auto pr-1">
+                <label
+                  v-for="f in kindFixtures"
+                  :key="f.id"
+                  class="flex items-center gap-2 text-sm text-slate-300"
+                >
+                  <UCheckbox
+                    :model-value="selectedFixtureIds.includes(f.id)"
+                    @update:model-value="
+                      (v: boolean | 'indeterminate') => toggle('fixture', f.id, v === true)
+                    "
+                  />
+                  <span class="truncate">{{ f.name }}</span>
+                </label>
+                <p v-if="!kindFixtures.length" class="text-xs text-slate-500">
+                  No fixtures for this agent.
+                </p>
+              </div>
+            </div>
+
+            <UFormField label="Judge model" hint="grades every cell">
+              <USelect v-model="selectedJudgeModel" :items="judgeModelItems" />
+            </UFormField>
+
+            <UFormField label="Name (optional)">
+              <UInput v-model="name" :placeholder="`${agentKind} — sandbox run`" />
+            </UFormField>
+
+            <div class="flex items-center justify-between">
+              <span class="text-xs text-slate-500">
+                {{ cellCount }} cell{{ cellCount === 1 ? '' : 's' }}
+                <span v-if="cellCount > store.maxCells" class="text-rose-400">
+                  (max {{ store.maxCells }})
+                </span>
+              </span>
+              <UButton
+                color="primary"
+                icon="i-lucide-play"
+                size="sm"
+                :loading="store.launching"
+                :disabled="!canRun"
+                @click="createAndRun()"
+              >
+                Run
+              </UButton>
+            </div>
+          </div>
+
+          <!-- history + results -->
+          <div class="space-y-3">
+            <div v-if="store.detail" class="rounded-lg border border-slate-700 bg-slate-900/40 p-3">
+              <div class="mb-2 flex items-center justify-between">
+                <p class="text-sm font-medium text-slate-200">
+                  {{ store.detail.experiment.name }}
+                </p>
+                <UBadge variant="soft" size="xs">{{ store.detail.experiment.status }}</UBadge>
+              </div>
+              <div class="overflow-auto">
+                <table class="w-full text-left text-xs">
+                  <thead class="text-slate-500">
+                    <tr>
+                      <th class="py-1 pr-2 font-medium">Prompt</th>
+                      <th class="py-1 pr-2 font-medium">Model</th>
+                      <th class="py-1 pr-2 font-medium">Fixture</th>
+                      <th class="py-1 pr-2 font-medium">Score</th>
+                      <th class="py-1 font-medium">Objective</th>
+                    </tr>
+                  </thead>
+                  <tbody>
+                    <tr
+                      v-for="run in store.detail.runs"
+                      :key="run.id"
+                      class="cursor-pointer border-t border-slate-800 hover:bg-slate-800/40"
+                      @click="selectedRun = run"
+                    >
+                      <td class="py-1 pr-2 text-slate-300">{{ run.promptLabel }}</td>
+                      <td class="py-1 pr-2 font-mono text-[11px] text-slate-400">
+                        {{ run.model }}
+                      </td>
+                      <td class="py-1 pr-2 text-slate-400">{{ fixtureName(run.fixtureId) }}</td>
+                      <td class="py-1 pr-2">
+                        <span
+                          v-if="gradeByRun.get(run.id)"
+                          :class="scoreColor(gradeByRun.get(run.id)!.weightedTotal)"
+                          class="font-semibold"
+                        >
+                          {{ gradeByRun.get(run.id)!.weightedTotal.toFixed(2) }}
+                        </span>
+                        <span v-else-if="run.status === 'failed'" class="text-rose-400"
+                          >failed</span
+                        >
+                        <span v-else class="text-slate-600">—</span>
+                      </td>
+                      <td class="py-1">
+                        <span
+                          v-if="gradeByRun.get(run.id)?.objective"
+                          :class="
+                            gradeByRun.get(run.id)!.objective!.pass
+                              ? 'text-emerald-400'
+                              : 'text-amber-400'
+                          "
+                        >
+                          {{ gradeByRun.get(run.id)!.objective!.caught }}/{{
+                            gradeByRun.get(run.id)!.objective!.total
+                          }}
+                        </span>
+                        <span v-else class="text-slate-600">—</span>
+                      </td>
+                    </tr>
+                  </tbody>
+                </table>
+              </div>
+
+              <!-- selected cell output -->
+              <div v-if="selectedRun" class="mt-3 border-t border-slate-800 pt-2">
+                <p class="mb-1 text-[11px] uppercase tracking-wide text-slate-500">
+                  {{ selectedRun.promptLabel }} · {{ selectedRun.model }}
+                </p>
+                <p v-if="selectedRun.error" class="text-xs text-rose-400">
+                  {{ selectedRun.error }}
+                </p>
+                <pre
+                  v-if="selectedRun.outputText"
+                  class="max-h-48 overflow-auto whitespace-pre-wrap rounded bg-slate-950/60 p-2 text-[11px] text-slate-300"
+                  >{{ selectedRun.outputText }}</pre
+                >
+                <div v-if="gradeByRun.get(selectedRun.id)" class="mt-2 space-y-0.5">
+                  <p
+                    v-for="d in gradeByRun.get(selectedRun.id)!.scores"
+                    :key="d.key"
+                    class="text-[11px] text-slate-400"
+                  >
+                    <span :class="scoreColor(d.score)" class="font-semibold">{{ d.score }}</span>
+                    <span class="ml-1 text-slate-300">{{ d.key }}</span>
+                    <span v-if="d.rationale" class="ml-1 text-slate-500">— {{ d.rationale }}</span>
+                  </p>
+                </div>
+              </div>
+            </div>
+
+            <p class="text-[11px] uppercase tracking-wide text-slate-500">Past experiments</p>
+            <div class="max-h-56 space-y-1 overflow-auto">
+              <button
+                v-for="x in store.experiments"
+                :key="x.id"
+                class="flex w-full items-center justify-between rounded-md border border-slate-800 bg-slate-900/40 px-2 py-1.5 text-left text-sm hover:bg-slate-800/50"
+                @click="store.openExperiment(x.id)"
+              >
+                <span class="truncate text-slate-300">{{ x.name }}</span>
+                <UBadge variant="soft" size="xs">{{ x.status }}</UBadge>
+              </button>
+              <p v-if="!store.experiments.length" class="text-xs text-slate-500">
+                No experiments yet.
+              </p>
+            </div>
+          </div>
+        </div>
+
+        <!-- ============================== PROMPTS ============================== -->
+        <div v-else-if="tab === 'prompts'" class="grid gap-4 lg:grid-cols-2">
+          <div class="max-h-[28rem] space-y-1.5 overflow-auto pr-1">
+            <div
+              v-for="p in store.prompts"
+              :key="p.id"
+              class="flex items-center justify-between rounded-md border border-slate-800 bg-slate-900/40 px-2.5 py-1.5 text-sm"
+            >
+              <div class="min-w-0">
+                <div class="flex items-center gap-2">
+                  <span class="truncate text-slate-200">{{ p.name }}</span>
+                  <UBadge
+                    :color="p.origin === 'baseline' ? 'neutral' : 'primary'"
+                    variant="soft"
+                    size="xs"
+                  >
+                    {{ p.origin === 'baseline' ? 'baseline' : `v${p.version}` }}
+                  </UBadge>
+                </div>
+                <span class="text-[11px] text-slate-500">{{ p.agentKind }}</span>
+              </div>
+              <div class="flex items-center gap-1">
+                <UButton
+                  icon="i-lucide-pencil"
+                  color="neutral"
+                  variant="ghost"
+                  size="xs"
+                  :title="p.origin === 'baseline' ? 'Fork into a candidate' : 'Edit / version'"
+                  @click="edit(p)"
+                />
+                <UButton
+                  v-if="p.origin === 'candidate'"
+                  icon="i-lucide-archive"
+                  color="error"
+                  variant="ghost"
+                  size="xs"
+                  @click="archive(p)"
+                />
+              </div>
+            </div>
+          </div>
+
+          <div
+            v-if="editing"
+            class="space-y-2 rounded-lg border border-slate-700 bg-slate-900/40 p-3"
+          >
+            <p class="text-[11px] uppercase tracking-wide text-slate-500">
+              {{ editing.origin === 'baseline' ? 'Fork' : 'New version of' }} · {{ editing.name }}
+            </p>
+            <UTextarea v-model="editText" :rows="16" class="w-full font-mono text-xs" autoresize />
+            <div class="flex justify-end gap-2">
+              <UButton color="neutral" variant="ghost" size="sm" @click="editing = null">
+                Cancel
+              </UButton>
+              <UButton
+                color="primary"
+                icon="i-lucide-save"
+                size="sm"
+                :loading="savingPrompt"
+                :disabled="!editText.trim()"
+                @click="saveVersion()"
+              >
+                Save new version
+              </UButton>
+            </div>
+          </div>
+          <p v-else class="self-start text-xs text-slate-500">
+            Pick a prompt to fork a shipped baseline or version a candidate. Each save appends an
+            immutable version you can put under test.
+          </p>
+        </div>
+
+        <!-- ============================== FIXTURES ============================== -->
+        <div v-else class="max-h-[28rem] space-y-1.5 overflow-auto pr-1">
+          <div
+            v-for="f in store.fixtures"
+            :key="f.id"
+            class="rounded-md border border-slate-800 bg-slate-900/40 px-2.5 py-2 text-sm"
+          >
+            <div class="flex items-center justify-between">
+              <span class="text-slate-200">{{ f.name }}</span>
+              <div class="flex items-center gap-1.5">
+                <UBadge variant="soft" size="xs">{{ f.kind }}</UBadge>
+                <UBadge
+                  :color="f.origin === 'builtin' ? 'neutral' : 'primary'"
+                  variant="soft"
+                  size="xs"
+                >
+                  {{ f.origin }}
+                </UBadge>
+              </div>
+            </div>
+            <p v-if="f.objective?.kind === 'findings'" class="mt-0.5 text-[11px] text-slate-500">
+              {{ f.objective.expectations.length }} graded expectation{{
+                f.objective.expectations.length === 1 ? '' : 's'
+              }}
+            </p>
+          </div>
+          <p v-if="!store.fixtures.length" class="text-xs text-slate-500">No fixtures.</p>
+        </div>
+      </div>
+    </template>
+  </UModal>
+</template>

package/app/composables/api/sandbox.ts

@@ -0,0 +1,57 @@
+import type {
+  CloneSandboxPromptInput,
+  CreateSandboxExperimentInput,
+  SandboxExperiment,
+  SandboxExperimentDetail,
+  SandboxFixture,
+  SandboxOverview,
+  SandboxPromptVersion,
+  SaveSandboxVersionInput,
+} from '~/types/sandbox'
+import type { ApiContext } from './context'
+
+/**
+ * The Sandbox API (the parallel prompt/model testing surface): manage versioned prompt
+ * candidates + the fixture library, define experiments (prompt × model × fixture), and
+ * launch one to run + grade every cell. Opt-in: every endpoint 503s when the deployment
+ * hasn't wired the Sandbox (its dedicated DB / schema).
+ */
+export function sandboxApi({ http, ws }: ApiContext) {
+  const base = (workspaceId: string) => `${ws(workspaceId)}/sandbox`
+  return {
+    getSandboxOverview: (workspaceId: string) =>
+      http<SandboxOverview>(`${base(workspaceId)}/overview`),
+
+    // ---- prompt versions -------------------------------------------------
+    cloneSandboxPrompt: (workspaceId: string, body: CloneSandboxPromptInput) =>
+      http<SandboxPromptVersion>(`${base(workspaceId)}/prompts/clone`, { method: 'POST', body }),
+    saveSandboxVersion: (workspaceId: string, body: SaveSandboxVersionInput) =>
+      http<SandboxPromptVersion>(`${base(workspaceId)}/prompts`, { method: 'POST', body }),
+    setSandboxPromptLabels: (workspaceId: string, promptId: string, labels: string[]) =>
+      http<SandboxPromptVersion>(
+        `${base(workspaceId)}/prompts/${encodeURIComponent(promptId)}/labels`,
+        { method: 'PATCH', body: { labels } },
+      ),
+    archiveSandboxPrompt: (workspaceId: string, promptId: string) =>
+      http(`${base(workspaceId)}/prompts/${encodeURIComponent(promptId)}`, { method: 'DELETE' }),
+
+    // ---- fixtures --------------------------------------------------------
+    createSandboxFixture: (workspaceId: string, body: Partial<SandboxFixture>) =>
+      http<SandboxFixture>(`${base(workspaceId)}/fixtures`, { method: 'POST', body }),
+    deleteSandboxFixture: (workspaceId: string, fixtureId: string) =>
+      http(`${base(workspaceId)}/fixtures/${encodeURIComponent(fixtureId)}`, { method: 'DELETE' }),
+
+    // ---- experiments -----------------------------------------------------
+    createSandboxExperiment: (workspaceId: string, body: CreateSandboxExperimentInput) =>
+      http<SandboxExperiment>(`${base(workspaceId)}/experiments`, { method: 'POST', body }),
+    getSandboxExperiment: (workspaceId: string, experimentId: string) =>
+      http<SandboxExperimentDetail>(
+        `${base(workspaceId)}/experiments/${encodeURIComponent(experimentId)}`,
+      ),
+    launchSandboxExperiment: (workspaceId: string, experimentId: string) =>
+      http<SandboxExperimentDetail>(
+        `${base(workspaceId)}/experiments/${encodeURIComponent(experimentId)}/launch`,
+        { method: 'POST' },
+      ),
+  }
+}

package/app/composables/useApi.ts

@@ -15,6 +15,7 @@ import { presetsApi } from './api/presets'
 import { providerConnectionsApi } from './api/providerConnections'
 import { recurringApi } from './api/recurring'
 import { releaseHealthApi } from './api/releaseHealth'
+import { sandboxApi } from './api/sandbox'
 import { reviewsApi } from './api/reviews'
 import { slackApi } from './api/slack'
 import { specApi } from './api/spec'
@@ -89,6 +90,7 @@ export function useApi() {
     ...providerConnectionsApi(ctx),
     ...releaseHealthApi(ctx),
     ...recurringApi(ctx),
+    ...sandboxApi(ctx),
     ...githubApi(ctx),
     ...slackApi(ctx),
     ...bootstrapApi(ctx),

package/app/pages/index.vue

@@ -33,6 +33,7 @@ import ProviderConnectionPanel from '~/components/settings/ProviderConnectionPan
 import ProviderConfigBanner from '~/components/layout/ProviderConfigBanner.vue'
 import ModelConfigurationPanel from '~/components/settings/ModelConfigurationPanel.vue'
 import LocalModelEndpointsPanel from '~/components/settings/LocalModelEndpointsPanel.vue'
+import SandboxPanel from '~/components/sandbox/SandboxPanel.vue'
 import UserSecretsSection from '~/components/settings/UserSecretsSection.vue'
 import OpenRouterCatalogPanel from '~/components/settings/OpenRouterCatalogPanel.vue'
 import VendorCredentialsModal from '~/components/providers/VendorCredentialsModal.vue'
@@ -189,6 +190,7 @@ watch(
       <ProviderConnectionPanel />
       <ModelConfigurationPanel />
       <LocalModelEndpointsPanel />
+      <SandboxPanel />
       <UserSecretsSection />
       <OpenRouterCatalogPanel />
       <VendorCredentialsModal />

package/app/stores/sandbox.ts

@@ -0,0 +1,174 @@
+import { defineStore } from 'pinia'
+import { computed, ref } from 'vue'
+import type { ModelOption } from '~/types/domain'
+import type {
+  CreateSandboxExperimentInput,
+  SandboxAgentKindMeta,
+  SandboxExperiment,
+  SandboxExperimentDetail,
+  SandboxFixture,
+  SandboxOverview,
+  SandboxPromptVersion,
+} from '~/types/sandbox'
+import { useWorkspaceStore } from '~/stores/workspace'
+
+/**
+ * The Sandbox (parallel prompt/model testing surface). Loaded on demand when the panel
+ * opens (it's an opt-in, secondary surface, not part of the board snapshot): the testable
+ * agent-kind catalog, the shipped baselines + stored candidate prompt versions, the
+ * fixture library, and experiment definitions. Running an experiment grades every cell
+ * with a judge model; `launch` returns the full result grid.
+ */
+export const useSandboxStore = defineStore('sandbox', () => {
+  const api = useApi()
+
+  const available = ref(true)
+  const loading = ref(false)
+  const error = ref<string | null>(null)
+
+  const agentKinds = ref<SandboxAgentKindMeta[]>([])
+  const prompts = ref<SandboxPromptVersion[]>([])
+  const fixtures = ref<SandboxFixture[]>([])
+  const experiments = ref<SandboxExperiment[]>([])
+  const models = ref<ModelOption[]>([])
+  /** The matrix cell cap (from the backend overview, so the builder gates on the same limit). */
+  const maxCells = ref(100)
+
+  /** The currently-opened experiment's full detail (result grid), if any. */
+  const detail = ref<SandboxExperimentDetail | null>(null)
+  const launching = ref(false)
+
+  function hydrate(overview: SandboxOverview) {
+    agentKinds.value = overview.agentKinds
+    prompts.value = overview.prompts
+    fixtures.value = overview.fixtures
+    experiments.value = [...overview.experiments].sort((a, b) => b.createdAt - a.createdAt)
+    maxCells.value = overview.maxCells
+  }
+
+  /** Patch one experiment into the list in place (newest-first), without a full reload. */
+  function upsertExperiment(experiment: SandboxExperiment) {
+    const next = experiments.value.filter((e) => e.id !== experiment.id)
+    next.push(experiment)
+    experiments.value = next.sort((a, b) => b.createdAt - a.createdAt)
+  }
+
+  /** Load the overview + the workspace model catalog. The 503 (feature off) is surfaced. */
+  async function load() {
+    const ws = useWorkspaceStore()
+    if (!ws.workspaceId) return
+    loading.value = true
+    error.value = null
+    try {
+      const [overview, modelList] = await Promise.all([
+        api.getSandboxOverview(ws.requireId()),
+        api.getWorkspaceModels(ws.requireId()),
+      ])
+      hydrate(overview)
+      models.value = modelList
+      available.value = true
+    } catch (e) {
+      const status =
+        (e as { statusCode?: number; response?: { status?: number } })?.statusCode ??
+        (e as { response?: { status?: number } })?.response?.status
+      if (status === 503) {
+        available.value = false
+      } else {
+        error.value = e instanceof Error ? e.message : String(e)
+      }
+    } finally {
+      loading.value = false
+    }
+  }
+
+  /** Selectable models for the experiment picker (the backend computed `available`). */
+  const selectableModels = computed(() => models.value.filter((m) => m.available !== false))
+
+  /** Prompt versions for one agent kind (baselines first, then candidates). */
+  function promptsForKind(agentKind: string): SandboxPromptVersion[] {
+    return prompts.value.filter((p) => p.agentKind === agentKind)
+  }
+
+  /** Fixtures authored for one agent kind, filtered by the catalog's `fixtureKinds`. */
+  function fixturesForKind(agentKind: string): SandboxFixture[] {
+    const meta = agentKinds.value.find((k) => k.agentKind === agentKind)
+    if (!meta) return fixtures.value
+    // The backend catalog is the source of truth for the fixture↔kind mapping.
+    const wanted = meta.fixtureKinds
+    return fixtures.value.filter((f) => wanted.includes(f.kind))
+  }
+
+  async function clonePrompt(agentKind: string, basePromptId: string | null, name?: string) {
+    const ws = useWorkspaceStore()
+    const created = await api.cloneSandboxPrompt(ws.requireId(), { agentKind, basePromptId, name })
+    await load()
+    return created
+  }
+
+  async function saveVersion(parentId: string, systemText: string) {
+    const ws = useWorkspaceStore()
+    const saved = await api.saveSandboxVersion(ws.requireId(), { parentId, systemText })
+    await load()
+    return saved
+  }
+
+  async function archivePrompt(promptId: string) {
+    const ws = useWorkspaceStore()
+    await api.archiveSandboxPrompt(ws.requireId(), promptId)
+    await load()
+  }
+
+  async function createExperiment(input: CreateSandboxExperimentInput) {
+    const ws = useWorkspaceStore()
+    const created = await api.createSandboxExperiment(ws.requireId(), input)
+    await load()
+    return created
+  }
+
+  async function openExperiment(experimentId: string) {
+    const ws = useWorkspaceStore()
+    detail.value = await api.getSandboxExperiment(ws.requireId(), experimentId)
+    return detail.value
+  }
+
+  async function launch(experimentId: string) {
+    const ws = useWorkspaceStore()
+    launching.value = true
+    try {
+      // `launch` returns the full graded grid AND the updated experiment, so patch both in
+      // place rather than calling `load()`: a transient failure in that follow-up fetch
+      // would otherwise set `error` and hide the freshly-returned result grid behind the
+      // error panel (and re-fetch the whole overview + model catalog for nothing).
+      const result = await api.launchSandboxExperiment(ws.requireId(), experimentId)
+      detail.value = result
+      upsertExperiment(result.experiment)
+      return result
+    } finally {
+      launching.value = false
+    }
+  }
+
+  return {
+    available,
+    loading,
+    error,
+    agentKinds,
+    prompts,
+    fixtures,
+    experiments,
+    models,
+    maxCells,
+    selectableModels,
+    detail,
+    launching,
+    load,
+    promptsForKind,
+    fixturesForKind,
+    clonePrompt,
+    saveVersion,
+    archivePrompt,
+    createExperiment,
+    openExperiment,
+    launch,
+  }
+})

package/app/stores/ui.ts

@@ -104,6 +104,8 @@ export const useUiStore = defineStore('ui', () => {
   const vendorCredentialsOpen = ref(false)
   // Per-user settings panel: the signed-in user's own-machine local model runners.
   const localModelsOpen = ref(false)
+  // The Sandbox (parallel prompt/model testing) surface — an opt-in, on-demand window.
+  const sandboxOpen = ref(false)
   const userSecretsOpen = ref(false)
   // Per-workspace settings panel: the OpenRouter dynamic catalog (browse/enable gateway models).
   const openRouterOpen = ref(false)
@@ -364,6 +366,12 @@ export const useUiStore = defineStore('ui', () => {
   function closeLocalModels() {
     localModelsOpen.value = false
   }
+  function openSandbox() {
+    sandboxOpen.value = true
+  }
+  function closeSandbox() {
+    sandboxOpen.value = false
+  }
   function openUserSecrets() {
     userSecretsOpen.value = true
   }
@@ -464,6 +472,7 @@ export const useUiStore = defineStore('ui', () => {
     modelConfigOpen,
     vendorCredentialsOpen,
     localModelsOpen,
+    sandboxOpen,
     userSecretsOpen,
     openRouterOpen,
     aiProviderSetupOpen,
@@ -528,6 +537,8 @@ export const useUiStore = defineStore('ui', () => {
     closeVendorCredentials,
     openLocalModels,
     closeLocalModels,
+    openSandbox,
+    closeSandbox,
     openUserSecrets,
     closeUserSecrets,
     openOpenRouter,

package/app/types/sandbox.ts

@@ -0,0 +1,183 @@
+// Sandbox (the parallel prompt/model testing surface) wire shapes, hand-mirrored from
+// `@cat-factory/contracts` (sandbox.ts) so a backend payload drops straight into the
+// store. Clone a shipped agent prompt into a versioned candidate, run an experiment
+// matrix (prompt versions × models × fixtures) for one agent kind, and grade every cell
+// with a judge model plus (where a fixture supports it) an objective findings score.
+
+export type SandboxPromptOrigin = 'baseline' | 'candidate'
+
+export interface SandboxPromptVersion {
+  id: string
+  lineageId: string
+  agentKind: string
+  name: string
+  origin: SandboxPromptOrigin
+  systemText: string
+  basePromptId: string | null
+  version: number
+  parentId: string | null
+  labels: string[]
+  createdAt: number
+  createdBy: string | null
+  archivedAt: number | null
+}
+
+export type SandboxFixtureKind =
+  | 'requirements'
+  | 'clarity'
+  | 'architecture'
+  | 'code-review'
+  | 'repo-feature'
+  | 'repo-bug'
+
+export interface SandboxExpectation {
+  id: string
+  summary: string
+  detail: string
+  trickiness: number
+  impact: number
+  matchHints: string[]
+}
+
+export type SandboxFixtureObjective =
+  | { kind: 'tests'; testCmd: string }
+  | { kind: 'findings'; expectations: SandboxExpectation[] }
+
+export interface SandboxFixture {
+  id: string
+  kind: SandboxFixtureKind
+  name: string
+  payload: Record<string, unknown> | null
+  repoRef: { owner: string; name: string; seedRef: string } | null
+  objective: SandboxFixtureObjective | null
+  origin: 'builtin' | 'custom'
+  createdAt: number
+}
+
+export type SandboxExperimentStatus = 'draft' | 'running' | 'done' | 'failed'
+
+export interface SandboxMatrix {
+  promptVersionIds: string[]
+  models: string[]
+  fixtureIds: string[]
+}
+
+export interface SandboxExperiment {
+  id: string
+  name: string
+  agentKind: string
+  judgeModel: string
+  repeats: number
+  status: SandboxExperimentStatus
+  matrix: SandboxMatrix
+  budgetTokens: number | null
+  createdAt: number
+  createdBy: string | null
+}
+
+export type SandboxRunStatus = 'queued' | 'running' | 'done' | 'failed'
+
+export interface SandboxTokenUsage {
+  inputTokens: number
+  outputTokens: number
+}
+
+export interface SandboxRun {
+  id: string
+  experimentId: string
+  promptVersionId: string
+  model: string
+  fixtureId: string
+  repeatIndex: number
+  status: SandboxRunStatus
+  outputText: string | null
+  usage: SandboxTokenUsage | null
+  latencyMs: number | null
+  branch: string | null
+  prUrl: string | null
+  diff: string | null
+  error: string | null
+  seedSha: string | null
+  promptLabel: string
+  startedAt: number | null
+  finishedAt: number | null
+}
+
+export interface SandboxGradeDimension {
+  key: string
+  score: number
+  rationale: string
+}
+
+export interface SandboxObjectiveResult {
+  kind: 'tests' | 'findings'
+  pass: boolean
+  detail: string
+  impactRecall: number | null
+  wowBonus: number | null
+  caught: number | null
+  total: number | null
+  missedHighImpact: string[] | null
+}
+
+export interface SandboxGrade {
+  id: string
+  runId: string
+  judgeModel: string
+  scores: SandboxGradeDimension[]
+  weightedTotal: number
+  objective: SandboxObjectiveResult | null
+  createdAt: number
+}
+
+/** The Sandbox catalog entry for a testable agent kind (from the overview). */
+export interface SandboxAgentKindMeta {
+  agentKind: string
+  label: string
+  bucket: 'inline' | 'container'
+  rubric: 'requirement-review' | 'code-review' | 'implementation'
+  /** Fixture kinds this agent is exercised against (the UI filters the library by these). */
+  fixtureKinds: SandboxFixtureKind[]
+  basePromptId: string | null
+}
+
+/** The composite the management surface loads on open (`GET /sandbox/overview`). */
+export interface SandboxOverview {
+  agentKinds: SandboxAgentKindMeta[]
+  prompts: SandboxPromptVersion[]
+  fixtures: SandboxFixture[]
+  experiments: SandboxExperiment[]
+  /** The matrix cell cap (the backend cost guard), so the builder gates on the same limit. */
+  maxCells: number
+}
+
+/** An experiment with its result grid (`GET /sandbox/experiments/:id`, also from launch). */
+export interface SandboxExperimentDetail {
+  experiment: SandboxExperiment
+  runs: SandboxRun[]
+  grades: SandboxGrade[]
+}
+
+// ---- request bodies --------------------------------------------------------
+
+export interface CloneSandboxPromptInput {
+  agentKind: string
+  basePromptId: string | null
+  name?: string
+  labels?: string[]
+}
+
+export interface SaveSandboxVersionInput {
+  parentId: string
+  systemText: string
+  labels?: string[]
+}
+
+export interface CreateSandboxExperimentInput {
+  name: string
+  agentKind: string
+  matrix: SandboxMatrix
+  judgeModel?: string
+  repeats?: number
+  budgetTokens?: number | null
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cat-factory/app",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "description": "Reusable Nuxt layer for the Agent Architecture Board SPA (components, stores, composables, pages). Consume it from a thin deployment app via `extends: ['@cat-factory/app']` and point it at your backend with NUXT_PUBLIC_API_BASE. See deploy/frontend for an example.",
   "repository": {
     "type": "git",