@cat-factory/app 0.26.7 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/components/layout/AccountDeploymentSettings.vue +236 -0
- package/app/components/layout/AccountTeamSettings.vue +6 -0
- package/app/components/layout/CommandBar.vue +8 -0
- package/app/components/layout/SideBar.vue +13 -0
- package/app/components/sandbox/SandboxPanel.vue +542 -0
- package/app/components/settings/ObservabilityConnectionPanel.vue +92 -0
- package/app/components/settings/WorkspaceSettingsPanel.vue +116 -1
- package/app/composables/api/accounts.ts +12 -0
- package/app/composables/api/releaseHealth.ts +17 -0
- package/app/composables/api/sandbox.ts +57 -0
- package/app/composables/useApi.ts +2 -0
- package/app/pages/index.vue +2 -0
- package/app/stores/accountSettings.ts +49 -0
- package/app/stores/releaseHealth.ts +37 -0
- package/app/stores/sandbox.ts +174 -0
- package/app/stores/ui.ts +11 -0
- package/app/stores/workspaceSettings.ts +3 -0
- package/app/types/accountSettings.ts +39 -0
- package/app/types/domain.ts +9 -0
- package/app/types/incidentEnrichment.ts +28 -0
- package/app/types/sandbox.ts +183 -0
- package/package.json +1 -1
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
<script setup lang="ts">
|
|
2
|
+
// The Sandbox surface: a parallel place to test prompts and models against graded
|
|
3
|
+
// fixtures, without touching the board. Three tabs — Experiments (define a matrix of
|
|
4
|
+
// prompt versions × models × fixtures for one agent kind, run it, read the graded grid),
|
|
5
|
+
// Prompts (clone a shipped baseline into an editable candidate lineage and version it),
|
|
6
|
+
// and Fixtures (the graded inputs each run is scored against). Loaded on demand when the
|
|
7
|
+
// window opens; 503 (the deployment hasn't provisioned the Sandbox DB) shows a notice.
|
|
8
|
+
import { computed, ref, watch } from 'vue'
|
|
9
|
+
import type { SandboxGrade, SandboxPromptVersion, SandboxRun } from '~/types/sandbox'
|
|
10
|
+
|
|
11
|
+
const ui = useUiStore()
|
|
12
|
+
const store = useSandboxStore()
|
|
13
|
+
const toast = useToast()
|
|
14
|
+
|
|
15
|
+
const open = computed({
|
|
16
|
+
get: () => ui.sandboxOpen,
|
|
17
|
+
set: (v: boolean) => (v ? ui.openSandbox() : ui.closeSandbox()),
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
const tab = ref<'experiments' | 'prompts' | 'fixtures'>('experiments')
|
|
21
|
+
|
|
22
|
+
watch(open, (isOpen) => {
|
|
23
|
+
if (isOpen) void store.load()
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
// ---- experiment builder ----------------------------------------------------
|
|
27
|
+
const agentKind = ref('requirements-review')
|
|
28
|
+
const name = ref('')
|
|
29
|
+
const selectedPromptIds = ref<string[]>([])
|
|
30
|
+
const selectedModelIds = ref<string[]>([])
|
|
31
|
+
const selectedFixtureIds = ref<string[]>([])
|
|
32
|
+
// The judge model. Empty string = the deployment's routing default (resolved server-side);
|
|
33
|
+
// picking one explicitly is the recourse on a deployment that has no default model wired,
|
|
34
|
+
// where leaving it on default makes every run fail at create time.
|
|
35
|
+
const selectedJudgeModel = ref<string>('')
|
|
36
|
+
|
|
37
|
+
const judgeModelItems = computed(() => [
|
|
38
|
+
{ label: 'Deployment default', value: '' },
|
|
39
|
+
...store.selectableModels.map((m) => ({ label: m.label, value: m.id })),
|
|
40
|
+
])
|
|
41
|
+
|
|
42
|
+
const kindPrompts = computed(() => store.promptsForKind(agentKind.value))
|
|
43
|
+
const kindFixtures = computed(() => store.fixturesForKind(agentKind.value))
|
|
44
|
+
|
|
45
|
+
// Reset the builder selections to sensible defaults when the agent kind (or loaded data)
|
|
46
|
+
// changes: every baseline prompt + every fixture for the kind, no models yet.
|
|
47
|
+
watch(
|
|
48
|
+
[agentKind, () => store.prompts, () => store.fixtures],
|
|
49
|
+
() => {
|
|
50
|
+
selectedPromptIds.value = kindPrompts.value
|
|
51
|
+
.filter((p) => p.origin === 'baseline')
|
|
52
|
+
.map((p) => p.id)
|
|
53
|
+
selectedFixtureIds.value = kindFixtures.value.map((f) => f.id)
|
|
54
|
+
},
|
|
55
|
+
{ immediate: true },
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
const cellCount = computed(
|
|
59
|
+
() =>
|
|
60
|
+
selectedPromptIds.value.length *
|
|
61
|
+
selectedModelIds.value.length *
|
|
62
|
+
selectedFixtureIds.value.length,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
const canRun = computed(() => cellCount.value > 0 && cellCount.value <= store.maxCells)
|
|
66
|
+
|
|
67
|
+
function toggle(which: 'prompt' | 'model' | 'fixture', id: string, on: boolean) {
|
|
68
|
+
const list =
|
|
69
|
+
which === 'prompt'
|
|
70
|
+
? selectedPromptIds
|
|
71
|
+
: which === 'model'
|
|
72
|
+
? selectedModelIds
|
|
73
|
+
: selectedFixtureIds
|
|
74
|
+
list.value = on ? [...new Set([...list.value, id])] : list.value.filter((x) => x !== id)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function createAndRun() {
|
|
78
|
+
if (!canRun.value) return
|
|
79
|
+
try {
|
|
80
|
+
const created = await store.createExperiment({
|
|
81
|
+
name: name.value.trim() || `${agentKind.value} — sandbox run`,
|
|
82
|
+
agentKind: agentKind.value,
|
|
83
|
+
judgeModel: selectedJudgeModel.value || undefined,
|
|
84
|
+
matrix: {
|
|
85
|
+
promptVersionIds: selectedPromptIds.value,
|
|
86
|
+
models: selectedModelIds.value,
|
|
87
|
+
fixtureIds: selectedFixtureIds.value,
|
|
88
|
+
},
|
|
89
|
+
})
|
|
90
|
+
name.value = ''
|
|
91
|
+
toast.add({ title: 'Running experiment…', icon: 'i-lucide-flask-conical', color: 'info' })
|
|
92
|
+
await store.launch(created.id)
|
|
93
|
+
toast.add({ title: 'Experiment complete', icon: 'i-lucide-check', color: 'success' })
|
|
94
|
+
} catch (e) {
|
|
95
|
+
toast.add({
|
|
96
|
+
title: 'Could not run the experiment',
|
|
97
|
+
description: e instanceof Error ? e.message : String(e),
|
|
98
|
+
icon: 'i-lucide-triangle-alert',
|
|
99
|
+
color: 'error',
|
|
100
|
+
})
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// ---- results grid ----------------------------------------------------------
|
|
105
|
+
const gradeByRun = computed(() => {
|
|
106
|
+
const map = new Map<string, SandboxGrade>()
|
|
107
|
+
for (const g of store.detail?.grades ?? []) map.set(g.runId, g)
|
|
108
|
+
return map
|
|
109
|
+
})
|
|
110
|
+
const selectedRun = ref<SandboxRun | null>(null)
|
|
111
|
+
|
|
112
|
+
function scoreColor(score: number): string {
|
|
113
|
+
if (score >= 4) return 'text-emerald-400'
|
|
114
|
+
if (score >= 3) return 'text-amber-400'
|
|
115
|
+
return 'text-rose-400'
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ---- prompt editor ---------------------------------------------------------
|
|
119
|
+
const editing = ref<SandboxPromptVersion | null>(null)
|
|
120
|
+
const editText = ref('')
|
|
121
|
+
const savingPrompt = ref(false)
|
|
122
|
+
|
|
123
|
+
function edit(prompt: SandboxPromptVersion) {
|
|
124
|
+
editing.value = prompt
|
|
125
|
+
editText.value = prompt.systemText
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async function saveVersion() {
|
|
129
|
+
if (!editing.value || !editText.value.trim()) return
|
|
130
|
+
savingPrompt.value = true
|
|
131
|
+
try {
|
|
132
|
+
await store.saveVersion(editing.value.id, editText.value)
|
|
133
|
+
toast.add({ title: 'Saved a new version', icon: 'i-lucide-check', color: 'success' })
|
|
134
|
+
editing.value = null
|
|
135
|
+
} catch (e) {
|
|
136
|
+
toast.add({
|
|
137
|
+
title: 'Could not save the version',
|
|
138
|
+
description: e instanceof Error ? e.message : String(e),
|
|
139
|
+
icon: 'i-lucide-triangle-alert',
|
|
140
|
+
color: 'error',
|
|
141
|
+
})
|
|
142
|
+
} finally {
|
|
143
|
+
savingPrompt.value = false
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async function archive(prompt: SandboxPromptVersion) {
|
|
148
|
+
try {
|
|
149
|
+
await store.archivePrompt(prompt.id)
|
|
150
|
+
if (editing.value?.id === prompt.id) editing.value = null
|
|
151
|
+
} catch (e) {
|
|
152
|
+
toast.add({
|
|
153
|
+
title: 'Could not archive',
|
|
154
|
+
description: e instanceof Error ? e.message : String(e),
|
|
155
|
+
icon: 'i-lucide-triangle-alert',
|
|
156
|
+
color: 'error',
|
|
157
|
+
})
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const fixtureName = (id: string) => store.fixtures.find((f) => f.id === id)?.name ?? id
|
|
162
|
+
</script>
|
|
163
|
+
|
|
164
|
+
<template>
|
|
165
|
+
<UModal
|
|
166
|
+
v-model:open="open"
|
|
167
|
+
title="Sandbox — prompt & model testing"
|
|
168
|
+
description="Try prompt versions and models against graded fixtures, scored by a judge model."
|
|
169
|
+
:ui="{ content: 'max-w-5xl' }"
|
|
170
|
+
>
|
|
171
|
+
<template #body>
|
|
172
|
+
<div v-if="store.loading" class="flex items-center justify-center py-12">
|
|
173
|
+
<UIcon name="i-lucide-loader-circle" class="h-6 w-6 animate-spin text-slate-400" />
|
|
174
|
+
</div>
|
|
175
|
+
|
|
176
|
+
<div
|
|
177
|
+
v-else-if="!store.available"
|
|
178
|
+
class="rounded-lg border border-slate-700 bg-slate-900/50 p-6 text-sm text-slate-300"
|
|
179
|
+
>
|
|
180
|
+
<p class="font-medium text-slate-200">The Sandbox isn't enabled for this deployment.</p>
|
|
181
|
+
<p class="mt-1 text-slate-400">
|
|
182
|
+
It needs its own database (a dedicated <code>SANDBOX_DB</code> on Cloudflare, or the
|
|
183
|
+
<code>sandbox</code> Postgres schema on Node). Provision it and reload.
|
|
184
|
+
</p>
|
|
185
|
+
</div>
|
|
186
|
+
|
|
187
|
+
<div
|
|
188
|
+
v-else-if="store.error"
|
|
189
|
+
class="rounded-lg border border-rose-800 bg-rose-950/40 p-6 text-sm text-rose-200"
|
|
190
|
+
>
|
|
191
|
+
<p class="font-medium text-rose-100">The Sandbox failed to load.</p>
|
|
192
|
+
<p class="mt-1 text-rose-300">{{ store.error }}</p>
|
|
193
|
+
<UButton class="mt-3" size="xs" color="neutral" variant="subtle" @click="store.load()">
|
|
194
|
+
Retry
|
|
195
|
+
</UButton>
|
|
196
|
+
</div>
|
|
197
|
+
|
|
198
|
+
<div v-else class="space-y-4">
|
|
199
|
+
<UTabs
|
|
200
|
+
v-model="tab"
|
|
201
|
+
:items="[
|
|
202
|
+
{ label: 'Experiments', value: 'experiments', icon: 'i-lucide-flask-conical' },
|
|
203
|
+
{ label: 'Prompts', value: 'prompts', icon: 'i-lucide-file-text' },
|
|
204
|
+
{ label: 'Fixtures', value: 'fixtures', icon: 'i-lucide-clipboard-list' },
|
|
205
|
+
]"
|
|
206
|
+
/>
|
|
207
|
+
|
|
208
|
+
<!-- ============================= EXPERIMENTS ============================= -->
|
|
209
|
+
<div v-if="tab === 'experiments'" class="grid gap-4 lg:grid-cols-2">
|
|
210
|
+
<!-- builder -->
|
|
211
|
+
<div class="space-y-3 rounded-lg border border-slate-700 bg-slate-900/40 p-3">
|
|
212
|
+
<p class="text-[11px] font-semibold uppercase tracking-wide text-slate-400">
|
|
213
|
+
New experiment
|
|
214
|
+
</p>
|
|
215
|
+
|
|
216
|
+
<UFormField label="Agent">
|
|
217
|
+
<USelect
|
|
218
|
+
v-model="agentKind"
|
|
219
|
+
:items="store.agentKinds.map((k) => ({ label: k.label, value: k.agentKind }))"
|
|
220
|
+
value-key="value"
|
|
221
|
+
class="w-full"
|
|
222
|
+
/>
|
|
223
|
+
</UFormField>
|
|
224
|
+
|
|
225
|
+
<div>
|
|
226
|
+
<span class="mb-1 block text-[10px] uppercase tracking-wide text-slate-500">
|
|
227
|
+
Prompt versions
|
|
228
|
+
</span>
|
|
229
|
+
<div class="max-h-28 space-y-1 overflow-auto pr-1">
|
|
230
|
+
<label
|
|
231
|
+
v-for="p in kindPrompts"
|
|
232
|
+
:key="p.id"
|
|
233
|
+
class="flex items-center gap-2 text-sm text-slate-300"
|
|
234
|
+
>
|
|
235
|
+
<UCheckbox
|
|
236
|
+
:model-value="selectedPromptIds.includes(p.id)"
|
|
237
|
+
@update:model-value="
|
|
238
|
+
(v: boolean | 'indeterminate') => toggle('prompt', p.id, v === true)
|
|
239
|
+
"
|
|
240
|
+
/>
|
|
241
|
+
<span class="truncate">{{ p.name }}</span>
|
|
242
|
+
<UBadge
|
|
243
|
+
:color="p.origin === 'baseline' ? 'neutral' : 'primary'"
|
|
244
|
+
variant="soft"
|
|
245
|
+
size="xs"
|
|
246
|
+
>
|
|
247
|
+
{{ p.origin === 'baseline' ? 'baseline' : `v${p.version}` }}
|
|
248
|
+
</UBadge>
|
|
249
|
+
</label>
|
|
250
|
+
</div>
|
|
251
|
+
</div>
|
|
252
|
+
|
|
253
|
+
<div>
|
|
254
|
+
<span class="mb-1 block text-[10px] uppercase tracking-wide text-slate-500">
|
|
255
|
+
Models
|
|
256
|
+
</span>
|
|
257
|
+
<div class="max-h-28 space-y-1 overflow-auto pr-1">
|
|
258
|
+
<label
|
|
259
|
+
v-for="m in store.selectableModels"
|
|
260
|
+
:key="m.id"
|
|
261
|
+
class="flex items-center gap-2 text-sm text-slate-300"
|
|
262
|
+
>
|
|
263
|
+
<UCheckbox
|
|
264
|
+
:model-value="selectedModelIds.includes(m.id)"
|
|
265
|
+
@update:model-value="
|
|
266
|
+
(v: boolean | 'indeterminate') => toggle('model', m.id, v === true)
|
|
267
|
+
"
|
|
268
|
+
/>
|
|
269
|
+
<span class="truncate">{{ m.label }}</span>
|
|
270
|
+
</label>
|
|
271
|
+
<p v-if="!store.selectableModels.length" class="text-xs text-slate-500">
|
|
272
|
+
No selectable models — configure a provider key or enable Cloudflare AI.
|
|
273
|
+
</p>
|
|
274
|
+
</div>
|
|
275
|
+
</div>
|
|
276
|
+
|
|
277
|
+
<div>
|
|
278
|
+
<span class="mb-1 block text-[10px] uppercase tracking-wide text-slate-500">
|
|
279
|
+
Fixtures
|
|
280
|
+
</span>
|
|
281
|
+
<div class="max-h-28 space-y-1 overflow-auto pr-1">
|
|
282
|
+
<label
|
|
283
|
+
v-for="f in kindFixtures"
|
|
284
|
+
:key="f.id"
|
|
285
|
+
class="flex items-center gap-2 text-sm text-slate-300"
|
|
286
|
+
>
|
|
287
|
+
<UCheckbox
|
|
288
|
+
:model-value="selectedFixtureIds.includes(f.id)"
|
|
289
|
+
@update:model-value="
|
|
290
|
+
(v: boolean | 'indeterminate') => toggle('fixture', f.id, v === true)
|
|
291
|
+
"
|
|
292
|
+
/>
|
|
293
|
+
<span class="truncate">{{ f.name }}</span>
|
|
294
|
+
</label>
|
|
295
|
+
<p v-if="!kindFixtures.length" class="text-xs text-slate-500">
|
|
296
|
+
No fixtures for this agent.
|
|
297
|
+
</p>
|
|
298
|
+
</div>
|
|
299
|
+
</div>
|
|
300
|
+
|
|
301
|
+
<UFormField label="Judge model" hint="grades every cell">
|
|
302
|
+
<USelect v-model="selectedJudgeModel" :items="judgeModelItems" />
|
|
303
|
+
</UFormField>
|
|
304
|
+
|
|
305
|
+
<UFormField label="Name (optional)">
|
|
306
|
+
<UInput v-model="name" :placeholder="`${agentKind} — sandbox run`" />
|
|
307
|
+
</UFormField>
|
|
308
|
+
|
|
309
|
+
<div class="flex items-center justify-between">
|
|
310
|
+
<span class="text-xs text-slate-500">
|
|
311
|
+
{{ cellCount }} cell{{ cellCount === 1 ? '' : 's' }}
|
|
312
|
+
<span v-if="cellCount > store.maxCells" class="text-rose-400">
|
|
313
|
+
(max {{ store.maxCells }})
|
|
314
|
+
</span>
|
|
315
|
+
</span>
|
|
316
|
+
<UButton
|
|
317
|
+
color="primary"
|
|
318
|
+
icon="i-lucide-play"
|
|
319
|
+
size="sm"
|
|
320
|
+
:loading="store.launching"
|
|
321
|
+
:disabled="!canRun"
|
|
322
|
+
@click="createAndRun()"
|
|
323
|
+
>
|
|
324
|
+
Run
|
|
325
|
+
</UButton>
|
|
326
|
+
</div>
|
|
327
|
+
</div>
|
|
328
|
+
|
|
329
|
+
<!-- history + results -->
|
|
330
|
+
<div class="space-y-3">
|
|
331
|
+
<div v-if="store.detail" class="rounded-lg border border-slate-700 bg-slate-900/40 p-3">
|
|
332
|
+
<div class="mb-2 flex items-center justify-between">
|
|
333
|
+
<p class="text-sm font-medium text-slate-200">
|
|
334
|
+
{{ store.detail.experiment.name }}
|
|
335
|
+
</p>
|
|
336
|
+
<UBadge variant="soft" size="xs">{{ store.detail.experiment.status }}</UBadge>
|
|
337
|
+
</div>
|
|
338
|
+
<div class="overflow-auto">
|
|
339
|
+
<table class="w-full text-left text-xs">
|
|
340
|
+
<thead class="text-slate-500">
|
|
341
|
+
<tr>
|
|
342
|
+
<th class="py-1 pr-2 font-medium">Prompt</th>
|
|
343
|
+
<th class="py-1 pr-2 font-medium">Model</th>
|
|
344
|
+
<th class="py-1 pr-2 font-medium">Fixture</th>
|
|
345
|
+
<th class="py-1 pr-2 font-medium">Score</th>
|
|
346
|
+
<th class="py-1 font-medium">Objective</th>
|
|
347
|
+
</tr>
|
|
348
|
+
</thead>
|
|
349
|
+
<tbody>
|
|
350
|
+
<tr
|
|
351
|
+
v-for="run in store.detail.runs"
|
|
352
|
+
:key="run.id"
|
|
353
|
+
class="cursor-pointer border-t border-slate-800 hover:bg-slate-800/40"
|
|
354
|
+
@click="selectedRun = run"
|
|
355
|
+
>
|
|
356
|
+
<td class="py-1 pr-2 text-slate-300">{{ run.promptLabel }}</td>
|
|
357
|
+
<td class="py-1 pr-2 font-mono text-[11px] text-slate-400">
|
|
358
|
+
{{ run.model }}
|
|
359
|
+
</td>
|
|
360
|
+
<td class="py-1 pr-2 text-slate-400">{{ fixtureName(run.fixtureId) }}</td>
|
|
361
|
+
<td class="py-1 pr-2">
|
|
362
|
+
<span
|
|
363
|
+
v-if="gradeByRun.get(run.id)"
|
|
364
|
+
:class="scoreColor(gradeByRun.get(run.id)!.weightedTotal)"
|
|
365
|
+
class="font-semibold"
|
|
366
|
+
>
|
|
367
|
+
{{ gradeByRun.get(run.id)!.weightedTotal.toFixed(2) }}
|
|
368
|
+
</span>
|
|
369
|
+
<span v-else-if="run.status === 'failed'" class="text-rose-400"
|
|
370
|
+
>failed</span
|
|
371
|
+
>
|
|
372
|
+
<span v-else class="text-slate-600">—</span>
|
|
373
|
+
</td>
|
|
374
|
+
<td class="py-1">
|
|
375
|
+
<span
|
|
376
|
+
v-if="gradeByRun.get(run.id)?.objective"
|
|
377
|
+
:class="
|
|
378
|
+
gradeByRun.get(run.id)!.objective!.pass
|
|
379
|
+
? 'text-emerald-400'
|
|
380
|
+
: 'text-amber-400'
|
|
381
|
+
"
|
|
382
|
+
>
|
|
383
|
+
{{ gradeByRun.get(run.id)!.objective!.caught }}/{{
|
|
384
|
+
gradeByRun.get(run.id)!.objective!.total
|
|
385
|
+
}}
|
|
386
|
+
</span>
|
|
387
|
+
<span v-else class="text-slate-600">—</span>
|
|
388
|
+
</td>
|
|
389
|
+
</tr>
|
|
390
|
+
</tbody>
|
|
391
|
+
</table>
|
|
392
|
+
</div>
|
|
393
|
+
|
|
394
|
+
<!-- selected cell output -->
|
|
395
|
+
<div v-if="selectedRun" class="mt-3 border-t border-slate-800 pt-2">
|
|
396
|
+
<p class="mb-1 text-[11px] uppercase tracking-wide text-slate-500">
|
|
397
|
+
{{ selectedRun.promptLabel }} · {{ selectedRun.model }}
|
|
398
|
+
</p>
|
|
399
|
+
<p v-if="selectedRun.error" class="text-xs text-rose-400">
|
|
400
|
+
{{ selectedRun.error }}
|
|
401
|
+
</p>
|
|
402
|
+
<pre
|
|
403
|
+
v-if="selectedRun.outputText"
|
|
404
|
+
class="max-h-48 overflow-auto whitespace-pre-wrap rounded bg-slate-950/60 p-2 text-[11px] text-slate-300"
|
|
405
|
+
>{{ selectedRun.outputText }}</pre
|
|
406
|
+
>
|
|
407
|
+
<div v-if="gradeByRun.get(selectedRun.id)" class="mt-2 space-y-0.5">
|
|
408
|
+
<p
|
|
409
|
+
v-for="d in gradeByRun.get(selectedRun.id)!.scores"
|
|
410
|
+
:key="d.key"
|
|
411
|
+
class="text-[11px] text-slate-400"
|
|
412
|
+
>
|
|
413
|
+
<span :class="scoreColor(d.score)" class="font-semibold">{{ d.score }}</span>
|
|
414
|
+
<span class="ml-1 text-slate-300">{{ d.key }}</span>
|
|
415
|
+
<span v-if="d.rationale" class="ml-1 text-slate-500">— {{ d.rationale }}</span>
|
|
416
|
+
</p>
|
|
417
|
+
</div>
|
|
418
|
+
</div>
|
|
419
|
+
</div>
|
|
420
|
+
|
|
421
|
+
<p class="text-[11px] uppercase tracking-wide text-slate-500">Past experiments</p>
|
|
422
|
+
<div class="max-h-56 space-y-1 overflow-auto">
|
|
423
|
+
<button
|
|
424
|
+
v-for="x in store.experiments"
|
|
425
|
+
:key="x.id"
|
|
426
|
+
class="flex w-full items-center justify-between rounded-md border border-slate-800 bg-slate-900/40 px-2 py-1.5 text-left text-sm hover:bg-slate-800/50"
|
|
427
|
+
@click="store.openExperiment(x.id)"
|
|
428
|
+
>
|
|
429
|
+
<span class="truncate text-slate-300">{{ x.name }}</span>
|
|
430
|
+
<UBadge variant="soft" size="xs">{{ x.status }}</UBadge>
|
|
431
|
+
</button>
|
|
432
|
+
<p v-if="!store.experiments.length" class="text-xs text-slate-500">
|
|
433
|
+
No experiments yet.
|
|
434
|
+
</p>
|
|
435
|
+
</div>
|
|
436
|
+
</div>
|
|
437
|
+
</div>
|
|
438
|
+
|
|
439
|
+
<!-- ============================== PROMPTS ============================== -->
|
|
440
|
+
<div v-else-if="tab === 'prompts'" class="grid gap-4 lg:grid-cols-2">
|
|
441
|
+
<div class="max-h-[28rem] space-y-1.5 overflow-auto pr-1">
|
|
442
|
+
<div
|
|
443
|
+
v-for="p in store.prompts"
|
|
444
|
+
:key="p.id"
|
|
445
|
+
class="flex items-center justify-between rounded-md border border-slate-800 bg-slate-900/40 px-2.5 py-1.5 text-sm"
|
|
446
|
+
>
|
|
447
|
+
<div class="min-w-0">
|
|
448
|
+
<div class="flex items-center gap-2">
|
|
449
|
+
<span class="truncate text-slate-200">{{ p.name }}</span>
|
|
450
|
+
<UBadge
|
|
451
|
+
:color="p.origin === 'baseline' ? 'neutral' : 'primary'"
|
|
452
|
+
variant="soft"
|
|
453
|
+
size="xs"
|
|
454
|
+
>
|
|
455
|
+
{{ p.origin === 'baseline' ? 'baseline' : `v${p.version}` }}
|
|
456
|
+
</UBadge>
|
|
457
|
+
</div>
|
|
458
|
+
<span class="text-[11px] text-slate-500">{{ p.agentKind }}</span>
|
|
459
|
+
</div>
|
|
460
|
+
<div class="flex items-center gap-1">
|
|
461
|
+
<UButton
|
|
462
|
+
icon="i-lucide-pencil"
|
|
463
|
+
color="neutral"
|
|
464
|
+
variant="ghost"
|
|
465
|
+
size="xs"
|
|
466
|
+
:title="p.origin === 'baseline' ? 'Fork into a candidate' : 'Edit / version'"
|
|
467
|
+
@click="edit(p)"
|
|
468
|
+
/>
|
|
469
|
+
<UButton
|
|
470
|
+
v-if="p.origin === 'candidate'"
|
|
471
|
+
icon="i-lucide-archive"
|
|
472
|
+
color="error"
|
|
473
|
+
variant="ghost"
|
|
474
|
+
size="xs"
|
|
475
|
+
@click="archive(p)"
|
|
476
|
+
/>
|
|
477
|
+
</div>
|
|
478
|
+
</div>
|
|
479
|
+
</div>
|
|
480
|
+
|
|
481
|
+
<div
|
|
482
|
+
v-if="editing"
|
|
483
|
+
class="space-y-2 rounded-lg border border-slate-700 bg-slate-900/40 p-3"
|
|
484
|
+
>
|
|
485
|
+
<p class="text-[11px] uppercase tracking-wide text-slate-500">
|
|
486
|
+
{{ editing.origin === 'baseline' ? 'Fork' : 'New version of' }} · {{ editing.name }}
|
|
487
|
+
</p>
|
|
488
|
+
<UTextarea v-model="editText" :rows="16" class="w-full font-mono text-xs" autoresize />
|
|
489
|
+
<div class="flex justify-end gap-2">
|
|
490
|
+
<UButton color="neutral" variant="ghost" size="sm" @click="editing = null">
|
|
491
|
+
Cancel
|
|
492
|
+
</UButton>
|
|
493
|
+
<UButton
|
|
494
|
+
color="primary"
|
|
495
|
+
icon="i-lucide-save"
|
|
496
|
+
size="sm"
|
|
497
|
+
:loading="savingPrompt"
|
|
498
|
+
:disabled="!editText.trim()"
|
|
499
|
+
@click="saveVersion()"
|
|
500
|
+
>
|
|
501
|
+
Save new version
|
|
502
|
+
</UButton>
|
|
503
|
+
</div>
|
|
504
|
+
</div>
|
|
505
|
+
<p v-else class="self-start text-xs text-slate-500">
|
|
506
|
+
Pick a prompt to fork a shipped baseline or version a candidate. Each save appends an
|
|
507
|
+
immutable version you can put under test.
|
|
508
|
+
</p>
|
|
509
|
+
</div>
|
|
510
|
+
|
|
511
|
+
<!-- ============================== FIXTURES ============================== -->
|
|
512
|
+
<div v-else class="max-h-[28rem] space-y-1.5 overflow-auto pr-1">
|
|
513
|
+
<div
|
|
514
|
+
v-for="f in store.fixtures"
|
|
515
|
+
:key="f.id"
|
|
516
|
+
class="rounded-md border border-slate-800 bg-slate-900/40 px-2.5 py-2 text-sm"
|
|
517
|
+
>
|
|
518
|
+
<div class="flex items-center justify-between">
|
|
519
|
+
<span class="text-slate-200">{{ f.name }}</span>
|
|
520
|
+
<div class="flex items-center gap-1.5">
|
|
521
|
+
<UBadge variant="soft" size="xs">{{ f.kind }}</UBadge>
|
|
522
|
+
<UBadge
|
|
523
|
+
:color="f.origin === 'builtin' ? 'neutral' : 'primary'"
|
|
524
|
+
variant="soft"
|
|
525
|
+
size="xs"
|
|
526
|
+
>
|
|
527
|
+
{{ f.origin }}
|
|
528
|
+
</UBadge>
|
|
529
|
+
</div>
|
|
530
|
+
</div>
|
|
531
|
+
<p v-if="f.objective?.kind === 'findings'" class="mt-0.5 text-[11px] text-slate-500">
|
|
532
|
+
{{ f.objective.expectations.length }} graded expectation{{
|
|
533
|
+
f.objective.expectations.length === 1 ? '' : 's'
|
|
534
|
+
}}
|
|
535
|
+
</p>
|
|
536
|
+
</div>
|
|
537
|
+
<p v-if="!store.fixtures.length" class="text-xs text-slate-500">No fixtures.</p>
|
|
538
|
+
</div>
|
|
539
|
+
</div>
|
|
540
|
+
</template>
|
|
541
|
+
</UModal>
|
|
542
|
+
</template>
|
|
@@ -25,6 +25,12 @@ const provider = ref<ObservabilityProviderKind>('datadog')
|
|
|
25
25
|
const datadog = reactive({ site: 'datadoghq.com', apiKey: '', appKey: '' })
|
|
26
26
|
const busy = ref(false)
|
|
27
27
|
|
|
28
|
+
// Incident enrichment (PagerDuty + incident.io) — write-only secrets; blank leaves the
|
|
29
|
+
// stored value unchanged. Paired with observability since it acts on the same regression.
|
|
30
|
+
const pagerDuty = reactive({ apiToken: '', fromEmail: '' })
|
|
31
|
+
const incidentIo = reactive({ apiKey: '' })
|
|
32
|
+
const incidentBusy = ref(false)
|
|
33
|
+
|
|
28
34
|
function notifyError(title: string, e: unknown) {
|
|
29
35
|
toast.add({
|
|
30
36
|
title,
|
|
@@ -41,11 +47,50 @@ watch(open, async (isOpen) => {
|
|
|
41
47
|
if (store.connection.provider) provider.value = store.connection.provider
|
|
42
48
|
const site = store.connection.summary?.site
|
|
43
49
|
if (site) datadog.site = site
|
|
50
|
+
await store.loadIncident()
|
|
44
51
|
} catch (e) {
|
|
45
52
|
notifyError('Could not load observability settings', e)
|
|
46
53
|
}
|
|
47
54
|
})
|
|
48
55
|
|
|
56
|
+
async function saveIncident() {
|
|
57
|
+
incidentBusy.value = true
|
|
58
|
+
try {
|
|
59
|
+
const input: Parameters<typeof store.saveIncident>[0] = {}
|
|
60
|
+
if (pagerDuty.apiToken.trim() && pagerDuty.fromEmail.trim()) {
|
|
61
|
+
input.pagerDuty = {
|
|
62
|
+
apiToken: pagerDuty.apiToken.trim(),
|
|
63
|
+
fromEmail: pagerDuty.fromEmail.trim(),
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
if (incidentIo.apiKey.trim()) input.incidentIo = { apiKey: incidentIo.apiKey.trim() }
|
|
67
|
+
if (!input.pagerDuty && !input.incidentIo) {
|
|
68
|
+
toast.add({ title: 'Enter PagerDuty or incident.io credentials', color: 'error' })
|
|
69
|
+
return
|
|
70
|
+
}
|
|
71
|
+
await store.saveIncident(input)
|
|
72
|
+
pagerDuty.apiToken = ''
|
|
73
|
+
pagerDuty.fromEmail = ''
|
|
74
|
+
incidentIo.apiKey = ''
|
|
75
|
+
toast.add({ title: 'Incident enrichment saved', icon: 'i-lucide-check', color: 'success' })
|
|
76
|
+
} catch (e) {
|
|
77
|
+
notifyError('Could not save incident enrichment', e)
|
|
78
|
+
} finally {
|
|
79
|
+
incidentBusy.value = false
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function disconnectIncident() {
|
|
84
|
+
incidentBusy.value = true
|
|
85
|
+
try {
|
|
86
|
+
await store.removeIncident()
|
|
87
|
+
} catch (e) {
|
|
88
|
+
notifyError('Could not disconnect incident enrichment', e)
|
|
89
|
+
} finally {
|
|
90
|
+
incidentBusy.value = false
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
49
94
|
async function saveConnection() {
|
|
50
95
|
busy.value = true
|
|
51
96
|
try {
|
|
@@ -145,6 +190,53 @@ const connectedLabel = computed(() => {
|
|
|
145
190
|
</UButton>
|
|
146
191
|
</div>
|
|
147
192
|
</section>
|
|
193
|
+
|
|
194
|
+
<!-- Incident enrichment (optional): annotate an incident PagerDuty / incident.io
|
|
195
|
+
already opened from the same monitors/SLOs. -->
|
|
196
|
+
<section
|
|
197
|
+
v-if="store.incidentAvailable !== false"
|
|
198
|
+
class="space-y-3 rounded-lg border border-slate-700 p-3"
|
|
199
|
+
>
|
|
200
|
+
<div class="flex items-center justify-between">
|
|
201
|
+
<h3 class="text-sm font-semibold">Incident enrichment</h3>
|
|
202
|
+
<UBadge :color="store.incident.connected ? 'success' : 'neutral'" variant="soft">
|
|
203
|
+
{{ store.incident.connected ? 'Configured' : 'Not set' }}
|
|
204
|
+
</UBadge>
|
|
205
|
+
</div>
|
|
206
|
+
<p class="text-[11px] text-slate-400">
|
|
207
|
+
Optional. On a regression, the on-call investigation is posted onto an incident these
|
|
208
|
+
systems ALREADY opened from the same signals (annotate, never re-alert). Secrets are
|
|
209
|
+
write-only — blank leaves a stored value unchanged.
|
|
210
|
+
</p>
|
|
211
|
+
|
|
212
|
+
<UFormField label="PagerDuty API token">
|
|
213
|
+
<UInput v-model="pagerDuty.apiToken" type="password" class="w-full" />
|
|
214
|
+
</UFormField>
|
|
215
|
+
<UFormField label="PagerDuty From email">
|
|
216
|
+
<UInput
|
|
217
|
+
v-model="pagerDuty.fromEmail"
|
|
218
|
+
type="email"
|
|
219
|
+
placeholder="oncall@example.com"
|
|
220
|
+
class="w-full"
|
|
221
|
+
/>
|
|
222
|
+
</UFormField>
|
|
223
|
+
<UFormField label="incident.io API key">
|
|
224
|
+
<UInput v-model="incidentIo.apiKey" type="password" class="w-full" />
|
|
225
|
+
</UFormField>
|
|
226
|
+
|
|
227
|
+
<div class="flex gap-2">
|
|
228
|
+
<UButton :loading="incidentBusy" @click="saveIncident">Save</UButton>
|
|
229
|
+
<UButton
|
|
230
|
+
v-if="store.incident.connected"
|
|
231
|
+
color="error"
|
|
232
|
+
variant="soft"
|
|
233
|
+
:loading="incidentBusy"
|
|
234
|
+
@click="disconnectIncident"
|
|
235
|
+
>
|
|
236
|
+
Clear
|
|
237
|
+
</UButton>
|
|
238
|
+
</div>
|
|
239
|
+
</section>
|
|
148
240
|
</div>
|
|
149
241
|
</template>
|
|
150
242
|
</UModal>
|