@cat-factory/app 0.232.2 → 0.234.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/components/observability/OutcomeFilterChips.vue +41 -0
- package/app/components/observability/RunFailureSummary.vue +243 -0
- package/app/components/observability/ToolCallList.vue +290 -0
- package/app/components/panels/ObservabilityPanel.vue +396 -129
- package/app/components/settings/TaskTypeSuppressionsPanel.vue +119 -0
- package/app/components/settings/WorkspaceSettingsPanel.vue +22 -0
- package/app/composables/api/execution.ts +22 -0
- package/app/composables/api/taskTypeSuppressions.ts +25 -0
- package/app/composables/useApi.ts +2 -0
- package/app/composables/usePipelineHealth.spec.ts +15 -1
- package/app/composables/usePipelineHealth.ts +13 -7
- package/app/docs/consumer-extensions.md +7 -1
- package/app/stores/observability/toolCalls.ts +173 -0
- package/app/stores/observability.ts +13 -0
- package/app/stores/pipelines.ts +22 -5
- package/app/stores/taskTypes.spec.ts +29 -0
- package/app/stores/taskTypes.ts +40 -1
- package/app/stores/workspace/hydrate.ts +5 -0
- package/app/types/domain.ts +3 -0
- package/app/types/execution.ts +6 -0
- package/app/utils/descriptorFields.ts +13 -27
- package/app/utils/observability.spec.ts +313 -2
- package/app/utils/observability.ts +232 -1
- package/i18n/locales/de.json +53 -0
- package/i18n/locales/en.json +53 -0
- package/i18n/locales/es.json +53 -0
- package/i18n/locales/fr.json +53 -0
- package/i18n/locales/he.json +53 -0
- package/i18n/locales/it.json +53 -0
- package/i18n/locales/ja.json +53 -0
- package/i18n/locales/pl.json +53 -0
- package/i18n/locales/tr.json +53 -0
- package/i18n/locales/uk.json +53 -0
- package/package.json +2 -2
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
<script setup lang="ts">
|
|
2
|
+
// Workspace settings: which of the deployment's REUSABLE OPERATIONS this board offers
|
|
3
|
+
// (`backend/docs/reusable-operations.md`). An org registers its operations process-wide, so a
|
|
4
|
+
// team that runs three of twenty needs a way to clear the rest out of its create picker.
|
|
5
|
+
//
|
|
6
|
+
// The list comes from its OWN read, not from the board snapshot's `customTaskTypes`: a suppressed
|
|
7
|
+
// operation is by construction absent from that catalog, so this screen is the only surface that
|
|
8
|
+
// can offer the way back. Every write answers with the whole list AND invalidates the catalog the
|
|
9
|
+
// picker renders, so a change is followed by a board refresh rather than a local patch.
|
|
10
|
+
//
|
|
11
|
+
// Labels and descriptions are DEPLOYMENT-authored English rendered verbatim (the descriptor
|
|
12
|
+
// convention); only the chrome around them is i18n.
|
|
13
|
+
import { onMounted, ref } from 'vue'
|
|
14
|
+
import type { TaskTypeSuppression } from '~/types/domain'
|
|
15
|
+
|
|
16
|
+
const { t } = useI18n()
|
|
17
|
+
const api = useApi()
|
|
18
|
+
const workspace = useWorkspaceStore()
|
|
19
|
+
const toast = useToast()
|
|
20
|
+
|
|
21
|
+
const rows = ref<TaskTypeSuppression[]>([])
|
|
22
|
+
const loading = ref(false)
|
|
23
|
+
/** The single id being written, so only its own switch shows the pending state. */
|
|
24
|
+
const busyId = ref<string | null>(null)
|
|
25
|
+
|
|
26
|
+
onMounted(() => void load())
|
|
27
|
+
|
|
28
|
+
async function load() {
|
|
29
|
+
if (!workspace.workspaceId) return
|
|
30
|
+
loading.value = true
|
|
31
|
+
try {
|
|
32
|
+
rows.value = (await api.listTaskTypeSuppressions(workspace.requireId())).taskTypes
|
|
33
|
+
} catch (e) {
|
|
34
|
+
fail(e)
|
|
35
|
+
} finally {
|
|
36
|
+
loading.value = false
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Flip one operation. The board snapshot's task-type catalog is derived from this state, so a
|
|
42
|
+
* successful write refreshes the workspace: leaving it stale would keep offering a hidden
|
|
43
|
+
* operation in the create picker until the next unrelated reload.
|
|
44
|
+
*/
|
|
45
|
+
async function toggle(row: TaskTypeSuppression, offered: boolean) {
|
|
46
|
+
const id = row.taskType.taskType
|
|
47
|
+
busyId.value = id
|
|
48
|
+
try {
|
|
49
|
+
const result = offered
|
|
50
|
+
? await api.restoreTaskType(workspace.requireId(), id)
|
|
51
|
+
: await api.suppressTaskType(workspace.requireId(), id)
|
|
52
|
+
rows.value = result.taskTypes
|
|
53
|
+
await workspace.refresh()
|
|
54
|
+
} catch (e) {
|
|
55
|
+
fail(e)
|
|
56
|
+
} finally {
|
|
57
|
+
busyId.value = null
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function fail(e: unknown) {
|
|
62
|
+
toast.add({
|
|
63
|
+
title: t('settings.taskTypeSuppressions.saveFailed'),
|
|
64
|
+
description: e instanceof Error ? e.message : String(e),
|
|
65
|
+
icon: 'i-lucide-triangle-alert',
|
|
66
|
+
color: 'error',
|
|
67
|
+
})
|
|
68
|
+
}
|
|
69
|
+
</script>
|
|
70
|
+
|
|
71
|
+
<template>
|
|
72
|
+
<div class="space-y-4">
|
|
73
|
+
<p class="text-xs text-slate-400">
|
|
74
|
+
{{ t('settings.taskTypeSuppressions.intro') }}
|
|
75
|
+
</p>
|
|
76
|
+
|
|
77
|
+
<p v-if="loading" class="text-[11px] text-slate-500">
|
|
78
|
+
{{ t('settings.taskTypeSuppressions.loading') }}
|
|
79
|
+
</p>
|
|
80
|
+
<p v-else-if="!rows.length" class="text-[11px] text-slate-500">
|
|
81
|
+
{{ t('settings.taskTypeSuppressions.empty') }}
|
|
82
|
+
</p>
|
|
83
|
+
<ul v-else class="space-y-2" data-testid="task-type-suppressions">
|
|
84
|
+
<li
|
|
85
|
+
v-for="row in rows"
|
|
86
|
+
:key="row.taskType.taskType"
|
|
87
|
+
class="flex items-start justify-between gap-3 rounded border border-slate-800 px-3 py-2"
|
|
88
|
+
data-testid="task-type-suppression"
|
|
89
|
+
:data-task-type="row.taskType.taskType"
|
|
90
|
+
>
|
|
91
|
+
<div class="min-w-0">
|
|
92
|
+
<div class="flex items-center gap-1.5">
|
|
93
|
+
<UIcon :name="row.taskType.presentation.icon" class="h-3.5 w-3.5 shrink-0" />
|
|
94
|
+
<span class="truncate text-xs font-medium text-slate-200">
|
|
95
|
+
{{ row.taskType.presentation.label }}
|
|
96
|
+
</span>
|
|
97
|
+
<UBadge
|
|
98
|
+
v-if="row.taskType.presentation.category"
|
|
99
|
+
color="neutral"
|
|
100
|
+
variant="subtle"
|
|
101
|
+
size="sm"
|
|
102
|
+
>
|
|
103
|
+
{{ row.taskType.presentation.category }}
|
|
104
|
+
</UBadge>
|
|
105
|
+
</div>
|
|
106
|
+
<p class="mt-0.5 text-[11px] text-slate-500">
|
|
107
|
+
{{ row.taskType.presentation.description }}
|
|
108
|
+
</p>
|
|
109
|
+
</div>
|
|
110
|
+
<USwitch
|
|
111
|
+
:model-value="!row.suppressed"
|
|
112
|
+
:loading="busyId === row.taskType.taskType"
|
|
113
|
+
:aria-label="t('settings.taskTypeSuppressions.offer')"
|
|
114
|
+
@update:model-value="(offered: boolean) => toggle(row, offered)"
|
|
115
|
+
/>
|
|
116
|
+
</li>
|
|
117
|
+
</ul>
|
|
118
|
+
</div>
|
|
119
|
+
</template>
|
|
@@ -15,6 +15,7 @@ import type { InputGateMode, ReviewFrictionMode, TaskLimitMode } from '~/types/d
|
|
|
15
15
|
import RiskPolicyPanel from '~/components/settings/RiskPolicyPanel.vue'
|
|
16
16
|
import IssueTrackerPanel from '~/components/settings/IssueTrackerPanel.vue'
|
|
17
17
|
import ServiceFragmentDefaultsPanel from '~/components/settings/ServiceFragmentDefaultsPanel.vue'
|
|
18
|
+
import TaskTypeSuppressionsPanel from '~/components/settings/TaskTypeSuppressionsPanel.vue'
|
|
18
19
|
import BudgetSettings from '~/components/settings/BudgetSettings.vue'
|
|
19
20
|
import UsageSettings from '~/components/settings/UsageSettings.vue'
|
|
20
21
|
import WorkspaceMembersSettings from '~/components/layout/WorkspaceMembersSettings.vue'
|
|
@@ -29,6 +30,10 @@ const workspace = useWorkspaceStore()
|
|
|
29
30
|
const access = useWorkspaceAccess()
|
|
30
31
|
const toast = useToast()
|
|
31
32
|
const slots = useReactiveSlots<AppSlots>()
|
|
33
|
+
// Whether the deployment registers any reusable operation at all, hidden or not, so the
|
|
34
|
+
// Operations tab exists only where there is something for it to manage. Not the OFFERED catalog:
|
|
35
|
+
// hiding the last operation would then take away the only screen that un-hides one.
|
|
36
|
+
const taskTypes = useTaskTypesStore()
|
|
32
37
|
|
|
33
38
|
// The Metadata tab exists only where the deployment DECLARES custom fields — an unwired
|
|
34
39
|
// capability is invisible, not an empty tab in every deployment. Declared-but-malformed fields
|
|
@@ -86,6 +91,18 @@ const tabs = computed(() => [
|
|
|
86
91
|
icon: 'i-lucide-book-open-check',
|
|
87
92
|
slot: 'fragments',
|
|
88
93
|
},
|
|
94
|
+
// Which of the deployment's reusable operations this board offers. Only where the deployment
|
|
95
|
+
// registers any: on the stock product the tab would name a catalog that does not exist.
|
|
96
|
+
...(taskTypes.hasRegisteredOperations
|
|
97
|
+
? [
|
|
98
|
+
{
|
|
99
|
+
value: 'operations',
|
|
100
|
+
label: t('settings.workspaceSettings.tabs.operations'),
|
|
101
|
+
icon: 'i-lucide-plug',
|
|
102
|
+
slot: 'operations',
|
|
103
|
+
},
|
|
104
|
+
]
|
|
105
|
+
: []),
|
|
89
106
|
...(hasMetadataFields.value
|
|
90
107
|
? [
|
|
91
108
|
{
|
|
@@ -605,6 +622,11 @@ async function save() {
|
|
|
605
622
|
<ServiceFragmentDefaultsPanel />
|
|
606
623
|
</template>
|
|
607
624
|
|
|
625
|
+
<!-- Reusable operations this board offers (only where the deployment registers any) -->
|
|
626
|
+
<template v-if="taskTypes.hasRegisteredOperations" #operations>
|
|
627
|
+
<TaskTypeSuppressionsPanel />
|
|
628
|
+
</template>
|
|
629
|
+
|
|
608
630
|
<!-- Custom workspace metadata (only where the deployment declares fields) -->
|
|
609
631
|
<template v-if="hasMetadataFields" #metadata>
|
|
610
632
|
<WorkspaceMetadataSettings />
|
|
@@ -5,6 +5,8 @@ import {
|
|
|
5
5
|
getExecutionAgentContextContract,
|
|
6
6
|
getExecutionLlmMetricsContract,
|
|
7
7
|
getExecutionSearchQueriesContract,
|
|
8
|
+
getExecutionToolCallFailuresContract,
|
|
9
|
+
getExecutionToolCallsContract,
|
|
8
10
|
getWorkspaceUsageContract,
|
|
9
11
|
mergeBlockContract,
|
|
10
12
|
rejectStepContract,
|
|
@@ -165,6 +167,26 @@ export function executionApi({ send, sendWith, ws, pwHeaders }: ApiContext) {
|
|
|
165
167
|
pathParams: { executionId },
|
|
166
168
|
}),
|
|
167
169
|
|
|
170
|
+
// The tool-call trajectory: what the run's agents DID, oldest first. The half of a
|
|
171
|
+
// failure no model call reports: a tool that errors leaves the call that asked for it
|
|
172
|
+
// reporting `ok`. Bounded, and says so via `truncated`. Empty when the sink is not wired /
|
|
173
|
+
// storing is off. The BROWSE read: fetched when the trajectory is opened, since it carries
|
|
174
|
+
// every argument and result the run captured.
|
|
175
|
+
getToolCalls: (workspaceId: string, executionId: string) =>
|
|
176
|
+
send(getExecutionToolCallsContract, {
|
|
177
|
+
pathPrefix: ws(workspaceId),
|
|
178
|
+
pathParams: { executionId },
|
|
179
|
+
}),
|
|
180
|
+
|
|
181
|
+
// The run's failing tool calls plus its exact `{ total, failed }`, counted in SQL rather
|
|
182
|
+
// than off any list. The panel's headline read, made on open: cheap enough to front the
|
|
183
|
+
// page, and exact enough that it never disagrees with the debug overview on a long run.
|
|
184
|
+
getToolCallFailures: (workspaceId: string, executionId: string) =>
|
|
185
|
+
send(getExecutionToolCallFailuresContract, {
|
|
186
|
+
pathPrefix: ws(workspaceId),
|
|
187
|
+
pathParams: { executionId },
|
|
188
|
+
}),
|
|
189
|
+
|
|
168
190
|
// ---- spend safeguard --------------------------------------------------
|
|
169
191
|
resumeSpend: (workspaceId: string) =>
|
|
170
192
|
send(resumeSpendContract, { pathPrefix: ws(workspaceId) }),
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import {
|
|
2
|
+
listTaskTypeSuppressionsContract,
|
|
3
|
+
restoreTaskTypeContract,
|
|
4
|
+
suppressTaskTypeContract,
|
|
5
|
+
} from '@cat-factory/contracts'
|
|
6
|
+
import type { ApiContext } from './context'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Which of the deployment's REUSABLE OPERATIONS this board offers
|
|
10
|
+
* (`backend/docs/reusable-operations.md`). Every call answers with the WHOLE list, because the
|
|
11
|
+
* board snapshot's `customTaskTypes` changes with it: hiding one removes it from the picker's own
|
|
12
|
+
* catalog, so a point response would leave the caller reconciling against data it just invalidated.
|
|
13
|
+
*/
|
|
14
|
+
export function taskTypeSuppressionsApi({ send, ws }: ApiContext) {
|
|
15
|
+
return {
|
|
16
|
+
listTaskTypeSuppressions: (workspaceId: string) =>
|
|
17
|
+
send(listTaskTypeSuppressionsContract, { pathPrefix: ws(workspaceId) }),
|
|
18
|
+
|
|
19
|
+
suppressTaskType: (workspaceId: string, taskType: string) =>
|
|
20
|
+
send(suppressTaskTypeContract, { pathPrefix: ws(workspaceId), pathParams: { taskType } }),
|
|
21
|
+
|
|
22
|
+
restoreTaskType: (workspaceId: string, taskType: string) =>
|
|
23
|
+
send(restoreTaskTypeContract, { pathPrefix: ws(workspaceId), pathParams: { taskType } }),
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -4,6 +4,7 @@ import type { ApiContext } from './api/context'
|
|
|
4
4
|
import { accountsApi } from './api/accounts'
|
|
5
5
|
import { agentPromptsApi } from './api/agentPrompts'
|
|
6
6
|
import { agentSettingsApi } from './api/agentSettings'
|
|
7
|
+
import { taskTypeSuppressionsApi } from './api/taskTypeSuppressions'
|
|
7
8
|
import { platformObservabilityApi } from './api/platformObservability'
|
|
8
9
|
import { reportsApi } from './api/reports'
|
|
9
10
|
import { authApi } from './api/auth'
|
|
@@ -144,6 +145,7 @@ export function useApi() {
|
|
|
144
145
|
...presetsApi(ctx),
|
|
145
146
|
...agentPromptsApi(ctx),
|
|
146
147
|
...agentSettingsApi(ctx),
|
|
148
|
+
...taskTypeSuppressionsApi(ctx),
|
|
147
149
|
...preflightsApi(ctx),
|
|
148
150
|
...publicApiKeysApi(ctx),
|
|
149
151
|
...sharedStacksApi(ctx),
|
|
@@ -37,6 +37,7 @@ function scan(
|
|
|
37
37
|
pipelines: Pipeline[],
|
|
38
38
|
versions: Record<string, number> = {},
|
|
39
39
|
retired: { id: string; replacedBy?: string }[] = [],
|
|
40
|
+
names: Record<string, string> = {},
|
|
40
41
|
) {
|
|
41
42
|
const store = usePipelinesStore()
|
|
42
43
|
const retiredIds = new Set(retired.map((r) => r.id))
|
|
@@ -46,7 +47,7 @@ function scan(
|
|
|
46
47
|
...versions,
|
|
47
48
|
}).filter(([id]) => !retiredIds.has(id)),
|
|
48
49
|
)
|
|
49
|
-
store.hydrate(pipelines, catalogVersions, retired)
|
|
50
|
+
store.hydrate(pipelines, catalogVersions, retired, names)
|
|
50
51
|
return usePipelineHealth()
|
|
51
52
|
}
|
|
52
53
|
|
|
@@ -202,6 +203,19 @@ describe('usePipelineHealth', () => {
|
|
|
202
203
|
expect(outdated.value).toHaveLength(0)
|
|
203
204
|
})
|
|
204
205
|
|
|
206
|
+
it("names an un-adopted catalog entry from the catalog's own name map, not its id", () => {
|
|
207
|
+
// The case that made the humanised fallback wrong: a deployment's registered pipeline behind a
|
|
208
|
+
// reusable operation. `pl_org_introduce_api` humanises to "org introduce api", a name that
|
|
209
|
+
// appears nowhere else in the product, and this advisory is shown on exactly the boards that
|
|
210
|
+
// predate the operation. With the map, the offer reads as the pipeline actually is.
|
|
211
|
+
const stored = builtin(['coder', 'reviewer'], { id: 'pl_full', version: 1 })
|
|
212
|
+
const { newPipelines } = scan([stored], { pl_full: 1, pl_org_introduce_api: 1 }, [], {
|
|
213
|
+
pl_full: 'Full build',
|
|
214
|
+
pl_org_introduce_api: 'Introduce API',
|
|
215
|
+
})
|
|
216
|
+
expect(newPipelines.value).toEqual([{ id: 'pl_org_introduce_api', name: 'Introduce API' }])
|
|
217
|
+
})
|
|
218
|
+
|
|
205
219
|
it('reports no new pipelines when every catalog id is already stored', () => {
|
|
206
220
|
const stored = builtin(['coder', 'reviewer'], { id: 'pl_full', version: 1 })
|
|
207
221
|
const { newPipelines, hasIssues } = scan([stored], { pl_full: 1 })
|
|
@@ -57,12 +57,17 @@ export interface NewPipeline {
|
|
|
57
57
|
}
|
|
58
58
|
|
|
59
59
|
/**
|
|
60
|
-
* A
|
|
61
|
-
*
|
|
62
|
-
*
|
|
60
|
+
* A catalog entry's display name for the "new pipeline" advisory, used only while the entry has no
|
|
61
|
+
* stored row to take a name off. The snapshot's companion name map answers it; the humanised id
|
|
62
|
+
* (`pl_review` -> "review", rendered capitalised) is the FALLBACK for a facade that ships no map.
|
|
63
|
+
*
|
|
64
|
+
* The map is not a nicety. Humanising was fine for the shipped built-ins, whose ids read as their
|
|
65
|
+
* names, and wrong the moment a deployment registers its own: a reusable operation's
|
|
66
|
+
* `pl_org_introduce_api` was offered as "org introduce api", a name appearing nowhere else in the
|
|
67
|
+
* product, on exactly the boards that predate the operation and therefore see this advisory.
|
|
63
68
|
*/
|
|
64
|
-
function builtinPipelineName(id: string): string {
|
|
65
|
-
return id.replace(/^pl_/, '').replace(/_/g, ' ')
|
|
69
|
+
function builtinPipelineName(id: string, names: Record<string, string>): string {
|
|
70
|
+
return names[id] ?? id.replace(/^pl_/, '').replace(/_/g, ' ')
|
|
66
71
|
}
|
|
67
72
|
|
|
68
73
|
/** Producers a companion kind is allowed to review (inverse of {@link COMPANION_FOR_PRODUCER}). */
|
|
@@ -207,7 +212,7 @@ export function usePipelineHealth() {
|
|
|
207
212
|
const storedIds = new Set(store.pipelines.map((p) => p.id))
|
|
208
213
|
return Object.keys(store.catalogVersions)
|
|
209
214
|
.filter((id) => !storedIds.has(id))
|
|
210
|
-
.map((id) => ({ id, name: builtinPipelineName(id) }))
|
|
215
|
+
.map((id) => ({ id, name: builtinPipelineName(id, store.catalogNames) }))
|
|
211
216
|
})
|
|
212
217
|
|
|
213
218
|
// Retired built-ins this workspace still stores: the ones seeded before the withdrawal. A
|
|
@@ -238,7 +243,8 @@ export function usePipelineHealth() {
|
|
|
238
243
|
function resolveReplacement(id: string): { id: string; name: string } | undefined {
|
|
239
244
|
const stored = store.getPipeline(id)
|
|
240
245
|
if (stored) return { id, name: stored.name }
|
|
241
|
-
if (id in store.catalogVersions)
|
|
246
|
+
if (id in store.catalogVersions)
|
|
247
|
+
return { id, name: builtinPipelineName(id, store.catalogNames) }
|
|
242
248
|
return undefined
|
|
243
249
|
}
|
|
244
250
|
|
|
@@ -206,7 +206,13 @@ stray `API delivery` / `API Delivery` pair does not split a category in half.
|
|
|
206
206
|
|
|
207
207
|
Your own strings (labels, category captions, descriptions) are rendered verbatim and never enter a
|
|
208
208
|
locale catalog; only the platform's own chrome around them is i18n, which is why the "Other" heading
|
|
209
|
-
is the one caption you do not supply.
|
|
209
|
+
is the one caption you do not supply.
|
|
210
|
+
|
|
211
|
+
**A workspace admin can HIDE any registered type from that board** (Workspace settings → Operations).
|
|
212
|
+
Only backend-REGISTERED types are hideable: a type your frontend module ships as a code
|
|
213
|
+
contribution has no backend row to suppress, so it is offered on every board. If your catalog is
|
|
214
|
+
large enough that teams will want to trim it, register the types on the backend rather than
|
|
215
|
+
contributing them here. Each row carries `data-testid="task-type-row"` plus
|
|
210
216
|
`data-task-type-row="<id>"`, and each choice `data-testid="task-type-<taskType>"`, so your own e2e
|
|
211
217
|
suite can address a row and the caption inside it.
|
|
212
218
|
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { ref } from 'vue'
|
|
2
|
+
import type { RunToolCallFailures, RunToolCallTrajectory } from '~/types/execution'
|
|
3
|
+
|
|
4
|
+
// The observability store's TOOL-CALL sink, extracted whole because it is one concern with two
|
|
5
|
+
// reads and its own coherence rule between them.
|
|
6
|
+
//
|
|
7
|
+
// The rule: the two reads answer at DIFFERENT BOUNDS and must never be mistaken for each other.
|
|
8
|
+
// `failures` is about the run — counts aggregated in SQL over every row it ever wrote. The
|
|
9
|
+
// trajectory is a bounded PREFIX of the run, carrying every captured argument and result, which
|
|
10
|
+
// is why it is loaded on demand rather than on open. Anything that counts, judges or headlines
|
|
11
|
+
// reads the first; only the browse view reads the second, and it renders under a flag saying so.
|
|
12
|
+
//
|
|
13
|
+
// Keeping that pairing in one module is the point of the split: a caller reaching for whichever
|
|
14
|
+
// list is nearest is exactly how a panel ends up reporting a long run's opening moves as
|
|
15
|
+
// everything it did.
|
|
16
|
+
|
|
17
|
+
/** What the sink needs from the API layer: the two reads, already bound to a workspace. */
|
|
18
|
+
export interface ToolCallSinkDeps {
|
|
19
|
+
/**
|
|
20
|
+
* Whether a workspace is resolved yet.
|
|
21
|
+
*
|
|
22
|
+
* Checked BEFORE either read rather than left to the binding throwing, because these loads
|
|
23
|
+
* record their failures as "this sink did not answer" — a state the panel reports to an
|
|
24
|
+
* operator — and "no workspace selected yet" is not that. It is nobody having asked.
|
|
25
|
+
*/
|
|
26
|
+
ready: () => boolean
|
|
27
|
+
fetchTrajectory: (executionId: string) => Promise<RunToolCallTrajectory & { executionId: string }>
|
|
28
|
+
fetchFailures: (executionId: string) => Promise<RunToolCallFailures & { executionId: string }>
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* What {@link ToolCallSinkState.toolCallsFor} answers for a run that was never loaded.
|
|
33
|
+
*
|
|
34
|
+
* A frozen shared value rather than a fresh object per call: this is read inside computeds, and a
|
|
35
|
+
* new identity on every evaluation re-triggers every one of them downstream.
|
|
36
|
+
*/
|
|
37
|
+
export const EMPTY_TRAJECTORY: RunToolCallTrajectory = Object.freeze({
|
|
38
|
+
toolCalls: Object.freeze([]) as never,
|
|
39
|
+
truncated: false,
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
/** Add or remove a key from a reactive `Set` ref, replacing it so the reactivity fires. */
|
|
43
|
+
function withFlag(set: ReturnType<typeof ref<Set<string>>>, key: string, on: boolean) {
|
|
44
|
+
const next = new Set(set.value)
|
|
45
|
+
if (on) next.add(key)
|
|
46
|
+
else next.delete(key)
|
|
47
|
+
set.value = next
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function createToolCallSinkState(deps: ToolCallSinkDeps) {
|
|
51
|
+
/**
|
|
52
|
+
* Per-execution-id trajectory PREFIX (oldest first, the order the agent worked in) with the
|
|
53
|
+
* flag saying whether the run made more calls than it holds.
|
|
54
|
+
*/
|
|
55
|
+
const toolCallsByExecution = ref<Record<string, RunToolCallTrajectory>>({})
|
|
56
|
+
/** Execution ids whose trajectory is currently loading. */
|
|
57
|
+
const toolCallsLoading = ref<Set<string>>(new Set())
|
|
58
|
+
/**
|
|
59
|
+
* Last trajectory-load error per execution id, or null. Recorded for the same reason the
|
|
60
|
+
* context load records its own: a swallowed failure would render as the "no tool calls
|
|
61
|
+
* recorded" empty state, which on this sink is a claim rather than a blank tab.
|
|
62
|
+
*/
|
|
63
|
+
const toolCallErrors = ref<Record<string, string | null>>({})
|
|
64
|
+
/**
|
|
65
|
+
* Per-execution-id FAILING tool calls plus the run's exact counts.
|
|
66
|
+
*
|
|
67
|
+
* Separate from the trajectory because the numbers here are SQL aggregates over the whole run
|
|
68
|
+
* while that list is a bounded prefix of it. Counting failures off the prefix is how a panel
|
|
69
|
+
* ends up printing "nothing failed" over a run whose failures came after its opening moves.
|
|
70
|
+
*/
|
|
71
|
+
const toolCallFailuresByExecution = ref<Record<string, RunToolCallFailures>>({})
|
|
72
|
+
/** Execution ids whose failure summary is currently loading. */
|
|
73
|
+
const toolCallFailuresLoading = ref<Set<string>>(new Set())
|
|
74
|
+
/**
|
|
75
|
+
* Last failure-summary load error per execution id, or null.
|
|
76
|
+
*
|
|
77
|
+
* The one error state the panel cannot afford to swallow: this read is what the pinned "what
|
|
78
|
+
* failed" section speaks from, so a failure here has to reach it as "this sink did not answer"
|
|
79
|
+
* rather than as the zero rows it would otherwise be indistinguishable from.
|
|
80
|
+
*/
|
|
81
|
+
const toolCallFailureErrors = ref<Record<string, string | null>>({})
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The loaded trajectory prefix, or the empty one.
|
|
85
|
+
*
|
|
86
|
+
* `truncated: false` on a run that was never loaded is not a claim that the run is short: a
|
|
87
|
+
* caller distinguishes the two through {@link hasToolCalls}, never by finding this empty.
|
|
88
|
+
*/
|
|
89
|
+
function toolCallsFor(executionId: string): RunToolCallTrajectory {
|
|
90
|
+
return toolCallsByExecution.value[executionId] ?? EMPTY_TRAJECTORY
|
|
91
|
+
}
|
|
92
|
+
function isToolCallsLoading(executionId: string): boolean {
|
|
93
|
+
return toolCallsLoading.value.has(executionId)
|
|
94
|
+
}
|
|
95
|
+
/** Whether the trajectory has ever been loaded for this run (an empty answer still counts). */
|
|
96
|
+
function hasToolCalls(executionId: string): boolean {
|
|
97
|
+
return executionId in toolCallsByExecution.value
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Load (or refresh) the tool-call trajectory for a run. */
|
|
101
|
+
async function loadToolCalls(executionId: string) {
|
|
102
|
+
if (!deps.ready()) return
|
|
103
|
+
withFlag(toolCallsLoading, executionId, true)
|
|
104
|
+
toolCallErrors.value = { ...toolCallErrors.value, [executionId]: null }
|
|
105
|
+
try {
|
|
106
|
+
const { toolCalls, truncated } = await deps.fetchTrajectory(executionId)
|
|
107
|
+
toolCallsByExecution.value = {
|
|
108
|
+
...toolCallsByExecution.value,
|
|
109
|
+
[executionId]: { toolCalls, truncated },
|
|
110
|
+
}
|
|
111
|
+
} catch (err) {
|
|
112
|
+
toolCallErrors.value = {
|
|
113
|
+
...toolCallErrors.value,
|
|
114
|
+
[executionId]: err instanceof Error ? err.message : 'Failed to load tool calls',
|
|
115
|
+
}
|
|
116
|
+
} finally {
|
|
117
|
+
withFlag(toolCallsLoading, executionId, false)
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** The loaded failure summary, or null when this run's has not answered (yet, or at all). */
|
|
122
|
+
function toolCallFailuresFor(executionId: string): RunToolCallFailures | null {
|
|
123
|
+
return toolCallFailuresByExecution.value[executionId] ?? null
|
|
124
|
+
}
|
|
125
|
+
function isToolCallFailuresLoading(executionId: string): boolean {
|
|
126
|
+
return toolCallFailuresLoading.value.has(executionId)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Load (or refresh) the run's failing tool calls and exact counts.
|
|
131
|
+
*
|
|
132
|
+
* Cleared on failure rather than left holding the previous answer: a stale summary beside a
|
|
133
|
+
* fresh error would let the panel keep asserting a failure count the backend just refused to
|
|
134
|
+
* confirm.
|
|
135
|
+
*/
|
|
136
|
+
async function loadToolCallFailures(executionId: string) {
|
|
137
|
+
if (!deps.ready()) return
|
|
138
|
+
withFlag(toolCallFailuresLoading, executionId, true)
|
|
139
|
+
toolCallFailureErrors.value = { ...toolCallFailureErrors.value, [executionId]: null }
|
|
140
|
+
try {
|
|
141
|
+
const { total, failed, failures, failuresTruncated } = await deps.fetchFailures(executionId)
|
|
142
|
+
toolCallFailuresByExecution.value = {
|
|
143
|
+
...toolCallFailuresByExecution.value,
|
|
144
|
+
[executionId]: { total, failed, failures, failuresTruncated },
|
|
145
|
+
}
|
|
146
|
+
} catch (err) {
|
|
147
|
+
const { [executionId]: _dropped, ...rest } = toolCallFailuresByExecution.value
|
|
148
|
+
toolCallFailuresByExecution.value = rest
|
|
149
|
+
toolCallFailureErrors.value = {
|
|
150
|
+
...toolCallFailureErrors.value,
|
|
151
|
+
[executionId]: err instanceof Error ? err.message : 'Failed to load tool-call failures',
|
|
152
|
+
}
|
|
153
|
+
} finally {
|
|
154
|
+
withFlag(toolCallFailuresLoading, executionId, false)
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
toolCallsByExecution,
|
|
160
|
+
toolCallErrors,
|
|
161
|
+
toolCallsFor,
|
|
162
|
+
hasToolCalls,
|
|
163
|
+
isToolCallsLoading,
|
|
164
|
+
loadToolCalls,
|
|
165
|
+
toolCallFailuresByExecution,
|
|
166
|
+
toolCallFailureErrors,
|
|
167
|
+
toolCallFailuresFor,
|
|
168
|
+
isToolCallFailuresLoading,
|
|
169
|
+
loadToolCallFailures,
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export type ToolCallSinkState = ReturnType<typeof createToolCallSinkState>
|
|
@@ -7,6 +7,7 @@ import type {
|
|
|
7
7
|
LlmCallMetric,
|
|
8
8
|
} from '~/types/execution'
|
|
9
9
|
import { useWorkspaceStore } from '~/stores/workspace'
|
|
10
|
+
import { createToolCallSinkState } from '~/stores/observability/toolCalls'
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* LLM observability state: the full per-call model activity for a run (prompts,
|
|
@@ -22,6 +23,17 @@ export const useObservabilityStore = defineStore('observability', () => {
|
|
|
22
23
|
const api = useApi()
|
|
23
24
|
const workspace = useWorkspaceStore()
|
|
24
25
|
|
|
26
|
+
/**
|
|
27
|
+
* The TOOL-CALL sink, extracted whole: two reads at two different bounds, plus the rule that
|
|
28
|
+
* keeps them apart (see `observability/toolCalls.ts`). The store owns the workspace binding and
|
|
29
|
+
* nothing else about it.
|
|
30
|
+
*/
|
|
31
|
+
const toolCalls = createToolCallSinkState({
|
|
32
|
+
ready: () => !!workspace.workspaceId,
|
|
33
|
+
fetchTrajectory: (executionId) => api.getToolCalls(workspace.requireId(), executionId),
|
|
34
|
+
fetchFailures: (executionId) => api.getToolCallFailures(workspace.requireId(), executionId),
|
|
35
|
+
})
|
|
36
|
+
|
|
25
37
|
/** Per-execution-id call list (newest first). */
|
|
26
38
|
const callsByExecution = ref<Record<string, LlmCallMetric[]>>({})
|
|
27
39
|
/** Per-execution-id provided-context snapshot list (newest first). */
|
|
@@ -223,5 +235,6 @@ export const useObservabilityStore = defineStore('observability', () => {
|
|
|
223
235
|
searchQueriesFor,
|
|
224
236
|
isSearchQueriesLoading,
|
|
225
237
|
loadSearchQueries,
|
|
238
|
+
...toolCalls,
|
|
226
239
|
}
|
|
227
240
|
})
|
package/app/stores/pipelines.ts
CHANGED
|
@@ -32,6 +32,13 @@ export const usePipelinesStore = defineStore('pipelines', () => {
|
|
|
32
32
|
* a newer definition available (see `usePipelineHealth`).
|
|
33
33
|
*/
|
|
34
34
|
const catalogVersions = ref<Record<string, number>>({})
|
|
35
|
+
/**
|
|
36
|
+
* The catalog's own NAME per id in {@link catalogVersions}, from the same snapshot field pair.
|
|
37
|
+
* Read only where a catalog entry has no stored row to take a name off (the "new built-ins"
|
|
38
|
+
* advisory and the replacement a retirement points at); everywhere else the stored row's authored
|
|
39
|
+
* name is the answer. Empty for a facade that ships no name map.
|
|
40
|
+
*/
|
|
41
|
+
const catalogNames = ref<Record<string, string>>({})
|
|
35
42
|
/**
|
|
36
43
|
* Built-in pipelines WITHDRAWN from the catalog (`retiredPipelines()`), from the workspace
|
|
37
44
|
* snapshot. A stored pipeline whose id appears here is no longer relevant and can be REMOVED —
|
|
@@ -75,18 +82,27 @@ export const usePipelinesStore = defineStore('pipelines', () => {
|
|
|
75
82
|
const editingId = ref<string | null>(null)
|
|
76
83
|
|
|
77
84
|
/**
|
|
78
|
-
* Replace the cached pipelines (and the current built-in catalog versions +
|
|
79
|
-
* snapshot. `retired` is applied even when EMPTY, unlike `versions`: an
|
|
80
|
-
* facade shipped no retirements, and carrying the previous board's forward
|
|
81
|
-
* for a pipeline this deployment still ships.
|
|
85
|
+
* Replace the cached pipelines (and the current built-in catalog versions + names +
|
|
86
|
+
* retirements) from a snapshot. `retired` is applied even when EMPTY, unlike `versions`: an
|
|
87
|
+
* absent list means the facade shipped no retirements, and carrying the previous board's forward
|
|
88
|
+
* would offer a delete for a pipeline this deployment still ships. `names` rides `versions`,
|
|
89
|
+
* being the other half of one read.
|
|
82
90
|
*/
|
|
83
91
|
function hydrate(
|
|
84
92
|
next: Pipeline[],
|
|
85
93
|
versions?: Record<string, number>,
|
|
86
94
|
retired?: RetiredPipelineWire[],
|
|
95
|
+
names?: Record<string, string>,
|
|
87
96
|
) {
|
|
88
97
|
pipelines.value = next
|
|
89
|
-
|
|
98
|
+
// The two catalog maps move TOGETHER, because they are one snapshot read split in two and are
|
|
99
|
+
// keyed identically by construction. Assigning names on their own truthiness would let a
|
|
100
|
+
// facade that ships versions and no names leave the previous board's names indexed against
|
|
101
|
+
// this board's ids, which is the one way the pair can disagree.
|
|
102
|
+
if (versions) {
|
|
103
|
+
catalogVersions.value = versions
|
|
104
|
+
catalogNames.value = names ?? {}
|
|
105
|
+
}
|
|
90
106
|
retiredPipelines.value = retired ?? []
|
|
91
107
|
}
|
|
92
108
|
|
|
@@ -133,6 +149,7 @@ export const usePipelinesStore = defineStore('pipelines', () => {
|
|
|
133
149
|
return {
|
|
134
150
|
pipelines,
|
|
135
151
|
catalogVersions,
|
|
152
|
+
catalogNames,
|
|
136
153
|
retiredPipelines,
|
|
137
154
|
gateConfigForms,
|
|
138
155
|
draft,
|
|
@@ -86,6 +86,35 @@ describe('taskTypes store — custom task-type catalog (extension slice B)', ()
|
|
|
86
86
|
expect(isKnownTaskType('acme:ws1')).toBe(false)
|
|
87
87
|
})
|
|
88
88
|
|
|
89
|
+
it('still reports registered operations when the board has hidden every one of them', () => {
|
|
90
|
+
// The regression this exists for: the workspace-settings Operations tab was gated on the
|
|
91
|
+
// OFFERED catalog, which suppression empties. Hiding the last operation therefore removed the
|
|
92
|
+
// only screen that un-hides one, and nothing failed: the tab simply was not there.
|
|
93
|
+
const store = useTaskTypesStore()
|
|
94
|
+
expect(store.hasRegisteredOperations).toBe(false)
|
|
95
|
+
|
|
96
|
+
hydrate(store, [backendType('acme:kept'), backendType('acme:hidden')])
|
|
97
|
+
store.hydrateSuppressed([])
|
|
98
|
+
expect(store.hasRegisteredOperations).toBe(true)
|
|
99
|
+
|
|
100
|
+
// Every registered operation hidden: the offered catalog is empty and the screen must remain.
|
|
101
|
+
hydrate(store, [])
|
|
102
|
+
store.hydrateSuppressed(['acme:kept', 'acme:hidden'])
|
|
103
|
+
expect(store.customTaskTypes).toEqual([])
|
|
104
|
+
expect(store.hasRegisteredOperations).toBe(true)
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
it('drops the previous board’s suppressions on re-hydrate (per-workspace state)', () => {
|
|
108
|
+
// Assigned unconditionally rather than on truthiness: carrying a hidden list forward would
|
|
109
|
+
// leave the Operations tab standing on a board that registers nothing.
|
|
110
|
+
const store = useTaskTypesStore()
|
|
111
|
+
hydrate(store, [])
|
|
112
|
+
store.hydrateSuppressed(['acme:ws1'])
|
|
113
|
+
expect(store.hasRegisteredOperations).toBe(true)
|
|
114
|
+
store.hydrateSuppressed([])
|
|
115
|
+
expect(store.hasRegisteredOperations).toBe(false)
|
|
116
|
+
})
|
|
117
|
+
|
|
89
118
|
it('get() returns the full registration (for the create-form field descriptors)', () => {
|
|
90
119
|
const store = useTaskTypesStore()
|
|
91
120
|
hydrate(store, [
|