@meistrari/tela-skills 1.6.2 → 1.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meistrari/tela-skills",
-  "version": "1.6.2",
+  "version": "1.7.0",
   "description": "Tela API skills for Claude Code",
   "type": "module",
   "bin": {

package/skill/TELA_SKILL.md

@@ -91,6 +91,30 @@ bun --preload ~/.claude/skills/tela/preload.ts -e "console.log(await tela.listPr
 - `tela.waitForTestCase(testCaseId)` - Wait for test case to complete
 - `tela.abortTestCase(testCaseId)` - Abort a running test case
 
+### Agent Test Cases
+
+Test cases for agents (FCC). Runs are keyed by `(testCaseId, commitHash)` — one run per test case per agent version. Run/continue/run-all return `202 { executionId }` immediately; poll with `waitForAgentTestCaseRun`. Derived statuses: `new`, `running`, `executed`, `success`, `failed`, `error` (terminal: `success`/`failed`/`error`).
+
+- `tela.listAgentTestCases(agentId, options?)` - List test cases (pass `commitHash` to attach runs/stats)
+- `tela.createAgentTestCase(agentId, payload)` - Create a test case
+- `tela.updateAgentTestCase(agentId, testId, payload)` - Update a test case
+- `tela.deleteAgentTestCase(agentId, testId)` - Delete a test case
+- `tela.runAgentTestCase(agentId, testId, commitHash)` - Run against an agent version
+- `tela.continueAgentTestCase(agentId, testId, commitHash, message)` - Continue session (multiturn)
+- `tela.runAllAgentTestCases(agentId, { commitHash, testCaseIds? | filters? })` - Batch run (returns `skipped` reasons)
+- `tela.getAgentTestCaseRun(agentId, testId, commitHash)` - Get the run for a commit
+- `tela.listAgentTestCaseRuns(agentId, testId, options?)` - Run history across commits (cursor pagination)
+- `tela.waitForAgentTestCaseRun(agentId, testId, commitHash, options?)` - Poll until terminal status
+- `tela.updateAgentTestCaseAttributeFeedback(agentId, testId, commitHash, attributes)` - Thumbs per attribute (feeds the answer bank)
+- `tela.getAgentTestCaseStats(agentId, commitHash, options?)` - Aggregated stats for a commit
+- `tela.listAgentTestMetrics(agentId)` / `createAgentTestMetric` / `updateAgentTestMetric` / `deleteAgentTestMetric` - Custom metrics over output attributes
+- `tela.listAgentTestCaseTags(agentId)` / `createAgentTestCaseTag` / `updateAgentTestCaseTag` / `deleteAgentTestCaseTag` - Tag CRUD
+- `tela.getAgentTestCaseTags(agentId, testId)` / `addAgentTestCaseTags` / `removeAgentTestCaseTag` - Tag assignment
+- `tela.bulkAddAgentTestCaseTags(agentId, testCaseIds, tagIds)` / `bulkRemoveAgentTestCaseTags(...)` - Bulk tagging
+- `tela.getAgentHistory(agentId, options?)` / `tela.getLatestAgentCommit(agentId)` - Resolve agent versions (commitHash)
+- `tela.buildAgentTestInputs(variables, options?)` / `tela.createAgentTestCasePayload(variables, options?)` - Payload helpers (vault:// values become file inputs)
+- `tela.getAgentTestCaseRunStatus(run)` - Derive run status locally
+
 ### Tasks
 
 - `tela.listTasks(options?)` - List tasks with pagination and filtering
@@ -192,6 +216,31 @@ console.log('Variables:', version?.variables?.map(v => v.name))
 "
 ```
 
+### Create and Run an Agent Test Case
+```bash
+bun --preload ~/.claude/skills/tela/preload.ts -e "
+const agentId = 'AGENT_ID'
+const vaultRef = await tela.uploadFile('./contract.pdf')
+
+const testCase = await tela.createAgentTestCase(agentId, tela.createAgentTestCasePayload({
+  document: vaultRef,
+  instructions: 'Extract the parties and total amount',
+}, {
+  title: 'Contract extraction',
+  evaluationInstructions: 'The output must list both parties and the correct total.',
+  fileNames: { document: 'contract.pdf' },
+}))
+
+const commitHash = await tela.getLatestAgentCommit(agentId)
+await tela.runAgentTestCase(agentId, testCase.id, commitHash)
+
+const { run, status } = await tela.waitForAgentTestCaseRun(agentId, testCase.id, commitHash)
+console.log('Status:', status)
+console.log('Output:', run.output)
+console.log('Validation:', run.validationSummary)
+"
+```
+
 ### List Tasks
 ```bash
 bun --preload ~/.claude/skills/tela/preload.ts -e "
@@ -332,7 +381,8 @@ bun localhost.ts "console.log(await tela.listProjects())"
 ├── prompts.ts            # Prompt/Canvas list functions
 ├── canvas.ts             # Canvas create/update functions
 ├── workflows.ts          # Workflow create/update/run/validation functions
-├── test-cases.ts         # Test case functions
+├── test-cases.ts         # Test case functions (prompt/canvas)
+├── agent-test-cases.ts   # Agent test case functions (FCC)
 ├── tasks.ts              # Task management functions
 ├── vault.ts              # Vault file management functions
 └── localhost-utils.ts    # Docker discovery, JWT signing, workspace fetching (internal, not on tela global)

package/skill/agent-test-cases.test.ts

@@ -0,0 +1,316 @@
+import { beforeEach, describe, expect, mock, test } from 'bun:test'
+
+const apiRequest = mock(async (_endpoint: string, _options?: RequestInit): Promise<unknown> => ({ data: {} }))
+
+class ApiRequestError extends Error {
+    constructor(public readonly status: number, message: string) {
+        super(message)
+        this.name = 'ApiRequestError'
+    }
+}
+
+void mock.module('./common.ts', () => ({
+    apiRequest,
+    ApiRequestError,
+}))
+
+describe('agent-test-cases', () => {
+    beforeEach(() => {
+        apiRequest.mockClear()
+        apiRequest.mockImplementation(async () => ({ data: {} }))
+    })
+
+    test('lists test cases with repeated array query params', async () => {
+        const { listAgentTestCases } = await import('./agent-test-cases.ts')
+        apiRequest.mockImplementation(async () => ({ data: [] }))
+
+        await listAgentTestCases('agent-1', {
+            commitHash: 'abc123',
+            title: 'search',
+            status: ['failed', 'error'],
+            tagIds: ['t1', 't2'],
+            createdBy: ['u1'],
+            createdAtSince: '2026-01-01T00:00:00Z',
+        })
+
+        const [endpoint] = apiRequest.mock.calls[0]!
+        const query = new URLSearchParams(String(endpoint).split('?')[1])
+
+        expect(String(endpoint).startsWith('/agent/agent-1/tests?')).toBe(true)
+        expect(query.get('commitHash')).toBe('abc123')
+        expect(query.get('title')).toBe('search')
+        expect(query.getAll('status')).toEqual(['failed', 'error'])
+        expect(query.getAll('tagIds')).toEqual(['t1', 't2'])
+        expect(query.getAll('createdBy')).toEqual(['u1'])
+        expect(query.get('createdAtSince')).toBe('2026-01-01T00:00:00Z')
+    })
+
+    test('omits the query string when no options are given', async () => {
+        const { listAgentTestCases } = await import('./agent-test-cases.ts')
+        apiRequest.mockImplementation(async () => ({ data: [] }))
+
+        await listAgentTestCases('agent-1')
+
+        expect(apiRequest.mock.calls[0]![0]).toBe('/agent/agent-1/tests')
+    })
+
+    test('pre-serializes bodies so vault refs are not rewritten by parsePayload', async () => {
+        const { createAgentTestCase } = await import('./agent-test-cases.ts')
+
+        await createAgentTestCase('agent-1', {
+            inputs: [{ type: 'file', name: 'doc', vaultRef: 'vault://abc', filename: 'doc.pdf' }],
+        })
+
+        const [endpoint, options] = apiRequest.mock.calls[0]!
+        expect(endpoint).toBe('/agent/agent-1/tests')
+        expect(options?.method).toBe('POST')
+        expect(typeof options?.body).toBe('string')
+
+        const body = JSON.parse(String(options?.body))
+        expect(body.inputs[0].vaultRef).toBe('vault://abc')
+    })
+
+    test('unwraps the { data } envelope', async () => {
+        const { createAgentTestCase } = await import('./agent-test-cases.ts')
+        apiRequest.mockImplementation(async () => ({ data: { id: 'tc-1', title: 'Test' } }))
+
+        const testCase = await createAgentTestCase('agent-1', { title: 'Test' })
+
+        expect(testCase).toEqual({ id: 'tc-1', title: 'Test' } as any)
+    })
+
+    test('runs a test case against a commit', async () => {
+        const { runAgentTestCase } = await import('./agent-test-cases.ts')
+        apiRequest.mockImplementation(async () => ({ data: { executionId: 'exec-1' } }))
+
+        const result = await runAgentTestCase('agent-1', 'tc-1', 'abc123')
+
+        const [endpoint, options] = apiRequest.mock.calls[0]!
+        expect(endpoint).toBe('/agent/agent-1/tests/tc-1/run')
+        expect(JSON.parse(String(options?.body))).toEqual({ commitHash: 'abc123' })
+        expect(result.executionId).toBe('exec-1')
+    })
+
+    test('continues a session with a message', async () => {
+        const { continueAgentTestCase } = await import('./agent-test-cases.ts')
+        apiRequest.mockImplementation(async () => ({ data: { executionId: 'exec-2' } }))
+
+        await continueAgentTestCase('agent-1', 'tc-1', 'abc123', 'next turn')
+
+        const [endpoint, options] = apiRequest.mock.calls[0]!
+        expect(endpoint).toBe('/agent/agent-1/tests/tc-1/continue')
+        expect(JSON.parse(String(options?.body))).toEqual({ commitHash: 'abc123', message: 'next turn' })
+    })
+
+    test('sends attribute feedback wrapped in attributes', async () => {
+        const { updateAgentTestCaseAttributeFeedback } = await import('./agent-test-cases.ts')
+
+        await updateAgentTestCaseAttributeFeedback('agent-1', 'tc-1', 'abc123', [
+            { path: 'summary', feedback: 1 },
+            { path: 'total', feedback: 0, message: 'wrong' },
+        ])
+
+        const [endpoint, options] = apiRequest.mock.calls[0]!
+        expect(endpoint).toBe('/agent/agent-1/tests/tc-1/runs/abc123/attribute-feedback')
+        expect(options?.method).toBe('PATCH')
+
+        const body = JSON.parse(String(options?.body))
+        expect(body.attributes).toHaveLength(2)
+        expect(body.attributes[1]).toEqual({ path: 'total', feedback: 0, message: 'wrong' })
+    })
+
+    test('requires commitHash on the stats endpoint query', async () => {
+        const { getAgentTestCaseStats } = await import('./agent-test-cases.ts')
+
+        await getAgentTestCaseStats('agent-1', 'abc123', { tagIds: ['t1'] })
+
+        const [endpoint] = apiRequest.mock.calls[0]!
+        const query = new URLSearchParams(String(endpoint).split('?')[1])
+
+        expect(String(endpoint).startsWith('/agent/agent-1/tests/stats?')).toBe(true)
+        expect(query.get('commitHash')).toBe('abc123')
+        expect(query.getAll('tagIds')).toEqual(['t1'])
+    })
+
+    test('run history keeps pagination fields from the response envelope', async () => {
+        const { listAgentTestCaseRuns } = await import('./agent-test-cases.ts')
+        // This endpoint's envelope is { data: entries, nextCursor, totalRuns } —
+        // pagination fields live beside data, not inside it
+        apiRequest.mockImplementation(async () => ({
+            data: [{ commitHash: 'abc123', score: 1 }],
+            nextCursor: 'cursor-1',
+            totalRuns: 5,
+        }))
+
+        const result = await listAgentTestCaseRuns('agent-1', 'tc-1', { limit: 1 })
+
+        expect(apiRequest.mock.calls[0]![0]).toBe('/agent/agent-1/tests/tc-1/runs?limit=1')
+        expect(result.data).toHaveLength(1)
+        expect(result.nextCursor).toBe('cursor-1')
+        expect(result.totalRuns).toBe(5)
+    })
+
+    describe('buildAgentTestInputs', () => {
+        test('splits text and vault file values', async () => {
+            const { buildAgentTestInputs } = await import('./agent-test-cases.ts')
+
+            const inputs = buildAgentTestInputs({
+                context: 'Some text',
+                document: 'vault://abc123',
+            }, { fileNames: { document: 'report.pdf' } })
+
+            expect(inputs).toEqual([
+                { type: 'text', name: 'context', content: 'Some text' },
+                { type: 'file', name: 'document', vaultRef: 'vault://abc123', filename: 'report.pdf' },
+            ])
+        })
+
+        test('defaults the filename to the input name', async () => {
+            const { buildAgentTestInputs } = await import('./agent-test-cases.ts')
+
+            const inputs = buildAgentTestInputs({ document: 'vault://abc123' })
+
+            expect(inputs[0]).toEqual({ type: 'file', name: 'document', vaultRef: 'vault://abc123', filename: 'document' })
+        })
+    })
+
+    describe('createAgentTestCasePayload', () => {
+        test('builds payload with expectations and derived title', async () => {
+            const { createAgentTestCasePayload } = await import('./agent-test-cases.ts')
+
+            const payload = createAgentTestCasePayload({ input: 'Hello world, this is a long input value' }, {
+                evaluationInstructions: 'Must greet back',
+            })
+
+            expect(payload.title).toBe(`Test: ${'Hello world, this is a long input value'.slice(0, 30)}...`)
+            expect(payload.evaluationInstructions).toBe('Must greet back')
+            expect(payload.inputs).toHaveLength(1)
+        })
+
+        test('falls back to the first filename for file-only cases', async () => {
+            const { createAgentTestCasePayload } = await import('./agent-test-cases.ts')
+
+            const payload = createAgentTestCasePayload({ doc: 'vault://abc' }, { fileNames: { doc: 'contract.pdf' } })
+
+            expect(payload.title).toBe('Test: contract.pdf')
+        })
+    })
+
+    describe('getAgentTestCaseRunStatus', () => {
+        const baseRun = {
+            executionSessionId: null as string | null,
+            executionStatus: null as any,
+            evaluationSessionId: null as string | null,
+            evaluationStatus: null as any,
+            feedback: null as 0 | 1 | null,
+        }
+
+        test('derives every status branch', async () => {
+            const { getAgentTestCaseRunStatus } = await import('./agent-test-cases.ts')
+
+            expect(getAgentTestCaseRunStatus(null)).toBe('new')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'error' })).toBe('error')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationStatus: 'error' })).toBe('error')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'running' })).toBe('running')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationSessionId: 's', evaluationStatus: 'running' })).toBe('running')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed' })).toBe('executed')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationSessionId: 's', evaluationStatus: 'completed', feedback: 1 })).toBe('success')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationSessionId: 's', evaluationStatus: 'completed', feedback: 0 })).toBe('failed')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionStatus: 'completed', evaluationSessionId: 's', evaluationStatus: 'failed' })).toBe('failed')
+            expect(getAgentTestCaseRunStatus({ ...baseRun, executionSessionId: 's' })).toBe('executed')
+        })
+    })
+
+    describe('waitForAgentTestCaseRun', () => {
+        test('retries 404s until the run appears, then resolves on terminal status', async () => {
+            const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts')
+
+            let calls = 0
+            apiRequest.mockImplementation(async () => {
+                calls++
+                if (calls === 1)
+                    throw new ApiRequestError(404, 'API request failed: 404 Not Found - {}')
+                if (calls === 2) {
+                    return { data: { executionStatus: 'running', executionSessionId: 's', evaluationSessionId: null, evaluationStatus: null, feedback: null } }
+                }
+                return { data: { executionStatus: 'completed', executionSessionId: 's', evaluationSessionId: 'e', evaluationStatus: 'completed', feedback: 1 } }
+            })
+
+            const { status } = await waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1 })
+
+            expect(status).toBe('success')
+            expect(calls).toBe(3)
+        })
+
+        test('stops at executed when until is executed', async () => {
+            const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts')
+
+            apiRequest.mockImplementation(async () => ({
+                data: { executionStatus: 'completed', executionSessionId: 's', evaluationSessionId: null, evaluationStatus: null, feedback: null },
+            }))
+
+            const { status } = await waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1, until: 'executed' })
+
+            expect(status).toBe('executed')
+        })
+
+        test('throws after maxAttempts without a terminal status', async () => {
+            const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts')
+
+            apiRequest.mockImplementation(async () => ({
+                data: { executionStatus: 'running', executionSessionId: 's', evaluationSessionId: null, evaluationStatus: null, feedback: null },
+            }))
+
+            expect(waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1, maxAttempts: 2 }))
+                .rejects
+                .toThrow('did not complete within timeout')
+        })
+
+        test('rethrows non-404 API errors', async () => {
+            const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts')
+
+            apiRequest.mockImplementation(async () => {
+                throw new ApiRequestError(500, 'API request failed: 500 Internal Server Error - {}')
+            })
+
+            expect(waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1 }))
+                .rejects
+                .toThrow('500')
+        })
+
+        test('rethrows untyped errors even when the message mentions 404', async () => {
+            const { waitForAgentTestCaseRun } = await import('./agent-test-cases.ts')
+
+            apiRequest.mockImplementation(async () => {
+                throw new Error('fetch failed with 404 somewhere')
+            })
+
+            expect(waitForAgentTestCaseRun('agent-1', 'tc-1', 'abc123', { intervalMs: 1 }))
+                .rejects
+                .toThrow('fetch failed')
+        })
+    })
+
+    describe('getLatestAgentCommit', () => {
+        test('returns the newest commit hash', async () => {
+            const { getLatestAgentCommit } = await import('./agent-test-cases.ts')
+            apiRequest.mockImplementation(async () => ({
+                data: { commits: [{ commitHash: 'abc123' }], branch: 'main', pagination: { page: 1, limit: 1, totalCount: 5, hasMore: true } },
+            }))
+
+            const hash = await getLatestAgentCommit('agent-1')
+
+            expect(apiRequest.mock.calls[0]![0]).toBe('/agent/agent-1/history?limit=1')
+            expect(hash).toBe('abc123')
+        })
+
+        test('throws when the agent has no commits', async () => {
+            const { getLatestAgentCommit } = await import('./agent-test-cases.ts')
+            apiRequest.mockImplementation(async () => ({
+                data: { commits: [], branch: 'main', pagination: { page: 1, limit: 1, totalCount: 0, hasMore: false } },
+            }))
+
+            expect(getLatestAgentCommit('agent-1')).rejects.toThrow('has no commits')
+        })
+    })
+})

package/skill/agent-test-cases.ts

@@ -0,0 +1,831 @@
+import { ApiRequestError, apiRequest } from './common.ts'
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * A single named input for an agent test case.
+ * Text inputs carry inline content; file inputs reference a Vault file.
+ */
+export type AgentTestInput
+    = | { type: 'text', name: string, content: string }
+        | { type: 'file', name: string, vaultRef: string, filename: string, metadata?: string }
+
+export interface AgentTestCaseValidationReferenceFile {
+    index: number
+    name: string
+    mimeType?: string | null
+    vaultRef: string
+    url?: string
+}
+
+export interface AgentTestCaseAnswers {
+    good: unknown[]
+    bad: unknown[]
+}
+
+export interface AgentTestCaseTag {
+    testCaseTagAssignmentId?: string
+    id: string
+    name: string
+    color: string
+    promptId: string | null
+    agentId: string | null
+    workspaceId: string
+    createdAt: string
+    updatedAt: string
+    deletedAt: string | null
+}
+
+export interface AgentTestCase {
+    id: string
+    agentId: string
+    workspaceId: string
+    title: string
+    inputs: AgentTestInput[]
+    expectedOutput: Record<string, unknown> | string | null
+    validationReferenceFiles: AgentTestCaseValidationReferenceFile[]
+    evaluationInstructions: string | null
+    answers: Record<string, AgentTestCaseAnswers>
+    metadata: Record<string, unknown>
+    tags: AgentTestCaseTag[]
+    createdBy: string
+    createdAt: string
+    updatedAt: string
+}
+
+export type AgentTestCaseFeedback = 0 | 1 | null
+export type AgentTestCaseValidationType = 'good' | 'bad' | 'warning' | 'missing'
+
+export interface AgentTestCaseValidationSummaryEntry {
+    type: AgentTestCaseValidationType
+    source?: 'llm-judge' | 'manual' | 'answer-match'
+    message?: string
+    matchedAnswer?: unknown
+}
+
+export interface AgentTestCaseRun {
+    id: string
+    testId: string
+    commitHash: string
+    executionSessionId: string | null
+    executionStatus: 'pending' | 'running' | 'completed' | 'error' | null
+    executionError: string | null
+    savedInputs: AgentTestInput[]
+    savedChatMessages: Array<{
+        content: string
+        stepEndIndex: number
+        attachments?: Array<{ vaultRef: string, fileName: string, fileType: string }>
+        output?: unknown
+        duration?: number
+    }>
+    output: Record<string, unknown> | string | null
+    executionUsage: Record<string, unknown> | null
+    evaluationSessionId: string | null
+    evaluationInputSnapshot: unknown
+    evaluationRequirements: string
+    evaluationStatus: 'pending' | 'running' | 'completed' | 'failed' | 'error' | null
+    evaluationError: string | null
+    feedback: AgentTestCaseFeedback
+    attributeFeedback: Record<string, 0 | 1>
+    validationSummary: Record<string, AgentTestCaseValidationSummaryEntry>
+    evaluatedAt: string | null
+    createdAt: string
+    updatedAt: string
+}
+
+/**
+ * Derived run status — not a stored column. Computed from execution/evaluation
+ * state by getAgentTestCaseRunStatus. Terminal: success, failed, error.
+ */
+export type AgentTestCaseStatus = 'new' | 'running' | 'executed' | 'success' | 'failed' | 'error'
+
+export interface AgentTestCaseCounters {
+    total: number
+    good: number
+    bad: number
+    pending: number
+}
+
+export interface AgentTestMetric {
+    id: string
+    agentId: string
+    workspaceId: string
+    name: string
+    attributeKeys: string[]
+    position: number
+    createdBy: string
+    updatedBy: string | null
+    createdAt: string
+    updatedAt: string
+}
+
+export type AgentTestMetricStatus = 'completed' | 'pending' | 'partial' | 'unavailable'
+
+export interface AgentTestMetricStats {
+    id: string
+    name: string
+    attributeKeys: string[]
+    availableAttributeKeys: string[]
+    missingAttributeKeys: string[]
+    stats: AgentTestCaseCounters
+    reviewedStats?: { total: number, good: number, bad: number }
+    score: number | null
+    status: AgentTestMetricStatus
+}
+
+export interface AgentTestCaseWithRun {
+    test: AgentTestCase
+    run: AgentTestCaseRun | null
+    stats?: AgentTestCaseCounters
+    metricStats?: AgentTestMetricStats[]
+}
+
+export interface AgentTestCaseStats {
+    agentId: string
+    commitHash: string
+    status: 'pending' | 'completed'
+    stats: AgentTestCaseCounters
+    reviewedStats: { total: number, good: number, bad: number }
+    testCases: {
+        total: number
+        executed: number
+        evaluated: number
+        passed: number
+        failed: number
+        pendingExecution: number
+        pendingEvaluation: number
+        running: number
+        error: number
+    }
+    metrics: AgentTestMetricStats[]
+}
+
+export interface AgentTestCaseRunHistoryEntry {
+    commitHash: string
+    createdAt: string
+    score: number
+    total: number
+    good: number
+    bad: number
+    pending: number
+    attributeOutcomes: Record<string, unknown>
+}
+
+export interface ListAgentTestCaseRunsResult {
+    data: AgentTestCaseRunHistoryEntry[]
+    nextCursor: string | null
+    totalRuns: number
+}
+
+export interface AgentCommit {
+    commitHash: string
+    shortHash: string
+    message: string
+    createdAt: string
+    author: { name: string, email: string, avatarUrl?: string }
+}
+
+export interface AgentHistory {
+    commits: AgentCommit[]
+    branch: string
+    pagination: { page: number, limit: number, totalCount: number, hasMore: boolean }
+}
+
+// ============================================================================
+// Payloads & Options
+// ============================================================================
+
+export interface CreateAgentTestCasePayload {
+    title?: string
+    inputs?: AgentTestInput[]
+    expectedOutput?: Record<string, unknown> | string | null
+    validationReferenceFiles?: AgentTestCaseValidationReferenceFile[]
+    evaluationInstructions?: string | null
+    answers?: Record<string, AgentTestCaseAnswers>
+    metadata?: Record<string, unknown>
+}
+
+export type UpdateAgentTestCasePayload = CreateAgentTestCasePayload
+
+export interface ListAgentTestCasesOptions {
+    /** Attach the run for this agent commit to each test case (enables stats) */
+    commitHash?: string
+    /** Search by test case title */
+    title?: string
+    /** Filter by creator user IDs */
+    createdBy?: string[]
+    createdAtSince?: string
+    createdAtUntil?: string
+    updatedAtSince?: string
+    updatedAtUntil?: string
+    /** Filter by derived run status (requires commitHash) */
+    status?: AgentTestCaseStatus[]
+    /** Filter by tag IDs */
+    tagIds?: string[]
+}
+
+export interface RunAllAgentTestCasesPayload {
+    commitHash: string
+    /** Explicit test case IDs — mutually exclusive with filters */
+    testCaseIds?: string[]
+    /** Filter-based selection — mutually exclusive with testCaseIds */
+    filters?: {
+        title?: string
+        createdBy?: string[]
+        createdAtSince?: string
+        createdAtUntil?: string
+        updatedAtSince?: string
+        updatedAtUntil?: string
+        tagIds?: string[]
+    }
+}
+
+export type AgentTestCaseSkipReason
+    = | 'already-running'
+        | 'already-passed'
+        | 'already-failed'
+        | 'missing-input'
+        | 'unknown-input'
+        | 'invalid-input'
+        | 'not-accessible'
+        | 'usage-limit'
+
+export interface RunAllAgentTestCasesResult {
+    executionId: string
+    executionIds: string[]
+    testCaseIds: string[]
+    skipped: Array<{ testCaseId: string, reason: AgentTestCaseSkipReason }>
+}
+
+export interface AgentTestCaseAttributeFeedback {
+    path: string
+    feedback: AgentTestCaseFeedback
+    type?: AgentTestCaseValidationType
+    message?: string
+}
+
+export interface CreateAgentTestMetricPayload {
+    name: string
+    attributeKeys: string[]
+    position?: number
+}
+
+export type UpdateAgentTestMetricPayload = Partial<CreateAgentTestMetricPayload>
+
+// ============================================================================
+// Internal helpers
+// ============================================================================
+
+// Agent test case endpoints use `vaultRef` fields that must NOT be rewritten
+// by parsePayload's vault:// → { file_url } expansion, so every body is
+// pre-serialized with JSON.stringify (string bodies skip parsePayload).
+function jsonBody(payload: unknown): string {
+    return JSON.stringify(payload)
+}
+
+function buildListQuery(options?: ListAgentTestCasesOptions): string {
+    const params = new URLSearchParams()
+
+    if (options?.commitHash)
+        params.set('commitHash', options.commitHash)
+    if (options?.title)
+        params.set('title', options.title)
+    if (options?.createdAtSince)
+        params.set('createdAtSince', options.createdAtSince)
+    if (options?.createdAtUntil)
+        params.set('createdAtUntil', options.createdAtUntil)
+    if (options?.updatedAtSince)
+        params.set('updatedAtSince', options.updatedAtSince)
+    if (options?.updatedAtUntil)
+        params.set('updatedAtUntil', options.updatedAtUntil)
+    // Arrays use repeated keys (Elysia query array format)
+    for (const v of options?.createdBy ?? []) params.append('createdBy', v)
+    for (const v of options?.status ?? []) params.append('status', v)
+    for (const v of options?.tagIds ?? []) params.append('tagIds', v)
+
+    const qs = params.toString()
+    return qs ? `?${qs}` : ''
+}
+
+// ============================================================================
+// Test Cases (CRUD)
+// ============================================================================
+
+/**
+ * List agent test cases with optional filters.
+ * Pass commitHash to attach that commit's run (and stats) to each test case.
+ * Returns all matching cases (no pagination on this endpoint).
+ */
+export async function listAgentTestCases(
+    agentId: string,
+    options?: ListAgentTestCasesOptions,
+): Promise<AgentTestCaseWithRun[]> {
+    const { data } = await apiRequest<{ data: AgentTestCaseWithRun[] }>(
+        `/agent/${agentId}/tests${buildListQuery(options)}`,
+    )
+    return data
+}
+
+/**
+ * Create a new test case for an agent.
+ * Use createAgentTestCasePayload/buildAgentTestInputs to build inputs.
+ */
+export async function createAgentTestCase(
+    agentId: string,
+    payload: CreateAgentTestCasePayload = {},
+): Promise<AgentTestCase> {
+    const { data } = await apiRequest<{ data: AgentTestCase }>(`/agent/${agentId}/tests`, {
+        method: 'POST',
+        body: jsonBody(payload),
+    })
+    return data
+}
+
+/**
+ * Update an existing agent test case (title, inputs, expectations, etc.)
+ */
+export async function updateAgentTestCase(
+    agentId: string,
+    testId: string,
+    payload: UpdateAgentTestCasePayload,
+): Promise<AgentTestCase> {
+    const { data } = await apiRequest<{ data: AgentTestCase }>(`/agent/${agentId}/tests/${testId}`, {
+        method: 'PATCH',
+        body: jsonBody(payload),
+    })
+    return data
+}
+
+/**
+ * Delete an agent test case (soft delete)
+ */
+export async function deleteAgentTestCase(agentId: string, testId: string): Promise<void> {
+    await apiRequest(`/agent/${agentId}/tests/${testId}`, { method: 'DELETE' })
+}
+
+// ============================================================================
+// Execution & Evaluation
+// ============================================================================
+
+/**
+ * Run a test case against an agent version (commitHash).
+ * Returns 202 immediately — poll getAgentTestCaseRun / waitForAgentTestCaseRun.
+ * One run exists per (testId, commitHash); re-running replaces it.
+ */
+export async function runAgentTestCase(
+    agentId: string,
+    testId: string,
+    commitHash: string,
+): Promise<{ executionId: string }> {
+    const { data } = await apiRequest<{ data: { executionId: string } }>(
+        `/agent/${agentId}/tests/${testId}/run`,
+        { method: 'POST', body: jsonBody({ commitHash }) },
+    )
+    return data
+}
+
+/**
+ * Continue a test case session with a new user message (multiturn).
+ * Requires an existing, non-running run for the commit. Re-evaluates after the turn.
+ */
+export async function continueAgentTestCase(
+    agentId: string,
+    testId: string,
+    commitHash: string,
+    message: string,
+): Promise<{ executionId: string }> {
+    const { data } = await apiRequest<{ data: { executionId: string } }>(
+        `/agent/${agentId}/tests/${testId}/continue`,
+        { method: 'POST', body: jsonBody({ commitHash, message }) },
+    )
+    return data
+}
+
+/**
+ * Run all (or a subset of) test cases against an agent version in batch.
+ * Select via explicit testCaseIds OR filters (mutually exclusive).
+ * Already-passed/failed/running cases are skipped with a reason.
+ */
+export async function runAllAgentTestCases(
+    agentId: string,
+    payload: RunAllAgentTestCasesPayload,
+): Promise<RunAllAgentTestCasesResult> {
+    const { data } = await apiRequest<{ data: RunAllAgentTestCasesResult }>(
+        `/agent/${agentId}/tests/run-all`,
+        { method: 'POST', body: jsonBody(payload) },
+    )
+    return data
+}
+
+/**
+ * Get the run of a test case for a specific agent commit
+ */
+export async function getAgentTestCaseRun(
+    agentId: string,
+    testId: string,
+    commitHash: string,
+): Promise<AgentTestCaseRun> {
+    const { data } = await apiRequest<{ data: AgentTestCaseRun }>(
+        `/agent/${agentId}/tests/${testId}/runs/${commitHash}`,
+    )
+    return data
+}
+
+/**
+ * List a test case's run history across commits (keyset pagination)
+ */
+export async function listAgentTestCaseRuns(
+    agentId: string,
+    testId: string,
+    options?: { limit?: number, cursor?: string },
+): Promise<ListAgentTestCaseRunsResult> {
+    const params = new URLSearchParams()
+    if (options?.limit !== undefined)
+        params.set('limit', String(options.limit))
+    if (options?.cursor)
+        params.set('cursor', options.cursor)
+    const qs = params.toString()
+
+    // Unlike the other endpoints, pagination fields live on the envelope itself:
+    // the API returns { data: entries, nextCursor, totalRuns }
+    return await apiRequest<ListAgentTestCaseRunsResult>(
+        `/agent/${agentId}/tests/${testId}/runs${qs ? `?${qs}` : ''}`,
+    )
+}
+
+/**
+ * Record manual thumbs up/down per output attribute path on a run.
+ * Also feeds the test case's answer bank (answers[path].good/bad), which
+ * auto-classifies future runs via answer-match. feedback: 1=good, 0=bad, null=clear.
+ */
+export async function updateAgentTestCaseAttributeFeedback(
+    agentId: string,
+    testId: string,
+    commitHash: string,
+    attributes: AgentTestCaseAttributeFeedback[],
+): Promise<AgentTestCaseRun> {
+    const { data } = await apiRequest<{ data: AgentTestCaseRun }>(
+        `/agent/${agentId}/tests/${testId}/runs/${commitHash}/attribute-feedback`,
+        { method: 'PATCH', body: jsonBody({ attributes }) },
+    )
+    return data
+}
+
+// ============================================================================
+// Stats & Metrics
+// ============================================================================
+
+/**
+ * Aggregated test case stats for an agent commit (scores per output attribute,
+ * counters, custom metrics). Accepts the same filters as listAgentTestCases.
+ */
+export async function getAgentTestCaseStats(
+    agentId: string,
+    commitHash: string,
+    options?: Omit<ListAgentTestCasesOptions, 'commitHash'>,
+): Promise<AgentTestCaseStats> {
+    const { data } = await apiRequest<{ data: AgentTestCaseStats }>(
+        `/agent/${agentId}/tests/stats${buildListQuery({ ...options, commitHash })}`,
+    )
+    return data
+}
+
+/**
+ * List custom test metrics for an agent
+ */
+export async function listAgentTestMetrics(agentId: string): Promise<AgentTestMetric[]> {
+    const { data } = await apiRequest<{ data: AgentTestMetric[] }>(`/agent/${agentId}/test-metrics`)
+    return data
+}
+
+/**
+ * Create a custom test metric grouping output attributes.
+ * attributeKeys must exist in the agent's output-format.json.
+ */
+export async function createAgentTestMetric(
+    agentId: string,
+    payload: CreateAgentTestMetricPayload,
+): Promise<AgentTestMetric> {
+    const { data } = await apiRequest<{ data: AgentTestMetric }>(`/agent/${agentId}/test-metrics`, {
+        method: 'POST',
+        body: jsonBody(payload),
+    })
+    return data
+}
+
+/**
+ * Update a custom test metric
+ */
+export async function updateAgentTestMetric(
+    agentId: string,
+    metricId: string,
+    payload: UpdateAgentTestMetricPayload,
+): Promise<AgentTestMetric> {
+    const { data } = await apiRequest<{ data: AgentTestMetric }>(
+        `/agent/${agentId}/test-metrics/${metricId}`,
+        { method: 'PATCH', body: jsonBody(payload) },
+    )
+    return data
+}
+
+/**
+ * Delete a custom test metric (soft delete)
+ */
+export async function deleteAgentTestMetric(agentId: string, metricId: string): Promise<void> {
+    await apiRequest(`/agent/${agentId}/test-metrics/${metricId}`, { method: 'DELETE' })
+}
+
+// ============================================================================
+// Tags
+// ============================================================================
+
+/**
+ * List an agent's test case tags
+ */
+export async function listAgentTestCaseTags(agentId: string): Promise<AgentTestCaseTag[]> {
+    const { data } = await apiRequest<{ data: AgentTestCaseTag[] }>(`/agent/${agentId}/tags`)
+    return data
+}
+
+/**
+ * Create a tag scoped to an agent
+ */
+export async function createAgentTestCaseTag(
+    agentId: string,
+    payload: { name: string, color: string },
+): Promise<AgentTestCaseTag> {
+    const { data } = await apiRequest<{ data: AgentTestCaseTag }>(`/agent/${agentId}/tags`, {
+        method: 'POST',
+        body: jsonBody(payload),
+    })
+    return data
+}
+
+/**
+ * Update a tag's name/color
+ */
+export async function updateAgentTestCaseTag(
+    agentId: string,
+    tagId: string,
+    payload: { name?: string, color?: string },
+): Promise<AgentTestCaseTag> {
+    const { data } = await apiRequest<{ data: AgentTestCaseTag }>(`/agent/${agentId}/tags/${tagId}`, {
+        method: 'PATCH',
+        body: jsonBody(payload),
+    })
+    return data
+}
+
+/**
+ * Delete a tag (removes all its assignments)
+ */
+export async function deleteAgentTestCaseTag(agentId: string, tagId: string): Promise<void> {
+    await apiRequest(`/agent/${agentId}/tags/${tagId}`, { method: 'DELETE' })
+}
+
+/**
+ * List tags assigned to a test case
+ */
+export async function getAgentTestCaseTags(
+    agentId: string,
+    testId: string,
+): Promise<AgentTestCaseTag[]> {
+    const { data } = await apiRequest<{ data: AgentTestCaseTag[] }>(
+        `/agent/${agentId}/tests/${testId}/tags`,
+    )
+    return data
+}
+
+/**
+ * Assign tags to a test case
+ */
+export async function addAgentTestCaseTags(
+    agentId: string,
+    testId: string,
+    tagIds: string[],
+): Promise<void> {
+    await apiRequest(`/agent/${agentId}/tests/${testId}/tags`, {
+        method: 'POST',
+        body: jsonBody({ tagIds }),
+    })
+}
+
+/**
+ * Remove one tag assignment from a test case
+ */
+export async function removeAgentTestCaseTag(
+    agentId: string,
+    testId: string,
+    tagId: string,
+): Promise<void> {
+    await apiRequest(`/agent/${agentId}/tests/${testId}/tags/${tagId}`, { method: 'DELETE' })
+}
+
+/**
+ * Assign tags to many test cases at once
+ */
+export async function bulkAddAgentTestCaseTags(
+    agentId: string,
+    testCaseIds: string[],
+    tagIds: string[],
+): Promise<void> {
+    await apiRequest(`/agent/${agentId}/tests/bulk-add-tags`, {
+        method: 'POST',
+        body: jsonBody({ testCaseIds, tagIds }),
+    })
+}
+
+/**
+ * Remove tags from many test cases at once
+ */
+export async function bulkRemoveAgentTestCaseTags(
+    agentId: string,
+    testCaseIds: string[],
+    tagIds: string[],
+): Promise<void> {
+    await apiRequest(`/agent/${agentId}/tests/bulk-remove-tags`, {
+        method: 'POST',
+        body: jsonBody({ testCaseIds, tagIds }),
+    })
+}
+
+// ============================================================================
+// Agent version helpers
+// ============================================================================
+
+/**
+ * Get an agent's commit history (agent versions)
+ */
+export async function getAgentHistory(
+    agentId: string,
+    options?: { branch?: string, page?: number, limit?: number },
+): Promise<AgentHistory> {
+    const params = new URLSearchParams()
+    if (options?.branch)
+        params.set('branch', options.branch)
+    if (options?.page !== undefined)
+        params.set('page', String(options.page))
+    if (options?.limit !== undefined)
+        params.set('limit', String(options.limit))
+    const qs = params.toString()
+
+    const { data } = await apiRequest<{ data: AgentHistory }>(
+        `/agent/${agentId}/history${qs ? `?${qs}` : ''}`,
+    )
+    return data
+}
+
+/**
+ * Resolve the agent's latest commitHash — the version test cases run against
+ */
+export async function getLatestAgentCommit(agentId: string): Promise<string> {
+    const history = await getAgentHistory(agentId, { limit: 1 })
+    const commitHash = history.commits[0]?.commitHash
+    if (!commitHash) {
+        throw new Error(`Agent ${agentId} has no commits`)
+    }
+    return commitHash
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/**
+ * Build the inputs array from a simple Record of variable values.
+ * Values starting with vault:// become file inputs; everything else is text.
+ * fileNames maps input names to display filenames (defaults to the input name).
+ */
+export function buildAgentTestInputs(
+    variables: Record<string, string>,
+    options?: { fileNames?: Record<string, string> },
+): AgentTestInput[] {
+    return Object.entries(variables).map(([name, value]) => {
+        if (typeof value === 'string' && value.startsWith('vault://')) {
+            return {
+                type: 'file' as const,
+                name,
+                vaultRef: value,
+                filename: options?.fileNames?.[name] ?? name,
+            }
+        }
+        return { type: 'text' as const, name, content: value }
+    })
+}
+
+/**
+ * Create an agent test case payload from variable values.
+ * The evaluator needs at least one expectation source to run:
+ * expectedOutput, validationReferenceFiles, or evaluationInstructions.
+ */
+export function createAgentTestCasePayload(
+    variables: Record<string, string>,
+    options?: {
+        title?: string
+        expectedOutput?: Record<string, unknown> | string
+        evaluationInstructions?: string
+        validationReferenceFiles?: AgentTestCaseValidationReferenceFile[]
+        metadata?: Record<string, unknown>
+        fileNames?: Record<string, string>
+    },
+): CreateAgentTestCasePayload {
+    const inputs = buildAgentTestInputs(variables, { fileNames: options?.fileNames })
+
+    const firstText = inputs.find(i => i.type === 'text' && i.content)
+    const firstFile = inputs.find(i => i.type === 'file')
+    const fallbackTitle = firstText?.type === 'text'
+        ? `Test: ${firstText.content.slice(0, 30)}...`
+        : firstFile?.type === 'file'
+            ? `Test: ${firstFile.filename}`
+            : 'Untitled Test'
+
+    return {
+        title: options?.title ?? fallbackTitle,
+        inputs,
+        expectedOutput: options?.expectedOutput,
+        evaluationInstructions: options?.evaluationInstructions,
+        validationReferenceFiles: options?.validationReferenceFiles,
+        metadata: options?.metadata,
+    }
+}
+
+/**
+ * Derive the run status. Intentionally a verbatim copy of the backend's
+ * getAgentTestCaseRunStatus (agent-test-case.controller.ts) so client-side
+ * status always matches API filtering and the UI — including completed
+ * evaluations without positive feedback counting as failed.
+ * Terminal: success, failed, error. `executed` = execution done, not evaluated.
+ */
+export function getAgentTestCaseRunStatus(
+    run: Pick<
+        AgentTestCaseRun,
+        'executionSessionId' | 'executionStatus' | 'evaluationSessionId' | 'evaluationStatus' | 'feedback'
+    > | null | undefined,
+): AgentTestCaseStatus {
+    if (!run)
+        return 'new'
+    if (run.executionStatus === 'error' || run.evaluationStatus === 'error')
+        return 'error'
+    if (
+        run.executionStatus === 'pending'
+        || run.executionStatus === 'running'
+        || (run.evaluationSessionId && (run.evaluationStatus === 'pending' || run.evaluationStatus === 'running'))
+    ) {
+        return 'running'
+    }
+    if (run.executionStatus === 'completed' && !run.evaluationSessionId)
+        return 'executed'
+    if (run.evaluationStatus === 'completed')
+        return run.feedback === 1 ? 'success' : 'failed'
+    if (run.evaluationStatus === 'failed' || run.feedback === 0)
+        return 'failed'
+    if (run.executionSessionId)
+        return 'executed'
+    return 'new'
+}
+
+/**
+ * Poll a test case run until it reaches a terminal status.
+ * The run row may not exist yet right after triggering — 404s are retried.
+ * By default waits for evaluation (success/failed/error); pass
+ * until: 'executed' to stop as soon as execution completes.
+ */
+export async function waitForAgentTestCaseRun(
+    agentId: string,
+    testId: string,
+    commitHash: string,
+    options?: { intervalMs?: number, maxAttempts?: number, until?: 'evaluated' | 'executed' },
+): Promise<{ run: AgentTestCaseRun, status: AgentTestCaseStatus }> {
+    const interval = options?.intervalMs ?? 2000
+    const maxAttempts = options?.maxAttempts ?? 300 // 10 minutes default
+    const until = options?.until ?? 'evaluated'
+
+    for (let i = 0; i < maxAttempts; i++) {
+        let run: AgentTestCaseRun | null = null
+        try {
+            run = await getAgentTestCaseRun(agentId, testId, commitHash)
+        }
+        catch (error) {
+            // Run row is created asynchronously by the trigger task
+            if (!(error instanceof ApiRequestError) || error.status !== 404) {
+                throw error
+            }
+        }
+
+        if (run) {
+            const status = getAgentTestCaseRunStatus(run)
+            if (status === 'success' || status === 'failed' || status === 'error') {
+                return { run, status }
+            }
+            if (status === 'executed' && until === 'executed') {
+                return { run, status }
+            }
+        }
+
+        await new Promise(resolve => setTimeout(resolve, interval))
+    }
+
+    throw new Error(`Agent test case run ${testId}@${commitHash} did not complete within timeout`)
+}

package/skill/common.ts

@@ -139,12 +139,19 @@ export async function apiRequestRaw(endpoint: string, options: Omit<RequestInit,
     })
 }
 
+export class ApiRequestError extends Error {
+    constructor(public readonly status: number, message: string) {
+        super(message)
+        this.name = 'ApiRequestError'
+    }
+}
+
 export async function apiRequest<T>(endpoint: string, options: Omit<RequestInit, 'body'> & { body?: unknown } = {}): Promise<T> {
     const response = await apiRequestRaw(endpoint, options)
 
     if (!response.ok) {
         const error = await response.text()
-        throw new Error(`API request failed: ${response.status} ${response.statusText} - ${error}`)
+        throw new ApiRequestError(response.status, `API request failed: ${response.status} ${response.statusText} - ${error}`)
     }
 
     return await (response.json() as Promise<T>)

package/skill/index.ts

@@ -1,3 +1,66 @@
+export {
+    addAgentTestCaseTags,
+    type AgentCommit,
+    type AgentHistory,
+    type AgentTestCase,
+    type AgentTestCaseAnswers,
+    type AgentTestCaseAttributeFeedback,
+    type AgentTestCaseCounters,
+    type AgentTestCaseFeedback,
+    type AgentTestCaseRun,
+    type AgentTestCaseRunHistoryEntry,
+    type AgentTestCaseSkipReason,
+    type AgentTestCaseStats,
+    type AgentTestCaseStatus,
+    type AgentTestCaseTag,
+    type AgentTestCaseValidationReferenceFile,
+    type AgentTestCaseValidationSummaryEntry,
+    type AgentTestCaseValidationType,
+    type AgentTestCaseWithRun,
+    type AgentTestInput,
+    type AgentTestMetric,
+    type AgentTestMetricStats,
+    type AgentTestMetricStatus,
+    // Helpers
+    buildAgentTestInputs,
+    bulkAddAgentTestCaseTags,
+    bulkRemoveAgentTestCaseTags,
+    continueAgentTestCase,
+    createAgentTestCase,
+    createAgentTestCasePayload,
+    type CreateAgentTestCasePayload,
+    createAgentTestCaseTag,
+    createAgentTestMetric,
+    type CreateAgentTestMetricPayload,
+    deleteAgentTestCase,
+    deleteAgentTestCaseTag,
+    deleteAgentTestMetric,
+    getAgentHistory,
+    getAgentTestCaseRun,
+    getAgentTestCaseRunStatus,
+    getAgentTestCaseStats,
+    getAgentTestCaseTags,
+    getLatestAgentCommit,
+    // Functions
+    listAgentTestCaseRuns,
+    type ListAgentTestCaseRunsResult,
+    listAgentTestCases,
+    type ListAgentTestCasesOptions,
+    listAgentTestCaseTags,
+    listAgentTestMetrics,
+    removeAgentTestCaseTag,
+    runAgentTestCase,
+    runAllAgentTestCases,
+    type RunAllAgentTestCasesPayload,
+    type RunAllAgentTestCasesResult,
+    updateAgentTestCase,
+    updateAgentTestCaseAttributeFeedback,
+    type UpdateAgentTestCasePayload,
+    updateAgentTestCaseTag,
+    updateAgentTestMetric,
+    type UpdateAgentTestMetricPayload,
+    waitForAgentTestCaseRun,
+} from './agent-test-cases.ts'
 export {
     type Canvas,
     type CanvasMessage,
@@ -27,7 +90,7 @@ export {
     // Types
     type Variable,
 } from './canvas.ts'
-export { apiRequest, getApiBaseUrl, getAppBaseUrl, getCanvasUrl, getProjectUrl, loadApiKey, parsePayload, SESSION_PATH } from './common.ts'
+export { apiRequest, ApiRequestError, getApiBaseUrl, getAppBaseUrl, getCanvasUrl, getProjectUrl, loadApiKey, parsePayload, SESSION_PATH } from './common.ts'
 export { createProject, type CreateProjectPayload, listProjects, type Project } from './projects.ts'
 export {
     getPromotedVersion,