@shipfox/api-workflows 2.0.0

package/drizzle/meta/_journal.json

@@ -0,0 +1,20 @@
+{
+  "version": "7",
+  "dialect": "postgresql",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "7",
+      "when": 1783159827882,
+      "tag": "0000_initial",
+      "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "7",
+      "when": 1783439600000,
+      "tag": "0001_agent_tool_materialization",
+      "breakpoints": true
+    }
+  ]
+}

package/drizzle.config.ts

@@ -0,0 +1,7 @@
+import {defineConfig} from 'drizzle-kit';
+
+export default defineConfig({
+  schema: './src/db/schema/*.ts',
+  out: './drizzle',
+  dialect: 'postgresql',
+});

package/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "@shipfox/api-workflows",
+  "license": "MIT",
+  "version": "2.0.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
+    "directory": "libs/api/workflows"
+  },
+  "private": false,
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "imports": {
+    "#test/*": "./test/*",
+    "#*": {
+      "workspace-source": "./src/*",
+      "development": "./src/*",
+      "default": "./dist/*"
+    }
+  },
+  "exports": {
+    ".": {
+      "development": {
+        "types": "./src/index.ts",
+        "default": "./src/index.ts"
+      },
+      "default": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    }
+  },
+  "dependencies": {
+    "@temporalio/common": "^1.16.1",
+    "@temporalio/workflow": "^1.16.1",
+    "drizzle-orm": "^0.45.2",
+    "zod": "^4.4.3",
+    "@shipfox/api-auth": "2.0.0",
+    "@shipfox/api-auth-context": "2.0.0",
+    "@shipfox/annotations": "2.0.0",
+    "@shipfox/api-integration-core": "2.0.0",
+    "@shipfox/api-projects": "2.0.0",
+    "@shipfox/api-agent-dto": "2.0.0",
+    "@shipfox/api-runners-dto": "2.0.0",
+    "@shipfox/api-runners": "2.0.0",
+    "@shipfox/api-secrets": "2.0.0",
+    "@shipfox/api-secrets-dto": "2.0.0",
+    "@shipfox/expression": "1.1.1",
+    "@shipfox/node-error-monitoring": "0.1.2",
+    "@shipfox/api-workflows-dto": "2.0.0",
+    "@shipfox/node-drizzle": "0.2.1",
+    "@shipfox/node-opentelemetry": "0.5.0",
+    "@shipfox/node-module": "0.2.0",
+    "@shipfox/node-outbox": "0.2.1",
+    "@shipfox/node-fastify": "0.2.1",
+    "@shipfox/node-postgres": "0.4.1",
+    "@shipfox/node-temporal": "0.2.0",
+    "@shipfox/runner-labels": "0.1.0",
+    "@shipfox/api-agent": "2.0.0",
+    "@shipfox/api-definitions": "2.0.0"
+  },
+  "devDependencies": {
+    "@temporalio/activity": "^1.16.1",
+    "@temporalio/client": "^1.16.1",
+    "@temporalio/testing": "^1.16.1",
+    "@temporalio/worker": "^1.16.1",
+    "@types/pg": "^8.15.5",
+    "drizzle-kit": "^0.31.10",
+    "fastify": "^5.3.3",
+    "fastify-type-provider-zod": "^6.0.0",
+    "fishery": "^2.2.2",
+    "@shipfox/api-auth-dto": "2.0.0",
+    "@shipfox/biome": "1.8.1",
+    "@shipfox/node-jwt": "0.2.0",
+    "@shipfox/swc": "1.2.5",
+    "@shipfox/vitest": "1.2.2",
+    "@shipfox/ts-config": "1.3.8",
+    "@shipfox/typescript": "1.1.6",
+    "@shipfox/node-log": "0.3.3"
+  },
+  "scripts": {
+    "build": "shipfox-swc",
+    "check": "shipfox-biome-check",
+    "check:fix": "shipfox-biome-check --write",
+    "test": "shipfox-vitest-run",
+    "test:watch": "shipfox-vitest-watch",
+    "type": "shipfox-tsc-check",
+    "type:emit": "shipfox-tsc-emit"
+  }
+}

package/src/core/README.md

@@ -0,0 +1,79 @@
+# Workflow Runtime Core
+
+Workflow runtime code is split into two independent subsystems: scheduling and step config.
+
+## Workflow Scheduling
+
+`core/workflow-scheduling` contains the pure DAG scheduler and run-progress helpers.
+
+- **`scheduleRuntimeDag`** reads a runtime DAG, completed job statuses, and currently running jobs, then returns scheduling commands.
+- **Runtime commands** describe host actions such as starting a job, skipping a job, cancelling a job, or completing a run.
+- **Runtime DAG entities** define the small job shape the scheduler needs.
+- **Run progress** tracks completed and skipped jobs while Temporal orchestration loops over scheduler decisions.
+
+The scheduler does not call Temporal, runners, the database, or external systems. The host loads durable state, calls the scheduler, applies commands, persists rows, and waits for job results.
+
+```ts
+import {scheduleRuntimeDag} from '#core/workflow-scheduling/schedule-runtime-dag.js';
+
+const commands = scheduleRuntimeDag({
+  jobs: [
+    {id: 'job-1', key: 'build', mode: 'one_shot', dependencies: [], version: 1},
+    {id: 'job-2', key: 'test', mode: 'one_shot', dependencies: ['build'], version: 1},
+  ],
+  completed: new Map(),
+  running: new Set(),
+});
+```
+
+## Step Config
+
+`core/step-config` owns workflow step config materialization and fill-site completion.
+
+- **Run and execution assembly** builds `WorkflowEvaluationContext` values for run creation, execution creation, and step dispatch.
+- **Materialization** converts a normalized workflow model into persisted job and step rows for a run or a single job execution.
+- **Step config resolution** evaluates and freezes planned step config fields at the earliest server site that has the required roots.
+- **Dispatch completion** fills any `step-dispatch` `config_plan` segments before a step is handed to a runner.
+
+Step config starts from the authored workflow model. Authoring validation records each expression's roots and produces a `config_plan` for fields that may need more than one fill site. At run creation or execution creation, the materializer assembles a phase-tagged context and resolves fields whose roots are available at that site.
+
+Fully resolved fields are frozen into `steps.config`. Deferred segments stay in the plan until their fill site. `completeStepDispatchConfig` handles the dispatch site, using the latest step context to finish server-side segments before the runner receives the step. Runner-only segments, such as secrets, remain references for runner fill and are never persisted as plaintext values.
+
+The context wrapper is:
+
+```ts
+type WorkflowEvaluationContext = {
+  site: AvailabilitySite;
+  values: WorkflowExpressionEvaluationContext;
+};
+```
+
+The `site` records where the fill is happening. The `values` object carries only the roots available there, such as `run`, `trigger`, `event`, `inputs`, `job`, `execution`, `executions`, and step data for dispatch.
+
+```ts
+import {assembleCreationContext} from '#core/step-config/assemble-run-context.js';
+import {materializeWorkflowModel} from '#core/step-config/materialize-workflow-model.js';
+
+const context = assembleCreationContext({
+  run,
+  triggerPayload,
+  inputs,
+});
+
+const materialized = await materializeWorkflowModel({
+  model,
+  context,
+  definitionId,
+  resolveAgentDefaults,
+});
+```
+
+## Development
+
+```sh
+turbo check --filter=@shipfox/api-workflows
+turbo type --filter=@shipfox/api-workflows
+turbo test --filter=@shipfox/api-workflows
+```
+
+DB-backed workflow tests need local Postgres from `docker compose up -d`.

package/src/core/agent-tool-capability-warning.test.ts

@@ -0,0 +1,280 @@
+import type {JobLeaseTokenClaims} from '@shipfox/api-auth';
+import {sql} from 'drizzle-orm';
+import {db} from '#db/db.js';
+import {warnAgentToolCapabilityMismatchOnDispatch} from './agent-tool-capability-warning.js';
+import type {Step} from './entities/step.js';
+
+function lease(params: Partial<JobLeaseTokenClaims> = {}): JobLeaseTokenClaims {
+  const now = Math.floor(Date.now() / 1000);
+  return {
+    aud: 'runner-job-lease',
+    iat: now,
+    exp: now + 60,
+    workspaceId: crypto.randomUUID(),
+    projectId: crypto.randomUUID(),
+    workflowRunId: crypto.randomUUID(),
+    workflowRunAttempt: 1,
+    workflowRunAttemptId: crypto.randomUUID(),
+    jobId: crypto.randomUUID(),
+    jobExecutionId: crypto.randomUUID(),
+    runnerSessionId: crypto.randomUUID(),
+    currentStepId: crypto.randomUUID(),
+    currentStepAttempt: 1,
+    ...params,
+  };
+}
+
+function agentStep(params: Partial<Step> = {}): Step {
+  return {
+    id: crypto.randomUUID(),
+    jobExecutionId: crypto.randomUUID(),
+    key: 'fix',
+    name: 'Fix',
+    sourceLocation: null,
+    status: 'running',
+    statusReason: null,
+    evaluationTrace: null,
+    type: 'agent',
+    config: {
+      harness: 'pi',
+      provider: 'anthropic',
+      model: 'claude-opus-4-8',
+      thinking: 'high',
+      tools: ['read', 'web_search'],
+      prompt: 'Fix it.',
+    },
+    condition: null,
+    configPlan: null,
+    authoredConfig: null,
+    error: null,
+    position: 0,
+    version: 1,
+    currentAttempt: 1,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+    ...params,
+  };
+}
+
+async function insertRunnerSession(params: {
+  runnerSessionId: string;
+  workspaceId: string;
+  toolCapabilities: Record<string, unknown> | null;
+  reportedAt?: 'fresh' | 'stale' | 'missing';
+}): Promise<void> {
+  const reportedAt =
+    params.reportedAt === 'missing'
+      ? sql`NULL`
+      : params.reportedAt === 'stale'
+        ? sql`now() - interval '1 hour'`
+        : sql`now()`;
+  const toolCapabilities =
+    params.toolCapabilities === null
+      ? sql`NULL`
+      : sql`${JSON.stringify(params.toolCapabilities)}::jsonb`;
+
+  await db().execute(sql`
+    INSERT INTO runners_runner_sessions (
+      id,
+      workspace_id,
+      scope,
+      registration_token_id,
+      registration_token_kind,
+      labels,
+      tool_capabilities,
+      tool_capabilities_reported_at
+    )
+    VALUES (
+      ${params.runnerSessionId},
+      ${params.workspaceId},
+      'workspace',
+      ${crypto.randomUUID()},
+      'manual',
+      ARRAY['linux'],
+      ${toolCapabilities},
+      ${reportedAt}
+    )
+  `);
+}
+
+async function annotationsFor(
+  jobExecutionId: string,
+): Promise<Array<{context: string; body: string}>> {
+  const result = await db().execute<{context: string; body: string}>(sql`
+    SELECT context, body
+    FROM annotations_annotations
+    WHERE job_execution_id = ${jobExecutionId}
+    ORDER BY sequence
+  `);
+  return [...result.rows];
+}
+
+describe('warnAgentToolCapabilityMismatchOnDispatch', () => {
+  it('writes a warning when the matched runner advertised a set without requested tools', async () => {
+    const identity = lease();
+    const step = agentStep({jobExecutionId: identity.jobExecutionId});
+    await insertRunnerSession({
+      runnerSessionId: identity.runnerSessionId,
+      workspaceId: identity.workspaceId,
+      toolCapabilities: {harnesses: {pi: {tools: ['read']}}},
+    });
+
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+
+    const rows = await annotationsFor(identity.jobExecutionId);
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.context).toBe(`agent-tool-capability:${step.id}`);
+    expect(rows[0]?.body).toContain('advertised a `pi` tool set without: `web_search`');
+    expect(rows[0]?.body).toContain('Execution is continuing');
+  });
+
+  it('writes unknown wording when capabilities are missing', async () => {
+    const identity = lease();
+    const step = agentStep({jobExecutionId: identity.jobExecutionId});
+    await insertRunnerSession({
+      runnerSessionId: identity.runnerSessionId,
+      workspaceId: identity.workspaceId,
+      toolCapabilities: null,
+      reportedAt: 'missing',
+    });
+
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+
+    const rows = await annotationsFor(identity.jobExecutionId);
+    expect(rows[0]?.body).toContain('reported no or stale tool capabilities');
+    expect(rows[0]?.body).toContain('`read`, `web_search`');
+  });
+
+  it('writes missing-harness wording when fresh capabilities omit the harness', async () => {
+    const identity = lease();
+    const step = agentStep({jobExecutionId: identity.jobExecutionId});
+    await insertRunnerSession({
+      runnerSessionId: identity.runnerSessionId,
+      workspaceId: identity.workspaceId,
+      toolCapabilities: {harnesses: {claude: {tools: ['read', 'web_search']}}},
+    });
+
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+
+    const rows = await annotationsFor(identity.jobExecutionId);
+    expect(rows[0]?.body).toContain('did not advertise a `pi` tool set');
+    expect(rows[0]?.body).not.toContain('reported no or stale tool capabilities');
+  });
+
+  it('uses Markdown-safe code spans for requested tool names', async () => {
+    const identity = lease();
+    const step = agentStep({
+      jobExecutionId: identity.jobExecutionId,
+      config: {
+        harness: 'pi',
+        provider: 'anthropic',
+        model: 'claude-opus-4-8',
+        thinking: 'high',
+        tools: ['read', 'we`ird'],
+        prompt: 'Fix it.',
+      },
+    });
+    await insertRunnerSession({
+      runnerSessionId: identity.runnerSessionId,
+      workspaceId: identity.workspaceId,
+      toolCapabilities: {harnesses: {pi: {tools: ['read']}}},
+    });
+
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+
+    const rows = await annotationsFor(identity.jobExecutionId);
+    expect(rows[0]?.body).toContain('``we`ird``');
+  });
+
+  it('removes an existing warning when a later dispatch has all requested tools', async () => {
+    const identity = lease();
+    const step = agentStep({jobExecutionId: identity.jobExecutionId});
+    await insertRunnerSession({
+      runnerSessionId: identity.runnerSessionId,
+      workspaceId: identity.workspaceId,
+      toolCapabilities: {harnesses: {pi: {tools: ['read']}}},
+    });
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+    await db().execute(sql`
+      UPDATE runners_runner_sessions
+      SET tool_capabilities = ${JSON.stringify({
+        harnesses: {pi: {tools: ['read', 'web_search']}},
+      })}::jsonb,
+      tool_capabilities_reported_at = now()
+      WHERE id = ${identity.runnerSessionId}
+    `);
+
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+
+    const rows = await annotationsFor(identity.jobExecutionId);
+    expect(rows).toEqual([]);
+  });
+
+  it('does nothing for agent steps without requested tools', async () => {
+    const identity = lease();
+    const step = agentStep({
+      jobExecutionId: identity.jobExecutionId,
+      config: {
+        harness: 'pi',
+        provider: 'anthropic',
+        model: 'claude-opus-4-8',
+        thinking: 'high',
+        prompt: 'Fix it.',
+      },
+    });
+    await insertRunnerSession({
+      runnerSessionId: identity.runnerSessionId,
+      workspaceId: identity.workspaceId,
+      toolCapabilities: null,
+    });
+
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+
+    expect(await annotationsFor(identity.jobExecutionId)).toEqual([]);
+  });
+
+  it('does nothing for non-agent steps', async () => {
+    const identity = lease();
+    const step = agentStep({
+      jobExecutionId: identity.jobExecutionId,
+      type: 'run',
+      config: {run: 'echo ok'},
+    });
+    await insertRunnerSession({
+      runnerSessionId: identity.runnerSessionId,
+      workspaceId: identity.workspaceId,
+      toolCapabilities: null,
+    });
+
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+
+    expect(await annotationsFor(identity.jobExecutionId)).toEqual([]);
+  });
+
+  it('caps long tool lists in the warning body', async () => {
+    const identity = lease();
+    const tools = Array.from({length: 12}, (_, index) => `tool_${index}`);
+    const step = agentStep({
+      jobExecutionId: identity.jobExecutionId,
+      config: {
+        harness: 'pi',
+        provider: 'anthropic',
+        model: 'claude-opus-4-8',
+        thinking: 'high',
+        tools,
+        prompt: 'Fix it.',
+      },
+    });
+    await insertRunnerSession({
+      runnerSessionId: identity.runnerSessionId,
+      workspaceId: identity.workspaceId,
+      toolCapabilities: {harnesses: {pi: {tools: []}}},
+    });
+
+    await warnAgentToolCapabilityMismatchOnDispatch({leaseIdentity: identity, step});
+
+    const rows = await annotationsFor(identity.jobExecutionId);
+    expect(rows[0]?.body).toContain('...and 2 more');
+    expect(rows[0]?.body).not.toContain('tool_10');
+  });
+});

package/src/core/agent-tool-capability-warning.ts

@@ -0,0 +1,200 @@
+import {
+  AnnotationBodyTooLargeError,
+  AnnotationCountLimitExceededError,
+  AnnotationTotalBytesLimitExceededError,
+  type WriteAnnotationsParams,
+  writeAnnotations,
+} from '@shipfox/annotations';
+import {
+  harnessSchema,
+  type MaterializedAgentStepConfigDto,
+  materializedAgentStepConfigSchema,
+} from '@shipfox/api-agent-dto';
+import type {LeasedJobContext} from '@shipfox/api-auth-context';
+import {getEffectiveRunnerToolCapabilities, unadvertisedRunnerTools} from '@shipfox/api-runners';
+import {logger} from '@shipfox/node-opentelemetry';
+import {recordWorkflowAgentToolWarningFailed} from '#metrics/instance.js';
+import type {Step} from './entities/step.js';
+
+const TOOL_LIST_LIMIT = 10;
+
+type WarningFailureReason = 'budget' | 'lookup' | 'write';
+type WarningReason = 'known-absence' | 'harness-not-advertised' | 'unknown-or-stale';
+
+export async function warnAgentToolCapabilityMismatchOnDispatch(params: {
+  leaseIdentity: LeasedJobContext;
+  step: Step;
+}): Promise<void> {
+  const config = parseAgentConfig(params.step);
+  if (!config) return;
+
+  const requestedTools = config.tools ?? [];
+  if (requestedTools.length === 0) return;
+
+  const harness = harnessSchema.safeParse(config.harness);
+  if (!harness.success) return;
+
+  let effective: Awaited<ReturnType<typeof getEffectiveRunnerToolCapabilities>>;
+  try {
+    effective = await getEffectiveRunnerToolCapabilities({
+      runnerSessionId: params.leaseIdentity.runnerSessionId,
+    });
+  } catch (error) {
+    recordWarningWriteFailure('lookup');
+    logger().warn(
+      {error, jobExecutionId: params.leaseIdentity.jobExecutionId, stepId: params.step.id},
+      'Failed to read runner tool capabilities for agent tool warning',
+    );
+    return;
+  }
+
+  const missing = unadvertisedRunnerTools({
+    harness: harness.data,
+    requestedTools,
+    capabilities: effective.capabilities,
+  });
+  const context = `agent-tool-capability:${params.step.id}`;
+
+  const operation =
+    missing.length === 0
+      ? {context, style: 'warning' as const, op: 'remove' as const}
+      : {
+          context,
+          style: 'warning' as const,
+          op: 'replace' as const,
+          body: warningBody({
+            harness: harness.data,
+            missing,
+            reason: warningReason({
+              reportFresh: effective.reportFresh,
+              harnessKnown: effective.harnessKnown(harness.data),
+            }),
+          }),
+        };
+
+  try {
+    await writeAnnotations({
+      ...writeParamsFromLease(params.leaseIdentity),
+      originStepId: params.step.id,
+      originStepAttempt: params.step.currentAttempt,
+      operations: [operation],
+    });
+  } catch (error) {
+    const reason = isAnnotationBudgetError(error) ? 'budget' : 'write';
+    recordWarningWriteFailure(reason);
+    logger().warn(
+      {
+        error,
+        reason,
+        jobExecutionId: params.leaseIdentity.jobExecutionId,
+        stepId: params.step.id,
+      },
+      'Failed to write agent tool capability warning annotation',
+    );
+    return;
+  }
+
+  if (missing.length > 0) {
+    logger().warn(
+      {
+        jobExecutionId: params.leaseIdentity.jobExecutionId,
+        stepId: params.step.id,
+        harness: harness.data,
+        missing,
+        reason: warningReason({
+          reportFresh: effective.reportFresh,
+          harnessKnown: effective.harnessKnown(harness.data),
+        }),
+      },
+      'Runner missing requested agent tools; dispatch is continuing',
+    );
+  }
+}
+
+function parseAgentConfig(step: Step): MaterializedAgentStepConfigDto | null {
+  if (step.type !== 'agent') return null;
+
+  const parsed = materializedAgentStepConfigSchema.safeParse(step.config);
+  return parsed.success ? parsed.data : null;
+}
+
+function warningBody(params: {
+  harness: string;
+  missing: readonly string[];
+  reason: WarningReason;
+}): string {
+  const tools = formatToolList(params.missing);
+  const harness = markdownInlineCode(params.harness);
+  if (params.reason === 'known-absence') {
+    return [
+      '**Runner missing requested agent tools**',
+      '',
+      `The matched runner advertised a ${harness} tool set without: ${tools}.`,
+      'Execution is continuing on this runner because labels matched; the step may fail if the harness cannot provide these tools.',
+    ].join('\n');
+  }
+
+  if (params.reason === 'harness-not-advertised') {
+    return [
+      '**Runner missing requested agent harness tools**',
+      '',
+      `The matched runner advertised fresh tool capabilities, but did not advertise a ${harness} tool set. Support for ${tools} could not be confirmed.`,
+      'Execution is continuing on this runner because labels matched; the step may fail if the harness cannot provide these tools.',
+    ].join('\n');
+  }
+
+  return [
+    '**Could not confirm runner agent tools**',
+    '',
+    `The matched runner reported no or stale tool capabilities, so support for ${tools} could not be confirmed.`,
+    'Execution is continuing on this runner because labels matched.',
+  ].join('\n');
+}
+
+function formatToolList(tools: readonly string[]): string {
+  const visible = tools.slice(0, TOOL_LIST_LIMIT).map(markdownInlineCode);
+  const remaining = tools.length - visible.length;
+  return remaining > 0 ? `${visible.join(', ')} ...and ${remaining} more` : visible.join(', ');
+}
+
+function markdownInlineCode(value: string): string {
+  const maxBacktickRun = Math.max(
+    0,
+    ...Array.from(value.matchAll(/`+/g), (match) => match[0].length),
+  );
+  const delimiter = '`'.repeat(maxBacktickRun + 1);
+  const needsPadding = value.startsWith('`') || value.endsWith('`');
+  const content = needsPadding ? ` ${value} ` : value;
+  return `${delimiter}${content}${delimiter}`;
+}
+
+function warningReason(params: {reportFresh: boolean; harnessKnown: boolean}): WarningReason {
+  if (params.harnessKnown) return 'known-absence';
+  return params.reportFresh ? 'harness-not-advertised' : 'unknown-or-stale';
+}
+
+function writeParamsFromLease(
+  lease: LeasedJobContext,
+): Omit<WriteAnnotationsParams, 'originStepId' | 'originStepAttempt' | 'operations'> {
+  return {
+    workspaceId: lease.workspaceId,
+    projectId: lease.projectId,
+    workflowRunId: lease.workflowRunId,
+    workflowRunAttempt: lease.workflowRunAttempt ?? 1,
+    workflowRunAttemptId: lease.workflowRunAttemptId,
+    jobId: lease.jobId,
+    jobExecutionId: lease.jobExecutionId,
+  };
+}
+
+function isAnnotationBudgetError(error: unknown): boolean {
+  return (
+    error instanceof AnnotationBodyTooLargeError ||
+    error instanceof AnnotationCountLimitExceededError ||
+    error instanceof AnnotationTotalBytesLimitExceededError
+  );
+}
+
+function recordWarningWriteFailure(reason: WarningFailureReason): void {
+  recordWorkflowAgentToolWarningFailed(reason);
+}