create-windy 0.6.11 → 0.6.14

package/template/apps/server/src/agent-workflow/evaluation-evidence-metrics.ts

@@ -0,0 +1,280 @@
+import {
+  agentWorkflowNodeRuns,
+  agentWorkflowRuns,
+  agentWorkflowSideEffectCommands,
+  agentWorkflowSideEffectReceipts,
+  auditLogs,
+} from "@southwind-ai/database";
+import type {
+  AgentWorkflowDefinition,
+  AgentWorkflowEvaluationFailure,
+  AgentWorkflowEvaluationMetrics,
+} from "@southwind-ai/shared";
+import type { GovernedToolRegistry } from "../agent/tool-registry.js";
+import { canonicalEvaluationDigest } from "./evaluation-repository.js";
+import {
+  resolveEvaluationTools,
+  type AgentWorkflowEvaluationToolRegistration,
+} from "./evaluation-tool-registration.js";
+
+type RunRow = typeof agentWorkflowRuns.$inferSelect;
+type NodeRunRow = typeof agentWorkflowNodeRuns.$inferSelect;
+type CommandRow = typeof agentWorkflowSideEffectCommands.$inferSelect;
+type EffectReceiptRow = typeof agentWorkflowSideEffectReceipts.$inferSelect;
+type AuditRow = typeof auditLogs.$inferSelect;
+
+export interface EvaluationMetricEvidenceFacts {
+  readonly rubricKey: string;
+  readonly rubricVersion: string;
+  readonly tools: GovernedToolRegistry;
+  readonly evaluatorTools?: AgentWorkflowEvaluationToolRegistration;
+  readonly commands: readonly CommandRow[];
+  readonly effectReceipts: readonly EffectReceiptRow[];
+  readonly auditLogs: readonly AuditRow[];
+}
+
+export function buildEvaluationMetrics(
+  run: RunRow,
+  nodes: readonly NodeRunRow[],
+  definition: AgentWorkflowDefinition,
+  facts: EvaluationMetricEvidenceFacts,
+): AgentWorkflowEvaluationMetrics {
+  return {
+    outcome: run.status as AgentWorkflowEvaluationMetrics["outcome"],
+    latencyMs: Math.max(0, (run.finishedAt ?? run.updatedAt) - run.createdAt),
+    cost: run.usageJson.cost,
+    inputTokens: run.usageJson.inputTokens,
+    outputTokens: run.usageJson.outputTokens,
+    failures: failures(run, nodes),
+    humanHandoffs: nodes
+      .filter(({ nodeType }) => nodeType === "approval")
+      .map((node) => ({
+        reason: node.safeErrorCode ?? "APPROVAL_REQUIRED",
+        outcome:
+          node.status === "completed"
+            ? "resolved"
+            : node.status === "cancelled"
+              ? "cancelled"
+              : "unresolved",
+        durationMs: Math.max(
+          0,
+          (node.finishedAt ?? run.finishedAt ?? run.updatedAt) -
+            (node.startedAt ?? run.createdAt),
+        ),
+      })),
+    evidenceQuality: rubricEvidence(run, nodes, definition, facts),
+    safety: safetyEvidence(run, nodes, definition, facts),
+    recovery:
+      run.recoveryCount > 0 || run.status === "completed"
+        ? {
+            status: "measured",
+            attemptCount: run.recoveryCount,
+            succeeded: run.status === "completed",
+            source: "workflow-run",
+          }
+        : { status: "unknown", reasonCode: "NO_RECOVERY_ATTEMPT" },
+  };
+}
+
+function rubricEvidence(
+  run: RunRow,
+  nodes: readonly NodeRunRow[],
+  definition: AgentWorkflowDefinition,
+  facts: EvaluationMetricEvidenceFacts,
+): AgentWorkflowEvaluationMetrics["evidenceQuality"] {
+  const evaluators = resolveEvaluationTools(facts.evaluatorTools, facts.tools);
+  const registered = evaluators?.rubric;
+  if (
+    !registered ||
+    facts.rubricKey !== registered.key ||
+    facts.rubricVersion !== registered.outputSchemaKey
+  ) {
+    return { status: "unknown", reasonCode: "RUBRIC_REGISTRY_MISMATCH" };
+  }
+  const value = governedEvidence(run, nodes, definition, facts, registered);
+  return value &&
+    typeof value.score === "number" &&
+    Number.isFinite(value.score) &&
+    value.score >= 0 &&
+    value.score <= 1
+    ? { status: "graded", score: value.score, source: "rubric-evidence" }
+    : { status: "unknown", reasonCode: "RUBRIC_EVIDENCE_NOT_VERIFIED" };
+}
+
+function safetyEvidence(
+  run: RunRow,
+  nodes: readonly NodeRunRow[],
+  definition: AgentWorkflowDefinition,
+  facts: EvaluationMetricEvidenceFacts,
+): AgentWorkflowEvaluationMetrics["safety"] {
+  const registered = resolveEvaluationTools(
+    facts.evaluatorTools,
+    facts.tools,
+  )?.safety;
+  if (!registered) {
+    return { status: "unknown", reasonCode: "SAFETY_EVALUATOR_UNREGISTERED" };
+  }
+  const value = governedEvidence(run, nodes, definition, facts, registered);
+  return value &&
+    Number.isSafeInteger(value.refusalCount) &&
+    Number(value.refusalCount) >= 0 &&
+    Number.isSafeInteger(value.unsafeAllowCount) &&
+    Number(value.unsafeAllowCount) >= 0
+    ? {
+        status: "measured",
+        refusalCount: Number(value.refusalCount),
+        unsafeAllowCount: Number(value.unsafeAllowCount),
+        source: "host-audit",
+      }
+    : { status: "unknown", reasonCode: "SAFETY_EVIDENCE_NOT_VERIFIED" };
+}
+
+function governedEvidence(
+  run: RunRow,
+  nodes: readonly NodeRunRow[],
+  definition: AgentWorkflowDefinition,
+  facts: EvaluationMetricEvidenceFacts,
+  registered: NonNullable<ReturnType<typeof resolveEvaluationTools>>["rubric"],
+) {
+  const toolKey = registered.key;
+  const releasedNodeIds = new Set(
+    definition.nodes
+      .filter(
+        (node) =>
+          node.type === "tool" &&
+          node.toolKey === toolKey &&
+          node.outputSchemaKey === registered.outputSchemaKey,
+      )
+      .map(({ id }) => id),
+  );
+  for (const node of nodes) {
+    if (
+      !releasedNodeIds.has(node.nodeId) ||
+      node.status !== "completed" ||
+      node.resultReceiptJson?.outputSchemaKey !== registered.outputSchemaKey
+    ) {
+      continue;
+    }
+    const command = facts.commands.find(
+      (item) =>
+        item.runId === run.runId &&
+        item.nodeRunId === node.nodeRunId &&
+        item.effectKind === "tool" &&
+        item.effectKey === node.nodeId &&
+        item.adapterKind === "agent-tool" &&
+        ["committed", "reconciled"].includes(item.status),
+    );
+    const receipt = command
+      ? facts.effectReceipts.find(
+          (item) =>
+            item.commandId === command.commandId &&
+            item.runId === run.runId &&
+            item.nodeRunId === node.nodeRunId &&
+            item.effectKind === command.effectKind &&
+            item.generation === command.generation &&
+            item.fenceTokenDigest === command.fenceTokenDigest &&
+            item.requestDigest === command.requestDigest &&
+            item.authorizationSnapshotDigest ===
+              command.authorizationSnapshotDigest &&
+            item.authorizationSnapshotDigest ===
+              run.authorizationSnapshotDigest &&
+            item.runPolicyDigest === command.runPolicyDigest &&
+            item.runPolicyDigest === run.executionPolicyDigest &&
+            item.resultDigest === node.resultReceiptJson?.resultDigest &&
+            verifiedSmallResult(item, node),
+        )
+      : undefined;
+    const audit = receipt
+      ? facts.auditLogs.find(
+          (item) =>
+            item.id === receipt.externalReceiptId &&
+            item.actorId === run.actorId &&
+            item.action === registered.auditAction &&
+            item.resource === "agent-tool" &&
+            item.resourceId === toolKey &&
+            auditSucceeded(item, registered.module) &&
+            auditWithinNode(item, node),
+        )
+      : undefined;
+    if (!audit) continue;
+    const value = receipt?.smallResultJson;
+    if (value && typeof value === "object" && !Array.isArray(value)) {
+      return value as Record<string, unknown>;
+    }
+  }
+  return undefined;
+}
+
+function verifiedSmallResult(receipt: EffectReceiptRow, node: NodeRunRow) {
+  const receiptValue = receipt.smallResultJson;
+  const nodeValue = node.resultReceiptJson?.smallResult;
+  return (
+    receiptValue !== null &&
+    nodeValue !== undefined &&
+    canonicalEvaluationDigest(receiptValue) ===
+      canonicalEvaluationDigest(nodeValue) &&
+    receipt.resultDigest === `sha256:${canonicalEvaluationDigest(receiptValue)}`
+  );
+}
+
+function auditWithinNode(row: AuditRow, node: NodeRunRow) {
+  return (
+    node.startedAt !== null &&
+    node.finishedAt !== null &&
+    row.createdAt.getTime() >= node.startedAt &&
+    row.createdAt.getTime() <= node.finishedAt
+  );
+}
+
+function auditSucceeded(row: AuditRow, moduleKey: string) {
+  const metadata = row.metadata;
+  return (
+    metadata !== null &&
+    typeof metadata === "object" &&
+    !Array.isArray(metadata) &&
+    metadata.outcome === "success" &&
+    metadata.module === moduleKey
+  );
+}
+
+export function collectEvaluationArtifacts(nodes: readonly NodeRunRow[]) {
+  return [
+    ...new Map(
+      nodes
+        .flatMap(({ outputArtifactsJson }) => outputArtifactsJson)
+        .map((item) => [`${item.artifactId}:${item.sha256}`, item]),
+    ).values(),
+  ];
+}
+
+function failures(run: RunRow, nodes: readonly NodeRunRow[]) {
+  const values: AgentWorkflowEvaluationFailure[] = nodes.flatMap((node) =>
+    node.safeErrorCode
+      ? [
+          {
+            category: failureCategory(node.nodeType),
+            stage: node.nodeId,
+            safeCode: node.safeErrorCode,
+          },
+        ]
+      : [],
+  );
+  if (run.safeErrorCode) {
+    values.push({
+      category: "runtime",
+      stage: "workflow-run",
+      safeCode: run.safeErrorCode,
+    });
+  }
+  return values;
+}
+
+function failureCategory(
+  nodeType: string,
+): AgentWorkflowEvaluationFailure["category"] {
+  if (nodeType === "tool") return "tool";
+  if (nodeType === "model" || nodeType === "subagent") return "model";
+  if (nodeType === "approval") return "approval";
+  if (nodeType === "job") return "job";
+  return "runtime";
+}

package/template/apps/server/src/agent-workflow/evaluation-evidence-reader.ts

@@ -0,0 +1,326 @@
+import {
+  agentWorkflowDelegations,
+  agentWorkflowNodeRuns,
+  agentWorkflowReleases,
+  agentWorkflowRuns,
+  agentWorkflowSideEffectCommands,
+  agentWorkflowSideEffectReceipts,
+  auditLogs,
+} from "@southwind-ai/database";
+import type {
+  AgentWorkflowEvaluationArmKind,
+  AgentWorkflowEvaluationMismatchEvidence,
+} from "@southwind-ai/shared";
+import { and, eq, inArray } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import type { GovernedToolRegistry } from "../agent/tool-registry.js";
+import {
+  lacksArmSemantics,
+  lacksGovernedToolFacts,
+  lacksProviderEvidence,
+  observedEvaluationCoordinates,
+  unavailableEvaluationCoordinates,
+} from "./evaluation-evidence-coordinates.js";
+import {
+  buildEvaluationMetrics,
+  collectEvaluationArtifacts,
+} from "./evaluation-evidence-metrics.js";
+import { hasCompleteEvaluationEvidence } from "./evaluation-metrics-validation.js";
+import { canonicalEvaluationDigest } from "./evaluation-repository.js";
+import {
+  freezeEvaluationToolRegistration,
+  type AgentWorkflowEvaluationToolRegistration,
+} from "./evaluation-tool-registration.js";
+import {
+  AgentWorkflowEvaluationServiceError,
+  type AgentWorkflowEvaluationArmRequest,
+  type AgentWorkflowEvaluationPairRequest,
+  type AgentWorkflowEvaluationRunEvidence,
+  type AgentWorkflowEvaluationRunEvidenceReader,
+} from "./evaluation-validation.js";
+
+type ReleaseRow = typeof agentWorkflowReleases.$inferSelect;
+type RunRow = typeof agentWorkflowRuns.$inferSelect;
+type NodeRunRow = typeof agentWorkflowNodeRuns.$inferSelect;
+type DelegationRow = typeof agentWorkflowDelegations.$inferSelect;
+type CommandRow = typeof agentWorkflowSideEffectCommands.$inferSelect;
+type EffectReceiptRow = typeof agentWorkflowSideEffectReceipts.$inferSelect;
+type AuditRow = typeof auditLogs.$inferSelect;
+
+interface EvaluationFacts {
+  readonly releases: readonly ReleaseRow[];
+  readonly runs: readonly RunRow[];
+  readonly nodes: readonly NodeRunRow[];
+  readonly delegations: readonly DelegationRow[];
+  readonly commands: readonly CommandRow[];
+  readonly effectReceipts: readonly EffectReceiptRow[];
+  readonly auditLogs: readonly AuditRow[];
+}
+
+export class DrizzleAgentWorkflowEvaluationEvidenceReader implements AgentWorkflowEvaluationRunEvidenceReader {
+  private readonly evaluatorTools:
+    | AgentWorkflowEvaluationToolRegistration
+    | undefined;
+
+  constructor(
+    private readonly db: NodePgDatabase,
+    private readonly tools: GovernedToolRegistry,
+    evaluatorTools?: AgentWorkflowEvaluationToolRegistration,
+  ) {
+    this.evaluatorTools = freezeEvaluationToolRegistration(evaluatorTools);
+  }
+
+  readPair(input: AgentWorkflowEvaluationPairRequest) {
+    return this.db.transaction(
+      async (tx) => {
+        const requests = [input.singleAgent, input.multiAgent];
+        const runIds = requests.flatMap(({ availability }) =>
+          availability.status === "run" ? [availability.runId] : [],
+        );
+        const runs = runIds.length
+          ? await tx
+              .select()
+              .from(agentWorkflowRuns)
+              .where(
+                and(
+                  inArray(agentWorkflowRuns.runId, runIds),
+                  eq(agentWorkflowRuns.tenantId, input.tenantId),
+                  eq(agentWorkflowRuns.actorId, input.createdBy),
+                  eq(agentWorkflowRuns.scopeDigest, input.scopeDigest),
+                ),
+              )
+          : [];
+        const releaseIds = [
+          ...new Set([
+            ...requests.map(({ releaseId }) => releaseId),
+            ...runs.map(({ releaseId }) => releaseId),
+          ]),
+        ];
+        const releases = await tx
+          .select()
+          .from(agentWorkflowReleases)
+          .where(inArray(agentWorkflowReleases.releaseId, releaseIds));
+        const authorizedRunIds = runs.map(({ runId }) => runId);
+        const nodes = await selectByRunIds(authorizedRunIds, () =>
+          tx
+            .select()
+            .from(agentWorkflowNodeRuns)
+            .where(inArray(agentWorkflowNodeRuns.runId, authorizedRunIds)),
+        );
+        const commands = await selectByRunIds(authorizedRunIds, () =>
+          tx
+            .select()
+            .from(agentWorkflowSideEffectCommands)
+            .where(
+              inArray(agentWorkflowSideEffectCommands.runId, authorizedRunIds),
+            ),
+        );
+        const delegations = await selectByRunIds(authorizedRunIds, () =>
+          tx
+            .select()
+            .from(agentWorkflowDelegations)
+            .where(
+              inArray(agentWorkflowDelegations.parentRunId, authorizedRunIds),
+            ),
+        );
+        const effectReceipts = await selectByRunIds(authorizedRunIds, () =>
+          tx
+            .select()
+            .from(agentWorkflowSideEffectReceipts)
+            .where(
+              inArray(agentWorkflowSideEffectReceipts.runId, authorizedRunIds),
+            ),
+        );
+        const auditIds = effectReceipts.map(
+          ({ externalReceiptId }) => externalReceiptId,
+        );
+        const persistedAuditLogs = auditIds.length
+          ? await tx
+              .select()
+              .from(auditLogs)
+              .where(inArray(auditLogs.id, auditIds))
+          : [];
+        const facts = {
+          releases,
+          runs,
+          nodes,
+          delegations,
+          commands,
+          effectReceipts,
+          auditLogs: persistedAuditLogs,
+        };
+        return {
+          singleAgent: evidenceFor(
+            input.singleAgent,
+            "single-agent",
+            input,
+            facts,
+            this.tools,
+            this.evaluatorTools,
+          ),
+          multiAgent: evidenceFor(
+            input.multiAgent,
+            "multi-agent",
+            input,
+            facts,
+            this.tools,
+            this.evaluatorTools,
+          ),
+        };
+      },
+      { isolationLevel: "repeatable read", accessMode: "read only" },
+    );
+  }
+}
+
+function evidenceFor(
+  requested: AgentWorkflowEvaluationArmRequest,
+  arm: AgentWorkflowEvaluationArmKind,
+  input: AgentWorkflowEvaluationPairRequest,
+  facts: EvaluationFacts,
+  tools: GovernedToolRegistry,
+  evaluatorTools: AgentWorkflowEvaluationToolRegistration | undefined,
+): AgentWorkflowEvaluationRunEvidence {
+  const requestedRelease = facts.releases.find(
+    ({ releaseId }) => releaseId === requested.releaseId,
+  );
+  if (!requestedRelease) invalid(`${arm}.releaseId`);
+  if (requested.availability.status !== "run") {
+    return unavailableEvidence(
+      arm,
+      requestedRelease,
+      input.scopeDigest,
+      "RUN_NOT_PROVIDED",
+      tools,
+      evaluatorTools,
+    );
+  }
+
+  const requestedRunId = requested.availability.runId;
+  const run = facts.runs.find(({ runId }) => runId === requestedRunId);
+  if (!run) {
+    return unavailableEvidence(
+      arm,
+      requestedRelease,
+      input.scopeDigest,
+      "RUN_EVIDENCE_UNAVAILABLE",
+      tools,
+      evaluatorTools,
+    );
+  }
+  const release = facts.releases.find(
+    ({ releaseId }) => releaseId === run.releaseId,
+  );
+  if (!release) invalid(`${arm}.run.releaseId`);
+  const nodeRuns = facts.nodes.filter(({ runId }) => runId === run.runId);
+  const coordinates = observedEvaluationCoordinates(
+    release,
+    run,
+    nodeRuns,
+    tools,
+    evaluatorTools,
+  );
+  const base = {
+    arm,
+    coordinates,
+    run: {
+      runId: run.runId,
+      releaseId: run.releaseId,
+      releaseDigest: release.releaseDigest,
+    },
+    evidenceArtifacts: collectEvaluationArtifacts(nodeRuns),
+    mismatchEvidence:
+      requested.releaseId === run.releaseId
+        ? []
+        : [mismatch("releaseId", requested.releaseId, run.releaseId)],
+  } as const;
+  if (!isTerminal(run.status)) return { ...base, status: "pending" };
+  if (run.finishedAt === null) {
+    return { ...base, status: "invalid", reasonCode: "TERMINAL_TIME_MISSING" };
+  }
+  const metrics = buildEvaluationMetrics(
+    run,
+    nodeRuns,
+    release.releaseJson.definition,
+    {
+      rubricKey: input.rubricKey,
+      rubricVersion: input.rubricVersion,
+      tools,
+      evaluatorTools,
+      commands: facts.commands.filter(({ runId }) => runId === run.runId),
+      effectReceipts: facts.effectReceipts.filter(
+        ({ runId }) => runId === run.runId,
+      ),
+      auditLogs: facts.auditLogs,
+    },
+  );
+  if (
+    !hasCompleteEvaluationEvidence(metrics) ||
+    lacksArmSemantics(release, nodeRuns, facts.delegations, run.runId, arm) ||
+    lacksProviderEvidence(release, nodeRuns) ||
+    lacksGovernedToolFacts(release, tools)
+  ) {
+    return {
+      ...base,
+      status: "unavailable",
+      reasonCode: "EVALUATION_EVIDENCE_INCOMPLETE",
+      metrics,
+    };
+  }
+  return { ...base, status: "terminal", metrics };
+}
+
+function unavailableEvidence(
+  arm: AgentWorkflowEvaluationArmKind,
+  release: ReleaseRow,
+  scopeDigest: string,
+  reasonCode: string,
+  tools: GovernedToolRegistry,
+  evaluatorTools: AgentWorkflowEvaluationToolRegistration | undefined,
+): AgentWorkflowEvaluationRunEvidence {
+  return {
+    arm,
+    status: "unavailable",
+    coordinates: unavailableEvaluationCoordinates(
+      release,
+      scopeDigest,
+      reasonCode,
+      tools,
+      evaluatorTools,
+    ),
+    reasonCode,
+    evidenceArtifacts: [],
+    mismatchEvidence: [],
+  };
+}
+
+async function selectByRunIds<T>(
+  runIds: readonly string[],
+  select: () => PromiseLike<T[]>,
+) {
+  return runIds.length ? Promise.resolve(select()) : Promise.resolve([] as T[]);
+}
+
+function mismatch(
+  field: string,
+  expected: unknown,
+  observed?: unknown,
+): AgentWorkflowEvaluationMismatchEvidence {
+  return {
+    source: "requested-vs-observed",
+    field,
+    reasonCode: "FROZEN_COORDINATE_MISMATCH",
+    expectedDigest: canonicalEvaluationDigest(expected),
+    ...(observed === undefined
+      ? {}
+      : { observedDigest: canonicalEvaluationDigest(observed) }),
+  };
+}
+
+function isTerminal(status: string) {
+  return ["completed", "failed", "cancelled", "timed-out"].includes(status);
+}
+
+function invalid(field: string): never {
+  throw new AgentWorkflowEvaluationServiceError("INVALID_RUN_EVIDENCE", field);
+}

package/template/apps/server/src/agent-workflow/evaluation-metrics-validation.ts

@@ -0,0 +1,85 @@
+import type { AgentWorkflowEvaluationMetrics } from "@southwind-ai/shared";
+
+export function invalidEvaluationMetricsField(
+  metrics: AgentWorkflowEvaluationMetrics,
+  field: string,
+): string | undefined {
+  const durations = [
+    metrics.latencyMs,
+    ...metrics.humanHandoffs.map(({ durationMs }) => durationMs),
+  ];
+  if (durations.some((value) => !Number.isFinite(value) || value < 0)) {
+    return `${field}.latency`;
+  }
+  if (!validMeasuredCount(metrics.inputTokens)) return `${field}.inputTokens`;
+  if (!validMeasuredCount(metrics.outputTokens)) return `${field}.outputTokens`;
+  if (
+    (metrics.cost.status === "measured" &&
+      (!nonNegativeInteger(metrics.cost.amountMicros) ||
+        !metrics.cost.currency.trim())) ||
+    (metrics.cost.status === "unknown" && !metrics.cost.reasonCode.trim())
+  ) {
+    return `${field}.cost`;
+  }
+  if (
+    metrics.failures.some(
+      ({ stage, safeCode }) => !stage.trim() || !safeCode.trim(),
+    ) ||
+    metrics.humanHandoffs.some(({ reason }) => !reason.trim())
+  ) {
+    return `${field}.facts`;
+  }
+  return invalidOptionalMetricsField(metrics, field);
+}
+
+export function hasCompleteEvaluationEvidence(
+  metrics: AgentWorkflowEvaluationMetrics,
+) {
+  return (
+    metrics.evidenceQuality.status === "graded" &&
+    metrics.safety.status === "measured" &&
+    metrics.recovery.status === "measured"
+  );
+}
+
+function validMeasuredCount(
+  value: AgentWorkflowEvaluationMetrics["inputTokens"],
+) {
+  return (
+    (value.status === "measured" && nonNegativeInteger(value.value)) ||
+    (value.status === "unknown" && Boolean(value.reasonCode.trim()))
+  );
+}
+
+function invalidOptionalMetricsField(
+  metrics: AgentWorkflowEvaluationMetrics,
+  field: string,
+) {
+  if (
+    (metrics.evidenceQuality.status === "graded" &&
+      (!Number.isFinite(metrics.evidenceQuality.score) ||
+        metrics.evidenceQuality.score < 0 ||
+        metrics.evidenceQuality.score > 1 ||
+        metrics.evidenceQuality.source !== "rubric-evidence")) ||
+    (metrics.evidenceQuality.status === "unknown" &&
+      !metrics.evidenceQuality.reasonCode.trim()) ||
+    (metrics.safety.status === "measured" &&
+      (!nonNegativeInteger(metrics.safety.refusalCount) ||
+        !nonNegativeInteger(metrics.safety.unsafeAllowCount) ||
+        metrics.safety.source !== "host-audit")) ||
+    (metrics.safety.status === "unknown" &&
+      !metrics.safety.reasonCode.trim()) ||
+    (metrics.recovery.status === "measured" &&
+      (!nonNegativeInteger(metrics.recovery.attemptCount) ||
+        metrics.recovery.source !== "workflow-run")) ||
+    (metrics.recovery.status === "unknown" &&
+      !metrics.recovery.reasonCode.trim())
+  ) {
+    return `${field}.optionalMetrics`;
+  }
+  return undefined;
+}
+
+function nonNegativeInteger(value: number) {
+  return Number.isSafeInteger(value) && value >= 0;
+}