create-windy 0.6.12 → 0.6.14

package/template/apps/server/src/agent-workflow/evaluation-validation.ts

@@ -0,0 +1,331 @@
+import type {
+  AgentWorkflowArtifactRef,
+  AgentWorkflowEvaluationArmCoordinates,
+  AgentWorkflowEvaluationArmKind,
+  AgentWorkflowEvaluationConditions,
+  AgentWorkflowEvaluationMetrics,
+  AgentWorkflowEvaluationMismatchEvidence,
+  AgentWorkflowEvaluationRecord,
+} from "@southwind-ai/shared";
+import { invalidEvaluationMetricsField } from "./evaluation-metrics-validation.js";
+import { immutableEvaluationClone } from "./evaluation-repository.js";
+
+export interface AgentWorkflowEvaluatedRunCoordinate {
+  readonly runId: string;
+  readonly releaseId: string;
+  readonly releaseDigest: string;
+}
+
+interface RunEvidenceBase {
+  readonly arm: AgentWorkflowEvaluationArmKind;
+  readonly coordinates: AgentWorkflowEvaluationArmCoordinates;
+  readonly evidenceArtifacts: readonly AgentWorkflowArtifactRef[];
+  readonly mismatchEvidence: readonly AgentWorkflowEvaluationMismatchEvidence[];
+}
+
+export type AgentWorkflowEvaluationRunEvidence = RunEvidenceBase &
+  (
+    | {
+        readonly status: "pending";
+        readonly run: AgentWorkflowEvaluatedRunCoordinate;
+      }
+    | {
+        readonly status: "unavailable";
+        readonly reasonCode: string;
+        readonly run?: AgentWorkflowEvaluatedRunCoordinate;
+        readonly metrics?: AgentWorkflowEvaluationMetrics;
+      }
+    | {
+        readonly status: "invalid";
+        readonly reasonCode: string;
+        readonly run?: AgentWorkflowEvaluatedRunCoordinate;
+      }
+    | {
+        readonly status: "terminal";
+        readonly run: AgentWorkflowEvaluatedRunCoordinate;
+        readonly metrics: AgentWorkflowEvaluationMetrics;
+      }
+  );
+
+export interface AgentWorkflowEvaluationRunEvidenceReader {
+  /** Reads both Runs and pinned Releases in one database transaction. */
+  readPair(input: AgentWorkflowEvaluationPairRequest): Promise<{
+    readonly singleAgent: AgentWorkflowEvaluationRunEvidence;
+    readonly multiAgent: AgentWorkflowEvaluationRunEvidence;
+  }>;
+}
+
+export interface AgentWorkflowEvaluationArmRequest {
+  readonly releaseId: string;
+  readonly availability:
+    | { readonly status: "run"; readonly runId: string }
+    | { readonly status: "unavailable" };
+}
+
+export interface AgentWorkflowEvaluationPairRequest {
+  /** Server-owned object authorization context. */
+  readonly tenantId: string;
+  readonly createdBy: string;
+  readonly scopeDigest: string;
+  /** Requested evaluator coordinate; evidence must resolve it from Server registry facts. */
+  readonly rubricKey: string;
+  readonly rubricVersion: string;
+  readonly singleAgent: AgentWorkflowEvaluationArmRequest;
+  readonly multiAgent: AgentWorkflowEvaluationArmRequest;
+}
+
+export interface EvaluateAgentWorkflowPairInput extends AgentWorkflowEvaluationPairRequest {
+  readonly evaluationSetKey: string;
+  readonly evaluationSetVersion: string;
+  readonly caseKey: string;
+  readonly caseVersion: string;
+  readonly trialKey: string;
+  readonly trialIndex: number;
+}
+
+export function normalizeArmCoordinates(
+  coordinates: AgentWorkflowEvaluationArmCoordinates,
+): AgentWorkflowEvaluationArmCoordinates {
+  return immutableEvaluationClone({
+    ...coordinates,
+    conditions: {
+      ...coordinates.conditions,
+      allowedToolKeys: [
+        ...new Set(coordinates.conditions.allowedToolKeys),
+      ].sort(),
+    },
+  });
+}
+
+export function validateEvaluationInput(input: EvaluateAgentWorkflowPairInput) {
+  const strings = [
+    input.tenantId,
+    input.createdBy,
+    input.scopeDigest,
+    input.evaluationSetKey,
+    input.evaluationSetVersion,
+    input.caseKey,
+    input.caseVersion,
+    input.trialKey,
+    input.rubricKey,
+    input.rubricVersion,
+  ];
+  if (
+    strings.some((value) => !value.trim()) ||
+    !nonNegativeInteger(input.trialIndex)
+  ) {
+    invalid("INVALID_INPUT", "trial");
+  }
+  validateArmRequest(input.singleAgent, "singleAgent");
+  validateArmRequest(input.multiAgent, "multiAgent");
+  const singleRun = runId(input.singleAgent);
+  const multiRun = runId(input.multiAgent);
+  if (singleRun && multiRun && singleRun === multiRun)
+    invalid("INVALID_INPUT", "runIds");
+}
+
+export function validateExistingEvaluation(
+  existing: AgentWorkflowEvaluationRecord,
+  expectedEvaluationId: string,
+  expectedIdentityDigest: string,
+) {
+  if (
+    existing.evaluationId !== expectedEvaluationId ||
+    existing.identityDigest !== expectedIdentityDigest
+  ) {
+    invalid("INVALID_INPUT", "frozenTrial");
+  }
+}
+
+export function validateEvaluationEvidence(
+  evidence: AgentWorkflowEvaluationRunEvidence,
+  expected: AgentWorkflowEvaluationArmRequest,
+  expectedArm: AgentWorkflowEvaluationArmKind,
+) {
+  if (evidence.arm !== expectedArm)
+    invalid("INVALID_RUN_EVIDENCE", expectedArm);
+  validateArmCoordinates(
+    evidence.coordinates,
+    expectedArm,
+    evidence.status === "unavailable",
+  );
+  validateMismatches(evidence.mismatchEvidence, expectedArm);
+  if (evidence.status === "unavailable") {
+    if (!evidence.reasonCode.trim())
+      invalid("INVALID_RUN_EVIDENCE", expectedArm);
+    if (evidence.run) {
+      validateRun(
+        evidence.run,
+        evidence.coordinates,
+        evidence.coordinates.availability.status === "run"
+          ? evidence.coordinates.availability.runId
+          : evidence.run.runId,
+      );
+    }
+    if (evidence.metrics) validateMetrics(evidence.metrics, expectedArm);
+    return;
+  }
+  if (expected.availability.status !== "run")
+    invalid("INVALID_RUN_EVIDENCE", expectedArm);
+  if (evidence.run)
+    validateRun(
+      evidence.run,
+      evidence.coordinates,
+      expected.availability.runId,
+    );
+  if (evidence.status === "invalid") {
+    if (!evidence.reasonCode.trim())
+      invalid("INVALID_RUN_EVIDENCE", expectedArm);
+    return;
+  }
+  validateRun(evidence.run, evidence.coordinates, expected.availability.runId);
+  if (evidence.status === "terminal")
+    validateMetrics(evidence.metrics, expectedArm);
+}
+
+function validateArmRequest(
+  value: AgentWorkflowEvaluationArmRequest,
+  field: string,
+) {
+  if (!value || typeof value !== "object" || !text(value.releaseId)) {
+    invalid("INVALID_INPUT", `${field}.releaseId`);
+  }
+  if (!value.availability || typeof value.availability !== "object") {
+    invalid("INVALID_INPUT", `${field}.availability`);
+  }
+  if (value.availability.status === "run") {
+    if (!text(value.availability.runId)) {
+      invalid("INVALID_INPUT", `${field}.availability`);
+    }
+    return;
+  }
+  if (value.availability.status !== "unavailable") {
+    invalid("INVALID_INPUT", `${field}.availability`);
+  }
+}
+
+function validateArmCoordinates(
+  value: AgentWorkflowEvaluationArmCoordinates,
+  field: string,
+  allowUnknown: boolean,
+) {
+  if (!value.releaseId.trim()) invalid("INVALID_INPUT", `${field}.releaseId`);
+  if (
+    Object.values(value.release).length !== 4 ||
+    Object.values(value.release).some((item) => !item.trim())
+  ) {
+    invalid("INVALID_INPUT", `${field}.release`);
+  }
+  validateConditions(
+    value.conditions,
+    field,
+    allowUnknown || value.availability.status === "unavailable",
+  );
+  if (value.availability.status === "run") {
+    if (!value.availability.runId.trim()) {
+      invalid("INVALID_INPUT", `${field}.availability`);
+    }
+    return;
+  }
+  if (
+    value.availability.status !== "unavailable" ||
+    !value.availability.reasonCode.trim()
+  ) {
+    invalid("INVALID_INPUT", `${field}.availability`);
+  }
+}
+
+function validateConditions(
+  value: AgentWorkflowEvaluationConditions,
+  field: string,
+  allowUnknown: boolean,
+) {
+  const strings = Object.entries(value).filter(
+    ([key]) => key !== "allowedToolKeys" && key !== "budget",
+  );
+  if (
+    Object.keys(value).length !== 14 ||
+    strings.some(([, item]) =>
+      item === null ? !allowUnknown : typeof item !== "string" || !item.trim(),
+    ) ||
+    Object.values(value.budget).length !== 8 ||
+    Object.values(value.budget).some((item) => !nonNegativeInteger(item)) ||
+    value.allowedToolKeys.some((key) => !key.trim())
+  ) {
+    invalid("INVALID_INPUT", `${field}.conditions`);
+  }
+}
+
+function validateRun(
+  run: AgentWorkflowEvaluatedRunCoordinate,
+  coordinates: AgentWorkflowEvaluationArmCoordinates,
+  expectedRunId: string,
+) {
+  if (
+    run.runId !== expectedRunId ||
+    run.releaseId !== coordinates.releaseId ||
+    run.releaseDigest !== coordinates.release.releaseDigest
+  ) {
+    invalid("INVALID_RUN_EVIDENCE", expectedRunId);
+  }
+}
+
+function validateMismatches(
+  values: readonly AgentWorkflowEvaluationMismatchEvidence[],
+  field: string,
+) {
+  if (
+    values.some(
+      (value) =>
+        !value.field.trim() ||
+        !value.reasonCode.trim() ||
+        !value.expectedDigest.trim() ||
+        (value.observedDigest !== undefined && !value.observedDigest.trim()),
+    )
+  ) {
+    invalid("INVALID_RUN_EVIDENCE", `${field}.mismatchEvidence`);
+  }
+}
+
+function validateMetrics(
+  metrics: AgentWorkflowEvaluationMetrics,
+  field: string,
+) {
+  const invalidField = invalidEvaluationMetricsField(metrics, field);
+  if (invalidField) invalid("INVALID_RUN_EVIDENCE", invalidField);
+}
+
+function text(value: unknown): value is string {
+  return typeof value === "string" && Boolean(value.trim());
+}
+
+function runId(value: AgentWorkflowEvaluationArmRequest) {
+  return value.availability.status === "run"
+    ? value.availability.runId
+    : undefined;
+}
+
+function nonNegativeInteger(value: number) {
+  return Number.isSafeInteger(value) && value >= 0;
+}
+
+export type AgentWorkflowEvaluationServiceErrorCode =
+  | "INVALID_INPUT"
+  | "INVALID_RUN_EVIDENCE";
+
+export class AgentWorkflowEvaluationServiceError extends Error {
+  constructor(
+    readonly code: AgentWorkflowEvaluationServiceErrorCode,
+    readonly field: string,
+  ) {
+    super(`${code}:${field}`);
+    this.name = "AgentWorkflowEvaluationServiceError";
+  }
+}
+
+function invalid(
+  code: AgentWorkflowEvaluationServiceErrorCode,
+  field: string,
+): never {
+  throw new AgentWorkflowEvaluationServiceError(code, field);
+}

package/template/apps/server/src/agent-workflow/event-validation.ts

@@ -0,0 +1,277 @@
+import type { AgentWorkflowEventEnvelope } from "@southwind-ai/shared";
+import { AGENT_WORKFLOW_EVENT_SCHEMA_VERSION } from "@southwind-ai/shared";
+import type { WorkflowEffectCommandReceipt } from "./effect-command-fence.js";
+import {
+  assertCanonicalWorkflowSafeValue,
+  WorkflowEventValidationError,
+} from "./workflow-value-validation.js";
+
+export {
+  assertCanonicalWorkflowSafeValue,
+  WorkflowEventValidationError,
+} from "./workflow-value-validation.js";
+
+const MAX_EVENT_BYTES = 64 * 1024;
+const RUN_TERMINAL = new Set([
+  "workflow.run-completed",
+  "workflow.run-failed",
+  "workflow.run-cancelled",
+  "workflow.run-timed-out",
+]);
+const DELEGATION_TYPES = new Set([
+  "workflow.delegation-created",
+  "workflow.delegation-child-linked",
+  "workflow.delegation-completed",
+  "workflow.delegation-failed",
+  "workflow.delegation-cancel-requested",
+]);
+
+export interface WorkflowEventLookupPort {
+  findCanonicalEvent(
+    eventId: string,
+  ): Promise<AgentWorkflowEventEnvelope | undefined>;
+}
+
+export function assertCanonicalWorkflowEvent(
+  event: AgentWorkflowEventEnvelope,
+): void {
+  if (
+    event.schemaVersion !== AGENT_WORKFLOW_EVENT_SCHEMA_VERSION ||
+    !coordinate(event.eventId) ||
+    !coordinate(event.workflowRunId) ||
+    !Number.isSafeInteger(event.sequence) ||
+    event.sequence < 1 ||
+    !Number.isFinite(Date.parse(event.occurredAt)) ||
+    (event.causationEventId !== undefined &&
+      !coordinate(event.causationEventId)) ||
+    (event.parentEventId !== undefined && !coordinate(event.parentEventId)) ||
+    (event.childWorkflowRunId !== undefined &&
+      !coordinate(event.childWorkflowRunId))
+  ) {
+    throw invalid("Canonical Event envelope 坐标无效");
+  }
+  if (
+    new TextEncoder().encode(JSON.stringify(event)).byteLength > MAX_EVENT_BYTES
+  ) {
+    throw invalid("Canonical Event 超过 64 KiB 上限");
+  }
+  assertCanonicalWorkflowSafeValue(event.data, "data", MAX_EVENT_BYTES);
+  assertParentChildShape(event);
+}
+
+export function assertCanonicalWorkflowEventHistory(
+  events: readonly AgentWorkflowEventEnvelope[],
+): void {
+  if (!events.length) return;
+  const runId = events[0]!.workflowRunId;
+  const ids = new Set<string>();
+  const byId = new Map<string, AgentWorkflowEventEnvelope>();
+  let terminal = false;
+  const delegations = new Map<
+    string,
+    {
+      parentNodeId: string;
+      parentNodeRunId: string;
+      childSessionId: string;
+      childAgentRunId: string | undefined;
+      linked: boolean;
+      terminal: boolean;
+    }
+  >();
+  for (const [index, event] of events.entries()) {
+    assertCanonicalWorkflowEvent(event);
+    if (
+      event.workflowRunId !== runId ||
+      event.sequence !== index + 1 ||
+      ids.has(event.eventId) ||
+      terminal
+    ) {
+      throw invalid("Canonical Event history 非连续、重复或在终态后追加");
+    }
+    if (index === 0 && event.data.type !== "workflow.run-created") {
+      throw invalid("Canonical Event history 必须从 Run Created 开始");
+    }
+    if (event.causationEventId) {
+      const cause = byId.get(event.causationEventId);
+      if (!cause || cause.sequence >= event.sequence) {
+        throw invalid("Causation Event 必须是同 Run 的既有事件");
+      }
+    }
+    assertDelegationLifecycle(event, delegations);
+    terminal = RUN_TERMINAL.has(event.data.type);
+    ids.add(event.eventId);
+    byId.set(event.eventId, event);
+  }
+}
+
+export async function assertParentChildEventCausality(input: {
+  readonly event: AgentWorkflowEventEnvelope;
+  readonly lookup: WorkflowEventLookupPort;
+}): Promise<void> {
+  const { event } = input;
+  assertCanonicalWorkflowEvent(event);
+  if (!event.parentEventId) return;
+  const parent = await input.lookup.findCanonicalEvent(event.parentEventId);
+  if (
+    !parent ||
+    parent.eventId === event.eventId ||
+    parent.workflowRunId === event.workflowRunId ||
+    parent.childWorkflowRunId !== event.workflowRunId ||
+    parent.data.type !== "workflow.delegation-child-linked" ||
+    event.data.type !== "workflow.run-created" ||
+    parent.workflowRunId !== event.data.parentRunId ||
+    parent.data.delegationId !== event.data.delegationId ||
+    parent.data.parentNodeId !== event.data.parentNodeId ||
+    parent.data.parentNodeRunId !== event.data.parentNodeRunId
+  ) {
+    throw invalid("Parent/Child Event 因果坐标无效");
+  }
+  assertCanonicalWorkflowEvent(parent);
+  if (
+    DELEGATION_TYPES.has(parent.data.type) &&
+    "childAgentRunId" in parent.data &&
+    parent.data.childAgentRunId !== undefined &&
+    !coordinate(parent.data.childAgentRunId)
+  ) {
+    throw invalid("Parent Event 的 Child Run 坐标不一致");
+  }
+}
+
+export function assertEffectReceiptEvent(
+  event: AgentWorkflowEventEnvelope,
+  receipt: WorkflowEffectCommandReceipt,
+): void {
+  assertCanonicalWorkflowEvent(event);
+  if (
+    event.workflowRunId !== receipt.runId ||
+    event.data.type !== "workflow.node-completed" ||
+    event.data.nodeRunId !== receipt.nodeRunId ||
+    event.data.receiptId !== receipt.receiptId ||
+    event.data.resultDigest !== receipt.resultDigest
+  ) {
+    throw invalid("Node Event 未引用匹配的 Effect Receipt");
+  }
+}
+
+function assertParentChildShape(event: AgentWorkflowEventEnvelope) {
+  const lineage =
+    event.data.type === "workflow.run-created"
+      ? [
+          event.data.parentRunId,
+          event.data.parentNodeId,
+          event.data.parentNodeRunId,
+          event.data.delegationId,
+          event.parentEventId,
+        ]
+      : [event.parentEventId];
+  const present = lineage.filter((value) => value !== undefined).length;
+  if (
+    (present !== 0 && present !== lineage.length) ||
+    lineage.some((value) => value !== undefined && !coordinate(value))
+  ) {
+    throw invalid("Parent/Child Event 因果坐标必须全有或全无");
+  }
+  if (
+    event.parentEventId &&
+    (event.sequence !== 1 || event.data.type !== "workflow.run-created")
+  ) {
+    throw invalid("只有 Child Run Created 可引用 Parent Event");
+  }
+  if (
+    event.data.type === "workflow.delegation-created" &&
+    (event.data.childAgentRunId !== undefined ||
+      event.childWorkflowRunId !== undefined)
+  ) {
+    throw invalid("Delegation Created 不能提前声明 Child Run");
+  }
+  if (
+    event.data.type === "workflow.delegation-child-linked" &&
+    !event.data.childAgentRunId
+  ) {
+    throw invalid("Delegation Child Linked 缺少 Child Agent Run 坐标");
+  }
+  if (event.childWorkflowRunId) {
+    if (
+      event.data.type !== "workflow.delegation-child-linked" ||
+      event.childWorkflowRunId === event.workflowRunId ||
+      ("childAgentRunId" in event.data &&
+        event.data.childAgentRunId !== undefined &&
+        !coordinate(event.data.childAgentRunId))
+    ) {
+      throw invalid("Parent Event 的 Child Run 坐标无效");
+    }
+  }
+}
+
+function assertDelegationLifecycle(
+  event: AgentWorkflowEventEnvelope,
+  states: Map<
+    string,
+    {
+      parentNodeId: string;
+      parentNodeRunId: string;
+      childSessionId: string;
+      childAgentRunId: string | undefined;
+      linked: boolean;
+      terminal: boolean;
+    }
+  >,
+) {
+  if (!isDelegationEvent(event.data)) return;
+  const data = event.data;
+  const state = states.get(data.delegationId);
+  if (data.type === "workflow.delegation-created") {
+    if (state) throw invalid("Delegation Created 重复");
+    states.set(data.delegationId, {
+      parentNodeId: data.parentNodeId,
+      parentNodeRunId: data.parentNodeRunId,
+      childSessionId: data.childSessionId,
+      childAgentRunId: data.childAgentRunId,
+      linked: Boolean(data.childAgentRunId),
+      terminal: false,
+    });
+    return;
+  }
+  if (
+    !state ||
+    state.terminal ||
+    state.childSessionId !== data.childSessionId ||
+    state.parentNodeId !== data.parentNodeId ||
+    state.parentNodeRunId !== data.parentNodeRunId
+  ) {
+    throw invalid("Delegation Event 顺序或 Child Session 坐标无效");
+  }
+  if (data.type === "workflow.delegation-child-linked") {
+    if (
+      !data.childAgentRunId ||
+      (state.childAgentRunId && state.childAgentRunId !== data.childAgentRunId)
+    ) {
+      throw invalid("Delegation Child Link 坐标无效");
+    }
+    state.childAgentRunId = data.childAgentRunId;
+    state.linked = true;
+  } else if (
+    data.type === "workflow.delegation-completed" ||
+    data.type === "workflow.delegation-failed"
+  ) {
+    if (!state.linked) throw invalid("Delegation 未关联 Child Run 就进入终态");
+    state.terminal = true;
+  }
+}
+
+function isDelegationEvent(
+  data: AgentWorkflowEventEnvelope["data"],
+): data is Extract<
+  AgentWorkflowEventEnvelope["data"],
+  { readonly delegationId: string }
+> {
+  return DELEGATION_TYPES.has(data.type) && "delegationId" in data;
+}
+
+function coordinate(value: string) {
+  return value.trim().length > 0 && value.length <= 256;
+}
+
+function invalid(message: string) {
+  return new WorkflowEventValidationError(message);
+}

package/template/apps/server/src/agent-workflow/governed-tool-schema.ts

@@ -0,0 +1,55 @@
+import type { AgentWorkflowDefinition } from "@southwind-ai/shared";
+import type {
+  GovernedToolRegistry,
+  RegisteredTool,
+} from "../agent/tool-registry.js";
+import { canonicalEvaluationDigest } from "./evaluation-repository.js";
+
+export function governedDefinitionToolSchemaVersion(
+  definition: AgentWorkflowDefinition,
+  tools: GovernedToolRegistry,
+) {
+  return `sha256:${canonicalEvaluationDigest(
+    definitionToolKeys(definition).map((key) => {
+      const registered = tools.definition(key);
+      return registered
+        ? registeredToolFact(registered)
+        : { key, status: "registry-definition-missing" };
+    }),
+  )}`;
+}
+
+export function definitionToolKeys(definition: AgentWorkflowDefinition) {
+  return [
+    ...new Set(
+      definition.nodes.flatMap((node) => {
+        if (node.type === "tool") return [node.toolKey];
+        if (node.type === "subagent") return node.allowedToolKeys;
+        return [];
+      }),
+    ),
+  ].sort();
+}
+
+function registeredToolFact(tool: RegisteredTool) {
+  return {
+    key: tool.key,
+    module: tool.module,
+    description: tool.description,
+    operation: tool.operation,
+    featureKey: tool.featureKey,
+    permissionKey: tool.permissionKey,
+    auditAction: tool.auditAction,
+    scopeMode: tool.scopeMode,
+    dataClassification: tool.dataClassification,
+    idempotency: tool.idempotency,
+    approval: tool.approval,
+    approvalRoles: tool.approvalRoles,
+    approvalTimeoutSeconds: tool.approvalTimeoutSeconds,
+    inputSchemaKey: tool.inputSchemaKey,
+    inputSchema: tool.inputSchema,
+    outputSchemaKey: tool.outputSchemaKey,
+    outputSchema: tool.outputSchema,
+    limits: tool.limits,
+  };
+}