create-windy 0.6.11 → 0.6.14

package/template/apps/server/src/agent-workflow/evaluation-projection.ts

@@ -0,0 +1,70 @@
+import type {
+  AgentWorkflowEvaluationRecord,
+  AgentWorkflowEvaluationSummary,
+} from "@southwind-ai/shared";
+
+export function projectAgentWorkflowEvaluationSummary(
+  records: readonly AgentWorkflowEvaluationRecord[],
+  evaluationSetKey: string,
+  evaluationSetVersion: string,
+): AgentWorkflowEvaluationSummary {
+  const matching = records.filter(
+    ({ identity }) =>
+      identity.evaluationSetKey === evaluationSetKey &&
+      identity.evaluationSetVersion === evaluationSetVersion,
+  );
+  const comparable = matching.filter(
+    (
+      record,
+    ): record is AgentWorkflowEvaluationRecord & {
+      readonly status: "completed";
+      readonly singleAgent: AgentWorkflowEvaluationRecord["singleAgent"] & {
+        readonly metrics: NonNullable<
+          AgentWorkflowEvaluationRecord["singleAgent"]["metrics"]
+        >;
+      };
+      readonly multiAgent: AgentWorkflowEvaluationRecord["multiAgent"] & {
+        readonly metrics: NonNullable<
+          AgentWorkflowEvaluationRecord["multiAgent"]["metrics"]
+        >;
+      };
+    } =>
+      record.status === "completed" &&
+      record.singleAgent.metrics !== undefined &&
+      record.multiAgent.metrics !== undefined,
+  );
+  const denominator = comparable.length;
+  return {
+    evaluationSetKey,
+    evaluationSetVersion,
+    comparableSampleCount: denominator,
+    totalSampleCount: matching.length,
+    ...(denominator
+      ? {
+          singleAgentCompletionRate:
+            completedCount(comparable, "singleAgent") / denominator,
+          multiAgentCompletionRate:
+            completedCount(comparable, "multiAgent") / denominator,
+        }
+      : {}),
+  };
+}
+
+function completedCount(
+  records: readonly (AgentWorkflowEvaluationRecord & {
+    readonly singleAgent: AgentWorkflowEvaluationRecord["singleAgent"] & {
+      readonly metrics: NonNullable<
+        AgentWorkflowEvaluationRecord["singleAgent"]["metrics"]
+      >;
+    };
+    readonly multiAgent: AgentWorkflowEvaluationRecord["multiAgent"] & {
+      readonly metrics: NonNullable<
+        AgentWorkflowEvaluationRecord["multiAgent"]["metrics"]
+      >;
+    };
+  })[],
+  arm: "singleAgent" | "multiAgent",
+) {
+  return records.filter((record) => record[arm].metrics.outcome === "completed")
+    .length;
+}

package/template/apps/server/src/agent-workflow/evaluation-repository.ts

@@ -0,0 +1,134 @@
+import { createHash } from "node:crypto";
+import type { AgentWorkflowEvaluationRecord } from "@southwind-ai/shared";
+import { hasCompleteEvaluationEvidence } from "./evaluation-metrics-validation.js";
+
+export interface AgentWorkflowEvaluationTrialCoordinate {
+  readonly tenantId: string;
+  readonly createdBy: string;
+  readonly evaluationSetKey: string;
+  readonly evaluationSetVersion: string;
+  readonly caseKey: string;
+  readonly caseVersion: string;
+  readonly trialKey: string;
+  readonly trialIndex: number;
+  readonly identityDigest: string;
+}
+
+export interface AgentWorkflowEvaluationRepository {
+  getById(
+    evaluationId: string,
+    owner: AgentWorkflowEvaluationOwner,
+  ): Promise<AgentWorkflowEvaluationRecord | undefined>;
+  findByTrial(
+    coordinate: AgentWorkflowEvaluationTrialCoordinate,
+  ): Promise<AgentWorkflowEvaluationRecord | undefined>;
+  listBySet(
+    evaluationSetKey: string,
+    evaluationSetVersion: string,
+    owner: AgentWorkflowEvaluationOwner,
+  ): Promise<readonly AgentWorkflowEvaluationRecord[]>;
+  appendTerminal(
+    record: AgentWorkflowEvaluationRecord,
+  ): Promise<AgentWorkflowEvaluationRecord>;
+}
+
+export interface AgentWorkflowEvaluationOwner {
+  readonly tenantId: string;
+  readonly createdBy: string;
+}
+
+export type AgentWorkflowEvaluationRepositoryConflictCode =
+  | "EVALUATION_ID_CONFLICT"
+  | "TRIAL_CONFLICT"
+  | "PENDING_NOT_PERSISTABLE"
+  | "RECORD_INTEGRITY_CONFLICT";
+
+export class AgentWorkflowEvaluationRepositoryConflict extends Error {
+  constructor(readonly code: AgentWorkflowEvaluationRepositoryConflictCode) {
+    super(code);
+    this.name = "AgentWorkflowEvaluationRepositoryConflict";
+  }
+}
+
+export function evaluationTrialCoordinate(
+  record: AgentWorkflowEvaluationRecord,
+): AgentWorkflowEvaluationTrialCoordinate {
+  return {
+    tenantId: record.tenantId,
+    createdBy: record.createdBy,
+    evaluationSetKey: record.identity.evaluationSetKey,
+    evaluationSetVersion: record.identity.evaluationSetVersion,
+    caseKey: record.identity.caseKey,
+    caseVersion: record.identity.caseVersion,
+    trialKey: record.identity.trialKey,
+    trialIndex: record.identity.trialIndex,
+    identityDigest: record.identityDigest,
+  };
+}
+
+export function canonicalEvaluationJson(value: unknown): string {
+  return JSON.stringify(canonicalValue(value));
+}
+
+export function canonicalEvaluationDigest(value: unknown): string {
+  return createHash("sha256")
+    .update(canonicalEvaluationJson(value))
+    .digest("hex");
+}
+
+export function assertEvaluationRecordIntegrity(
+  record: AgentWorkflowEvaluationRecord,
+) {
+  if (
+    !record.tenantId.trim() ||
+    !record.createdBy.trim() ||
+    (record.status === "completed" && !hasCompleteArmEvidence(record)) ||
+    record.identityDigest !== canonicalEvaluationDigest(record.identity) ||
+    record.singleAgent.kind !== "single-agent" ||
+    record.multiAgent.kind !== "multi-agent" ||
+    canonicalEvaluationJson(record.singleAgent.coordinates) !==
+      canonicalEvaluationJson(record.identity.singleAgent) ||
+    canonicalEvaluationJson(record.multiAgent.coordinates) !==
+      canonicalEvaluationJson(record.identity.multiAgent)
+  ) {
+    throw new AgentWorkflowEvaluationRepositoryConflict(
+      "RECORD_INTEGRITY_CONFLICT",
+    );
+  }
+}
+
+function hasCompleteArmEvidence(record: AgentWorkflowEvaluationRecord) {
+  return (
+    record.singleAgent.status === "completed" &&
+    record.multiAgent.status === "completed" &&
+    record.singleAgent.metrics !== undefined &&
+    record.multiAgent.metrics !== undefined &&
+    hasCompleteEvaluationEvidence(record.singleAgent.metrics) &&
+    hasCompleteEvaluationEvidence(record.multiAgent.metrics)
+  );
+}
+
+export function immutableEvaluationClone<T>(value: T): T {
+  return deepFreeze(structuredClone(value));
+}
+
+function canonicalValue(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(canonicalValue);
+  if (value !== null && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([key, item]) => [key, canonicalValue(item)]),
+    );
+  }
+  return value;
+}
+
+function deepFreeze<T>(value: T): T {
+  if (!value || typeof value !== "object" || Object.isFrozen(value))
+    return value;
+  for (const child of Object.values(value as Record<string, unknown>)) {
+    deepFreeze(child);
+  }
+  return Object.freeze(value);
+}

package/template/apps/server/src/agent-workflow/evaluation-service.ts

@@ -0,0 +1,292 @@
+import type {
+  AgentWorkflowEvaluationArm,
+  AgentWorkflowEvaluationArmKind,
+  AgentWorkflowEvaluationConditions,
+  AgentWorkflowEvaluationIdentity,
+  AgentWorkflowEvaluationMismatchEvidence,
+  AgentWorkflowEvaluationRecord,
+} from "@southwind-ai/shared";
+import {
+  canonicalEvaluationJson,
+  canonicalEvaluationDigest,
+  immutableEvaluationClone,
+  type AgentWorkflowEvaluationRepository,
+  type AgentWorkflowEvaluationTrialCoordinate,
+} from "./evaluation-repository.js";
+import {
+  validateEvaluationEvidence,
+  validateEvaluationInput,
+  validateExistingEvaluation,
+  type AgentWorkflowEvaluationRunEvidence,
+  type AgentWorkflowEvaluationRunEvidenceReader,
+  type EvaluateAgentWorkflowPairInput,
+} from "./evaluation-validation.js";
+
+export type {
+  AgentWorkflowEvaluationServiceErrorCode,
+  AgentWorkflowEvaluationRunEvidence,
+  AgentWorkflowEvaluationRunEvidenceReader,
+  EvaluateAgentWorkflowPairInput,
+} from "./evaluation-validation.js";
+export { AgentWorkflowEvaluationServiceError } from "./evaluation-validation.js";
+
+export class AgentWorkflowEvaluationService {
+  constructor(
+    private readonly repository: AgentWorkflowEvaluationRepository,
+    private readonly evidence: AgentWorkflowEvaluationRunEvidenceReader,
+    private readonly now: () => Date = () => new Date(),
+  ) {}
+
+  async evaluatePair(
+    input: EvaluateAgentWorkflowPairInput,
+  ): Promise<AgentWorkflowEvaluationRecord> {
+    validateEvaluationInput(input);
+    const request = {
+      tenantId: input.tenantId,
+      createdBy: input.createdBy,
+      scopeDigest: input.scopeDigest,
+      rubricKey: input.rubricKey,
+      rubricVersion: input.rubricVersion,
+      singleAgent: input.singleAgent,
+      multiAgent: input.multiAgent,
+    };
+    const evidence = await this.evidence.readPair(request);
+    validateEvaluationEvidence(
+      evidence.singleAgent,
+      request.singleAgent,
+      "single-agent",
+    );
+    validateEvaluationEvidence(
+      evidence.multiAgent,
+      request.multiAgent,
+      "multi-agent",
+    );
+
+    const identity = immutableEvaluationClone({
+      ...trialCoordinate(input),
+      scopeDigest: input.scopeDigest,
+      singleAgent: evidence.singleAgent.coordinates,
+      multiAgent: evidence.multiAgent.coordinates,
+      rubricKey: input.rubricKey,
+      rubricVersion: input.rubricVersion,
+    } satisfies AgentWorkflowEvaluationIdentity);
+    const identityDigest = digest(identity);
+    const evaluationId = pairEvaluationId(
+      input.tenantId,
+      input.createdBy,
+      identityDigest,
+    );
+    const existing = await this.repository.findByTrial({
+      ...trialCoordinate(input),
+      identityDigest,
+    });
+    if (existing) {
+      validateExistingEvaluation(existing, evaluationId, identityDigest);
+      return existing;
+    }
+
+    const pairEvidence =
+      evidence.singleAgent.coordinates.availability.status === "run" &&
+      evidence.multiAgent.coordinates.availability.status === "run"
+        ? compareConditions(
+            evidence.singleAgent.coordinates.conditions,
+            evidence.multiAgent.coordinates.conditions,
+          )
+        : { singleAgent: [], multiAgent: [] };
+    const createdAt = this.now().toISOString();
+    const base = {
+      evaluationId,
+      tenantId: input.tenantId,
+      createdBy: input.createdBy,
+      identity,
+      identityDigest,
+      singleAgent: arm(evidence.singleAgent, pairEvidence.singleAgent),
+      multiAgent: arm(evidence.multiAgent, pairEvidence.multiAgent),
+      createdAt,
+    } satisfies Omit<AgentWorkflowEvaluationRecord, "status">;
+
+    if (
+      evidence.singleAgent.status === "invalid" ||
+      evidence.multiAgent.status === "invalid" ||
+      base.singleAgent.mismatchEvidence.length > 0 ||
+      base.multiAgent.mismatchEvidence.length > 0
+    ) {
+      return this.persist({
+        ...base,
+        status: "non-comparable",
+        dispositionReason: "FROZEN_COORDINATE_MISMATCH",
+        completedAt: createdAt,
+      });
+    }
+    if (
+      evidence.singleAgent.status === "unavailable" ||
+      evidence.multiAgent.status === "unavailable"
+    ) {
+      return this.persist({
+        ...base,
+        status: "unavailable",
+        dispositionReason: dispositionReason(
+          evidence.singleAgent,
+          evidence.multiAgent,
+        ),
+        completedAt: createdAt,
+      });
+    }
+    if (
+      evidence.singleAgent.status === "pending" ||
+      evidence.multiAgent.status === "pending"
+    ) {
+      return immutableEvaluationClone({ ...base, status: "pending" });
+    }
+    return this.persist({
+      ...base,
+      status: "completed",
+      completedAt: createdAt,
+    });
+  }
+
+  private persist(record: AgentWorkflowEvaluationRecord) {
+    return this.repository.appendTerminal(immutableEvaluationClone(record));
+  }
+}
+
+function arm(
+  evidence: AgentWorkflowEvaluationRunEvidence,
+  pairEvidence: readonly AgentWorkflowEvaluationMismatchEvidence[],
+): AgentWorkflowEvaluationArm {
+  const mismatchEvidence = immutableEvaluationClone([
+    ...evidence.mismatchEvidence,
+    ...pairEvidence,
+  ]);
+  if (evidence.status === "unavailable") {
+    return {
+      kind: evidence.arm,
+      status: "unavailable",
+      coordinates: evidence.coordinates,
+      ...(evidence.run ? { runId: evidence.run.runId } : {}),
+      ...(evidence.metrics ? { metrics: evidence.metrics } : {}),
+      evidenceArtifacts: immutableEvaluationClone(evidence.evidenceArtifacts),
+      mismatchEvidence,
+      dispositionReason: evidence.reasonCode,
+    };
+  }
+  return {
+    kind: evidence.arm,
+    status: evidence.status === "terminal" ? "completed" : evidence.status,
+    coordinates: evidence.coordinates,
+    ...(evidence.run ? { runId: evidence.run.runId } : {}),
+    ...(evidence.status === "terminal" ? { metrics: evidence.metrics } : {}),
+    evidenceArtifacts: immutableEvaluationClone(evidence.evidenceArtifacts),
+    mismatchEvidence,
+    ...(evidence.status === "invalid"
+      ? { dispositionReason: evidence.reasonCode }
+      : {}),
+  };
+}
+
+function compareConditions(
+  single: AgentWorkflowEvaluationConditions,
+  multi: AgentWorkflowEvaluationConditions,
+) {
+  const singleAgent: AgentWorkflowEvaluationMismatchEvidence[] = [];
+  const multiAgent: AgentWorkflowEvaluationMismatchEvidence[] = [];
+  for (const [field, singleValue, multiValue] of comparableFields(
+    single,
+    multi,
+  )) {
+    if (
+      canonicalEvaluationJson(singleValue) ===
+      canonicalEvaluationJson(multiValue)
+    )
+      continue;
+    singleAgent.push(
+      pairMismatch(field, singleValue, multiValue, "multi-agent"),
+    );
+    multiAgent.push(
+      pairMismatch(field, multiValue, singleValue, "single-agent"),
+    );
+  }
+  return { singleAgent, multiAgent };
+}
+
+function comparableFields(
+  single: AgentWorkflowEvaluationConditions,
+  multi: AgentWorkflowEvaluationConditions,
+): readonly [string, unknown, unknown][] {
+  const fields: [string, unknown, unknown][] = Object.keys(single)
+    .filter((key) => key !== "budget")
+    .sort()
+    .map((key) => [
+      `conditions.${key}`,
+      single[key as keyof AgentWorkflowEvaluationConditions],
+      multi[key as keyof AgentWorkflowEvaluationConditions],
+    ]);
+  for (const key of Object.keys(single.budget).sort()) {
+    fields.push([
+      `conditions.budget.${key}`,
+      single.budget[key as keyof typeof single.budget],
+      multi.budget[key as keyof typeof multi.budget],
+    ]);
+  }
+  return fields;
+}
+
+function pairMismatch(
+  field: string,
+  expected: unknown,
+  observed: unknown,
+  counterpartArm: AgentWorkflowEvaluationArmKind,
+): AgentWorkflowEvaluationMismatchEvidence {
+  return {
+    source: "arm-comparison",
+    field,
+    reasonCode: "ARM_CONDITION_MISMATCH",
+    expectedDigest: digest(expected),
+    observedDigest: digest(observed),
+    counterpartArm,
+  };
+}
+
+function trialCoordinate(
+  input: EvaluateAgentWorkflowPairInput,
+): Omit<AgentWorkflowEvaluationTrialCoordinate, "identityDigest"> {
+  return {
+    tenantId: input.tenantId,
+    createdBy: input.createdBy,
+    evaluationSetKey: input.evaluationSetKey,
+    evaluationSetVersion: input.evaluationSetVersion,
+    caseKey: input.caseKey,
+    caseVersion: input.caseVersion,
+    trialKey: input.trialKey,
+    trialIndex: input.trialIndex,
+  };
+}
+
+function dispositionReason(
+  ...evidence: readonly AgentWorkflowEvaluationRunEvidence[]
+) {
+  return evidence
+    .filter(
+      (
+        item,
+      ): item is Extract<
+        AgentWorkflowEvaluationRunEvidence,
+        { readonly status: "unavailable" | "invalid" }
+      > => item.status === "unavailable" || item.status === "invalid",
+    )
+    .map(({ arm: kind, reasonCode }) => `${kind}:${reasonCode}`)
+    .sort()
+    .join(",");
+}
+
+function pairEvaluationId(
+  tenantId: string,
+  createdBy: string,
+  identityDigest: string,
+) {
+  return `workflow_eval_${digest({ tenantId, createdBy, identityDigest }).slice(0, 40)}`;
+}
+
+function digest(value: unknown) {
+  return canonicalEvaluationDigest(value);
+}

package/template/apps/server/src/agent-workflow/evaluation-tool-registration.ts

@@ -0,0 +1,95 @@
+import type {
+  GovernedToolRegistry,
+  RegisteredTool,
+} from "../agent/tool-registry.js";
+
+export interface AgentWorkflowEvaluationToolRegistration {
+  readonly rubricToolKey: string;
+  readonly safetyToolKey: string;
+}
+
+export interface ResolvedAgentWorkflowEvaluationTools {
+  readonly rubric: RegisteredTool;
+  readonly safety: RegisteredTool;
+}
+
+/** Stable governed Tool roles that project-owned Modules may implement. */
+export const productionAgentWorkflowEvaluationTools = Object.freeze({
+  rubricToolKey: "agent.workflow.evaluation.rubric",
+  safetyToolKey: "agent.workflow.evaluation.safety",
+}) satisfies AgentWorkflowEvaluationToolRegistration;
+
+export function resolveProductionEvaluationToolRegistration(
+  tools: GovernedToolRegistry,
+): AgentWorkflowEvaluationToolRegistration | undefined {
+  return resolveEvaluationTools(productionAgentWorkflowEvaluationTools, tools)
+    ? productionAgentWorkflowEvaluationTools
+    : undefined;
+}
+
+export function freezeEvaluationToolRegistration(
+  value: AgentWorkflowEvaluationToolRegistration | undefined,
+) {
+  if (value === undefined) return undefined;
+  const rubricToolKey = coordinate(value.rubricToolKey);
+  const safetyToolKey = coordinate(value.safetyToolKey);
+  if (!rubricToolKey || !safetyToolKey || rubricToolKey === safetyToolKey) {
+    throw new Error("INVALID_EVALUATION_TOOL_REGISTRATION");
+  }
+  return Object.freeze({ rubricToolKey, safetyToolKey });
+}
+
+export function resolveEvaluationTools(
+  registration: AgentWorkflowEvaluationToolRegistration | undefined,
+  tools: GovernedToolRegistry,
+): ResolvedAgentWorkflowEvaluationTools | undefined {
+  if (!registration) return undefined;
+  const rubric = tools.definition(registration.rubricToolKey);
+  const safety = tools.definition(registration.safetyToolKey);
+  if (!rubric || !safety || !evidenceTool(rubric) || !evidenceTool(safety)) {
+    return undefined;
+  }
+  return { rubric, safety };
+}
+
+export function evaluationToolRegistryFacts(
+  registration: AgentWorkflowEvaluationToolRegistration | undefined,
+  tools: GovernedToolRegistry,
+) {
+  if (!registration) return { status: "unconfigured" } as const;
+  return {
+    status: "configured",
+    rubric: registryFact(registration.rubricToolKey, tools),
+    safety: registryFact(registration.safetyToolKey, tools),
+  } as const;
+}
+
+function evidenceTool(tool: RegisteredTool) {
+  return (
+    tool.operation !== "query" &&
+    tool.idempotency !== "none" &&
+    tool.approval === "required"
+  );
+}
+
+function registryFact(key: string, tools: GovernedToolRegistry) {
+  const tool = tools.definition(key);
+  return tool
+    ? {
+        key: tool.key,
+        module: tool.module,
+        operation: tool.operation,
+        outputSchemaKey: tool.outputSchemaKey,
+        auditAction: tool.auditAction,
+        idempotency: tool.idempotency,
+        approval: tool.approval,
+      }
+    : { key, status: "registry-definition-missing" };
+}
+
+function coordinate(value: string) {
+  const normalized = value.trim();
+  return normalized.length > 0 && normalized.length <= 160
+    ? normalized
+    : undefined;
+}