@evalops/maestro 0.10.18 → 0.10.20

package/dist/platform/fermata-eval-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fermata-eval-client.js","sourceRoot":"","sources":["../../src/platform/fermata-eval-client.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,6BAA6B,EAC7B,gCAAgC,EAChC,6BAA6B,GAC7B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAEN,gBAAgB,EAChB,mBAAmB,EACnB,4BAA4B,EAC5B,UAAU,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EACN,wBAAwB,EACxB,yBAAyB,EACzB,0BAA0B,GAC1B,MAAM,oBAAoB,CAAC;AAE5B,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,MAAM,sBAAsB,GAAG,yBAAyB,CACvD,wBAAwB,CAAC,OAAO,CAAC,eAAe,CAChD,CAAC;AACF,MAAM,mBAAmB,GAAG,yBAAyB,CACpD,wBAAwB,CAAC,gBAAgB,CAAC,YAAY,CACtD,CAAC;AAEF,MAAM,yBAAyB,GAAG;IACjC,sBAAsB;IACtB,mBAAmB;IACnB,0BAA0B,CACzB,wBAAwB,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CACxD;IACD,0BAA0B,CACzB,wBAAwB,CAAC,gBAAgB,CAAC,YAAY,CAAC,OAAO,CAC9D;CACQ,CAAC;AAEX,MAAM,yBAAyB,GAAG;IACjC,qBAAqB;IACrB,6BAA6B;IAC7B,uCAAuC;IACvC,2BAA2B;IAC3B,0BAA0B;IAC1B,kBAAkB;CACT,CAAC;AAEX,MAAM,sBAAsB,GAAG;IAC9B,uBAAuB;IACvB,+BAA+B;IAC/B,yCAAyC;IACzC,GAAG,6BAA6B;CACvB,CAAC;AAEX,MAAM,6BAA6B,GAAG;IACrC,iCAAiC;IACjC,iCAAiC;IACjC,2CAA2C;IAC3C,GAAG,gCAAgC;CAC1B,CAAC;AAEX,MAAM,0BAA0B,GAAG;IAClC,8BAA8B;IAC9B,8BAA8B;IAC9B,wCAAwC;IACxC,GAAG,6BAA6B;CACvB,CAAC;AAEX,MAAM,wBAAwB,GAAG;IAChC,4BAA4B;IAC5B,oCAAoC;IACpC,8CAA8C;CACrC,CAAC;AAEX,MAAM,6BAA6B,GAAG;IACrC,8BAA8B;IAC9B,sCAAsC;IACtC,gDAAgD;CACvC,CAAC;AA2OX,SAAS,oBAAoB,CAC5B,MAA+B;IAE/B,OAAO,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CACjE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC/B,QAAkB,EAClB,WAAmB;IAEnB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CACd,GAAG,WAAW,aAAa,QAAQ,CAAC,MAAM,KAAK,IAAI,IAAI,QAAQ,CAAC,UAAU,EAAE,CAC5E,CAAC;IACH,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,GAAG,WAAW,0BAA0B,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;AACpD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACpD,YAA+C,EAAE;IAEjD,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC;QACjD,cAAc,EAAE,yBAAyB;QACzC,YAAY,EAAE,sBAAsB;QACpC,mBAAmB,EAAE,6BAA6B;QAClD,gBAAgB,EAAE,0BAA0B;QAC5C,cAAc,EAAE,wBAAwB;QACxC,kBAAkB,EAAE,6BAA6B;QACjD,eAAe,EAAE,yBAAyB;QAC1C,gBAAgB,EAAE,kBAAkB;QACpC,kBAAkB,EAAE,oBAAoB;QACxC,cAAc,EAAE,IAAI;QACpB,qBAAqB,EAAE,IAAI;QAC3B,YAAY,EAAE,IAAI;QAClB,uBAAuB,EAAE,KAAK;KAC9B,CAAC,CAAC;IACH,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC7C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO;QACN,GAAG,MAAM;QACT,OAAO,EAAE,gBAAgB,CACxB,UAAU,CAAC,SAAS,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,EACjE,yBAAyB,CACzB;QACD,cAAc,EACb,UAAU,CAAC,SAAS,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC;YAC7D,MAAM,CAAC,cAAc;QACtB,WAAW,EACV,UAAU,CAAC,SAAS,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;YACvD,MAAM,CAAC,WAAW;QACnB,KAAK,EAAE,UAAU,CAAC,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK;QAClE,SAAS,EAAE,SAAS,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;QAClD,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW;QACxD,MAAM,EAAE,UAAU,CAAC,SAAS,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;KACrD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACpD,YAA+C,EAAE;IAEjD,MAAM,MAAM,GAAG,MAAM,+BAA+B,CAAC,SAAS,CAAC,CAAC;IAChE,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACd,iHAAiH,CACjH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kCAAkC,CACvD,MAAgC,EAChC,OAAsC,EACtC,MAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CACzC,MAAM,EACN,sBAAsB,EACtB,oBAAoB,CAAC;QACpB,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC1B,CAAC,EACF;QACC,WAAW,EAAE,4BAA4B;QACzC,WAAW,EAAE,UAAU;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,MAAM;KACN,CACD,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,iBAAiB,CACtC,QAAQ,EACR,4BAA4B,CAC5B,CAAC;IACF,OAAO,OAAoD,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACpD,MAAgC,EAChC,OAAmC,EACnC,MAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CACzC,MAAM,EACN,mBAAmB,EACnB,oBAAoB,CAAC;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,UAAU,EAAE,OAAO,CAAC,UAAU;KAC9B,CAAC,EACF;QACC,WAAW,EAAE,sCAAsC;QACnD,WAAW,EAAE,UAAU;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,MAAM;KACN,CACD,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,iBAAiB,CACtC,QAAQ,EACR,sCAAsC,CACtC,CAAC;IACF,OAAO,OAAiD,CAAC;AAC1D,CAAC","sourcesContent":["import {\n\tEVALOPS_ACCESS_TOKEN_ENV_VARS,\n\tEVALOPS_ORGANIZATION_ID_ENV_VARS,\n\tEVALOPS_WORKSPACE_ID_ENV_VARS,\n} from \"../evalops/env-aliases.js\";\nimport {\n\ttype PlatformServiceConfig,\n\tnormalizeBaseUrl,\n\tpostPlatformConnect,\n\tresolvePlatformServiceConfig,\n\ttrimString,\n} from \"./client.js\";\nimport {\n\tPLATFORM_CONNECT_METHODS,\n\tplatformConnectMethodPath,\n\tplatformConnectServicePath,\n} from \"./core-services.js\";\n\nconst DEFAULT_TIMEOUT_MS = 5_000;\nconst DEFAULT_MAX_ATTEMPTS = 1;\n\nconst CREATE_TEST_SUITE_PATH = platformConnectMethodPath(\n\tPLATFORM_CONNECT_METHODS.fermata.createTestSuite,\n);\nconst RUN_TEST_SUITE_PATH = platformConnectMethodPath(\n\tPLATFORM_CONNECT_METHODS.fermataExecution.runTestSuite,\n);\n\nconst FERMATA_BASE_URL_SUFFIXES = [\n\tCREATE_TEST_SUITE_PATH,\n\tRUN_TEST_SUITE_PATH,\n\tplatformConnectServicePath(\n\t\tPLATFORM_CONNECT_METHODS.fermata.createTestSuite.service,\n\t),\n\tplatformConnectServicePath(\n\t\tPLATFORM_CONNECT_METHODS.fermataExecution.runTestSuite.service,\n\t),\n] as const;\n\nconst FERMATA_BASE_URL_ENV_VARS = [\n\t\"FERMATA_SERVICE_URL\",\n\t\"MAESTRO_FERMATA_SERVICE_URL\",\n\t\"MAESTRO_FERMATA_EXECUTION_SERVICE_URL\",\n\t\"MAESTRO_PLATFORM_BASE_URL\",\n\t\"MAESTRO_EVALOPS_BASE_URL\",\n\t\"EVALOPS_BASE_URL\",\n] as const;\n\nconst FERMATA_TOKEN_ENV_VARS = [\n\t\"FERMATA_SERVICE_TOKEN\",\n\t\"MAESTRO_FERMATA_SERVICE_TOKEN\",\n\t\"MAESTRO_FERMATA_EXECUTION_SERVICE_TOKEN\",\n\t...EVALOPS_ACCESS_TOKEN_ENV_VARS,\n] as const;\n\nconst FERMATA_ORGANIZATION_ENV_VARS = [\n\t\"FERMATA_SERVICE_ORGANIZATION_ID\",\n\t\"MAESTRO_FERMATA_ORGANIZATION_ID\",\n\t\"MAESTRO_FERMATA_EXECUTION_ORGANIZATION_ID\",\n\t...EVALOPS_ORGANIZATION_ID_ENV_VARS,\n] as const;\n\nconst FERMATA_WORKSPACE_ENV_VARS = [\n\t\"FERMATA_SERVICE_WORKSPACE_ID\",\n\t\"MAESTRO_FERMATA_WORKSPACE_ID\",\n\t\"MAESTRO_FERMATA_EXECUTION_WORKSPACE_ID\",\n\t...EVALOPS_WORKSPACE_ID_ENV_VARS,\n] as const;\n\nconst FERMATA_TIMEOUT_ENV_VARS = [\n\t\"FERMATA_SERVICE_TIMEOUT_MS\",\n\t\"MAESTRO_FERMATA_SERVICE_TIMEOUT_MS\",\n\t\"MAESTRO_FERMATA_EXECUTION_SERVICE_TIMEOUT_MS\",\n] as const;\n\nconst FERMATA_MAX_ATTEMPTS_ENV_VARS = [\n\t\"FERMATA_SERVICE_MAX_ATTEMPTS\",\n\t\"MAESTRO_FERMATA_SERVICE_MAX_ATTEMPTS\",\n\t\"MAESTRO_FERMATA_EXECUTION_SERVICE_MAX_ATTEMPTS\",\n] as const;\n\nexport type FermataAssertionKind =\n\t| \"ASSERTION_KIND_CONTAINS\"\n\t| \"ASSERTION_KIND_NOT_CONTAINS\"\n\t| \"ASSERTION_KIND_REGEX\"\n\t| \"ASSERTION_KIND_SIMILARITY\"\n\t| \"ASSERTION_KIND_CUSTOM_JUDGE\"\n\t| \"ASSERTION_KIND_JSON_SCHEMA\"\n\t| \"ASSERTION_KIND_EQUALS\"\n\t| \"ASSERTION_KIND_STARTS_WITH\"\n\t| \"ASSERTION_KIND_CONTAINS_ANY\"\n\t| \"ASSERTION_KIND_CONTAINS_ALL\"\n\t| \"ASSERTION_KIND_WORD_COUNT\"\n\t| \"ASSERTION_KIND_NUMERIC_GTE\"\n\t| \"ASSERTION_KIND_NUMERIC_LTE\"\n\t| \"ASSERTION_KIND_TOOL_CALLED\"\n\t| \"ASSERTION_KIND_TOOL_NOT_CALLED\"\n\t| \"ASSERTION_KIND_TOOL_SEQUENCE\"\n\t| \"ASSERTION_KIND_TRACE_SPAN_COUNT\"\n\t| \"ASSERTION_KIND_TOOL_ARGS_MATCH\"\n\t| \"ASSERTION_KIND_TRACE_ERROR_SPANS\"\n\t| \"ASSERTION_KIND_TRACE_SPAN_DURATION\"\n\t| \"ASSERTION_KIND_TRACE_STEP_COUNT\"\n\t| \"ASSERTION_KIND_TOOL_CALL_F1\"\n\t| \"ASSERTION_KIND_LLM_RUBRIC\"\n\t| \"ASSERTION_KIND_LLM_PAIRWISE_RUBRIC\"\n\t| \"ASSERTION_KIND_AGENT_TRAJECTORY\";\n\nexport interface FermataLLMRubricAssertion {\n\tjudgeId: string;\n\tverifierJudgeId?: string;\n\trubric: string;\n\tminScore?: number;\n\trepeat?: number;\n\tquorum?: number;\n\trecordJudgeValidation?: boolean;\n\trequireCalibratedJudge?: boolean;\n\tminJudgeValidationAccuracy?: number;\n\tminJudgeValidationCount?: number;\n\trubricVersion?: string;\n\tcalibrationCohort?: string;\n\tadvisoryOnly?: boolean;\n}\n\nexport interface FermataLLMPairwiseRubricAssertion {\n\tjudgeId: string;\n\tverifierJudgeId?: string;\n\trubric: string;\n\tbaselineLabel?: string;\n\tcandidateLabel?: string;\n\tminScore?: number;\n\trepeat?: number;\n\tquorum?: number;\n\trecordJudgeValidation?: boolean;\n\trequireCalibratedJudge?: boolean;\n\tminJudgeValidationAccuracy?: number;\n\tminJudgeValidationCount?: number;\n\trubricVersion?: string;\n\tcalibrationCohort?: string;\n\tadvisoryOnly?: boolean;\n}\n\nexport interface FermataAgentTrajectoryStatusExpectation {\n\tid: string;\n\tstatus: string;\n}\n\nexport interface FermataAgentTrajectoryStateExpectation {\n\tpath: string;\n\tvalueJson?: string;\n}\n\nexport interface FermataAgentTrajectoryAssertion {\n\trequiredTools?: string[];\n\tforbiddenTools?: string[];\n\trequiredEvents?: string[];\n\tforbiddenEvents?: string[];\n\trequiredAssertionStatuses?: FermataAgentTrajectoryStatusExpectation[];\n\trequiredStateWrites?: FermataAgentTrajectoryStateExpectation[];\n\tforbiddenStateWrites?: FermataAgentTrajectoryStateExpectation[];\n\tmaxEvents?: number;\n\tmaxToolCalls?: number;\n\tmaxReplayDeltas?: number;\n\tmaxScoreFailures?: number;\n\tmaxScoreWarnings?: number;\n\tmaxLatencyMs?: number;\n\tmaxCostMicros?: number;\n\tmaxRetries?: number;\n\trequireIdempotentReplay?: boolean;\n\tforbidDuplicateExternalActions?: boolean;\n\tforbiddenExternalActions?: string[];\n\trequiredTraceJoinKeys?: string[];\n}\n\nexport interface FermataAssertion {\n\tid?: string;\n\tkind: FermataAssertionKind;\n\ttarget?: string;\n\texpected?: string;\n\tdescription?: string;\n\tmetadata?: Record<string, unknown>;\n\tllmRubric?: FermataLLMRubricAssertion;\n\tllmPairwiseRubric?: FermataLLMPairwiseRubricAssertion;\n\tagentTrajectory?: FermataAgentTrajectoryAssertion;\n}\n\nexport interface FermataTestCase {\n\tid?: string;\n\tname: string;\n\tinput: string;\n\texpectedOutput?: string;\n\tassertions: FermataAssertion[];\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface FermataTestSuite {\n\tid?: string;\n\torganizationId?: string;\n\tworkspaceId?: string;\n\tname?: string;\n\tdescription?: string;\n\tcases?: FermataTestCase[];\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface CreateFermataTestSuiteRequest {\n\torganizationId: string;\n\tworkspaceId: string;\n\tname: string;\n\tdescription?: string;\n\tcases: FermataTestCase[];\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface CreateFermataTestSuiteResponse {\n\tsuite: FermataTestSuite;\n}\n\nexport interface FermataProviderRef {\n\tprovider?: string;\n\tenvironment?: string;\n\tcredentialName?: string;\n\tteamId?: string;\n}\n\nexport interface FermataTestSuiteCandidate {\n\tcandidateId: string;\n\tlabel?: string;\n\tmodel: string;\n\tproviderRef?: FermataProviderRef;\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface RunFermataTestSuiteRequest {\n\tsuiteId: string;\n\torganizationId: string;\n\tworkspaceId: string;\n\tevaluationId?: string;\n\tmodel?: string;\n\trunContext?: Record<string, unknown>;\n\tmetadata?: Record<string, unknown>;\n\trunIdPrefix?: string;\n\ttraceId?: string;\n\tlineageId?: string;\n\tcaseIds?: string[];\n\tmaxConcurrency?: number;\n\tstopOnFirstFailure?: boolean;\n\tcandidates?: FermataTestSuiteCandidate[];\n}\n\nexport interface FermataAssertionResult {\n\tassertionId?: string;\n\tpassed?: boolean;\n\tscore?: number;\n\tactual?: string;\n\tmessage?: string;\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface FermataTestCaseRun {\n\ttestCaseId?: string;\n\ttestCaseName?: string;\n\tpassed?: boolean;\n\tscore?: number;\n\toutput?: string;\n\tassertionResults?: FermataAssertionResult[];\n\tcandidateId?: string;\n\tcandidateLabel?: string;\n\tmodel?: string;\n\tproviderRef?: FermataProviderRef;\n}\n\nexport interface FermataTestSuiteCandidateSummary {\n\tcandidateId?: string;\n\tcandidateLabel?: string;\n\tmodel?: string;\n\tproviderRef?: FermataProviderRef;\n\ttotalCases?: number;\n\tpassedCases?: number;\n\tfailedCases?: number;\n\tscore?: number;\n\tpassRate?: number;\n\tpassed?: boolean;\n}\n\nexport interface FermataTestSuiteAssertionSummary {\n\tassertionId?: string;\n\ttotalResults?: number;\n\tpassedResults?: number;\n\tfailedResults?: number;\n\tscore?: number;\n\tpassRate?: number;\n\tpassed?: boolean;\n\tfailedTestCaseIds?: string[];\n\tfailedCandidateIds?: string[];\n\tfailedModels?: string[];\n}\n\nexport interface RunFermataTestSuiteResponse {\n\tevaluationId?: string;\n\tsuite?: FermataTestSuite;\n\tcaseResults?: FermataTestCaseRun[];\n\tpassed?: boolean;\n\tscore?: number;\n\ttotalCases?: number;\n\tpassedCases?: number;\n\tfailedCases?: number;\n\tlineageId?: string;\n\tcandidateSummaries?: FermataTestSuiteCandidateSummary[];\n\tassertionSummaries?: FermataTestSuiteAssertionSummary[];\n}\n\nexport interface FermataEvalServiceConfig extends PlatformServiceConfig {}\n\nfunction stripUndefinedValues(\n\trecord: Record<string, unknown>,\n): Record<string, unknown> {\n\treturn Object.fromEntries(\n\t\tObject.entries(record).filter(([, value]) => value !== undefined),\n\t);\n}\n\nasync function parseJsonResponse(\n\tresponse: Response,\n\tserviceName: string,\n): Promise<Record<string, unknown>> {\n\tconst text = await response.text();\n\tif (!response.ok) {\n\t\tthrow new Error(\n\t\t\t`${serviceName} returned ${response.status}: ${text || response.statusText}`,\n\t\t);\n\t}\n\tif (!text.trim()) {\n\t\tthrow new Error(`${serviceName} returned empty response`);\n\t}\n\treturn JSON.parse(text) as Record<string, unknown>;\n}\n\nexport async function resolveFermataEvalServiceConfig(\n\toverrides: Partial<FermataEvalServiceConfig> = {},\n): Promise<FermataEvalServiceConfig | null> {\n\tconst config = await resolvePlatformServiceConfig({\n\t\tbaseUrlEnvVars: FERMATA_BASE_URL_ENV_VARS,\n\t\ttokenEnvVars: FERMATA_TOKEN_ENV_VARS,\n\t\torganizationEnvVars: FERMATA_ORGANIZATION_ENV_VARS,\n\t\tworkspaceEnvVars: FERMATA_WORKSPACE_ENV_VARS,\n\t\ttimeoutEnvVars: FERMATA_TIMEOUT_ENV_VARS,\n\t\tmaxAttemptsEnvVars: FERMATA_MAX_ATTEMPTS_ENV_VARS,\n\t\tbaseUrlSuffixes: FERMATA_BASE_URL_SUFFIXES,\n\t\tdefaultTimeoutMs: DEFAULT_TIMEOUT_MS,\n\t\tdefaultMaxAttempts: DEFAULT_MAX_ATTEMPTS,\n\t\trequireBaseUrl: true,\n\t\trequireOrganizationId: true,\n\t\trequireToken: true,\n\t\tallowOAuthTokenFallback: false,\n\t});\n\tif (!config?.baseUrl || !config.workspaceId) {\n\t\treturn null;\n\t}\n\treturn {\n\t\t...config,\n\t\tbaseUrl: normalizeBaseUrl(\n\t\t\ttrimString(overrides.baseUrl ?? config.baseUrl) ?? config.baseUrl,\n\t\t\tFERMATA_BASE_URL_SUFFIXES,\n\t\t),\n\t\torganizationId:\n\t\t\ttrimString(overrides.organizationId ?? config.organizationId) ??\n\t\t\tconfig.organizationId,\n\t\tworkspaceId:\n\t\t\ttrimString(overrides.workspaceId ?? config.workspaceId) ??\n\t\t\tconfig.workspaceId,\n\t\ttoken: trimString(overrides.token ?? config.token) ?? config.token,\n\t\ttimeoutMs: overrides.timeoutMs ?? config.timeoutMs,\n\t\tmaxAttempts: overrides.maxAttempts ?? config.maxAttempts,\n\t\tteamId: trimString(overrides.teamId ?? config.teamId),\n\t};\n}\n\nexport async function requireFermataEvalServiceConfig(\n\toverrides: Partial<FermataEvalServiceConfig> = {},\n): Promise<FermataEvalServiceConfig> {\n\tconst config = await resolveFermataEvalServiceConfig(overrides);\n\tif (!config) {\n\t\tthrow new Error(\n\t\t\t\"Fermata eval integration requires FERMATA_SERVICE_URL, FERMATA_SERVICE_TOKEN, organization id, and workspace id\",\n\t\t);\n\t}\n\treturn config;\n}\n\nexport async function createFermataTestSuiteWithPlatform(\n\tconfig: FermataEvalServiceConfig,\n\trequest: CreateFermataTestSuiteRequest,\n\tsignal?: AbortSignal,\n): Promise<CreateFermataTestSuiteResponse> {\n\tconst response = await postPlatformConnect(\n\t\tconfig,\n\t\tCREATE_TEST_SUITE_PATH,\n\t\tstripUndefinedValues({\n\t\t\torganizationId: request.organizationId,\n\t\t\tworkspaceId: request.workspaceId,\n\t\t\tname: request.name,\n\t\t\tdescription: request.description,\n\t\t\tcases: request.cases,\n\t\t\tmetadata: request.metadata,\n\t\t}),\n\t\t{\n\t\t\tserviceName: \"fermata test suite service\",\n\t\t\tfailureMode: \"required\",\n\t\t\ttimeoutMs: config.timeoutMs,\n\t\t\tmaxAttempts: config.maxAttempts,\n\t\t\tsignal,\n\t\t},\n\t);\n\tconst payload = await parseJsonResponse(\n\t\tresponse,\n\t\t\"fermata test suite service\",\n\t);\n\treturn payload as unknown as CreateFermataTestSuiteResponse;\n}\n\nexport async function runFermataTestSuiteWithPlatform(\n\tconfig: FermataEvalServiceConfig,\n\trequest: RunFermataTestSuiteRequest,\n\tsignal?: AbortSignal,\n): Promise<RunFermataTestSuiteResponse> {\n\tconst response = await postPlatformConnect(\n\t\tconfig,\n\t\tRUN_TEST_SUITE_PATH,\n\t\tstripUndefinedValues({\n\t\t\tsuiteId: request.suiteId,\n\t\t\torganizationId: request.organizationId,\n\t\t\tworkspaceId: request.workspaceId,\n\t\t\tevaluationId: request.evaluationId,\n\t\t\tmodel: request.model,\n\t\t\trunContext: request.runContext,\n\t\t\tmetadata: request.metadata,\n\t\t\trunIdPrefix: request.runIdPrefix,\n\t\t\ttraceId: request.traceId,\n\t\t\tlineageId: request.lineageId,\n\t\t\tcaseIds: request.caseIds,\n\t\t\tmaxConcurrency: request.maxConcurrency,\n\t\t\tstopOnFirstFailure: request.stopOnFirstFailure,\n\t\t\tcandidates: request.candidates,\n\t\t}),\n\t\t{\n\t\t\tserviceName: \"fermata test suite execution service\",\n\t\t\tfailureMode: \"required\",\n\t\t\ttimeoutMs: config.timeoutMs,\n\t\t\tmaxAttempts: config.maxAttempts,\n\t\t\tsignal,\n\t\t},\n\t);\n\tconst payload = await parseJsonResponse(\n\t\tresponse,\n\t\t\"fermata test suite execution service\",\n\t);\n\treturn payload as unknown as RunFermataTestSuiteResponse;\n}\n"]}

package/dist/platform/fermata-scenario-suite.d.ts

@@ -0,0 +1,99 @@
+import type { CreateFermataTestSuiteRequest, FermataEvalServiceConfig, RunFermataTestSuiteRequest } from "./fermata-eval-client.js";
+type AgentTrajectoryScenarioAssertionStatus = "pass" | "fail" | "warn";
+interface AgentTrajectoryScenarioEvidence {
+    kind: string;
+    id: string;
+    source: string;
+    label: string;
+}
+interface AgentTrajectoryScenarioAssertionResult {
+    id: string;
+    kind: string;
+    status: AgentTrajectoryScenarioAssertionStatus;
+    severity: string;
+    message: string;
+    evidence: AgentTrajectoryScenarioEvidence[];
+}
+interface AgentTrajectoryScenarioResult {
+    schemaVersion: string;
+    scenarioSchemaVersion: string;
+    scenario: {
+        id: string;
+        title: string;
+        expectedOutcome: string;
+        observedOutcome: string;
+        reviewLabels: string[];
+    };
+    run: {
+        id: string;
+        sessionId: string;
+        source?: string;
+        platformBacked?: boolean;
+        scenarioId?: string;
+        replay?: boolean;
+        generatedAt?: string;
+    };
+    counts: {
+        assertions: number;
+        passed: number;
+        failed: number;
+        warnings: number;
+        events: number;
+        toolCalls: number;
+        replayDeltas: number;
+        scoreFailures: number;
+        scoreWarnings: number;
+    };
+    platform: {
+        primitive: string;
+        evidenceEventType: string;
+        traceJoinKeys: string[];
+        eventType?: string;
+        rationale?: string;
+    };
+    assertions: AgentTrajectoryScenarioAssertionResult[];
+    provenance?: Array<{
+        eventId: string;
+        eventType: string;
+        phase?: string;
+        actor?: string;
+    }>;
+}
+export interface FermataScenarioSuiteLLMJudgeOptions {
+    judgeId: string;
+    verifierJudgeId?: string;
+    rubric: string;
+    minScore?: number;
+    repeat?: number;
+    quorum?: number;
+    recordJudgeValidation?: boolean;
+    requireCalibratedJudge?: boolean;
+    minJudgeValidationAccuracy?: number;
+    minJudgeValidationCount?: number;
+    rubricVersion?: string;
+    calibrationCohort?: string;
+    advisoryOnly?: boolean;
+}
+export interface FermataScenarioSuiteLLMPairwiseJudgeOptions extends FermataScenarioSuiteLLMJudgeOptions {
+    baselineLabel?: string;
+    candidateLabel?: string;
+}
+export interface FermataScenarioSuiteOptions {
+    name?: string;
+    description?: string;
+    model?: string;
+    candidateId?: string;
+    candidateLabel?: string;
+    evaluationId?: string;
+    runIdPrefix?: string;
+    lineageId?: string;
+    traceId?: string;
+    maxConcurrency?: number;
+    metadata?: Record<string, unknown>;
+    llmJudge?: FermataScenarioSuiteLLMJudgeOptions;
+    llmPairwiseJudge?: FermataScenarioSuiteLLMPairwiseJudgeOptions;
+}
+export declare function buildFermataCreateScenarioSuiteRequest(config: Pick<FermataEvalServiceConfig, "organizationId" | "workspaceId">, expected: AgentTrajectoryScenarioResult, actual: AgentTrajectoryScenarioResult, options?: FermataScenarioSuiteOptions): CreateFermataTestSuiteRequest;
+export declare function buildFermataRunScenarioSuiteRequest(config: Pick<FermataEvalServiceConfig, "organizationId" | "workspaceId">, suiteId: string, expected: AgentTrajectoryScenarioResult, actual: AgentTrajectoryScenarioResult, options?: FermataScenarioSuiteOptions): RunFermataTestSuiteRequest;
+export {};
+//# sourceMappingURL=fermata-scenario-suite.d.ts.map

package/dist/platform/fermata-scenario-suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fermata-scenario-suite.d.ts","sourceRoot":"","sources":["../../src/platform/fermata-scenario-suite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,6BAA6B,EAE7B,wBAAwB,EAIxB,0BAA0B,EAC1B,MAAM,0BAA0B,CAAC;AAElC,KAAK,sCAAsC,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAEvE,UAAU,+BAA+B;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AAED,UAAU,sCAAsC;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,sCAAsC,CAAC;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,+BAA+B,EAAE,CAAC;CAC5C;AAED,UAAU,6BAA6B;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,QAAQ,EAAE;QACT,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,GAAG,EAAE;QACJ,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,QAAQ,EAAE;QACT,SAAS,EAAE,MAAM,CAAC;QAClB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,EAAE,sCAAsC,EAAE,CAAC;IACrD,UAAU,CAAC,EAAE,KAAK,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;CACH;AAcD,MAAM,WAAW,mCAAmC;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,2CAChB,SAAQ,mCAAmC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,2BAA2B;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,QAAQ,CAAC,EAAE,mCAAmC,CAAC;IAC/C,gBAAgB,CAAC,EAAE,2CAA2C,CAAC;CAC/D;AAmhBD,wBAAgB,sCAAsC,CACrD,MAAM,EAAE,IAAI,CAAC,wBAAwB,EAAE,gBAAgB,GAAG,aAAa,CAAC,EACxE,QAAQ,EAAE,6BAA6B,EACvC,MAAM,EAAE,6BAA6B,EACrC,OAAO,GAAE,2BAAgC,GACvC,6BAA6B,CA6B/B;AAED,wBAAgB,mCAAmC,CAClD,MAAM,EAAE,IAAI,CAAC,wBAAwB,EAAE,gBAAgB,GAAG,aAAa,CAAC,EACxE,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,6BAA6B,EACvC,MAAM,EAAE,6BAA6B,EACrC,OAAO,GAAE,2BAAgC,GACvC,0BAA0B,CA0C5B"}

package/dist/platform/fermata-scenario-suite.js

@@ -0,0 +1,459 @@
+const COUNT_KEYS = [
+    "assertions",
+    "passed",
+    "failed",
+    "warnings",
+    "events",
+    "toolCalls",
+    "replayDeltas",
+    "scoreFailures",
+    "scoreWarnings",
+];
+function isObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function stableId(...parts) {
+    const id = parts
+        .join(".")
+        .toLowerCase()
+        .replace(/[^a-z0-9._-]+/gu, "-")
+        .replace(/^-+|-+$/gu, "")
+        .replace(/--+/gu, "-");
+    return id || "maestro-scenario";
+}
+function scenarioResultLabel(result) {
+    return `${result.scenario.id} (${result.run.id})`;
+}
+function requireSameScenario(expected, actual) {
+    if (expected.scenario.id !== actual.scenario.id) {
+        throw new Error(`Fermata scenario suite requires matching scenario ids; expected ${scenarioResultLabel(expected)}, received ${scenarioResultLabel(actual)}`);
+    }
+    if (expected.schemaVersion !== actual.schemaVersion) {
+        throw new Error(`Fermata scenario suite requires matching result schema versions for ${expected.scenario.id}`);
+    }
+}
+function serializeCaseInput(value) {
+    return JSON.stringify(value, null, "\t");
+}
+function assertionMap(result) {
+    return new Map(result.assertions.map((assertion) => [assertion.id, assertion]));
+}
+function evidenceLabels(assertion) {
+    return (assertion?.evidence ?? []).map((evidence) => `${evidence.source}:${evidence.kind}:${evidence.id}`);
+}
+function requireNonEmpty(value, label) {
+    const trimmed = value?.trim();
+    if (!trimmed) {
+        throw new Error(`Fermata scenario suite LLM judge requires ${label}`);
+    }
+    return trimmed;
+}
+function optionalPositiveNumber(value, label, max) {
+    if (value === undefined)
+        return undefined;
+    if (!Number.isFinite(value) ||
+        value <= 0 ||
+        (max !== undefined && value > max)) {
+        throw new Error(`Fermata scenario suite LLM judge ${label} is invalid`);
+    }
+    return value;
+}
+function optionalPositiveInteger(value, label) {
+    if (value === undefined)
+        return undefined;
+    if (!Number.isInteger(value) || value <= 0) {
+        throw new Error(`Fermata scenario suite LLM judge ${label} is invalid`);
+    }
+    return value;
+}
+function optionalTrimmed(value) {
+    const trimmed = value?.trim();
+    return trimmed || undefined;
+}
+function llmRubricConfig(options) {
+    const repeat = optionalPositiveInteger(options.repeat, "repeat");
+    const quorum = optionalPositiveInteger(options.quorum, "quorum");
+    if (repeat !== undefined && quorum !== undefined && quorum > repeat) {
+        throw new Error("Fermata scenario suite LLM judge quorum cannot exceed repeat");
+    }
+    return {
+        judgeId: requireNonEmpty(options.judgeId, "judgeId"),
+        verifierJudgeId: options.verifierJudgeId?.trim() || undefined,
+        rubric: requireNonEmpty(options.rubric, "rubric"),
+        minScore: optionalPositiveNumber(options.minScore, "minScore", 1),
+        repeat,
+        quorum,
+        recordJudgeValidation: options.recordJudgeValidation ?? true,
+        requireCalibratedJudge: options.requireCalibratedJudge,
+        minJudgeValidationAccuracy: optionalPositiveNumber(options.minJudgeValidationAccuracy, "minJudgeValidationAccuracy", 1),
+        minJudgeValidationCount: optionalPositiveInteger(options.minJudgeValidationCount, "minJudgeValidationCount"),
+        rubricVersion: optionalTrimmed(options.rubricVersion),
+        calibrationCohort: optionalTrimmed(options.calibrationCohort),
+        advisoryOnly: options.advisoryOnly,
+    };
+}
+function llmPairwiseRubricConfig(options) {
+    const repeat = optionalPositiveInteger(options.repeat, "repeat");
+    const quorum = optionalPositiveInteger(options.quorum, "quorum");
+    if (repeat !== undefined && quorum !== undefined && quorum > repeat) {
+        throw new Error("Fermata scenario suite LLM judge quorum cannot exceed repeat");
+    }
+    return {
+        judgeId: requireNonEmpty(options.judgeId, "judgeId"),
+        verifierJudgeId: options.verifierJudgeId?.trim() || undefined,
+        rubric: requireNonEmpty(options.rubric, "rubric"),
+        baselineLabel: options.baselineLabel?.trim() || undefined,
+        candidateLabel: options.candidateLabel?.trim() || undefined,
+        minScore: optionalPositiveNumber(options.minScore, "minScore", 1),
+        repeat,
+        quorum,
+        recordJudgeValidation: options.recordJudgeValidation ?? true,
+        requireCalibratedJudge: options.requireCalibratedJudge,
+        minJudgeValidationAccuracy: optionalPositiveNumber(options.minJudgeValidationAccuracy, "minJudgeValidationAccuracy", 1),
+        minJudgeValidationCount: optionalPositiveInteger(options.minJudgeValidationCount, "minJudgeValidationCount"),
+        rubricVersion: optionalTrimmed(options.rubricVersion),
+        calibrationCohort: optionalTrimmed(options.calibrationCohort),
+        advisoryOnly: options.advisoryOnly,
+    };
+}
+function llmJudgeMetadata(options) {
+    const rubricVersion = optionalTrimmed(options.rubricVersion);
+    const calibrationCohort = optionalTrimmed(options.calibrationCohort);
+    if (options.advisoryOnly === undefined &&
+        !rubricVersion &&
+        !calibrationCohort) {
+        return {};
+    }
+    return {
+        judge_mode: options.advisoryOnly ? "advisory" : "blocking",
+        ...(rubricVersion ? { rubric_version: rubricVersion } : {}),
+        ...(calibrationCohort ? { calibration_cohort: calibrationCohort } : {}),
+    };
+}
+function realScenarioResultJSON(result) {
+    return JSON.stringify(result, null, "\t");
+}
+function uniqueValues(values) {
+    const seen = new Set();
+    for (const value of values) {
+        const trimmed = value?.trim();
+        if (trimmed)
+            seen.add(trimmed);
+    }
+    return [...seen];
+}
+function agentTrajectoryConfig(expected) {
+    return {
+        requiredEvents: uniqueValues(expected.provenance?.map((event) => event.eventType) ?? []),
+        requiredAssertionStatuses: expected.assertions.map((assertion) => ({
+            id: assertion.id,
+            status: assertion.status,
+        })),
+        maxEvents: expected.counts.events,
+        maxToolCalls: expected.counts.toolCalls,
+        maxReplayDeltas: expected.counts.replayDeltas,
+        maxScoreFailures: expected.counts.scoreFailures,
+        maxScoreWarnings: expected.counts.scoreWarnings,
+        requireIdempotentReplay: true,
+        forbidDuplicateExternalActions: true,
+        requiredTraceJoinKeys: expected.platform.traceJoinKeys,
+    };
+}
+function agentTrajectoryCase(expected, actual) {
+    const scenarioId = expected.scenario.id;
+    const id = stableId(scenarioId, "agent-trajectory", "native-trajectory-guard");
+    return {
+        id,
+        name: `${scenarioId} native trajectory guard`,
+        input: serializeCaseInput({
+            scenarioId,
+            title: expected.scenario.title,
+            platformPrimitive: actual.platform.primitive,
+            expectedRunId: expected.run.id,
+            actualRunId: actual.run.id,
+            traceJoinKeys: actual.platform.traceJoinKeys,
+            expectedAssertionIds: expected.assertions.map((assertion) => assertion.id),
+            actualAssertionIds: actual.assertions.map((assertion) => assertion.id),
+        }),
+        expectedOutput: realScenarioResultJSON(expected),
+        metadata: {
+            actual_output: realScenarioResultJSON(actual),
+            case_kind: "scenario_agent_trajectory",
+            ci_tier: "core-regression",
+            scenario_id: scenarioId,
+            expected_run_id: expected.run.id,
+            actual_run_id: actual.run.id,
+            expected_assertion_count: expected.counts.assertions,
+            actual_assertion_count: actual.counts.assertions,
+            source: "maestro.agent_trajectory_scenario_result",
+        },
+        assertions: [
+            {
+                id: `${id}.agent-trajectory`,
+                kind: "ASSERTION_KIND_AGENT_TRAJECTORY",
+                target: "response",
+                description: "Native Fermata trajectory assertion evaluates real Maestro replay results for event, assertion, budget, idempotency, and trace-link invariants.",
+                agentTrajectory: agentTrajectoryConfig(expected),
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                    case_kind: "scenario_agent_trajectory",
+                    ci_tier: "core-regression",
+                },
+            },
+        ],
+    };
+}
+function llmRubricCase(expected, actual, options) {
+    const scenarioId = expected.scenario.id;
+    const id = stableId(scenarioId, "llm-rubric", "semantic-trajectory-quality");
+    return {
+        id,
+        name: `${scenarioId} semantic trajectory quality`,
+        input: serializeCaseInput({
+            scenarioId,
+            title: expected.scenario.title,
+            platformPrimitive: actual.platform.primitive,
+            expectedRunId: expected.run.id,
+            actualRunId: actual.run.id,
+            traceJoinKeys: actual.platform.traceJoinKeys,
+            expectedAssertionIds: expected.assertions.map((assertion) => assertion.id),
+            actualAssertionIds: actual.assertions.map((assertion) => assertion.id),
+        }),
+        expectedOutput: realScenarioResultJSON(expected),
+        metadata: {
+            actual_output: realScenarioResultJSON(actual),
+            case_kind: "scenario_llm_rubric",
+            ...llmJudgeMetadata(options),
+            scenario_id: scenarioId,
+            expected_run_id: expected.run.id,
+            actual_run_id: actual.run.id,
+            expected_assertion_count: expected.counts.assertions,
+            actual_assertion_count: actual.counts.assertions,
+            source: "maestro.agent_trajectory_scenario_result",
+        },
+        assertions: [
+            {
+                id: `${id}.llm-rubric`,
+                kind: "ASSERTION_KIND_LLM_RUBRIC",
+                target: "response",
+                description: "Cataloged Fermata LLM judge evaluates semantic trajectory quality against the committed real Maestro scenario result.",
+                llmRubric: llmRubricConfig(options),
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                    case_kind: "scenario_llm_rubric",
+                    ...llmJudgeMetadata(options),
+                },
+            },
+        ],
+    };
+}
+function llmPairwiseRubricCase(expected, actual, options) {
+    const scenarioId = expected.scenario.id;
+    const id = stableId(scenarioId, "llm-pairwise-rubric", "semantic-trajectory-preference");
+    return {
+        id,
+        name: `${scenarioId} semantic trajectory preference`,
+        input: serializeCaseInput({
+            scenarioId,
+            title: expected.scenario.title,
+            platformPrimitive: actual.platform.primitive,
+            baselineRunId: expected.run.id,
+            candidateRunId: actual.run.id,
+            traceJoinKeys: actual.platform.traceJoinKeys,
+            baselineAssertionIds: expected.assertions.map((assertion) => assertion.id),
+            candidateAssertionIds: actual.assertions.map((assertion) => assertion.id),
+        }),
+        expectedOutput: realScenarioResultJSON(expected),
+        metadata: {
+            actual_output: realScenarioResultJSON(actual),
+            case_kind: "scenario_llm_pairwise_rubric",
+            ...llmJudgeMetadata(options),
+            scenario_id: scenarioId,
+            baseline_run_id: expected.run.id,
+            candidate_run_id: actual.run.id,
+            baseline_assertion_count: expected.counts.assertions,
+            candidate_assertion_count: actual.counts.assertions,
+            source: "maestro.agent_trajectory_scenario_result",
+        },
+        assertions: [
+            {
+                id: `${id}.llm-pairwise-rubric`,
+                kind: "ASSERTION_KIND_LLM_PAIRWISE_RUBRIC",
+                target: "response",
+                description: "Cataloged Fermata LLM judge compares real Maestro trajectory outputs and prefers the stronger candidate.",
+                llmPairwiseRubric: llmPairwiseRubricConfig(options),
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                    case_kind: "scenario_llm_pairwise_rubric",
+                    ...llmJudgeMetadata(options),
+                },
+            },
+        ],
+    };
+}
+function equalsCase(id, name, input, expected, actual, metadata) {
+    return {
+        id,
+        name,
+        input: serializeCaseInput(input),
+        expectedOutput: expected,
+        metadata: {
+            ...metadata,
+            actual_output: actual,
+        },
+        assertions: [
+            {
+                id: `${id}.equals`,
+                kind: "ASSERTION_KIND_EQUALS",
+                target: "response",
+                expected,
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                },
+            },
+        ],
+    };
+}
+function outcomeCase(expected, actual) {
+    const scenarioId = expected.scenario.id;
+    return equalsCase(stableId(scenarioId, "observed-outcome"), `${scenarioId} observed outcome`, {
+        scenarioId,
+        title: expected.scenario.title,
+        expectedOutcome: expected.scenario.expectedOutcome,
+        committedObservedOutcome: expected.scenario.observedOutcome,
+        actualObservedOutcome: actual.scenario.observedOutcome,
+    }, expected.scenario.observedOutcome, actual.scenario.observedOutcome, {
+        case_kind: "scenario_observed_outcome",
+        scenario_id: scenarioId,
+        expected_outcome: expected.scenario.expectedOutcome,
+        expected_observed_outcome: expected.scenario.observedOutcome,
+        actual_observed_outcome: actual.scenario.observedOutcome,
+        review_labels: actual.scenario.reviewLabels,
+    });
+}
+function countCases(expected, actual) {
+    return COUNT_KEYS.map((key) => equalsCase(stableId(expected.scenario.id, "count", key), `${expected.scenario.id} ${key} count`, {
+        scenarioId: expected.scenario.id,
+        count: key,
+        expected: expected.counts[key],
+        actual: actual.counts[key],
+    }, String(expected.counts[key]), String(actual.counts[key]), {
+        case_kind: "scenario_count",
+        scenario_id: expected.scenario.id,
+        count: key,
+        expected_count: expected.counts[key],
+        actual_count: actual.counts[key],
+    }));
+}
+function assertionCases(expected, actual) {
+    const actualById = assertionMap(actual);
+    return expected.assertions.map((expectedAssertion) => {
+        const actualAssertion = actualById.get(expectedAssertion.id);
+        const actualStatus = actualAssertion?.status ?? "missing";
+        return equalsCase(stableId(expected.scenario.id, "assertion", expectedAssertion.id), `${expected.scenario.id} ${expectedAssertion.id}`, {
+            scenarioId: expected.scenario.id,
+            assertionId: expectedAssertion.id,
+            kind: expectedAssertion.kind,
+            expectedStatus: expectedAssertion.status,
+            actualStatus,
+            expectedMessage: expectedAssertion.message,
+            actualMessage: actualAssertion?.message ?? null,
+        }, expectedAssertion.status, actualStatus, {
+            case_kind: "scenario_assertion_status",
+            scenario_id: expected.scenario.id,
+            assertion_id: expectedAssertion.id,
+            assertion_kind: expectedAssertion.kind,
+            severity: expectedAssertion.severity,
+            expected_message: expectedAssertion.message,
+            actual_message: actualAssertion?.message,
+            expected_evidence: evidenceLabels(expectedAssertion),
+            actual_evidence: evidenceLabels(actualAssertion),
+        });
+    });
+}
+function suiteMetadata(expected, actual, extra) {
+    return {
+        source: "maestro.agent_trajectory_scenario_result",
+        schema_version: actual.schemaVersion,
+        scenario_schema_version: actual.scenarioSchemaVersion,
+        scenario_id: actual.scenario.id,
+        scenario_title: actual.scenario.title,
+        expected_run_id: expected.run.id,
+        actual_run_id: actual.run.id,
+        expected_outcome: expected.scenario.expectedOutcome,
+        committed_observed_outcome: expected.scenario.observedOutcome,
+        actual_observed_outcome: actual.scenario.observedOutcome,
+        platform_primitive: actual.platform.primitive,
+        evidence_event_type: actual.platform.evidenceEventType,
+        trace_join_keys: actual.platform.traceJoinKeys,
+        ...(isObject(extra) ? extra : {}),
+    };
+}
+export function buildFermataCreateScenarioSuiteRequest(config, expected, actual, options = {}) {
+    requireSameScenario(expected, actual);
+    if (!config.organizationId || !config.workspaceId) {
+        throw new Error("Fermata scenario suite requires organizationId and workspaceId");
+    }
+    const scenarioId = expected.scenario.id;
+    return {
+        organizationId: config.organizationId,
+        workspaceId: config.workspaceId,
+        name: options.name ?? `Maestro scenario: ${scenarioId}`,
+        description: options.description ??
+            `Recorded Maestro agent trajectory scenario suite for ${scenarioId}.`,
+        cases: [
+            outcomeCase(expected, actual),
+            ...countCases(expected, actual),
+            ...assertionCases(expected, actual),
+            agentTrajectoryCase(expected, actual),
+            ...(options.llmJudge
+                ? [llmRubricCase(expected, actual, options.llmJudge)]
+                : []),
+            ...(options.llmPairwiseJudge
+                ? [llmPairwiseRubricCase(expected, actual, options.llmPairwiseJudge)]
+                : []),
+        ],
+        metadata: suiteMetadata(expected, actual, options.metadata),
+    };
+}
+export function buildFermataRunScenarioSuiteRequest(config, suiteId, expected, actual, options = {}) {
+    if (!config.organizationId || !config.workspaceId) {
+        throw new Error("Fermata scenario suite run requires organizationId and workspaceId");
+    }
+    const scenarioId = actual.scenario.id;
+    const runId = stableId(scenarioId, actual.run.id);
+    const model = options.model ?? "maestro-agent-trajectory-replay";
+    return {
+        suiteId,
+        organizationId: config.organizationId,
+        workspaceId: config.workspaceId,
+        evaluationId: options.evaluationId ?? `maestro-${runId}`,
+        runIdPrefix: options.runIdPrefix ?? stableId("maestro", scenarioId),
+        lineageId: options.lineageId ?? actual.run.id,
+        traceId: options.traceId,
+        maxConcurrency: options.maxConcurrency ?? 1,
+        stopOnFirstFailure: false,
+        runContext: {
+            source: "maestro.agent_trajectory_scenario_result",
+            scenarioId,
+            runId: actual.run.id,
+            sessionId: actual.run.sessionId,
+            platformPrimitive: actual.platform.primitive,
+            traceJoinKeys: actual.platform.traceJoinKeys,
+        },
+        metadata: suiteMetadata(expected, actual, options.metadata),
+        candidates: [
+            {
+                candidateId: options.candidateId ?? stableId("maestro-recorded", scenarioId),
+                label: options.candidateLabel ?? "Maestro recorded trajectory",
+                model,
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                    scenario_id: scenarioId,
+                    run_id: actual.run.id,
+                },
+            },
+        ],
+    };
+}
+//# sourceMappingURL=fermata-scenario-suite.js.map