@evalops/maestro 0.10.17 → 0.10.20

package/dist/platform/fermata-scenario-suite.js

@@ -0,0 +1,459 @@
+const COUNT_KEYS = [
+    "assertions",
+    "passed",
+    "failed",
+    "warnings",
+    "events",
+    "toolCalls",
+    "replayDeltas",
+    "scoreFailures",
+    "scoreWarnings",
+];
+function isObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function stableId(...parts) {
+    const id = parts
+        .join(".")
+        .toLowerCase()
+        .replace(/[^a-z0-9._-]+/gu, "-")
+        .replace(/^-+|-+$/gu, "")
+        .replace(/--+/gu, "-");
+    return id || "maestro-scenario";
+}
+function scenarioResultLabel(result) {
+    return `${result.scenario.id} (${result.run.id})`;
+}
+function requireSameScenario(expected, actual) {
+    if (expected.scenario.id !== actual.scenario.id) {
+        throw new Error(`Fermata scenario suite requires matching scenario ids; expected ${scenarioResultLabel(expected)}, received ${scenarioResultLabel(actual)}`);
+    }
+    if (expected.schemaVersion !== actual.schemaVersion) {
+        throw new Error(`Fermata scenario suite requires matching result schema versions for ${expected.scenario.id}`);
+    }
+}
+function serializeCaseInput(value) {
+    return JSON.stringify(value, null, "\t");
+}
+function assertionMap(result) {
+    return new Map(result.assertions.map((assertion) => [assertion.id, assertion]));
+}
+function evidenceLabels(assertion) {
+    return (assertion?.evidence ?? []).map((evidence) => `${evidence.source}:${evidence.kind}:${evidence.id}`);
+}
+function requireNonEmpty(value, label) {
+    const trimmed = value?.trim();
+    if (!trimmed) {
+        throw new Error(`Fermata scenario suite LLM judge requires ${label}`);
+    }
+    return trimmed;
+}
+function optionalPositiveNumber(value, label, max) {
+    if (value === undefined)
+        return undefined;
+    if (!Number.isFinite(value) ||
+        value <= 0 ||
+        (max !== undefined && value > max)) {
+        throw new Error(`Fermata scenario suite LLM judge ${label} is invalid`);
+    }
+    return value;
+}
+function optionalPositiveInteger(value, label) {
+    if (value === undefined)
+        return undefined;
+    if (!Number.isInteger(value) || value <= 0) {
+        throw new Error(`Fermata scenario suite LLM judge ${label} is invalid`);
+    }
+    return value;
+}
+function optionalTrimmed(value) {
+    const trimmed = value?.trim();
+    return trimmed || undefined;
+}
+function llmRubricConfig(options) {
+    const repeat = optionalPositiveInteger(options.repeat, "repeat");
+    const quorum = optionalPositiveInteger(options.quorum, "quorum");
+    if (repeat !== undefined && quorum !== undefined && quorum > repeat) {
+        throw new Error("Fermata scenario suite LLM judge quorum cannot exceed repeat");
+    }
+    return {
+        judgeId: requireNonEmpty(options.judgeId, "judgeId"),
+        verifierJudgeId: options.verifierJudgeId?.trim() || undefined,
+        rubric: requireNonEmpty(options.rubric, "rubric"),
+        minScore: optionalPositiveNumber(options.minScore, "minScore", 1),
+        repeat,
+        quorum,
+        recordJudgeValidation: options.recordJudgeValidation ?? true,
+        requireCalibratedJudge: options.requireCalibratedJudge,
+        minJudgeValidationAccuracy: optionalPositiveNumber(options.minJudgeValidationAccuracy, "minJudgeValidationAccuracy", 1),
+        minJudgeValidationCount: optionalPositiveInteger(options.minJudgeValidationCount, "minJudgeValidationCount"),
+        rubricVersion: optionalTrimmed(options.rubricVersion),
+        calibrationCohort: optionalTrimmed(options.calibrationCohort),
+        advisoryOnly: options.advisoryOnly,
+    };
+}
+function llmPairwiseRubricConfig(options) {
+    const repeat = optionalPositiveInteger(options.repeat, "repeat");
+    const quorum = optionalPositiveInteger(options.quorum, "quorum");
+    if (repeat !== undefined && quorum !== undefined && quorum > repeat) {
+        throw new Error("Fermata scenario suite LLM judge quorum cannot exceed repeat");
+    }
+    return {
+        judgeId: requireNonEmpty(options.judgeId, "judgeId"),
+        verifierJudgeId: options.verifierJudgeId?.trim() || undefined,
+        rubric: requireNonEmpty(options.rubric, "rubric"),
+        baselineLabel: options.baselineLabel?.trim() || undefined,
+        candidateLabel: options.candidateLabel?.trim() || undefined,
+        minScore: optionalPositiveNumber(options.minScore, "minScore", 1),
+        repeat,
+        quorum,
+        recordJudgeValidation: options.recordJudgeValidation ?? true,
+        requireCalibratedJudge: options.requireCalibratedJudge,
+        minJudgeValidationAccuracy: optionalPositiveNumber(options.minJudgeValidationAccuracy, "minJudgeValidationAccuracy", 1),
+        minJudgeValidationCount: optionalPositiveInteger(options.minJudgeValidationCount, "minJudgeValidationCount"),
+        rubricVersion: optionalTrimmed(options.rubricVersion),
+        calibrationCohort: optionalTrimmed(options.calibrationCohort),
+        advisoryOnly: options.advisoryOnly,
+    };
+}
+function llmJudgeMetadata(options) {
+    const rubricVersion = optionalTrimmed(options.rubricVersion);
+    const calibrationCohort = optionalTrimmed(options.calibrationCohort);
+    if (options.advisoryOnly === undefined &&
+        !rubricVersion &&
+        !calibrationCohort) {
+        return {};
+    }
+    return {
+        judge_mode: options.advisoryOnly ? "advisory" : "blocking",
+        ...(rubricVersion ? { rubric_version: rubricVersion } : {}),
+        ...(calibrationCohort ? { calibration_cohort: calibrationCohort } : {}),
+    };
+}
+function realScenarioResultJSON(result) {
+    return JSON.stringify(result, null, "\t");
+}
+function uniqueValues(values) {
+    const seen = new Set();
+    for (const value of values) {
+        const trimmed = value?.trim();
+        if (trimmed)
+            seen.add(trimmed);
+    }
+    return [...seen];
+}
+function agentTrajectoryConfig(expected) {
+    return {
+        requiredEvents: uniqueValues(expected.provenance?.map((event) => event.eventType) ?? []),
+        requiredAssertionStatuses: expected.assertions.map((assertion) => ({
+            id: assertion.id,
+            status: assertion.status,
+        })),
+        maxEvents: expected.counts.events,
+        maxToolCalls: expected.counts.toolCalls,
+        maxReplayDeltas: expected.counts.replayDeltas,
+        maxScoreFailures: expected.counts.scoreFailures,
+        maxScoreWarnings: expected.counts.scoreWarnings,
+        requireIdempotentReplay: true,
+        forbidDuplicateExternalActions: true,
+        requiredTraceJoinKeys: expected.platform.traceJoinKeys,
+    };
+}
+function agentTrajectoryCase(expected, actual) {
+    const scenarioId = expected.scenario.id;
+    const id = stableId(scenarioId, "agent-trajectory", "native-trajectory-guard");
+    return {
+        id,
+        name: `${scenarioId} native trajectory guard`,
+        input: serializeCaseInput({
+            scenarioId,
+            title: expected.scenario.title,
+            platformPrimitive: actual.platform.primitive,
+            expectedRunId: expected.run.id,
+            actualRunId: actual.run.id,
+            traceJoinKeys: actual.platform.traceJoinKeys,
+            expectedAssertionIds: expected.assertions.map((assertion) => assertion.id),
+            actualAssertionIds: actual.assertions.map((assertion) => assertion.id),
+        }),
+        expectedOutput: realScenarioResultJSON(expected),
+        metadata: {
+            actual_output: realScenarioResultJSON(actual),
+            case_kind: "scenario_agent_trajectory",
+            ci_tier: "core-regression",
+            scenario_id: scenarioId,
+            expected_run_id: expected.run.id,
+            actual_run_id: actual.run.id,
+            expected_assertion_count: expected.counts.assertions,
+            actual_assertion_count: actual.counts.assertions,
+            source: "maestro.agent_trajectory_scenario_result",
+        },
+        assertions: [
+            {
+                id: `${id}.agent-trajectory`,
+                kind: "ASSERTION_KIND_AGENT_TRAJECTORY",
+                target: "response",
+                description: "Native Fermata trajectory assertion evaluates real Maestro replay results for event, assertion, budget, idempotency, and trace-link invariants.",
+                agentTrajectory: agentTrajectoryConfig(expected),
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                    case_kind: "scenario_agent_trajectory",
+                    ci_tier: "core-regression",
+                },
+            },
+        ],
+    };
+}
+function llmRubricCase(expected, actual, options) {
+    const scenarioId = expected.scenario.id;
+    const id = stableId(scenarioId, "llm-rubric", "semantic-trajectory-quality");
+    return {
+        id,
+        name: `${scenarioId} semantic trajectory quality`,
+        input: serializeCaseInput({
+            scenarioId,
+            title: expected.scenario.title,
+            platformPrimitive: actual.platform.primitive,
+            expectedRunId: expected.run.id,
+            actualRunId: actual.run.id,
+            traceJoinKeys: actual.platform.traceJoinKeys,
+            expectedAssertionIds: expected.assertions.map((assertion) => assertion.id),
+            actualAssertionIds: actual.assertions.map((assertion) => assertion.id),
+        }),
+        expectedOutput: realScenarioResultJSON(expected),
+        metadata: {
+            actual_output: realScenarioResultJSON(actual),
+            case_kind: "scenario_llm_rubric",
+            ...llmJudgeMetadata(options),
+            scenario_id: scenarioId,
+            expected_run_id: expected.run.id,
+            actual_run_id: actual.run.id,
+            expected_assertion_count: expected.counts.assertions,
+            actual_assertion_count: actual.counts.assertions,
+            source: "maestro.agent_trajectory_scenario_result",
+        },
+        assertions: [
+            {
+                id: `${id}.llm-rubric`,
+                kind: "ASSERTION_KIND_LLM_RUBRIC",
+                target: "response",
+                description: "Cataloged Fermata LLM judge evaluates semantic trajectory quality against the committed real Maestro scenario result.",
+                llmRubric: llmRubricConfig(options),
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                    case_kind: "scenario_llm_rubric",
+                    ...llmJudgeMetadata(options),
+                },
+            },
+        ],
+    };
+}
+function llmPairwiseRubricCase(expected, actual, options) {
+    const scenarioId = expected.scenario.id;
+    const id = stableId(scenarioId, "llm-pairwise-rubric", "semantic-trajectory-preference");
+    return {
+        id,
+        name: `${scenarioId} semantic trajectory preference`,
+        input: serializeCaseInput({
+            scenarioId,
+            title: expected.scenario.title,
+            platformPrimitive: actual.platform.primitive,
+            baselineRunId: expected.run.id,
+            candidateRunId: actual.run.id,
+            traceJoinKeys: actual.platform.traceJoinKeys,
+            baselineAssertionIds: expected.assertions.map((assertion) => assertion.id),
+            candidateAssertionIds: actual.assertions.map((assertion) => assertion.id),
+        }),
+        expectedOutput: realScenarioResultJSON(expected),
+        metadata: {
+            actual_output: realScenarioResultJSON(actual),
+            case_kind: "scenario_llm_pairwise_rubric",
+            ...llmJudgeMetadata(options),
+            scenario_id: scenarioId,
+            baseline_run_id: expected.run.id,
+            candidate_run_id: actual.run.id,
+            baseline_assertion_count: expected.counts.assertions,
+            candidate_assertion_count: actual.counts.assertions,
+            source: "maestro.agent_trajectory_scenario_result",
+        },
+        assertions: [
+            {
+                id: `${id}.llm-pairwise-rubric`,
+                kind: "ASSERTION_KIND_LLM_PAIRWISE_RUBRIC",
+                target: "response",
+                description: "Cataloged Fermata LLM judge compares real Maestro trajectory outputs and prefers the stronger candidate.",
+                llmPairwiseRubric: llmPairwiseRubricConfig(options),
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                    case_kind: "scenario_llm_pairwise_rubric",
+                    ...llmJudgeMetadata(options),
+                },
+            },
+        ],
+    };
+}
+function equalsCase(id, name, input, expected, actual, metadata) {
+    return {
+        id,
+        name,
+        input: serializeCaseInput(input),
+        expectedOutput: expected,
+        metadata: {
+            ...metadata,
+            actual_output: actual,
+        },
+        assertions: [
+            {
+                id: `${id}.equals`,
+                kind: "ASSERTION_KIND_EQUALS",
+                target: "response",
+                expected,
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                },
+            },
+        ],
+    };
+}
+function outcomeCase(expected, actual) {
+    const scenarioId = expected.scenario.id;
+    return equalsCase(stableId(scenarioId, "observed-outcome"), `${scenarioId} observed outcome`, {
+        scenarioId,
+        title: expected.scenario.title,
+        expectedOutcome: expected.scenario.expectedOutcome,
+        committedObservedOutcome: expected.scenario.observedOutcome,
+        actualObservedOutcome: actual.scenario.observedOutcome,
+    }, expected.scenario.observedOutcome, actual.scenario.observedOutcome, {
+        case_kind: "scenario_observed_outcome",
+        scenario_id: scenarioId,
+        expected_outcome: expected.scenario.expectedOutcome,
+        expected_observed_outcome: expected.scenario.observedOutcome,
+        actual_observed_outcome: actual.scenario.observedOutcome,
+        review_labels: actual.scenario.reviewLabels,
+    });
+}
+function countCases(expected, actual) {
+    return COUNT_KEYS.map((key) => equalsCase(stableId(expected.scenario.id, "count", key), `${expected.scenario.id} ${key} count`, {
+        scenarioId: expected.scenario.id,
+        count: key,
+        expected: expected.counts[key],
+        actual: actual.counts[key],
+    }, String(expected.counts[key]), String(actual.counts[key]), {
+        case_kind: "scenario_count",
+        scenario_id: expected.scenario.id,
+        count: key,
+        expected_count: expected.counts[key],
+        actual_count: actual.counts[key],
+    }));
+}
+function assertionCases(expected, actual) {
+    const actualById = assertionMap(actual);
+    return expected.assertions.map((expectedAssertion) => {
+        const actualAssertion = actualById.get(expectedAssertion.id);
+        const actualStatus = actualAssertion?.status ?? "missing";
+        return equalsCase(stableId(expected.scenario.id, "assertion", expectedAssertion.id), `${expected.scenario.id} ${expectedAssertion.id}`, {
+            scenarioId: expected.scenario.id,
+            assertionId: expectedAssertion.id,
+            kind: expectedAssertion.kind,
+            expectedStatus: expectedAssertion.status,
+            actualStatus,
+            expectedMessage: expectedAssertion.message,
+            actualMessage: actualAssertion?.message ?? null,
+        }, expectedAssertion.status, actualStatus, {
+            case_kind: "scenario_assertion_status",
+            scenario_id: expected.scenario.id,
+            assertion_id: expectedAssertion.id,
+            assertion_kind: expectedAssertion.kind,
+            severity: expectedAssertion.severity,
+            expected_message: expectedAssertion.message,
+            actual_message: actualAssertion?.message,
+            expected_evidence: evidenceLabels(expectedAssertion),
+            actual_evidence: evidenceLabels(actualAssertion),
+        });
+    });
+}
+function suiteMetadata(expected, actual, extra) {
+    return {
+        source: "maestro.agent_trajectory_scenario_result",
+        schema_version: actual.schemaVersion,
+        scenario_schema_version: actual.scenarioSchemaVersion,
+        scenario_id: actual.scenario.id,
+        scenario_title: actual.scenario.title,
+        expected_run_id: expected.run.id,
+        actual_run_id: actual.run.id,
+        expected_outcome: expected.scenario.expectedOutcome,
+        committed_observed_outcome: expected.scenario.observedOutcome,
+        actual_observed_outcome: actual.scenario.observedOutcome,
+        platform_primitive: actual.platform.primitive,
+        evidence_event_type: actual.platform.evidenceEventType,
+        trace_join_keys: actual.platform.traceJoinKeys,
+        ...(isObject(extra) ? extra : {}),
+    };
+}
+export function buildFermataCreateScenarioSuiteRequest(config, expected, actual, options = {}) {
+    requireSameScenario(expected, actual);
+    if (!config.organizationId || !config.workspaceId) {
+        throw new Error("Fermata scenario suite requires organizationId and workspaceId");
+    }
+    const scenarioId = expected.scenario.id;
+    return {
+        organizationId: config.organizationId,
+        workspaceId: config.workspaceId,
+        name: options.name ?? `Maestro scenario: ${scenarioId}`,
+        description: options.description ??
+            `Recorded Maestro agent trajectory scenario suite for ${scenarioId}.`,
+        cases: [
+            outcomeCase(expected, actual),
+            ...countCases(expected, actual),
+            ...assertionCases(expected, actual),
+            agentTrajectoryCase(expected, actual),
+            ...(options.llmJudge
+                ? [llmRubricCase(expected, actual, options.llmJudge)]
+                : []),
+            ...(options.llmPairwiseJudge
+                ? [llmPairwiseRubricCase(expected, actual, options.llmPairwiseJudge)]
+                : []),
+        ],
+        metadata: suiteMetadata(expected, actual, options.metadata),
+    };
+}
+export function buildFermataRunScenarioSuiteRequest(config, suiteId, expected, actual, options = {}) {
+    if (!config.organizationId || !config.workspaceId) {
+        throw new Error("Fermata scenario suite run requires organizationId and workspaceId");
+    }
+    const scenarioId = actual.scenario.id;
+    const runId = stableId(scenarioId, actual.run.id);
+    const model = options.model ?? "maestro-agent-trajectory-replay";
+    return {
+        suiteId,
+        organizationId: config.organizationId,
+        workspaceId: config.workspaceId,
+        evaluationId: options.evaluationId ?? `maestro-${runId}`,
+        runIdPrefix: options.runIdPrefix ?? stableId("maestro", scenarioId),
+        lineageId: options.lineageId ?? actual.run.id,
+        traceId: options.traceId,
+        maxConcurrency: options.maxConcurrency ?? 1,
+        stopOnFirstFailure: false,
+        runContext: {
+            source: "maestro.agent_trajectory_scenario_result",
+            scenarioId,
+            runId: actual.run.id,
+            sessionId: actual.run.sessionId,
+            platformPrimitive: actual.platform.primitive,
+            traceJoinKeys: actual.platform.traceJoinKeys,
+        },
+        metadata: suiteMetadata(expected, actual, options.metadata),
+        candidates: [
+            {
+                candidateId: options.candidateId ?? stableId("maestro-recorded", scenarioId),
+                label: options.candidateLabel ?? "Maestro recorded trajectory",
+                model,
+                metadata: {
+                    source: "maestro.agent_trajectory_scenario_result",
+                    scenario_id: scenarioId,
+                    run_id: actual.run.id,
+                },
+            },
+        ],
+    };
+}
+//# sourceMappingURL=fermata-scenario-suite.js.map

package/dist/platform/fermata-scenario-suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fermata-scenario-suite.js","sourceRoot":"","sources":["../../src/platform/fermata-scenario-suite.ts"],"names":[],"mappings":"AA0EA,MAAM,UAAU,GAAG;IAClB,YAAY;IACZ,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,QAAQ;IACR,WAAW;IACX,cAAc;IACd,eAAe;IACf,eAAe;CACN,CAAC;AAwCX,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,QAAQ,CAAC,GAAG,KAAe;IACnC,MAAM,EAAE,GAAG,KAAK;SACd,IAAI,CAAC,GAAG,CAAC;SACT,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACxB,OAAO,EAAE,IAAI,kBAAkB,CAAC;AACjC,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAqC;IACjE,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC;AACnD,CAAC;AAED,SAAS,mBAAmB,CAC3B,QAAuC,EACvC,MAAqC;IAErC,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CACd,mEAAmE,mBAAmB,CACrF,QAAQ,CACR,cAAc,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAC5C,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,aAAa,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CACd,uEAAuE,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,CAC7F,CAAC;IACH,CAAC;AACF,CAAC;AAED,SAAS,kBAAkB,CAAC,KAA8B;IACzD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,YAAY,CACpB,MAAqC;IAErC,OAAO,IAAI,GAAG,CACb,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC,CAC/D,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACtB,SAA6D;IAE7D,OAAO,CAAC,SAAS,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CACrC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,EAAE,EAAE,CAClE,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB,EAAE,KAAa;IAChE,MAAM,OAAO,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,6CAA6C,KAAK,EAAE,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,sBAAsB,CAC9B,KAAyB,EACzB,KAAa,EACb,GAAY;IAEZ,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IACC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QACvB,KAAK,IAAI,CAAC;QACV,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,GAAG,CAAC,EACjC,CAAC;QACF,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,aAAa,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,uBAAuB,CAC/B,KAAyB,EACzB,KAAa;IAEb,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,aAAa,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IACjD,MAAM,OAAO,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC;IAC9B,OAAO,OAAO,IAAI,SAAS,CAAC;AAC7B,CAAC;AAED,SAAS,eAAe,CACvB,OAA4C;IAE5C,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjE,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,GAAG,MAAM,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CACd,8DAA8D,CAC9D,CAAC;IACH,CAAC;IACD,OAAO;QACN,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC;QACpD,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE,IAAI,EAAE,IAAI,SAAS;QAC7D,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC;QACjD,QAAQ,EAAE,sBAAsB,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;QACjE,MAAM;QACN,MAAM;QACN,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,IAAI,IAAI;QAC5D,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;QACtD,0BAA0B,EAAE,sBAAsB,CACjD,OAAO,CAAC,0BAA0B,EAClC,4BAA4B,EAC5B,CAAC,CACD;QACD,uBAAuB,EAAE,uBAAuB,CAC/C,OAAO,CAAC,uBAAuB,EAC/B,yBAAyB,CACzB;QACD,aAAa,EAAE,eAAe,CAAC,OAAO,CAAC,aAAa,CAAC;QACrD,iBAAiB,EAAE,eAAe,CAAC,OAAO,CAAC,iBAAiB,CAAC;QAC7D,YAAY,EAAE,OAAO,CAAC,YAAY;KAClC,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAC/B,OAAoD;IAEpD,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjE,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,GAAG,MAAM,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CACd,8DAA8D,CAC9D,CAAC;IACH,CAAC;IACD,OAAO;QACN,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC;QACpD,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE,IAAI,EAAE,IAAI,SAAS;QAC7D,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC;QACjD,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,SAAS;QACzD,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,SAAS;QAC3D,QAAQ,EAAE,sBAAsB,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;QACjE,MAAM;QACN,MAAM;QACN,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,IAAI,IAAI;QAC5D,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;QACtD,0BAA0B,EAAE,sBAAsB,CACjD,OAAO,CAAC,0BAA0B,EAClC,4BAA4B,EAC5B,CAAC,CACD;QACD,uBAAuB,EAAE,uBAAuB,CAC/C,OAAO,CAAC,uBAAuB,EAC/B,yBAAyB,CACzB;QACD,aAAa,EAAE,eAAe,CAAC,OAAO,CAAC,aAAa,CAAC;QACrD,iBAAiB,EAAE,eAAe,CAAC,OAAO,CAAC,iBAAiB,CAAC;QAC7D,YAAY,EAAE,OAAO,CAAC,YAAY;KAClC,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CACxB,OAA4C;IAE5C,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,iBAAiB,GAAG,eAAe,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACrE,IACC,OAAO,CAAC,YAAY,KAAK,SAAS;QAClC,CAAC,aAAa;QACd,CAAC,iBAAiB,EACjB,CAAC;QACF,OAAO,EAAE,CAAC;IACX,CAAC;IACD,OAAO;QACN,UAAU,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU;QAC1D,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAqC;IACpE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAAoC;IACzD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC;QAC9B,IAAI,OAAO;YAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,qBAAqB,CAC7B,QAAuC;IAEvC,OAAO;QACN,cAAc,EAAE,YAAY,CAC3B,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAC1D;QACD,yBAAyB,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YAClE,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,MAAM,EAAE,SAAS,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM;QACjC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS;QACvC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,YAAY;QAC7C,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,aAAa;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,aAAa;QAC/C,uBAAuB,EAAE,IAAI;QAC7B,8BAA8B,EAAE,IAAI;QACpC,qBAAqB,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa;KACtD,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC3B,QAAuC,EACvC,MAAqC;IAErC,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxC,MAAM,EAAE,GAAG,QAAQ,CAClB,UAAU,EACV,kBAAkB,EAClB,yBAAyB,CACzB,CAAC;IACF,OAAO;QACN,EAAE;QACF,IAAI,EAAE,GAAG,UAAU,0BAA0B;QAC7C,KAAK,EAAE,kBAAkB,CAAC;YACzB,UAAU;YACV,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK;YAC9B,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;YAC5C,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC9B,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;YAC1B,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa;YAC5C,oBAAoB,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAC5C,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAC3B;YACD,kBAAkB,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;SACtE,CAAC;QACF,cAAc,EAAE,sBAAsB,CAAC,QAAQ,CAAC;QAChD,QAAQ,EAAE;YACT,aAAa,EAAE,sBAAsB,CAAC,MAAM,CAAC;YAC7C,SAAS,EAAE,2BAA2B;YACtC,OAAO,EAAE,iBAAiB;YAC1B,WAAW,EAAE,UAAU;YACvB,eAAe,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YAChC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;YAC5B,wBAAwB,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;YACpD,sBAAsB,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU;YAChD,MAAM,EAAE,0CAA0C;SAClD;QACD,UAAU,EAAE;YACX;gBACC,EAAE,EAAE,GAAG,EAAE,mBAAmB;gBAC5B,IAAI,EAAE,iCAAiC;gBACvC,MAAM,EAAE,UAAU;gBAClB,WAAW,EACV,iJAAiJ;gBAClJ,eAAe,EAAE,qBAAqB,CAAC,QAAQ,CAAC;gBAChD,QAAQ,EAAE;oBACT,MAAM,EAAE,0CAA0C;oBAClD,SAAS,EAAE,2BAA2B;oBACtC,OAAO,EAAE,iBAAiB;iBAC1B;aACD;SACD;KACD,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CACrB,QAAuC,EACvC,MAAqC,EACrC,OAA4C;IAE5C,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxC,MAAM,EAAE,GAAG,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,6BAA6B,CAAC,CAAC;IAC7E,OAAO;QACN,EAAE;QACF,IAAI,EAAE,GAAG,UAAU,8BAA8B;QACjD,KAAK,EAAE,kBAAkB,CAAC;YACzB,UAAU;YACV,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK;YAC9B,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;YAC5C,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC9B,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;YAC1B,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa;YAC5C,oBAAoB,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAC5C,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAC3B;YACD,kBAAkB,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;SACtE,CAAC;QACF,cAAc,EAAE,sBAAsB,CAAC,QAAQ,CAAC;QAChD,QAAQ,EAAE;YACT,aAAa,EAAE,sBAAsB,CAAC,MAAM,CAAC;YAC7C,SAAS,EAAE,qBAAqB;YAChC,GAAG,gBAAgB,CAAC,OAAO,CAAC;YAC5B,WAAW,EAAE,UAAU;YACvB,eAAe,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YAChC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;YAC5B,wBAAwB,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;YACpD,sBAAsB,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU;YAChD,MAAM,EAAE,0CAA0C;SAClD;QACD,UAAU,EAAE;YACX;gBACC,EAAE,EAAE,GAAG,EAAE,aAAa;gBACtB,IAAI,EAAE,2BAA2B;gBACjC,MAAM,EAAE,UAAU;gBAClB,WAAW,EACV,uHAAuH;gBACxH,SAAS,EAAE,eAAe,CAAC,OAAO,CAAC;gBACnC,QAAQ,EAAE;oBACT,MAAM,EAAE,0CAA0C;oBAClD,SAAS,EAAE,qBAAqB;oBAChC,GAAG,gBAAgB,CAAC,OAAO,CAAC;iBAC5B;aACD;SACD;KACD,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC7B,QAAuC,EACvC,MAAqC,EACrC,OAAoD;IAEpD,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxC,MAAM,EAAE,GAAG,QAAQ,CAClB,UAAU,EACV,qBAAqB,EACrB,gCAAgC,CAChC,CAAC;IACF,OAAO;QACN,EAAE;QACF,IAAI,EAAE,GAAG,UAAU,iCAAiC;QACpD,KAAK,EAAE,kBAAkB,CAAC;YACzB,UAAU;YACV,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK;YAC9B,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;YAC5C,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC9B,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7B,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa;YAC5C,oBAAoB,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAC5C,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAC3B;YACD,qBAAqB,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;SACzE,CAAC;QACF,cAAc,EAAE,sBAAsB,CAAC,QAAQ,CAAC;QAChD,QAAQ,EAAE;YACT,aAAa,EAAE,sBAAsB,CAAC,MAAM,CAAC;YAC7C,SAAS,EAAE,8BAA8B;YACzC,GAAG,gBAAgB,CAAC,OAAO,CAAC;YAC5B,WAAW,EAAE,UAAU;YACvB,eAAe,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YAChC,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;YAC/B,wBAAwB,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;YACpD,yBAAyB,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU;YACnD,MAAM,EAAE,0CAA0C;SAClD;QACD,UAAU,EAAE;YACX;gBACC,EAAE,EAAE,GAAG,EAAE,sBAAsB;gBAC/B,IAAI,EAAE,oCAAoC;gBAC1C,MAAM,EAAE,UAAU;gBAClB,WAAW,EACV,0GAA0G;gBAC3G,iBAAiB,EAAE,uBAAuB,CAAC,OAAO,CAAC;gBACnD,QAAQ,EAAE;oBACT,MAAM,EAAE,0CAA0C;oBAClD,SAAS,EAAE,8BAA8B;oBACzC,GAAG,gBAAgB,CAAC,OAAO,CAAC;iBAC5B;aACD;SACD;KACD,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAClB,EAAU,EACV,IAAY,EACZ,KAA8B,EAC9B,QAAgB,EAChB,MAAc,EACd,QAAiC;IAEjC,OAAO;QACN,EAAE;QACF,IAAI;QACJ,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC;QAChC,cAAc,EAAE,QAAQ;QACxB,QAAQ,EAAE;YACT,GAAG,QAAQ;YACX,aAAa,EAAE,MAAM;SACrB;QACD,UAAU,EAAE;YACX;gBACC,EAAE,EAAE,GAAG,EAAE,SAAS;gBAClB,IAAI,EAAE,uBAAuB;gBAC7B,MAAM,EAAE,UAAU;gBAClB,QAAQ;gBACR,QAAQ,EAAE;oBACT,MAAM,EAAE,0CAA0C;iBAClD;aACD;SACD;KACD,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CACnB,QAAuC,EACvC,MAAqC;IAErC,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxC,OAAO,UAAU,CAChB,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAC,EACxC,GAAG,UAAU,mBAAmB,EAChC;QACC,UAAU;QACV,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK;QAC9B,eAAe,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe;QAClD,wBAAwB,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe;QAC3D,qBAAqB,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe;KACtD,EACD,QAAQ,CAAC,QAAQ,CAAC,eAAe,EACjC,MAAM,CAAC,QAAQ,CAAC,eAAe,EAC/B;QACC,SAAS,EAAE,2BAA2B;QACtC,WAAW,EAAE,UAAU;QACvB,gBAAgB,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe;QACnD,yBAAyB,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe;QAC5D,uBAAuB,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe;QACxD,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY;KAC3C,CACD,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAClB,QAAuC,EACvC,MAAqC;IAErC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC7B,UAAU,CACT,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,EAC5C,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,IAAI,GAAG,QAAQ,EACtC;QACC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAChC,KAAK,EAAE,GAAG;QACV,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC;QAC9B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;KAC1B,EACD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAC5B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAC1B;QACC,SAAS,EAAE,gBAAgB;QAC3B,WAAW,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE;QACjC,KAAK,EAAE,GAAG;QACV,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC;QACpC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;KAChC,CACD,CACD,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACtB,QAAuC,EACvC,MAAqC;IAErC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,EAAE;QACpD,MAAM,eAAe,GAAG,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,eAAe,EAAE,MAAM,IAAI,SAAS,CAAC;QAC1D,OAAO,UAAU,CAChB,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,WAAW,EAAE,iBAAiB,CAAC,EAAE,CAAC,EACjE,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,IAAI,iBAAiB,CAAC,EAAE,EAAE,EACjD;YACC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE;YAChC,WAAW,EAAE,iBAAiB,CAAC,EAAE;YACjC,IAAI,EAAE,iBAAiB,CAAC,IAAI;YAC5B,cAAc,EAAE,iBAAiB,CAAC,MAAM;YACxC,YAAY;YACZ,eAAe,EAAE,iBAAiB,CAAC,OAAO;YAC1C,aAAa,EAAE,eAAe,EAAE,OAAO,IAAI,IAAI;SAC/C,EACD,iBAAiB,CAAC,MAAM,EACxB,YAAY,EACZ;YACC,SAAS,EAAE,2BAA2B;YACtC,WAAW,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACjC,YAAY,EAAE,iBAAiB,CAAC,EAAE;YAClC,cAAc,EAAE,iBAAiB,CAAC,IAAI;YACtC,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;YACpC,gBAAgB,EAAE,iBAAiB,CAAC,OAAO;YAC3C,cAAc,EAAE,eAAe,EAAE,OAAO;YACxC,iBAAiB,EAAE,cAAc,CAAC,iBAAiB,CAAC;YACpD,eAAe,EAAE,cAAc,CAAC,eAAe,CAAC;SAChD,CACD,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACrB,QAAuC,EACvC,MAAqC,EACrC,KAA0C;IAE1C,OAAO;QACN,MAAM,EAAE,0CAA0C;QAClD,cAAc,EAAE,MAAM,CAAC,aAAa;QACpC,uBAAuB,EAAE,MAAM,CAAC,qBAAqB;QACrD,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE;QAC/B,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK;QACrC,eAAe,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;QAChC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;QAC5B,gBAAgB,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe;QACnD,0BAA0B,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe;QAC7D,uBAAuB,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe;QACxD,kBAAkB,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;QAC7C,mBAAmB,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB;QACtD,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa;QAC9C,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;KACjC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sCAAsC,CACrD,MAAwE,EACxE,QAAuC,EACvC,MAAqC,EACrC,UAAuC,EAAE;IAEzC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACd,gEAAgE,CAChE,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxC,OAAO;QACN,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,qBAAqB,UAAU,EAAE;QACvD,WAAW,EACV,OAAO,CAAC,WAAW;YACnB,wDAAwD,UAAU,GAAG;QACtE,KAAK,EAAE;YACN,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC7B,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC/B,GAAG,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC;YACnC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC;YACrC,GAAG,CAAC,OAAO,CAAC,QAAQ;gBACnB,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;gBACrD,CAAC,CAAC,EAAE,CAAC;YACN,GAAG,CAAC,OAAO,CAAC,gBAAgB;gBAC3B,CAAC,CAAC,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;gBACrE,CAAC,CAAC,EAAE,CAAC;SACN;QACD,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC;KAC3D,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mCAAmC,CAClD,MAAwE,EACxE,OAAe,EACf,QAAuC,EACvC,MAAqC,EACrC,UAAuC,EAAE;IAEzC,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACd,oEAAoE,CACpE,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,iCAAiC,CAAC;IACjE,OAAO;QACN,OAAO;QACP,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,WAAW,KAAK,EAAE;QACxD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;QACnE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;QAC7C,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,CAAC;QAC3C,kBAAkB,EAAE,KAAK;QACzB,UAAU,EAAE;YACX,MAAM,EAAE,0CAA0C;YAClD,UAAU;YACV,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;YACpB,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS;YAC/B,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;YAC5C,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa;SAC5C;QACD,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC;QAC3D,UAAU,EAAE;YACX;gBACC,WAAW,EACV,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAC,kBAAkB,EAAE,UAAU,CAAC;gBAChE,KAAK,EAAE,OAAO,CAAC,cAAc,IAAI,6BAA6B;gBAC9D,KAAK;gBACL,QAAQ,EAAE;oBACT,MAAM,EAAE,0CAA0C;oBAClD,WAAW,EAAE,UAAU;oBACvB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;iBACrB;aACD;SACD;KACD,CAAC;AACH,CAAC","sourcesContent":["import type {\n\tCreateFermataTestSuiteRequest,\n\tFermataAgentTrajectoryAssertion,\n\tFermataEvalServiceConfig,\n\tFermataLLMPairwiseRubricAssertion,\n\tFermataLLMRubricAssertion,\n\tFermataTestCase,\n\tRunFermataTestSuiteRequest,\n} from \"./fermata-eval-client.js\";\n\ntype AgentTrajectoryScenarioAssertionStatus = \"pass\" | \"fail\" | \"warn\";\n\ninterface AgentTrajectoryScenarioEvidence {\n\tkind: string;\n\tid: string;\n\tsource: string;\n\tlabel: string;\n}\n\ninterface AgentTrajectoryScenarioAssertionResult {\n\tid: string;\n\tkind: string;\n\tstatus: AgentTrajectoryScenarioAssertionStatus;\n\tseverity: string;\n\tmessage: string;\n\tevidence: AgentTrajectoryScenarioEvidence[];\n}\n\ninterface AgentTrajectoryScenarioResult {\n\tschemaVersion: string;\n\tscenarioSchemaVersion: string;\n\tscenario: {\n\t\tid: string;\n\t\ttitle: string;\n\t\texpectedOutcome: string;\n\t\tobservedOutcome: string;\n\t\treviewLabels: string[];\n\t};\n\trun: {\n\t\tid: string;\n\t\tsessionId: string;\n\t\tsource?: string;\n\t\tplatformBacked?: boolean;\n\t\tscenarioId?: string;\n\t\treplay?: boolean;\n\t\tgeneratedAt?: string;\n\t};\n\tcounts: {\n\t\tassertions: number;\n\t\tpassed: number;\n\t\tfailed: number;\n\t\twarnings: number;\n\t\tevents: number;\n\t\ttoolCalls: number;\n\t\treplayDeltas: number;\n\t\tscoreFailures: number;\n\t\tscoreWarnings: number;\n\t};\n\tplatform: {\n\t\tprimitive: string;\n\t\tevidenceEventType: string;\n\t\ttraceJoinKeys: string[];\n\t\teventType?: string;\n\t\trationale?: string;\n\t};\n\tassertions: AgentTrajectoryScenarioAssertionResult[];\n\tprovenance?: Array<{\n\t\teventId: string;\n\t\teventType: string;\n\t\tphase?: string;\n\t\tactor?: string;\n\t}>;\n}\n\nconst COUNT_KEYS = [\n\t\"assertions\",\n\t\"passed\",\n\t\"failed\",\n\t\"warnings\",\n\t\"events\",\n\t\"toolCalls\",\n\t\"replayDeltas\",\n\t\"scoreFailures\",\n\t\"scoreWarnings\",\n] as const;\n\nexport interface FermataScenarioSuiteLLMJudgeOptions {\n\tjudgeId: string;\n\tverifierJudgeId?: string;\n\trubric: string;\n\tminScore?: number;\n\trepeat?: number;\n\tquorum?: number;\n\trecordJudgeValidation?: boolean;\n\trequireCalibratedJudge?: boolean;\n\tminJudgeValidationAccuracy?: number;\n\tminJudgeValidationCount?: number;\n\trubricVersion?: string;\n\tcalibrationCohort?: string;\n\tadvisoryOnly?: boolean;\n}\n\nexport interface FermataScenarioSuiteLLMPairwiseJudgeOptions\n\textends FermataScenarioSuiteLLMJudgeOptions {\n\tbaselineLabel?: string;\n\tcandidateLabel?: string;\n}\n\nexport interface FermataScenarioSuiteOptions {\n\tname?: string;\n\tdescription?: string;\n\tmodel?: string;\n\tcandidateId?: string;\n\tcandidateLabel?: string;\n\tevaluationId?: string;\n\trunIdPrefix?: string;\n\tlineageId?: string;\n\ttraceId?: string;\n\tmaxConcurrency?: number;\n\tmetadata?: Record<string, unknown>;\n\tllmJudge?: FermataScenarioSuiteLLMJudgeOptions;\n\tllmPairwiseJudge?: FermataScenarioSuiteLLMPairwiseJudgeOptions;\n}\n\nfunction isObject(value: unknown): value is Record<string, unknown> {\n\treturn typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction stableId(...parts: string[]): string {\n\tconst id = parts\n\t\t.join(\".\")\n\t\t.toLowerCase()\n\t\t.replace(/[^a-z0-9._-]+/gu, \"-\")\n\t\t.replace(/^-+|-+$/gu, \"\")\n\t\t.replace(/--+/gu, \"-\");\n\treturn id || \"maestro-scenario\";\n}\n\nfunction scenarioResultLabel(result: AgentTrajectoryScenarioResult): string {\n\treturn `${result.scenario.id} (${result.run.id})`;\n}\n\nfunction requireSameScenario(\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n): void {\n\tif (expected.scenario.id !== actual.scenario.id) {\n\t\tthrow new Error(\n\t\t\t`Fermata scenario suite requires matching scenario ids; expected ${scenarioResultLabel(\n\t\t\t\texpected,\n\t\t\t)}, received ${scenarioResultLabel(actual)}`,\n\t\t);\n\t}\n\tif (expected.schemaVersion !== actual.schemaVersion) {\n\t\tthrow new Error(\n\t\t\t`Fermata scenario suite requires matching result schema versions for ${expected.scenario.id}`,\n\t\t);\n\t}\n}\n\nfunction serializeCaseInput(value: Record<string, unknown>): string {\n\treturn JSON.stringify(value, null, \"\\t\");\n}\n\nfunction assertionMap(\n\tresult: AgentTrajectoryScenarioResult,\n): Map<string, AgentTrajectoryScenarioAssertionResult> {\n\treturn new Map(\n\t\tresult.assertions.map((assertion) => [assertion.id, assertion]),\n\t);\n}\n\nfunction evidenceLabels(\n\tassertion: AgentTrajectoryScenarioAssertionResult | undefined,\n): string[] {\n\treturn (assertion?.evidence ?? []).map(\n\t\t(evidence) => `${evidence.source}:${evidence.kind}:${evidence.id}`,\n\t);\n}\n\nfunction requireNonEmpty(value: string | undefined, label: string): string {\n\tconst trimmed = value?.trim();\n\tif (!trimmed) {\n\t\tthrow new Error(`Fermata scenario suite LLM judge requires ${label}`);\n\t}\n\treturn trimmed;\n}\n\nfunction optionalPositiveNumber(\n\tvalue: number | undefined,\n\tlabel: string,\n\tmax?: number,\n): number | undefined {\n\tif (value === undefined) return undefined;\n\tif (\n\t\t!Number.isFinite(value) ||\n\t\tvalue <= 0 ||\n\t\t(max !== undefined && value > max)\n\t) {\n\t\tthrow new Error(`Fermata scenario suite LLM judge ${label} is invalid`);\n\t}\n\treturn value;\n}\n\nfunction optionalPositiveInteger(\n\tvalue: number | undefined,\n\tlabel: string,\n): number | undefined {\n\tif (value === undefined) return undefined;\n\tif (!Number.isInteger(value) || value <= 0) {\n\t\tthrow new Error(`Fermata scenario suite LLM judge ${label} is invalid`);\n\t}\n\treturn value;\n}\n\nfunction optionalTrimmed(value: string | undefined): string | undefined {\n\tconst trimmed = value?.trim();\n\treturn trimmed || undefined;\n}\n\nfunction llmRubricConfig(\n\toptions: FermataScenarioSuiteLLMJudgeOptions,\n): FermataLLMRubricAssertion {\n\tconst repeat = optionalPositiveInteger(options.repeat, \"repeat\");\n\tconst quorum = optionalPositiveInteger(options.quorum, \"quorum\");\n\tif (repeat !== undefined && quorum !== undefined && quorum > repeat) {\n\t\tthrow new Error(\n\t\t\t\"Fermata scenario suite LLM judge quorum cannot exceed repeat\",\n\t\t);\n\t}\n\treturn {\n\t\tjudgeId: requireNonEmpty(options.judgeId, \"judgeId\"),\n\t\tverifierJudgeId: options.verifierJudgeId?.trim() || undefined,\n\t\trubric: requireNonEmpty(options.rubric, \"rubric\"),\n\t\tminScore: optionalPositiveNumber(options.minScore, \"minScore\", 1),\n\t\trepeat,\n\t\tquorum,\n\t\trecordJudgeValidation: options.recordJudgeValidation ?? true,\n\t\trequireCalibratedJudge: options.requireCalibratedJudge,\n\t\tminJudgeValidationAccuracy: optionalPositiveNumber(\n\t\t\toptions.minJudgeValidationAccuracy,\n\t\t\t\"minJudgeValidationAccuracy\",\n\t\t\t1,\n\t\t),\n\t\tminJudgeValidationCount: optionalPositiveInteger(\n\t\t\toptions.minJudgeValidationCount,\n\t\t\t\"minJudgeValidationCount\",\n\t\t),\n\t\trubricVersion: optionalTrimmed(options.rubricVersion),\n\t\tcalibrationCohort: optionalTrimmed(options.calibrationCohort),\n\t\tadvisoryOnly: options.advisoryOnly,\n\t};\n}\n\nfunction llmPairwiseRubricConfig(\n\toptions: FermataScenarioSuiteLLMPairwiseJudgeOptions,\n): FermataLLMPairwiseRubricAssertion {\n\tconst repeat = optionalPositiveInteger(options.repeat, \"repeat\");\n\tconst quorum = optionalPositiveInteger(options.quorum, \"quorum\");\n\tif (repeat !== undefined && quorum !== undefined && quorum > repeat) {\n\t\tthrow new Error(\n\t\t\t\"Fermata scenario suite LLM judge quorum cannot exceed repeat\",\n\t\t);\n\t}\n\treturn {\n\t\tjudgeId: requireNonEmpty(options.judgeId, \"judgeId\"),\n\t\tverifierJudgeId: options.verifierJudgeId?.trim() || undefined,\n\t\trubric: requireNonEmpty(options.rubric, \"rubric\"),\n\t\tbaselineLabel: options.baselineLabel?.trim() || undefined,\n\t\tcandidateLabel: options.candidateLabel?.trim() || undefined,\n\t\tminScore: optionalPositiveNumber(options.minScore, \"minScore\", 1),\n\t\trepeat,\n\t\tquorum,\n\t\trecordJudgeValidation: options.recordJudgeValidation ?? true,\n\t\trequireCalibratedJudge: options.requireCalibratedJudge,\n\t\tminJudgeValidationAccuracy: optionalPositiveNumber(\n\t\t\toptions.minJudgeValidationAccuracy,\n\t\t\t\"minJudgeValidationAccuracy\",\n\t\t\t1,\n\t\t),\n\t\tminJudgeValidationCount: optionalPositiveInteger(\n\t\t\toptions.minJudgeValidationCount,\n\t\t\t\"minJudgeValidationCount\",\n\t\t),\n\t\trubricVersion: optionalTrimmed(options.rubricVersion),\n\t\tcalibrationCohort: optionalTrimmed(options.calibrationCohort),\n\t\tadvisoryOnly: options.advisoryOnly,\n\t};\n}\n\nfunction llmJudgeMetadata(\n\toptions: FermataScenarioSuiteLLMJudgeOptions,\n): Record<string, unknown> {\n\tconst rubricVersion = optionalTrimmed(options.rubricVersion);\n\tconst calibrationCohort = optionalTrimmed(options.calibrationCohort);\n\tif (\n\t\toptions.advisoryOnly === undefined &&\n\t\t!rubricVersion &&\n\t\t!calibrationCohort\n\t) {\n\t\treturn {};\n\t}\n\treturn {\n\t\tjudge_mode: options.advisoryOnly ? \"advisory\" : \"blocking\",\n\t\t...(rubricVersion ? { rubric_version: rubricVersion } : {}),\n\t\t...(calibrationCohort ? { calibration_cohort: calibrationCohort } : {}),\n\t};\n}\n\nfunction realScenarioResultJSON(result: AgentTrajectoryScenarioResult): string {\n\treturn JSON.stringify(result, null, \"\\t\");\n}\n\nfunction uniqueValues(values: Iterable<string | undefined>): string[] {\n\tconst seen = new Set<string>();\n\tfor (const value of values) {\n\t\tconst trimmed = value?.trim();\n\t\tif (trimmed) seen.add(trimmed);\n\t}\n\treturn [...seen];\n}\n\nfunction agentTrajectoryConfig(\n\texpected: AgentTrajectoryScenarioResult,\n): FermataAgentTrajectoryAssertion {\n\treturn {\n\t\trequiredEvents: uniqueValues(\n\t\t\texpected.provenance?.map((event) => event.eventType) ?? [],\n\t\t),\n\t\trequiredAssertionStatuses: expected.assertions.map((assertion) => ({\n\t\t\tid: assertion.id,\n\t\t\tstatus: assertion.status,\n\t\t})),\n\t\tmaxEvents: expected.counts.events,\n\t\tmaxToolCalls: expected.counts.toolCalls,\n\t\tmaxReplayDeltas: expected.counts.replayDeltas,\n\t\tmaxScoreFailures: expected.counts.scoreFailures,\n\t\tmaxScoreWarnings: expected.counts.scoreWarnings,\n\t\trequireIdempotentReplay: true,\n\t\tforbidDuplicateExternalActions: true,\n\t\trequiredTraceJoinKeys: expected.platform.traceJoinKeys,\n\t};\n}\n\nfunction agentTrajectoryCase(\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n): FermataTestCase {\n\tconst scenarioId = expected.scenario.id;\n\tconst id = stableId(\n\t\tscenarioId,\n\t\t\"agent-trajectory\",\n\t\t\"native-trajectory-guard\",\n\t);\n\treturn {\n\t\tid,\n\t\tname: `${scenarioId} native trajectory guard`,\n\t\tinput: serializeCaseInput({\n\t\t\tscenarioId,\n\t\t\ttitle: expected.scenario.title,\n\t\t\tplatformPrimitive: actual.platform.primitive,\n\t\t\texpectedRunId: expected.run.id,\n\t\t\tactualRunId: actual.run.id,\n\t\t\ttraceJoinKeys: actual.platform.traceJoinKeys,\n\t\t\texpectedAssertionIds: expected.assertions.map(\n\t\t\t\t(assertion) => assertion.id,\n\t\t\t),\n\t\t\tactualAssertionIds: actual.assertions.map((assertion) => assertion.id),\n\t\t}),\n\t\texpectedOutput: realScenarioResultJSON(expected),\n\t\tmetadata: {\n\t\t\tactual_output: realScenarioResultJSON(actual),\n\t\t\tcase_kind: \"scenario_agent_trajectory\",\n\t\t\tci_tier: \"core-regression\",\n\t\t\tscenario_id: scenarioId,\n\t\t\texpected_run_id: expected.run.id,\n\t\t\tactual_run_id: actual.run.id,\n\t\t\texpected_assertion_count: expected.counts.assertions,\n\t\t\tactual_assertion_count: actual.counts.assertions,\n\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t},\n\t\tassertions: [\n\t\t\t{\n\t\t\t\tid: `${id}.agent-trajectory`,\n\t\t\t\tkind: \"ASSERTION_KIND_AGENT_TRAJECTORY\",\n\t\t\t\ttarget: \"response\",\n\t\t\t\tdescription:\n\t\t\t\t\t\"Native Fermata trajectory assertion evaluates real Maestro replay results for event, assertion, budget, idempotency, and trace-link invariants.\",\n\t\t\t\tagentTrajectory: agentTrajectoryConfig(expected),\n\t\t\t\tmetadata: {\n\t\t\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t\t\t\tcase_kind: \"scenario_agent_trajectory\",\n\t\t\t\t\tci_tier: \"core-regression\",\n\t\t\t\t},\n\t\t\t},\n\t\t],\n\t};\n}\n\nfunction llmRubricCase(\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n\toptions: FermataScenarioSuiteLLMJudgeOptions,\n): FermataTestCase {\n\tconst scenarioId = expected.scenario.id;\n\tconst id = stableId(scenarioId, \"llm-rubric\", \"semantic-trajectory-quality\");\n\treturn {\n\t\tid,\n\t\tname: `${scenarioId} semantic trajectory quality`,\n\t\tinput: serializeCaseInput({\n\t\t\tscenarioId,\n\t\t\ttitle: expected.scenario.title,\n\t\t\tplatformPrimitive: actual.platform.primitive,\n\t\t\texpectedRunId: expected.run.id,\n\t\t\tactualRunId: actual.run.id,\n\t\t\ttraceJoinKeys: actual.platform.traceJoinKeys,\n\t\t\texpectedAssertionIds: expected.assertions.map(\n\t\t\t\t(assertion) => assertion.id,\n\t\t\t),\n\t\t\tactualAssertionIds: actual.assertions.map((assertion) => assertion.id),\n\t\t}),\n\t\texpectedOutput: realScenarioResultJSON(expected),\n\t\tmetadata: {\n\t\t\tactual_output: realScenarioResultJSON(actual),\n\t\t\tcase_kind: \"scenario_llm_rubric\",\n\t\t\t...llmJudgeMetadata(options),\n\t\t\tscenario_id: scenarioId,\n\t\t\texpected_run_id: expected.run.id,\n\t\t\tactual_run_id: actual.run.id,\n\t\t\texpected_assertion_count: expected.counts.assertions,\n\t\t\tactual_assertion_count: actual.counts.assertions,\n\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t},\n\t\tassertions: [\n\t\t\t{\n\t\t\t\tid: `${id}.llm-rubric`,\n\t\t\t\tkind: \"ASSERTION_KIND_LLM_RUBRIC\",\n\t\t\t\ttarget: \"response\",\n\t\t\t\tdescription:\n\t\t\t\t\t\"Cataloged Fermata LLM judge evaluates semantic trajectory quality against the committed real Maestro scenario result.\",\n\t\t\t\tllmRubric: llmRubricConfig(options),\n\t\t\t\tmetadata: {\n\t\t\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t\t\t\tcase_kind: \"scenario_llm_rubric\",\n\t\t\t\t\t...llmJudgeMetadata(options),\n\t\t\t\t},\n\t\t\t},\n\t\t],\n\t};\n}\n\nfunction llmPairwiseRubricCase(\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n\toptions: FermataScenarioSuiteLLMPairwiseJudgeOptions,\n): FermataTestCase {\n\tconst scenarioId = expected.scenario.id;\n\tconst id = stableId(\n\t\tscenarioId,\n\t\t\"llm-pairwise-rubric\",\n\t\t\"semantic-trajectory-preference\",\n\t);\n\treturn {\n\t\tid,\n\t\tname: `${scenarioId} semantic trajectory preference`,\n\t\tinput: serializeCaseInput({\n\t\t\tscenarioId,\n\t\t\ttitle: expected.scenario.title,\n\t\t\tplatformPrimitive: actual.platform.primitive,\n\t\t\tbaselineRunId: expected.run.id,\n\t\t\tcandidateRunId: actual.run.id,\n\t\t\ttraceJoinKeys: actual.platform.traceJoinKeys,\n\t\t\tbaselineAssertionIds: expected.assertions.map(\n\t\t\t\t(assertion) => assertion.id,\n\t\t\t),\n\t\t\tcandidateAssertionIds: actual.assertions.map((assertion) => assertion.id),\n\t\t}),\n\t\texpectedOutput: realScenarioResultJSON(expected),\n\t\tmetadata: {\n\t\t\tactual_output: realScenarioResultJSON(actual),\n\t\t\tcase_kind: \"scenario_llm_pairwise_rubric\",\n\t\t\t...llmJudgeMetadata(options),\n\t\t\tscenario_id: scenarioId,\n\t\t\tbaseline_run_id: expected.run.id,\n\t\t\tcandidate_run_id: actual.run.id,\n\t\t\tbaseline_assertion_count: expected.counts.assertions,\n\t\t\tcandidate_assertion_count: actual.counts.assertions,\n\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t},\n\t\tassertions: [\n\t\t\t{\n\t\t\t\tid: `${id}.llm-pairwise-rubric`,\n\t\t\t\tkind: \"ASSERTION_KIND_LLM_PAIRWISE_RUBRIC\",\n\t\t\t\ttarget: \"response\",\n\t\t\t\tdescription:\n\t\t\t\t\t\"Cataloged Fermata LLM judge compares real Maestro trajectory outputs and prefers the stronger candidate.\",\n\t\t\t\tllmPairwiseRubric: llmPairwiseRubricConfig(options),\n\t\t\t\tmetadata: {\n\t\t\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t\t\t\tcase_kind: \"scenario_llm_pairwise_rubric\",\n\t\t\t\t\t...llmJudgeMetadata(options),\n\t\t\t\t},\n\t\t\t},\n\t\t],\n\t};\n}\n\nfunction equalsCase(\n\tid: string,\n\tname: string,\n\tinput: Record<string, unknown>,\n\texpected: string,\n\tactual: string,\n\tmetadata: Record<string, unknown>,\n): FermataTestCase {\n\treturn {\n\t\tid,\n\t\tname,\n\t\tinput: serializeCaseInput(input),\n\t\texpectedOutput: expected,\n\t\tmetadata: {\n\t\t\t...metadata,\n\t\t\tactual_output: actual,\n\t\t},\n\t\tassertions: [\n\t\t\t{\n\t\t\t\tid: `${id}.equals`,\n\t\t\t\tkind: \"ASSERTION_KIND_EQUALS\",\n\t\t\t\ttarget: \"response\",\n\t\t\t\texpected,\n\t\t\t\tmetadata: {\n\t\t\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t\t\t},\n\t\t\t},\n\t\t],\n\t};\n}\n\nfunction outcomeCase(\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n): FermataTestCase {\n\tconst scenarioId = expected.scenario.id;\n\treturn equalsCase(\n\t\tstableId(scenarioId, \"observed-outcome\"),\n\t\t`${scenarioId} observed outcome`,\n\t\t{\n\t\t\tscenarioId,\n\t\t\ttitle: expected.scenario.title,\n\t\t\texpectedOutcome: expected.scenario.expectedOutcome,\n\t\t\tcommittedObservedOutcome: expected.scenario.observedOutcome,\n\t\t\tactualObservedOutcome: actual.scenario.observedOutcome,\n\t\t},\n\t\texpected.scenario.observedOutcome,\n\t\tactual.scenario.observedOutcome,\n\t\t{\n\t\t\tcase_kind: \"scenario_observed_outcome\",\n\t\t\tscenario_id: scenarioId,\n\t\t\texpected_outcome: expected.scenario.expectedOutcome,\n\t\t\texpected_observed_outcome: expected.scenario.observedOutcome,\n\t\t\tactual_observed_outcome: actual.scenario.observedOutcome,\n\t\t\treview_labels: actual.scenario.reviewLabels,\n\t\t},\n\t);\n}\n\nfunction countCases(\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n): FermataTestCase[] {\n\treturn COUNT_KEYS.map((key) =>\n\t\tequalsCase(\n\t\t\tstableId(expected.scenario.id, \"count\", key),\n\t\t\t`${expected.scenario.id} ${key} count`,\n\t\t\t{\n\t\t\t\tscenarioId: expected.scenario.id,\n\t\t\t\tcount: key,\n\t\t\t\texpected: expected.counts[key],\n\t\t\t\tactual: actual.counts[key],\n\t\t\t},\n\t\t\tString(expected.counts[key]),\n\t\t\tString(actual.counts[key]),\n\t\t\t{\n\t\t\t\tcase_kind: \"scenario_count\",\n\t\t\t\tscenario_id: expected.scenario.id,\n\t\t\t\tcount: key,\n\t\t\t\texpected_count: expected.counts[key],\n\t\t\t\tactual_count: actual.counts[key],\n\t\t\t},\n\t\t),\n\t);\n}\n\nfunction assertionCases(\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n): FermataTestCase[] {\n\tconst actualById = assertionMap(actual);\n\treturn expected.assertions.map((expectedAssertion) => {\n\t\tconst actualAssertion = actualById.get(expectedAssertion.id);\n\t\tconst actualStatus = actualAssertion?.status ?? \"missing\";\n\t\treturn equalsCase(\n\t\t\tstableId(expected.scenario.id, \"assertion\", expectedAssertion.id),\n\t\t\t`${expected.scenario.id} ${expectedAssertion.id}`,\n\t\t\t{\n\t\t\t\tscenarioId: expected.scenario.id,\n\t\t\t\tassertionId: expectedAssertion.id,\n\t\t\t\tkind: expectedAssertion.kind,\n\t\t\t\texpectedStatus: expectedAssertion.status,\n\t\t\t\tactualStatus,\n\t\t\t\texpectedMessage: expectedAssertion.message,\n\t\t\t\tactualMessage: actualAssertion?.message ?? null,\n\t\t\t},\n\t\t\texpectedAssertion.status,\n\t\t\tactualStatus,\n\t\t\t{\n\t\t\t\tcase_kind: \"scenario_assertion_status\",\n\t\t\t\tscenario_id: expected.scenario.id,\n\t\t\t\tassertion_id: expectedAssertion.id,\n\t\t\t\tassertion_kind: expectedAssertion.kind,\n\t\t\t\tseverity: expectedAssertion.severity,\n\t\t\t\texpected_message: expectedAssertion.message,\n\t\t\t\tactual_message: actualAssertion?.message,\n\t\t\t\texpected_evidence: evidenceLabels(expectedAssertion),\n\t\t\t\tactual_evidence: evidenceLabels(actualAssertion),\n\t\t\t},\n\t\t);\n\t});\n}\n\nfunction suiteMetadata(\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n\textra: Record<string, unknown> | undefined,\n): Record<string, unknown> {\n\treturn {\n\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\tschema_version: actual.schemaVersion,\n\t\tscenario_schema_version: actual.scenarioSchemaVersion,\n\t\tscenario_id: actual.scenario.id,\n\t\tscenario_title: actual.scenario.title,\n\t\texpected_run_id: expected.run.id,\n\t\tactual_run_id: actual.run.id,\n\t\texpected_outcome: expected.scenario.expectedOutcome,\n\t\tcommitted_observed_outcome: expected.scenario.observedOutcome,\n\t\tactual_observed_outcome: actual.scenario.observedOutcome,\n\t\tplatform_primitive: actual.platform.primitive,\n\t\tevidence_event_type: actual.platform.evidenceEventType,\n\t\ttrace_join_keys: actual.platform.traceJoinKeys,\n\t\t...(isObject(extra) ? extra : {}),\n\t};\n}\n\nexport function buildFermataCreateScenarioSuiteRequest(\n\tconfig: Pick<FermataEvalServiceConfig, \"organizationId\" | \"workspaceId\">,\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n\toptions: FermataScenarioSuiteOptions = {},\n): CreateFermataTestSuiteRequest {\n\trequireSameScenario(expected, actual);\n\tif (!config.organizationId || !config.workspaceId) {\n\t\tthrow new Error(\n\t\t\t\"Fermata scenario suite requires organizationId and workspaceId\",\n\t\t);\n\t}\n\tconst scenarioId = expected.scenario.id;\n\treturn {\n\t\torganizationId: config.organizationId,\n\t\tworkspaceId: config.workspaceId,\n\t\tname: options.name ?? `Maestro scenario: ${scenarioId}`,\n\t\tdescription:\n\t\t\toptions.description ??\n\t\t\t`Recorded Maestro agent trajectory scenario suite for ${scenarioId}.`,\n\t\tcases: [\n\t\t\toutcomeCase(expected, actual),\n\t\t\t...countCases(expected, actual),\n\t\t\t...assertionCases(expected, actual),\n\t\t\tagentTrajectoryCase(expected, actual),\n\t\t\t...(options.llmJudge\n\t\t\t\t? [llmRubricCase(expected, actual, options.llmJudge)]\n\t\t\t\t: []),\n\t\t\t...(options.llmPairwiseJudge\n\t\t\t\t? [llmPairwiseRubricCase(expected, actual, options.llmPairwiseJudge)]\n\t\t\t\t: []),\n\t\t],\n\t\tmetadata: suiteMetadata(expected, actual, options.metadata),\n\t};\n}\n\nexport function buildFermataRunScenarioSuiteRequest(\n\tconfig: Pick<FermataEvalServiceConfig, \"organizationId\" | \"workspaceId\">,\n\tsuiteId: string,\n\texpected: AgentTrajectoryScenarioResult,\n\tactual: AgentTrajectoryScenarioResult,\n\toptions: FermataScenarioSuiteOptions = {},\n): RunFermataTestSuiteRequest {\n\tif (!config.organizationId || !config.workspaceId) {\n\t\tthrow new Error(\n\t\t\t\"Fermata scenario suite run requires organizationId and workspaceId\",\n\t\t);\n\t}\n\tconst scenarioId = actual.scenario.id;\n\tconst runId = stableId(scenarioId, actual.run.id);\n\tconst model = options.model ?? \"maestro-agent-trajectory-replay\";\n\treturn {\n\t\tsuiteId,\n\t\torganizationId: config.organizationId,\n\t\tworkspaceId: config.workspaceId,\n\t\tevaluationId: options.evaluationId ?? `maestro-${runId}`,\n\t\trunIdPrefix: options.runIdPrefix ?? stableId(\"maestro\", scenarioId),\n\t\tlineageId: options.lineageId ?? actual.run.id,\n\t\ttraceId: options.traceId,\n\t\tmaxConcurrency: options.maxConcurrency ?? 1,\n\t\tstopOnFirstFailure: false,\n\t\trunContext: {\n\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t\tscenarioId,\n\t\t\trunId: actual.run.id,\n\t\t\tsessionId: actual.run.sessionId,\n\t\t\tplatformPrimitive: actual.platform.primitive,\n\t\t\ttraceJoinKeys: actual.platform.traceJoinKeys,\n\t\t},\n\t\tmetadata: suiteMetadata(expected, actual, options.metadata),\n\t\tcandidates: [\n\t\t\t{\n\t\t\t\tcandidateId:\n\t\t\t\t\toptions.candidateId ?? stableId(\"maestro-recorded\", scenarioId),\n\t\t\t\tlabel: options.candidateLabel ?? \"Maestro recorded trajectory\",\n\t\t\t\tmodel,\n\t\t\t\tmetadata: {\n\t\t\t\t\tsource: \"maestro.agent_trajectory_scenario_result\",\n\t\t\t\t\tscenario_id: scenarioId,\n\t\t\t\t\trun_id: actual.run.id,\n\t\t\t\t},\n\t\t\t},\n\t\t],\n\t};\n}\n"]}

package/dist/platform/maestro-timeline-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"maestro-timeline-client.d.ts","sourceRoot":"","sources":["../../src/platform/maestro-timeline-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAIX,2BAA2B,EAG3B,MAAM,oBAAoB,CAAC;AAM5B,OAAO,EACN,KAAK,qBAAqB,EAI1B,MAAM,aAAa,CAAC;AAwDrB,MAAM,WAAW,4BAA6B,SAAQ,qBAAqB;CAAG;AAE9E,MAAM,WAAW,4BAA4B;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAiCD,wBAAsB,mCAAmC,CACxD,SAAS,GAAE,OAAO,CAAC,4BAA4B,CAAM,GACnD,OAAO,CAAC,4BAA4B,GAAG,IAAI,CAAC,CAoC9C;AAuUD,wBAAsB,+BAA+B,CACpD,MAAM,EAAE,4BAA4B,EACpC,KAAK,EAAE,4BAA4B,EACnC,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,2BAA2B,CAAC,CA+CtC;AAED,wBAAsB,kCAAkC,CACvD,KAAK,EAAE,4BAA4B,EACnC,OAAO,CAAC,EAAE;IACT,MAAM,CAAC,EAAE,4BAA4B,CAAC;IACtC,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB,GACC,OAAO,CAAC,2BAA2B,GAAG,IAAI,CAAC,CAkB7C"}
+{"version":3,"file":"maestro-timeline-client.d.ts","sourceRoot":"","sources":["../../src/platform/maestro-timeline-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAIX,2BAA2B,EAG3B,MAAM,oBAAoB,CAAC;AAY5B,OAAO,EACN,KAAK,qBAAqB,EAI1B,MAAM,aAAa,CAAC;AAmDrB,MAAM,WAAW,4BAA6B,SAAQ,qBAAqB;CAAG;AAE9E,MAAM,WAAW,4BAA4B;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAiCD,wBAAsB,mCAAmC,CACxD,SAAS,GAAE,OAAO,CAAC,4BAA4B,CAAM,GACnD,OAAO,CAAC,4BAA4B,GAAG,IAAI,CAAC,CAoC9C;AAmUD,wBAAsB,+BAA+B,CACpD,MAAM,EAAE,4BAA4B,EACpC,KAAK,EAAE,4BAA4B,EACnC,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,2BAA2B,CAAC,CA+CtC;AAED,wBAAsB,kCAAkC,CACvD,KAAK,EAAE,4BAA4B,EACnC,OAAO,CAAC,EAAE;IACT,MAAM,CAAC,EAAE,4BAA4B,CAAC;IACtC,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB,GACC,OAAO,CAAC,2BAA2B,GAAG,IAAI,CAAC,CAkB7C"}

package/dist/platform/maestro-timeline-client.js

@@ -1,4 +1,6 @@
+import { EVALOPS_ACCESS_TOKEN_ENV_VARS, EVALOPS_ORGANIZATION_ID_ENV_VARS, EVALOPS_WORKSPACE_ID_ENV_VARS, } from "../evalops/env-aliases.js";
 import { compactTimelineMetadata, compactTimelineSummary, redactTimelineMetadata, } from "../timeline/redaction.js";
+import { isAbortError } from "../utils/abort-error.js";
 import { postPlatformConnect, resolvePlatformServiceConfig, trimString, } from "./client.js";
 import { PLATFORM_CONNECT_METHODS, platformConnectMethodPath, platformConnectServicePath, } from "./core-services.js";
 const DEFAULT_TIMEOUT_MS = 2_500;
@@ -14,23 +16,18 @@ const TIMELINE_BASE_URL_ENV_VARS = [
 const TIMELINE_TOKEN_ENV_VARS = [
     "MAESTRO_TIMELINE_SERVICE_TOKEN",
     "MAESTRO_PLATFORM_TIMELINE_SERVICE_TOKEN",
-    "MAESTRO_EVALOPS_ACCESS_TOKEN",
-    "EVALOPS_TOKEN",
+    ...EVALOPS_ACCESS_TOKEN_ENV_VARS,
 ];
 const TIMELINE_ORGANIZATION_ENV_VARS = [
     "MAESTRO_TIMELINE_ORG_ID",
     "MAESTRO_PLATFORM_TIMELINE_ORG_ID",
-    "MAESTRO_EVALOPS_ORG_ID",
-    "EVALOPS_ORGANIZATION_ID",
-    "MAESTRO_ENTERPRISE_ORG_ID",
+    ...EVALOPS_ORGANIZATION_ID_ENV_VARS,
 ];
 const TIMELINE_WORKSPACE_ENV_VARS = [
     "MAESTRO_TIMELINE_WORKSPACE_ID",
     "MAESTRO_PLATFORM_TIMELINE_WORKSPACE_ID",
     "MAESTRO_REMOTE_RUNNER_WORKSPACE_ID",
-    "MAESTRO_EVALOPS_WORKSPACE_ID",
-    "EVALOPS_WORKSPACE_ID",
-    "MAESTRO_WORKSPACE_ID",
+    ...EVALOPS_WORKSPACE_ID_ENV_VARS,
 ];
 const TIMELINE_TIMEOUT_ENV_VARS = [
     "MAESTRO_TIMELINE_SERVICE_TIMEOUT_MS",
@@ -78,9 +75,6 @@ function stringValue(value) {
         ? value.trim()
         : undefined;
 }
-function isAbortError(error) {
-    return error instanceof Error && error.name === "AbortError";
-}
 function relatedString(relatedIds, ...keys) {
     for (const key of keys) {
         const value = stringValue(relatedIds?.[key]);