@evalops/maestro 0.10.17 → 0.10.20

package/dist/platform/fermata-eval-client.js

@@ -0,0 +1,146 @@
+import { EVALOPS_ACCESS_TOKEN_ENV_VARS, EVALOPS_ORGANIZATION_ID_ENV_VARS, EVALOPS_WORKSPACE_ID_ENV_VARS, } from "../evalops/env-aliases.js";
+import { normalizeBaseUrl, postPlatformConnect, resolvePlatformServiceConfig, trimString, } from "./client.js";
+import { PLATFORM_CONNECT_METHODS, platformConnectMethodPath, platformConnectServicePath, } from "./core-services.js";
+const DEFAULT_TIMEOUT_MS = 5_000;
+const DEFAULT_MAX_ATTEMPTS = 1;
+const CREATE_TEST_SUITE_PATH = platformConnectMethodPath(PLATFORM_CONNECT_METHODS.fermata.createTestSuite);
+const RUN_TEST_SUITE_PATH = platformConnectMethodPath(PLATFORM_CONNECT_METHODS.fermataExecution.runTestSuite);
+const FERMATA_BASE_URL_SUFFIXES = [
+    CREATE_TEST_SUITE_PATH,
+    RUN_TEST_SUITE_PATH,
+    platformConnectServicePath(PLATFORM_CONNECT_METHODS.fermata.createTestSuite.service),
+    platformConnectServicePath(PLATFORM_CONNECT_METHODS.fermataExecution.runTestSuite.service),
+];
+const FERMATA_BASE_URL_ENV_VARS = [
+    "FERMATA_SERVICE_URL",
+    "MAESTRO_FERMATA_SERVICE_URL",
+    "MAESTRO_FERMATA_EXECUTION_SERVICE_URL",
+    "MAESTRO_PLATFORM_BASE_URL",
+    "MAESTRO_EVALOPS_BASE_URL",
+    "EVALOPS_BASE_URL",
+];
+const FERMATA_TOKEN_ENV_VARS = [
+    "FERMATA_SERVICE_TOKEN",
+    "MAESTRO_FERMATA_SERVICE_TOKEN",
+    "MAESTRO_FERMATA_EXECUTION_SERVICE_TOKEN",
+    ...EVALOPS_ACCESS_TOKEN_ENV_VARS,
+];
+const FERMATA_ORGANIZATION_ENV_VARS = [
+    "FERMATA_SERVICE_ORGANIZATION_ID",
+    "MAESTRO_FERMATA_ORGANIZATION_ID",
+    "MAESTRO_FERMATA_EXECUTION_ORGANIZATION_ID",
+    ...EVALOPS_ORGANIZATION_ID_ENV_VARS,
+];
+const FERMATA_WORKSPACE_ENV_VARS = [
+    "FERMATA_SERVICE_WORKSPACE_ID",
+    "MAESTRO_FERMATA_WORKSPACE_ID",
+    "MAESTRO_FERMATA_EXECUTION_WORKSPACE_ID",
+    ...EVALOPS_WORKSPACE_ID_ENV_VARS,
+];
+const FERMATA_TIMEOUT_ENV_VARS = [
+    "FERMATA_SERVICE_TIMEOUT_MS",
+    "MAESTRO_FERMATA_SERVICE_TIMEOUT_MS",
+    "MAESTRO_FERMATA_EXECUTION_SERVICE_TIMEOUT_MS",
+];
+const FERMATA_MAX_ATTEMPTS_ENV_VARS = [
+    "FERMATA_SERVICE_MAX_ATTEMPTS",
+    "MAESTRO_FERMATA_SERVICE_MAX_ATTEMPTS",
+    "MAESTRO_FERMATA_EXECUTION_SERVICE_MAX_ATTEMPTS",
+];
+function stripUndefinedValues(record) {
+    return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
+}
+async function parseJsonResponse(response, serviceName) {
+    const text = await response.text();
+    if (!response.ok) {
+        throw new Error(`${serviceName} returned ${response.status}: ${text || response.statusText}`);
+    }
+    if (!text.trim()) {
+        throw new Error(`${serviceName} returned empty response`);
+    }
+    return JSON.parse(text);
+}
+export async function resolveFermataEvalServiceConfig(overrides = {}) {
+    const config = await resolvePlatformServiceConfig({
+        baseUrlEnvVars: FERMATA_BASE_URL_ENV_VARS,
+        tokenEnvVars: FERMATA_TOKEN_ENV_VARS,
+        organizationEnvVars: FERMATA_ORGANIZATION_ENV_VARS,
+        workspaceEnvVars: FERMATA_WORKSPACE_ENV_VARS,
+        timeoutEnvVars: FERMATA_TIMEOUT_ENV_VARS,
+        maxAttemptsEnvVars: FERMATA_MAX_ATTEMPTS_ENV_VARS,
+        baseUrlSuffixes: FERMATA_BASE_URL_SUFFIXES,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+        defaultMaxAttempts: DEFAULT_MAX_ATTEMPTS,
+        requireBaseUrl: true,
+        requireOrganizationId: true,
+        requireToken: true,
+        allowOAuthTokenFallback: false,
+    });
+    if (!config?.baseUrl || !config.workspaceId) {
+        return null;
+    }
+    return {
+        ...config,
+        baseUrl: normalizeBaseUrl(trimString(overrides.baseUrl ?? config.baseUrl) ?? config.baseUrl, FERMATA_BASE_URL_SUFFIXES),
+        organizationId: trimString(overrides.organizationId ?? config.organizationId) ??
+            config.organizationId,
+        workspaceId: trimString(overrides.workspaceId ?? config.workspaceId) ??
+            config.workspaceId,
+        token: trimString(overrides.token ?? config.token) ?? config.token,
+        timeoutMs: overrides.timeoutMs ?? config.timeoutMs,
+        maxAttempts: overrides.maxAttempts ?? config.maxAttempts,
+        teamId: trimString(overrides.teamId ?? config.teamId),
+    };
+}
+export async function requireFermataEvalServiceConfig(overrides = {}) {
+    const config = await resolveFermataEvalServiceConfig(overrides);
+    if (!config) {
+        throw new Error("Fermata eval integration requires FERMATA_SERVICE_URL, FERMATA_SERVICE_TOKEN, organization id, and workspace id");
+    }
+    return config;
+}
+export async function createFermataTestSuiteWithPlatform(config, request, signal) {
+    const response = await postPlatformConnect(config, CREATE_TEST_SUITE_PATH, stripUndefinedValues({
+        organizationId: request.organizationId,
+        workspaceId: request.workspaceId,
+        name: request.name,
+        description: request.description,
+        cases: request.cases,
+        metadata: request.metadata,
+    }), {
+        serviceName: "fermata test suite service",
+        failureMode: "required",
+        timeoutMs: config.timeoutMs,
+        maxAttempts: config.maxAttempts,
+        signal,
+    });
+    const payload = await parseJsonResponse(response, "fermata test suite service");
+    return payload;
+}
+export async function runFermataTestSuiteWithPlatform(config, request, signal) {
+    const response = await postPlatformConnect(config, RUN_TEST_SUITE_PATH, stripUndefinedValues({
+        suiteId: request.suiteId,
+        organizationId: request.organizationId,
+        workspaceId: request.workspaceId,
+        evaluationId: request.evaluationId,
+        model: request.model,
+        runContext: request.runContext,
+        metadata: request.metadata,
+        runIdPrefix: request.runIdPrefix,
+        traceId: request.traceId,
+        lineageId: request.lineageId,
+        caseIds: request.caseIds,
+        maxConcurrency: request.maxConcurrency,
+        stopOnFirstFailure: request.stopOnFirstFailure,
+        candidates: request.candidates,
+    }), {
+        serviceName: "fermata test suite execution service",
+        failureMode: "required",
+        timeoutMs: config.timeoutMs,
+        maxAttempts: config.maxAttempts,
+        signal,
+    });
+    const payload = await parseJsonResponse(response, "fermata test suite execution service");
+    return payload;
+}
+//# sourceMappingURL=fermata-eval-client.js.map

package/dist/platform/fermata-eval-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fermata-eval-client.js","sourceRoot":"","sources":["../../src/platform/fermata-eval-client.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,6BAA6B,EAC7B,gCAAgC,EAChC,6BAA6B,GAC7B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAEN,gBAAgB,EAChB,mBAAmB,EACnB,4BAA4B,EAC5B,UAAU,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EACN,wBAAwB,EACxB,yBAAyB,EACzB,0BAA0B,GAC1B,MAAM,oBAAoB,CAAC;AAE5B,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,MAAM,sBAAsB,GAAG,yBAAyB,CACvD,wBAAwB,CAAC,OAAO,CAAC,eAAe,CAChD,CAAC;AACF,MAAM,mBAAmB,GAAG,yBAAyB,CACpD,wBAAwB,CAAC,gBAAgB,CAAC,YAAY,CACtD,CAAC;AAEF,MAAM,yBAAyB,GAAG;IACjC,sBAAsB;IACtB,mBAAmB;IACnB,0BAA0B,CACzB,wBAAwB,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CACxD;IACD,0BAA0B,CACzB,wBAAwB,CAAC,gBAAgB,CAAC,YAAY,CAAC,OAAO,CAC9D;CACQ,CAAC;AAEX,MAAM,yBAAyB,GAAG;IACjC,qBAAqB;IACrB,6BAA6B;IAC7B,uCAAuC;IACvC,2BAA2B;IAC3B,0BAA0B;IAC1B,kBAAkB;CACT,CAAC;AAEX,MAAM,sBAAsB,GAAG;IAC9B,uBAAuB;IACvB,+BAA+B;IAC/B,yCAAyC;IACzC,GAAG,6BAA6B;CACvB,CAAC;AAEX,MAAM,6BAA6B,GAAG;IACrC,iCAAiC;IACjC,iCAAiC;IACjC,2CAA2C;IAC3C,GAAG,gCAAgC;CAC1B,CAAC;AAEX,MAAM,0BAA0B,GAAG;IAClC,8BAA8B;IAC9B,8BAA8B;IAC9B,wCAAwC;IACxC,GAAG,6BAA6B;CACvB,CAAC;AAEX,MAAM,wBAAwB,GAAG;IAChC,4BAA4B;IAC5B,oCAAoC;IACpC,8CAA8C;CACrC,CAAC;AAEX,MAAM,6BAA6B,GAAG;IACrC,8BAA8B;IAC9B,sCAAsC;IACtC,gDAAgD;CACvC,CAAC;AA2OX,SAAS,oBAAoB,CAC5B,MAA+B;IAE/B,OAAO,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CACjE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC/B,QAAkB,EAClB,WAAmB;IAEnB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CACd,GAAG,WAAW,aAAa,QAAQ,CAAC,MAAM,KAAK,IAAI,IAAI,QAAQ,CAAC,UAAU,EAAE,CAC5E,CAAC;IACH,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,GAAG,WAAW,0BAA0B,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;AACpD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACpD,YAA+C,EAAE;IAEjD,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC;QACjD,cAAc,EAAE,yBAAyB;QACzC,YAAY,EAAE,sBAAsB;QACpC,mBAAmB,EAAE,6BAA6B;QAClD,gBAAgB,EAAE,0BAA0B;QAC5C,cAAc,EAAE,wBAAwB;QACxC,kBAAkB,EAAE,6BAA6B;QACjD,eAAe,EAAE,yBAAyB;QAC1C,gBAAgB,EAAE,kBAAkB;QACpC,kBAAkB,EAAE,oBAAoB;QACxC,cAAc,EAAE,IAAI;QACpB,qBAAqB,EAAE,IAAI;QAC3B,YAAY,EAAE,IAAI;QAClB,uBAAuB,EAAE,KAAK;KAC9B,CAAC,CAAC;IACH,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC7C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO;QACN,GAAG,MAAM;QACT,OAAO,EAAE,gBAAgB,CACxB,UAAU,CAAC,SAAS,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,EACjE,yBAAyB,CACzB;QACD,cAAc,EACb,UAAU,CAAC,SAAS,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC;YAC7D,MAAM,CAAC,cAAc;QACtB,WAAW,EACV,UAAU,CAAC,SAAS,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;YACvD,MAAM,CAAC,WAAW;QACnB,KAAK,EAAE,UAAU,CAAC,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK;QAClE,SAAS,EAAE,SAAS,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;QAClD,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW;QACxD,MAAM,EAAE,UAAU,CAAC,SAAS,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;KACrD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACpD,YAA+C,EAAE;IAEjD,MAAM,MAAM,GAAG,MAAM,+BAA+B,CAAC,SAAS,CAAC,CAAC;IAChE,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACd,iHAAiH,CACjH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kCAAkC,CACvD,MAAgC,EAChC,OAAsC,EACtC,MAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CACzC,MAAM,EACN,sBAAsB,EACtB,oBAAoB,CAAC;QACpB,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC1B,CAAC,EACF;QACC,WAAW,EAAE,4BAA4B;QACzC,WAAW,EAAE,UAAU;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,MAAM;KACN,CACD,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,iBAAiB,CACtC,QAAQ,EACR,4BAA4B,CAC5B,CAAC;IACF,OAAO,OAAoD,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACpD,MAAgC,EAChC,OAAmC,EACnC,MAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CACzC,MAAM,EACN,mBAAmB,EACnB,oBAAoB,CAAC;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,UAAU,EAAE,OAAO,CAAC,UAAU;KAC9B,CAAC,EACF;QACC,WAAW,EAAE,sCAAsC;QACnD,WAAW,EAAE,UAAU;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,MAAM;KACN,CACD,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,iBAAiB,CACtC,QAAQ,EACR,sCAAsC,CACtC,CAAC;IACF,OAAO,OAAiD,CAAC;AAC1D,CAAC","sourcesContent":["import {\n\tEVALOPS_ACCESS_TOKEN_ENV_VARS,\n\tEVALOPS_ORGANIZATION_ID_ENV_VARS,\n\tEVALOPS_WORKSPACE_ID_ENV_VARS,\n} from \"../evalops/env-aliases.js\";\nimport {\n\ttype PlatformServiceConfig,\n\tnormalizeBaseUrl,\n\tpostPlatformConnect,\n\tresolvePlatformServiceConfig,\n\ttrimString,\n} from \"./client.js\";\nimport {\n\tPLATFORM_CONNECT_METHODS,\n\tplatformConnectMethodPath,\n\tplatformConnectServicePath,\n} from \"./core-services.js\";\n\nconst DEFAULT_TIMEOUT_MS = 5_000;\nconst DEFAULT_MAX_ATTEMPTS = 1;\n\nconst CREATE_TEST_SUITE_PATH = platformConnectMethodPath(\n\tPLATFORM_CONNECT_METHODS.fermata.createTestSuite,\n);\nconst RUN_TEST_SUITE_PATH = platformConnectMethodPath(\n\tPLATFORM_CONNECT_METHODS.fermataExecution.runTestSuite,\n);\n\nconst FERMATA_BASE_URL_SUFFIXES = [\n\tCREATE_TEST_SUITE_PATH,\n\tRUN_TEST_SUITE_PATH,\n\tplatformConnectServicePath(\n\t\tPLATFORM_CONNECT_METHODS.fermata.createTestSuite.service,\n\t),\n\tplatformConnectServicePath(\n\t\tPLATFORM_CONNECT_METHODS.fermataExecution.runTestSuite.service,\n\t),\n] as const;\n\nconst FERMATA_BASE_URL_ENV_VARS = [\n\t\"FERMATA_SERVICE_URL\",\n\t\"MAESTRO_FERMATA_SERVICE_URL\",\n\t\"MAESTRO_FERMATA_EXECUTION_SERVICE_URL\",\n\t\"MAESTRO_PLATFORM_BASE_URL\",\n\t\"MAESTRO_EVALOPS_BASE_URL\",\n\t\"EVALOPS_BASE_URL\",\n] as const;\n\nconst FERMATA_TOKEN_ENV_VARS = [\n\t\"FERMATA_SERVICE_TOKEN\",\n\t\"MAESTRO_FERMATA_SERVICE_TOKEN\",\n\t\"MAESTRO_FERMATA_EXECUTION_SERVICE_TOKEN\",\n\t...EVALOPS_ACCESS_TOKEN_ENV_VARS,\n] as const;\n\nconst FERMATA_ORGANIZATION_ENV_VARS = [\n\t\"FERMATA_SERVICE_ORGANIZATION_ID\",\n\t\"MAESTRO_FERMATA_ORGANIZATION_ID\",\n\t\"MAESTRO_FERMATA_EXECUTION_ORGANIZATION_ID\",\n\t...EVALOPS_ORGANIZATION_ID_ENV_VARS,\n] as const;\n\nconst FERMATA_WORKSPACE_ENV_VARS = [\n\t\"FERMATA_SERVICE_WORKSPACE_ID\",\n\t\"MAESTRO_FERMATA_WORKSPACE_ID\",\n\t\"MAESTRO_FERMATA_EXECUTION_WORKSPACE_ID\",\n\t...EVALOPS_WORKSPACE_ID_ENV_VARS,\n] as const;\n\nconst FERMATA_TIMEOUT_ENV_VARS = [\n\t\"FERMATA_SERVICE_TIMEOUT_MS\",\n\t\"MAESTRO_FERMATA_SERVICE_TIMEOUT_MS\",\n\t\"MAESTRO_FERMATA_EXECUTION_SERVICE_TIMEOUT_MS\",\n] as const;\n\nconst FERMATA_MAX_ATTEMPTS_ENV_VARS = [\n\t\"FERMATA_SERVICE_MAX_ATTEMPTS\",\n\t\"MAESTRO_FERMATA_SERVICE_MAX_ATTEMPTS\",\n\t\"MAESTRO_FERMATA_EXECUTION_SERVICE_MAX_ATTEMPTS\",\n] as const;\n\nexport type FermataAssertionKind =\n\t| \"ASSERTION_KIND_CONTAINS\"\n\t| \"ASSERTION_KIND_NOT_CONTAINS\"\n\t| \"ASSERTION_KIND_REGEX\"\n\t| \"ASSERTION_KIND_SIMILARITY\"\n\t| \"ASSERTION_KIND_CUSTOM_JUDGE\"\n\t| \"ASSERTION_KIND_JSON_SCHEMA\"\n\t| \"ASSERTION_KIND_EQUALS\"\n\t| \"ASSERTION_KIND_STARTS_WITH\"\n\t| \"ASSERTION_KIND_CONTAINS_ANY\"\n\t| \"ASSERTION_KIND_CONTAINS_ALL\"\n\t| \"ASSERTION_KIND_WORD_COUNT\"\n\t| \"ASSERTION_KIND_NUMERIC_GTE\"\n\t| \"ASSERTION_KIND_NUMERIC_LTE\"\n\t| \"ASSERTION_KIND_TOOL_CALLED\"\n\t| \"ASSERTION_KIND_TOOL_NOT_CALLED\"\n\t| \"ASSERTION_KIND_TOOL_SEQUENCE\"\n\t| \"ASSERTION_KIND_TRACE_SPAN_COUNT\"\n\t| \"ASSERTION_KIND_TOOL_ARGS_MATCH\"\n\t| \"ASSERTION_KIND_TRACE_ERROR_SPANS\"\n\t| \"ASSERTION_KIND_TRACE_SPAN_DURATION\"\n\t| \"ASSERTION_KIND_TRACE_STEP_COUNT\"\n\t| \"ASSERTION_KIND_TOOL_CALL_F1\"\n\t| \"ASSERTION_KIND_LLM_RUBRIC\"\n\t| \"ASSERTION_KIND_LLM_PAIRWISE_RUBRIC\"\n\t| \"ASSERTION_KIND_AGENT_TRAJECTORY\";\n\nexport interface FermataLLMRubricAssertion {\n\tjudgeId: string;\n\tverifierJudgeId?: string;\n\trubric: string;\n\tminScore?: number;\n\trepeat?: number;\n\tquorum?: number;\n\trecordJudgeValidation?: boolean;\n\trequireCalibratedJudge?: boolean;\n\tminJudgeValidationAccuracy?: number;\n\tminJudgeValidationCount?: number;\n\trubricVersion?: string;\n\tcalibrationCohort?: string;\n\tadvisoryOnly?: boolean;\n}\n\nexport interface FermataLLMPairwiseRubricAssertion {\n\tjudgeId: string;\n\tverifierJudgeId?: string;\n\trubric: string;\n\tbaselineLabel?: string;\n\tcandidateLabel?: string;\n\tminScore?: number;\n\trepeat?: number;\n\tquorum?: number;\n\trecordJudgeValidation?: boolean;\n\trequireCalibratedJudge?: boolean;\n\tminJudgeValidationAccuracy?: number;\n\tminJudgeValidationCount?: number;\n\trubricVersion?: string;\n\tcalibrationCohort?: string;\n\tadvisoryOnly?: boolean;\n}\n\nexport interface FermataAgentTrajectoryStatusExpectation {\n\tid: string;\n\tstatus: string;\n}\n\nexport interface FermataAgentTrajectoryStateExpectation {\n\tpath: string;\n\tvalueJson?: string;\n}\n\nexport interface FermataAgentTrajectoryAssertion {\n\trequiredTools?: string[];\n\tforbiddenTools?: string[];\n\trequiredEvents?: string[];\n\tforbiddenEvents?: string[];\n\trequiredAssertionStatuses?: FermataAgentTrajectoryStatusExpectation[];\n\trequiredStateWrites?: FermataAgentTrajectoryStateExpectation[];\n\tforbiddenStateWrites?: FermataAgentTrajectoryStateExpectation[];\n\tmaxEvents?: number;\n\tmaxToolCalls?: number;\n\tmaxReplayDeltas?: number;\n\tmaxScoreFailures?: number;\n\tmaxScoreWarnings?: number;\n\tmaxLatencyMs?: number;\n\tmaxCostMicros?: number;\n\tmaxRetries?: number;\n\trequireIdempotentReplay?: boolean;\n\tforbidDuplicateExternalActions?: boolean;\n\tforbiddenExternalActions?: string[];\n\trequiredTraceJoinKeys?: string[];\n}\n\nexport interface FermataAssertion {\n\tid?: string;\n\tkind: FermataAssertionKind;\n\ttarget?: string;\n\texpected?: string;\n\tdescription?: string;\n\tmetadata?: Record<string, unknown>;\n\tllmRubric?: FermataLLMRubricAssertion;\n\tllmPairwiseRubric?: FermataLLMPairwiseRubricAssertion;\n\tagentTrajectory?: FermataAgentTrajectoryAssertion;\n}\n\nexport interface FermataTestCase {\n\tid?: string;\n\tname: string;\n\tinput: string;\n\texpectedOutput?: string;\n\tassertions: FermataAssertion[];\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface FermataTestSuite {\n\tid?: string;\n\torganizationId?: string;\n\tworkspaceId?: string;\n\tname?: string;\n\tdescription?: string;\n\tcases?: FermataTestCase[];\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface CreateFermataTestSuiteRequest {\n\torganizationId: string;\n\tworkspaceId: string;\n\tname: string;\n\tdescription?: string;\n\tcases: FermataTestCase[];\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface CreateFermataTestSuiteResponse {\n\tsuite: FermataTestSuite;\n}\n\nexport interface FermataProviderRef {\n\tprovider?: string;\n\tenvironment?: string;\n\tcredentialName?: string;\n\tteamId?: string;\n}\n\nexport interface FermataTestSuiteCandidate {\n\tcandidateId: string;\n\tlabel?: string;\n\tmodel: string;\n\tproviderRef?: FermataProviderRef;\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface RunFermataTestSuiteRequest {\n\tsuiteId: string;\n\torganizationId: string;\n\tworkspaceId: string;\n\tevaluationId?: string;\n\tmodel?: string;\n\trunContext?: Record<string, unknown>;\n\tmetadata?: Record<string, unknown>;\n\trunIdPrefix?: string;\n\ttraceId?: string;\n\tlineageId?: string;\n\tcaseIds?: string[];\n\tmaxConcurrency?: number;\n\tstopOnFirstFailure?: boolean;\n\tcandidates?: FermataTestSuiteCandidate[];\n}\n\nexport interface FermataAssertionResult {\n\tassertionId?: string;\n\tpassed?: boolean;\n\tscore?: number;\n\tactual?: string;\n\tmessage?: string;\n\tmetadata?: Record<string, unknown>;\n}\n\nexport interface FermataTestCaseRun {\n\ttestCaseId?: string;\n\ttestCaseName?: string;\n\tpassed?: boolean;\n\tscore?: number;\n\toutput?: string;\n\tassertionResults?: FermataAssertionResult[];\n\tcandidateId?: string;\n\tcandidateLabel?: string;\n\tmodel?: string;\n\tproviderRef?: FermataProviderRef;\n}\n\nexport interface FermataTestSuiteCandidateSummary {\n\tcandidateId?: string;\n\tcandidateLabel?: string;\n\tmodel?: string;\n\tproviderRef?: FermataProviderRef;\n\ttotalCases?: number;\n\tpassedCases?: number;\n\tfailedCases?: number;\n\tscore?: number;\n\tpassRate?: number;\n\tpassed?: boolean;\n}\n\nexport interface FermataTestSuiteAssertionSummary {\n\tassertionId?: string;\n\ttotalResults?: number;\n\tpassedResults?: number;\n\tfailedResults?: number;\n\tscore?: number;\n\tpassRate?: number;\n\tpassed?: boolean;\n\tfailedTestCaseIds?: string[];\n\tfailedCandidateIds?: string[];\n\tfailedModels?: string[];\n}\n\nexport interface RunFermataTestSuiteResponse {\n\tevaluationId?: string;\n\tsuite?: FermataTestSuite;\n\tcaseResults?: FermataTestCaseRun[];\n\tpassed?: boolean;\n\tscore?: number;\n\ttotalCases?: number;\n\tpassedCases?: number;\n\tfailedCases?: number;\n\tlineageId?: string;\n\tcandidateSummaries?: FermataTestSuiteCandidateSummary[];\n\tassertionSummaries?: FermataTestSuiteAssertionSummary[];\n}\n\nexport interface FermataEvalServiceConfig extends PlatformServiceConfig {}\n\nfunction stripUndefinedValues(\n\trecord: Record<string, unknown>,\n): Record<string, unknown> {\n\treturn Object.fromEntries(\n\t\tObject.entries(record).filter(([, value]) => value !== undefined),\n\t);\n}\n\nasync function parseJsonResponse(\n\tresponse: Response,\n\tserviceName: string,\n): Promise<Record<string, unknown>> {\n\tconst text = await response.text();\n\tif (!response.ok) {\n\t\tthrow new Error(\n\t\t\t`${serviceName} returned ${response.status}: ${text || response.statusText}`,\n\t\t);\n\t}\n\tif (!text.trim()) {\n\t\tthrow new Error(`${serviceName} returned empty response`);\n\t}\n\treturn JSON.parse(text) as Record<string, unknown>;\n}\n\nexport async function resolveFermataEvalServiceConfig(\n\toverrides: Partial<FermataEvalServiceConfig> = {},\n): Promise<FermataEvalServiceConfig | null> {\n\tconst config = await resolvePlatformServiceConfig({\n\t\tbaseUrlEnvVars: FERMATA_BASE_URL_ENV_VARS,\n\t\ttokenEnvVars: FERMATA_TOKEN_ENV_VARS,\n\t\torganizationEnvVars: FERMATA_ORGANIZATION_ENV_VARS,\n\t\tworkspaceEnvVars: FERMATA_WORKSPACE_ENV_VARS,\n\t\ttimeoutEnvVars: FERMATA_TIMEOUT_ENV_VARS,\n\t\tmaxAttemptsEnvVars: FERMATA_MAX_ATTEMPTS_ENV_VARS,\n\t\tbaseUrlSuffixes: FERMATA_BASE_URL_SUFFIXES,\n\t\tdefaultTimeoutMs: DEFAULT_TIMEOUT_MS,\n\t\tdefaultMaxAttempts: DEFAULT_MAX_ATTEMPTS,\n\t\trequireBaseUrl: true,\n\t\trequireOrganizationId: true,\n\t\trequireToken: true,\n\t\tallowOAuthTokenFallback: false,\n\t});\n\tif (!config?.baseUrl || !config.workspaceId) {\n\t\treturn null;\n\t}\n\treturn {\n\t\t...config,\n\t\tbaseUrl: normalizeBaseUrl(\n\t\t\ttrimString(overrides.baseUrl ?? config.baseUrl) ?? config.baseUrl,\n\t\t\tFERMATA_BASE_URL_SUFFIXES,\n\t\t),\n\t\torganizationId:\n\t\t\ttrimString(overrides.organizationId ?? config.organizationId) ??\n\t\t\tconfig.organizationId,\n\t\tworkspaceId:\n\t\t\ttrimString(overrides.workspaceId ?? config.workspaceId) ??\n\t\t\tconfig.workspaceId,\n\t\ttoken: trimString(overrides.token ?? config.token) ?? config.token,\n\t\ttimeoutMs: overrides.timeoutMs ?? config.timeoutMs,\n\t\tmaxAttempts: overrides.maxAttempts ?? config.maxAttempts,\n\t\tteamId: trimString(overrides.teamId ?? config.teamId),\n\t};\n}\n\nexport async function requireFermataEvalServiceConfig(\n\toverrides: Partial<FermataEvalServiceConfig> = {},\n): Promise<FermataEvalServiceConfig> {\n\tconst config = await resolveFermataEvalServiceConfig(overrides);\n\tif (!config) {\n\t\tthrow new Error(\n\t\t\t\"Fermata eval integration requires FERMATA_SERVICE_URL, FERMATA_SERVICE_TOKEN, organization id, and workspace id\",\n\t\t);\n\t}\n\treturn config;\n}\n\nexport async function createFermataTestSuiteWithPlatform(\n\tconfig: FermataEvalServiceConfig,\n\trequest: CreateFermataTestSuiteRequest,\n\tsignal?: AbortSignal,\n): Promise<CreateFermataTestSuiteResponse> {\n\tconst response = await postPlatformConnect(\n\t\tconfig,\n\t\tCREATE_TEST_SUITE_PATH,\n\t\tstripUndefinedValues({\n\t\t\torganizationId: request.organizationId,\n\t\t\tworkspaceId: request.workspaceId,\n\t\t\tname: request.name,\n\t\t\tdescription: request.description,\n\t\t\tcases: request.cases,\n\t\t\tmetadata: request.metadata,\n\t\t}),\n\t\t{\n\t\t\tserviceName: \"fermata test suite service\",\n\t\t\tfailureMode: \"required\",\n\t\t\ttimeoutMs: config.timeoutMs,\n\t\t\tmaxAttempts: config.maxAttempts,\n\t\t\tsignal,\n\t\t},\n\t);\n\tconst payload = await parseJsonResponse(\n\t\tresponse,\n\t\t\"fermata test suite service\",\n\t);\n\treturn payload as unknown as CreateFermataTestSuiteResponse;\n}\n\nexport async function runFermataTestSuiteWithPlatform(\n\tconfig: FermataEvalServiceConfig,\n\trequest: RunFermataTestSuiteRequest,\n\tsignal?: AbortSignal,\n): Promise<RunFermataTestSuiteResponse> {\n\tconst response = await postPlatformConnect(\n\t\tconfig,\n\t\tRUN_TEST_SUITE_PATH,\n\t\tstripUndefinedValues({\n\t\t\tsuiteId: request.suiteId,\n\t\t\torganizationId: request.organizationId,\n\t\t\tworkspaceId: request.workspaceId,\n\t\t\tevaluationId: request.evaluationId,\n\t\t\tmodel: request.model,\n\t\t\trunContext: request.runContext,\n\t\t\tmetadata: request.metadata,\n\t\t\trunIdPrefix: request.runIdPrefix,\n\t\t\ttraceId: request.traceId,\n\t\t\tlineageId: request.lineageId,\n\t\t\tcaseIds: request.caseIds,\n\t\t\tmaxConcurrency: request.maxConcurrency,\n\t\t\tstopOnFirstFailure: request.stopOnFirstFailure,\n\t\t\tcandidates: request.candidates,\n\t\t}),\n\t\t{\n\t\t\tserviceName: \"fermata test suite execution service\",\n\t\t\tfailureMode: \"required\",\n\t\t\ttimeoutMs: config.timeoutMs,\n\t\t\tmaxAttempts: config.maxAttempts,\n\t\t\tsignal,\n\t\t},\n\t);\n\tconst payload = await parseJsonResponse(\n\t\tresponse,\n\t\t\"fermata test suite execution service\",\n\t);\n\treturn payload as unknown as RunFermataTestSuiteResponse;\n}\n"]}

package/dist/platform/fermata-scenario-suite.d.ts

@@ -0,0 +1,99 @@
+import type { CreateFermataTestSuiteRequest, FermataEvalServiceConfig, RunFermataTestSuiteRequest } from "./fermata-eval-client.js";
+type AgentTrajectoryScenarioAssertionStatus = "pass" | "fail" | "warn";
+interface AgentTrajectoryScenarioEvidence {
+    kind: string;
+    id: string;
+    source: string;
+    label: string;
+}
+interface AgentTrajectoryScenarioAssertionResult {
+    id: string;
+    kind: string;
+    status: AgentTrajectoryScenarioAssertionStatus;
+    severity: string;
+    message: string;
+    evidence: AgentTrajectoryScenarioEvidence[];
+}
+interface AgentTrajectoryScenarioResult {
+    schemaVersion: string;
+    scenarioSchemaVersion: string;
+    scenario: {
+        id: string;
+        title: string;
+        expectedOutcome: string;
+        observedOutcome: string;
+        reviewLabels: string[];
+    };
+    run: {
+        id: string;
+        sessionId: string;
+        source?: string;
+        platformBacked?: boolean;
+        scenarioId?: string;
+        replay?: boolean;
+        generatedAt?: string;
+    };
+    counts: {
+        assertions: number;
+        passed: number;
+        failed: number;
+        warnings: number;
+        events: number;
+        toolCalls: number;
+        replayDeltas: number;
+        scoreFailures: number;
+        scoreWarnings: number;
+    };
+    platform: {
+        primitive: string;
+        evidenceEventType: string;
+        traceJoinKeys: string[];
+        eventType?: string;
+        rationale?: string;
+    };
+    assertions: AgentTrajectoryScenarioAssertionResult[];
+    provenance?: Array<{
+        eventId: string;
+        eventType: string;
+        phase?: string;
+        actor?: string;
+    }>;
+}
+export interface FermataScenarioSuiteLLMJudgeOptions {
+    judgeId: string;
+    verifierJudgeId?: string;
+    rubric: string;
+    minScore?: number;
+    repeat?: number;
+    quorum?: number;
+    recordJudgeValidation?: boolean;
+    requireCalibratedJudge?: boolean;
+    minJudgeValidationAccuracy?: number;
+    minJudgeValidationCount?: number;
+    rubricVersion?: string;
+    calibrationCohort?: string;
+    advisoryOnly?: boolean;
+}
+export interface FermataScenarioSuiteLLMPairwiseJudgeOptions extends FermataScenarioSuiteLLMJudgeOptions {
+    baselineLabel?: string;
+    candidateLabel?: string;
+}
+export interface FermataScenarioSuiteOptions {
+    name?: string;
+    description?: string;
+    model?: string;
+    candidateId?: string;
+    candidateLabel?: string;
+    evaluationId?: string;
+    runIdPrefix?: string;
+    lineageId?: string;
+    traceId?: string;
+    maxConcurrency?: number;
+    metadata?: Record<string, unknown>;
+    llmJudge?: FermataScenarioSuiteLLMJudgeOptions;
+    llmPairwiseJudge?: FermataScenarioSuiteLLMPairwiseJudgeOptions;
+}
+export declare function buildFermataCreateScenarioSuiteRequest(config: Pick<FermataEvalServiceConfig, "organizationId" | "workspaceId">, expected: AgentTrajectoryScenarioResult, actual: AgentTrajectoryScenarioResult, options?: FermataScenarioSuiteOptions): CreateFermataTestSuiteRequest;
+export declare function buildFermataRunScenarioSuiteRequest(config: Pick<FermataEvalServiceConfig, "organizationId" | "workspaceId">, suiteId: string, expected: AgentTrajectoryScenarioResult, actual: AgentTrajectoryScenarioResult, options?: FermataScenarioSuiteOptions): RunFermataTestSuiteRequest;
+export {};
+//# sourceMappingURL=fermata-scenario-suite.d.ts.map

package/dist/platform/fermata-scenario-suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fermata-scenario-suite.d.ts","sourceRoot":"","sources":["../../src/platform/fermata-scenario-suite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,6BAA6B,EAE7B,wBAAwB,EAIxB,0BAA0B,EAC1B,MAAM,0BAA0B,CAAC;AAElC,KAAK,sCAAsC,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAEvE,UAAU,+BAA+B;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AAED,UAAU,sCAAsC;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,sCAAsC,CAAC;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,+BAA+B,EAAE,CAAC;CAC5C;AAED,UAAU,6BAA6B;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,QAAQ,EAAE;QACT,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,GAAG,EAAE;QACJ,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,QAAQ,EAAE;QACT,SAAS,EAAE,MAAM,CAAC;QAClB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,EAAE,sCAAsC,EAAE,CAAC;IACrD,UAAU,CAAC,EAAE,KAAK,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;CACH;AAcD,MAAM,WAAW,mCAAmC;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,2CAChB,SAAQ,mCAAmC;IAC3C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,2BAA2B;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,QAAQ,CAAC,EAAE,mCAAmC,CAAC;IAC/C,gBAAgB,CAAC,EAAE,2CAA2C,CAAC;CAC/D;AAmhBD,wBAAgB,sCAAsC,CACrD,MAAM,EAAE,IAAI,CAAC,wBAAwB,EAAE,gBAAgB,GAAG,aAAa,CAAC,EACxE,QAAQ,EAAE,6BAA6B,EACvC,MAAM,EAAE,6BAA6B,EACrC,OAAO,GAAE,2BAAgC,GACvC,6BAA6B,CA6B/B;AAED,wBAAgB,mCAAmC,CAClD,MAAM,EAAE,IAAI,CAAC,wBAAwB,EAAE,gBAAgB,GAAG,aAAa,CAAC,EACxE,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,6BAA6B,EACvC,MAAM,EAAE,6BAA6B,EACrC,OAAO,GAAE,2BAAgC,GACvC,0BAA0B,CA0C5B"}