@plumloom/cli 0.1.0

package/dist/run.js

@@ -0,0 +1,134 @@
+import { CommanderError } from 'commander';
+import { validateCliKey } from './auth/credentials.js';
+import { KeyringCredentialStore } from './auth/keyring-store.js';
+import { redactText } from './auth/redact.js';
+import { RuntimeCommandExecutor } from './commands/executor.js';
+import { configureSafeProgramOutput, createProgram, normalizeGlobalCliFlags, } from './commands/program.js';
+import { loadConfiguration } from './config.js';
+import { asAutoevalError } from './errors/autoeval-error.js';
+import { hintForError } from './errors/hints.js';
+import { OutputWriter } from './output/writer.js';
+import { renderSplash, renderWelcome, resolveColorMode, resolveBoltSize, shouldShowSplash, supportsUnicode, } from './output/splash.js';
+import { CLI_VERSION } from './version.js';
+function hasPrimaryCommand(userArgs) {
+    for (const token of userArgs) {
+        if (token === '--')
+            return false;
+        if (token.startsWith('-'))
+            continue;
+        return true;
+    }
+    return false;
+}
+function hasRootHelpFlag(userArgs) {
+    for (const token of userArgs) {
+        if (token === '--')
+            return false;
+        if (token === '--help' || token === '-h')
+            return true;
+        if (!token.startsWith('-'))
+            return false;
+    }
+    return false;
+}
+function hasJsonFlag(userArgs) {
+    return userArgs.includes('--json');
+}
+export function shouldRenderCliSplash(argv, io, environment) {
+    const userArgs = argv.slice(2);
+    const rootInteractive = !hasPrimaryCommand(userArgs) && (userArgs.length === 0 || hasRootHelpFlag(userArgs));
+    return shouldShowSplash({
+        interactive: rootInteractive,
+        json: hasJsonFlag(userArgs),
+        stdinIsTTY: io.stdinIsTTY,
+        stdoutIsTTY: io.stdoutIsTTY,
+        stderrIsTTY: io.stderrIsTTY,
+        environment,
+    });
+}
+async function resolveSplashSignedIn(environment, apiOrigin) {
+    const environmentKey = environment.AUTOEVAL_API_KEY;
+    if (environmentKey !== undefined && environmentKey.trim() !== '') {
+        try {
+            validateCliKey(environmentKey);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    try {
+        const storedKey = await new KeyringCredentialStore(apiOrigin).read();
+        if (storedKey === undefined || storedKey.trim() === '')
+            return false;
+        validateCliKey(storedKey);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Builds and runs the CLI program, translating thrown errors into redacted
+ * output and process exit codes.
+ */
+export async function runCli(options = {}) {
+    const abortController = new AbortController();
+    const onInterrupt = () => abortController.abort(new Error('interrupted'));
+    const onTerminate = () => abortController.abort(new Error('terminated'));
+    process.once('SIGINT', onInterrupt);
+    process.once('SIGTERM', onTerminate);
+    const writer = new OutputWriter(process.stdout, process.stderr);
+    const sourceArgv = [...(options.argv ?? process.argv)];
+    const argv = [...sourceArgv.slice(0, 2), ...normalizeGlobalCliFlags(sourceArgv.slice(2))];
+    const environment = process.env;
+    try {
+        const configuration = loadConfiguration();
+        const commandExecutor = new RuntimeCommandExecutor({
+            configuration,
+            signal: abortController.signal,
+        });
+        const extensions = options.extend
+            ? await options.extend({
+                configuration,
+                commandExecutor,
+                signal: abortController.signal,
+            })
+            : [];
+        const program = createProgram(commandExecutor, extensions);
+        configureSafeProgramOutput(program, process.stdout, process.stderr);
+        if (shouldRenderCliSplash(argv, {
+            stdinIsTTY: process.stdin.isTTY,
+            stdoutIsTTY: process.stdout.isTTY,
+            stderrIsTTY: process.stderr.isTTY,
+        }, environment)) {
+            const signedIn = await resolveSplashSignedIn(environment, configuration.apiBaseUrl.origin);
+            const unicode = supportsUnicode(environment);
+            const splashContext = {
+                version: CLI_VERSION,
+                signedIn,
+                colorMode: resolveColorMode(environment),
+                unicode,
+                boltSize: resolveBoltSize(environment, unicode),
+            };
+            process.stderr.write(renderSplash(splashContext));
+            if (argv.slice(2).length === 0) {
+                process.stderr.write(`${renderWelcome(splashContext)}\n`);
+                return;
+            }
+        }
+        await program.parseAsync(argv);
+    }
+    catch (error) {
+        if (error instanceof CommanderError) {
+            if (error.code === 'commander.helpDisplayed' || error.code === 'commander.version')
+                return;
+            process.exitCode = error.exitCode;
+            return;
+        }
+        const normalized = asAutoevalError(error);
+        writer.writeError(redactText(normalized.message), hasJsonFlag(argv.slice(2)), normalized.code, hintForError(normalized));
+        process.exitCode = normalized.exitCode;
+    }
+}
+//# sourceMappingURL=run.js.map

package/dist/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EACL,0BAA0B,EAC1B,aAAa,EACb,uBAAuB,GAExB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,iBAAiB,EAA8B,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAQ3C,SAAS,iBAAiB,CAAC,QAA2B;IACpD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QACjC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QACpC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,QAA2B;IAClD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QACjC,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,QAA2B;IAC9C,OAAO,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,IAAuB,EACvB,EAAe,EACf,WAAyD;IAEzD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,eAAe,GACnB,CAAC,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvF,OAAO,gBAAgB,CAAC;QACtB,WAAW,EAAE,eAAe;QAC5B,IAAI,EAAE,WAAW,CAAC,QAAQ,CAAC;QAC3B,UAAU,EAAE,EAAE,CAAC,UAAU;QACzB,WAAW,EAAE,EAAE,CAAC,WAAW;QAC3B,WAAW,EAAE,EAAE,CAAC,WAAW;QAC3B,WAAW;KACZ,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,WAAyD,EACzD,SAAiB;IAEjB,MAAM,cAAc,GAAG,WAAW,CAAC,gBAAgB,CAAC;IACpD,IAAI,cAAc,KAAK,SAAS,IAAI,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACjE,IAAI,CAAC;YACH,cAAc,CAAC,cAAc,CAAC,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,IAAI,sBAAsB,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,KAAK,CAAC;QACrE,cAAc,CAAC,SAAS,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAeD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,UAAyB,EAAE;IACtD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAC9C,MAAM,WAAW,GAAG,GAAS,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;IAChF,MAAM,WAAW,GAAG,GAAS,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;IAC/E,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAErC,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,uBAAuB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,iBAAiB,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,IAAI,sBAAsB,CAAC;YACjD,aAAa;YACb,MAAM,EAAE,eAAe,CAAC,MAAM;SAC/B,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM;YAC/B,CAAC,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC;gBACnB,aAAa;gBACb,eAAe;gBACf,MAAM,EAAE,eAAe,CAAC,MAAM;aAC/B,CAAC;YACJ,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,OAAO,GAAG,aAAa,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;QAC3D,0BAA0B,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACpE,IACE,qBAAqB,CACnB,IAAI,EACJ;YACE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK;YAC/B,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK;YACjC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK;SAClC,EACD,WAAW,CACZ,EACD,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC3F,MAAM,OAAO,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG;gBACpB,OAAO,EAAE,WAAW;gBACpB,QAAQ;gBACR,SAAS,EAAE,gBAAgB,CAAC,WAAW,CAAC;gBACxC,OAAO;gBACP,QAAQ,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC;aAChD,CAAC;YACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC;YAElD,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC1D,OAAO;YACT,CAAC;QACH,CAAC;QACD,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,IAAI,KAAK,yBAAyB,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB;gBAAE,OAAO;YAC3F,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;YAClC,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,CAAC,UAAU,CACf,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,EAC9B,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAC1B,UAAU,CAAC,IAAI,EACf,YAAY,CAAC,UAAU,CAAC,CACzB,CAAC;QACF,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;IACzC,CAAC;AACH,CAAC"}

package/dist/runtime/action-context.d.ts

@@ -0,0 +1,19 @@
+import type { ActionContext } from '../actions/context.js';
+import type { CredentialStore } from '../auth/credential-store.js';
+import type { AutoevalConfiguration } from '../config.js';
+import { type FetchImplementation, type ApiDiagnostic } from '../api/client.js';
+export type ApiActionContextInput = {
+    configuration: AutoevalConfiguration;
+    apiKey: string;
+    fetchImplementation?: FetchImplementation;
+    onDiagnostic?: (diagnostic: ApiDiagnostic) => void;
+};
+export declare function createApiActionContext(input: ApiActionContextInput): ActionContext;
+export declare function createAuthenticatedActionContext(input: {
+    configuration: AutoevalConfiguration;
+    environment?: Readonly<Record<string, string | undefined>>;
+    store?: CredentialStore;
+    fetchImplementation?: FetchImplementation;
+    onDiagnostic?: (diagnostic: ApiDiagnostic) => void;
+}): Promise<ActionContext>;
+//# sourceMappingURL=action-context.d.ts.map

package/dist/runtime/action-context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"action-context.d.ts","sourceRoot":"","sources":["../../src/runtime/action-context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,6BAA6B,CAAC;AAGjF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAG1D,OAAO,EAAa,KAAK,mBAAmB,EAAE,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAY3F,MAAM,MAAM,qBAAqB,GAAG;IAClC,aAAa,EAAE,qBAAqB,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,aAAa,KAAK,IAAI,CAAC;CACpD,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,qBAAqB,GAAG,aAAa,CAalF;AAED,wBAAsB,gCAAgC,CAAC,KAAK,EAAE;IAC5D,aAAa,EAAE,qBAAqB,CAAC;IACrC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IAC3D,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,aAAa,KAAK,IAAI,CAAC;CACpD,GAAG,OAAO,CAAC,aAAa,CAAC,CAgBzB"}

package/dist/runtime/action-context.js

@@ -0,0 +1,40 @@
+import { resolveCredential } from '../auth/credentials.js';
+import { KeyringCredentialStore } from '../auth/keyring-store.js';
+import { AutoevalError } from '../errors/autoeval-error.js';
+import { AutoevalApiClient } from '../api/api.js';
+import { ApiClient } from '../api/client.js';
+const nonInteractivePrompt = {
+    request: () => Promise.reject(new AutoevalError('Interactive credential prompts are unavailable in this runtime.', {
+        kind: 'authentication',
+        code: 'CREDENTIAL_REQUIRED',
+    })),
+};
+export function createApiActionContext(input) {
+    return {
+        api: new AutoevalApiClient(new ApiClient({
+            baseUrl: input.configuration.apiBaseUrl,
+            apiKey: input.apiKey,
+            requestTimeoutMs: input.configuration.requestTimeoutMs,
+            maxResponseBytes: input.configuration.maxResponseBytes,
+            ...(input.fetchImplementation ? { fetchImplementation: input.fetchImplementation } : {}),
+            ...(input.onDiagnostic ? { onDiagnostic: input.onDiagnostic } : {}),
+        })),
+    };
+}
+export async function createAuthenticatedActionContext(input) {
+    const environment = input.environment ?? process.env;
+    const store = input.store ?? new KeyringCredentialStore(input.configuration.apiBaseUrl.origin);
+    const credential = await resolveCredential({
+        environment,
+        store,
+        prompt: nonInteractivePrompt,
+        allowPrompt: false,
+    });
+    return createApiActionContext({
+        configuration: input.configuration,
+        apiKey: credential.apiKey,
+        ...(input.fetchImplementation ? { fetchImplementation: input.fetchImplementation } : {}),
+        ...(input.onDiagnostic ? { onDiagnostic: input.onDiagnostic } : {}),
+    });
+}
+//# sourceMappingURL=action-context.js.map

package/dist/runtime/action-context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"action-context.js","sourceRoot":"","sources":["../../src/runtime/action-context.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAElE,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,SAAS,EAAgD,MAAM,kBAAkB,CAAC;AAE3F,MAAM,oBAAoB,GAAiB;IACzC,OAAO,EAAE,GAAG,EAAE,CACZ,OAAO,CAAC,MAAM,CACZ,IAAI,aAAa,CAAC,iEAAiE,EAAE;QACnF,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,qBAAqB;KAC5B,CAAC,CACH;CACJ,CAAC;AASF,MAAM,UAAU,sBAAsB,CAAC,KAA4B;IACjE,OAAO;QACL,GAAG,EAAE,IAAI,iBAAiB,CACxB,IAAI,SAAS,CAAC;YACZ,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,UAAU;YACvC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,gBAAgB,EAAE,KAAK,CAAC,aAAa,CAAC,gBAAgB;YACtD,gBAAgB,EAAE,KAAK,CAAC,aAAa,CAAC,gBAAgB;YACtD,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxF,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC,CACH;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gCAAgC,CAAC,KAMtD;IACC,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC;IACrD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,IAAI,sBAAsB,CAAC,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC/F,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC;QACzC,WAAW;QACX,KAAK;QACL,MAAM,EAAE,oBAAoB;QAC5B,WAAW,EAAE,KAAK;KACnB,CAAC,CAAC;IAEH,OAAO,sBAAsB,CAAC;QAC5B,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpE,CAAC,CAAC;AACL,CAAC"}

package/dist/suite/execution.d.ts

@@ -0,0 +1,46 @@
+import type { PollClock } from '../polling/clock.js';
+/**
+ * Two-plane suite execution.
+ *
+ * Plane 1 (execution) runs evals with bounded concurrency and an optional
+ * start stagger. A slot is occupied only while an eval is executing: the moment
+ * an eval reaches a terminal execution state the slot is released and the next
+ * queued eval starts.
+ *
+ * Plane 2 (results) fetches and normalizes results for evals that already
+ * finished, concurrently with evals that are still executing. It has its own
+ * concurrency budget and never occupies an execution slot.
+ *
+ * A failure in either plane is recorded against that eval only; the remaining
+ * evals keep going.
+ */
+export type SuiteEvalStatus = 'completed' | 'execution_failed' | 'result_failed';
+export type SuiteItem = {
+    index: number;
+    inputFile: string;
+};
+export type SuiteEvalRecord<TExecution, TResults> = {
+    index: number;
+    inputFile: string;
+    status: SuiteEvalStatus;
+    execution?: TExecution;
+    results?: TResults;
+    error?: string;
+};
+export type TwoPlaneSuiteInput<TExecution, TResults> = {
+    inputFiles: readonly string[];
+    /** Maximum evals executing at the same time. */
+    executionConcurrency: number;
+    /** Maximum result fetches at the same time. Defaults to the execution budget. */
+    resultConcurrency?: number;
+    /** Delay applied between successive execution starts. */
+    staggerMs: number;
+    clock: PollClock;
+    signal?: AbortSignal;
+    /** Create + run + poll one eval to a terminal execution state. */
+    execute: (item: SuiteItem) => Promise<TExecution>;
+    /** Fetch and normalize results for a terminal eval. */
+    fetchResults: (execution: TExecution, item: SuiteItem) => Promise<TResults>;
+};
+export declare function runTwoPlaneSuite<TExecution, TResults>(input: TwoPlaneSuiteInput<TExecution, TResults>): Promise<SuiteEvalRecord<TExecution, TResults>[]>;
+//# sourceMappingURL=execution.d.ts.map

package/dist/suite/execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution.d.ts","sourceRoot":"","sources":["../../src/suite/execution.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAErD;;;;;;;;;;;;;;GAcG;AAEH,MAAM,MAAM,eAAe,GAAG,WAAW,GAAG,kBAAkB,GAAG,eAAe,CAAC;AAEjF,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,eAAe,CAAC,UAAU,EAAE,QAAQ,IAAI;IAClD,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,kBAAkB,CAAC,UAAU,EAAE,QAAQ,IAAI;IACrD,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iFAAiF;IACjF,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,CAAC;IACjB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,kEAAkE;IAClE,OAAO,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;IAClD,uDAAuD;IACvD,YAAY,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CAC7E,CAAC;AAqBF,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,QAAQ,EACzD,KAAK,EAAE,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,GAC9C,OAAO,CAAC,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,CAgFlD"}

package/dist/suite/execution.js

@@ -0,0 +1,97 @@
+function describeError(error) {
+    if (error instanceof Error && error.message.trim() !== '')
+        return error.message;
+    return String(error);
+}
+/** Bounded worker pool over a shared queue of tasks. */
+async function drain(taskCount, workers, run) {
+    let next = 0;
+    const worker = async () => {
+        while (true) {
+            const index = next;
+            if (index >= taskCount)
+                return;
+            next += 1;
+            await run(index);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(workers, taskCount)) }, worker));
+}
+export async function runTwoPlaneSuite(input) {
+    const items = input.inputFiles.map((inputFile, index) => ({ index, inputFile }));
+    const records = items.map((item) => ({
+        index: item.index,
+        inputFile: item.inputFile,
+        status: 'execution_failed',
+        error: 'Eval did not start.',
+    }));
+    // Results plane: its own bounded pool, fed as evals reach terminal state.
+    const resultBudget = Math.max(1, input.resultConcurrency ?? input.executionConcurrency);
+    let activeResultFetches = 0;
+    const resultWaiters = [];
+    const resultFetches = [];
+    const acquireResultSlot = async () => {
+        if (activeResultFetches < resultBudget) {
+            activeResultFetches += 1;
+            return;
+        }
+        await new Promise((resolveWaiter) => resultWaiters.push(resolveWaiter));
+        activeResultFetches += 1;
+    };
+    const releaseResultSlot = () => {
+        activeResultFetches -= 1;
+        resultWaiters.shift()?.();
+    };
+    const enqueueResultFetch = (item, execution) => {
+        const record = records[item.index];
+        if (record === undefined)
+            return;
+        resultFetches.push((async () => {
+            await acquireResultSlot();
+            try {
+                record.results = await input.fetchResults(execution, item);
+                record.status = 'completed';
+                delete record.error;
+            }
+            catch (error) {
+                record.status = 'result_failed';
+                record.error = describeError(error);
+            }
+            finally {
+                releaseResultSlot();
+            }
+        })());
+    };
+    // Execution plane.
+    let nextStartAtMs = input.clock.now();
+    const executionWorkers = Math.max(1, input.executionConcurrency);
+    await drain(items.length, executionWorkers, async (index) => {
+        const item = items[index];
+        if (item === undefined)
+            return;
+        const record = records[index];
+        if (record === undefined)
+            return;
+        const scheduledStartMs = nextStartAtMs;
+        nextStartAtMs += input.staggerMs;
+        const delayMs = Math.max(0, scheduledStartMs - input.clock.now());
+        if (delayMs > 0) {
+            await input.clock.sleep(delayMs, input.signal);
+        }
+        try {
+            const execution = await input.execute(item);
+            record.execution = execution;
+            record.status = 'result_failed';
+            record.error = 'Results were not fetched.';
+            // Slot is released as soon as this returns; result fetching runs off-plane.
+            enqueueResultFetch(item, execution);
+        }
+        catch (error) {
+            record.status = 'execution_failed';
+            record.error = describeError(error);
+        }
+    });
+    await Promise.all(resultFetches);
+    return records;
+}
+//# sourceMappingURL=execution.js.map

package/dist/suite/execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution.js","sourceRoot":"","sources":["../../src/suite/execution.ts"],"names":[],"mappings":"AAkDA,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAChF,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,wDAAwD;AACxD,KAAK,UAAU,KAAK,CAAC,SAAiB,EAAE,OAAe,EAAE,GAAqC;IAC5F,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,IAAI,KAAK,IAAI,SAAS;gBAAE,OAAO;YAC/B,IAAI,IAAI,CAAC,CAAC;YACV,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;AAC/F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAA+C;IAE/C,MAAM,KAAK,GAAgB,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IAC9F,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAwC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1E,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,MAAM,EAAE,kBAAkB;QAC1B,KAAK,EAAE,qBAAqB;KAC7B,CAAC,CAAC,CAAC;IAEJ,0EAA0E;IAC1E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,iBAAiB,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxF,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,MAAM,aAAa,GAAmB,EAAE,CAAC;IACzC,MAAM,aAAa,GAAoB,EAAE,CAAC;IAE1C,MAAM,iBAAiB,GAAG,KAAK,IAAmB,EAAE;QAClD,IAAI,mBAAmB,GAAG,YAAY,EAAE,CAAC;YACvC,mBAAmB,IAAI,CAAC,CAAC;YACzB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,OAAO,CAAO,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;QAC9E,mBAAmB,IAAI,CAAC,CAAC;IAC3B,CAAC,CAAC;IACF,MAAM,iBAAiB,GAAG,GAAS,EAAE;QACnC,mBAAmB,IAAI,CAAC,CAAC;QACzB,aAAa,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC;IAC5B,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG,CAAC,IAAe,EAAE,SAAqB,EAAQ,EAAE;QAC1E,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QACjC,aAAa,CAAC,IAAI,CAChB,CAAC,KAAK,IAAI,EAAE;YACV,MAAM,iBAAiB,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,CAAC,OAAO,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBAC3D,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;gBAC5B,OAAO,MAAM,CAAC,KAAK,CAAC;YACtB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,MAAM,GAAG,eAAe,CAAC;gBAChC,MAAM,CAAC,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACtC,CAAC;oBAAS,CAAC;gBACT,iBAAiB,EAAE,CAAC;YACtB,CAAC;QACH,CAAC,CAAC,EAAE,CACL,CAAC;IACJ,CAAC,CAAC;IAEF,mBAAmB;IACnB,IAAI,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IACtC,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAEjE,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO;QAC/B,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QAEjC,MAAM,gBAAgB,GAAG,aAAa,CAAC;QACvC,aAAa,IAAI,KAAK,CAAC,SAAS,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;QAClE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;YAC7B,MAAM,CAAC,MAAM,GAAG,eAAe,CAAC;YAChC,MAAM,CAAC,KAAK,GAAG,2BAA2B,CAAC;YAC3C,4EAA4E;YAC5E,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC;YACnC,MAAM,CAAC,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACjC,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/suite/manifest.d.ts

@@ -0,0 +1,16 @@
+import { type SuiteGatePolicy } from '../gate/policy.js';
+export type SuiteManifestEntry = {
+    /** Eval file path, resolved against the manifest's directory. */
+    file: string;
+    /** Suite default merged with the per-eval override. */
+    gate: SuiteGatePolicy;
+};
+export type SuiteManifest = {
+    workspaceId: string;
+    entries: readonly SuiteManifestEntry[];
+    /** Eval file paths in manifest order. */
+    evalFiles: readonly string[];
+};
+export declare function parseSuiteManifest(source: string, manifestPath: string): SuiteManifest;
+export declare function readSuiteManifest(manifestPath: string): Promise<SuiteManifest>;
+//# sourceMappingURL=manifest.d.ts.map

package/dist/suite/manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/suite/manifest.ts"],"names":[],"mappings":"AAOA,OAAO,EAAwC,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAyC/F,MAAM,MAAM,kBAAkB,GAAG;IAC/B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,IAAI,EAAE,eAAe,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACvC,yCAAyC;IACzC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAwCtF;AAED,wBAAsB,iBAAiB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAYpF"}

package/dist/suite/manifest.js

@@ -0,0 +1,92 @@
+import { readFile } from 'node:fs/promises';
+import { dirname, isAbsolute, resolve } from 'node:path';
+import { parse as parseYaml } from 'yaml';
+import { z } from 'zod';
+import { AutoevalError } from '../errors/autoeval-error.js';
+import { mergePolicies, suiteGatePolicySchema } from '../gate/policy.js';
+/**
+ * A suite manifest is the CI unit of work: one file listing the eval files a
+ * release is gated on. It carries no eval configuration of its own — artifact
+ * and reference-document paths live inside each eval file — so the manifest
+ * stays reviewable and stable across runs.
+ *
+ *   workspace: 4c1c8a3e-...
+ *   gate:
+ *     minOverall: 4.0
+ *   evals:
+ *     - ./evals/refund-policy.autoeval.json
+ *     - file: ./evals/support-conversation.autoeval.json
+ *       gate:
+ *         metrics:
+ *           factuality: 4.0
+ *
+ * YAML and JSON are both accepted; paths resolve relative to the manifest.
+ * The optional `gate` blocks configure the release gate only; they are never
+ * sent to the backend.
+ */
+const evalEntrySchema = z.union([
+    z.string().trim().min(1),
+    z
+        .object({
+        file: z.string().trim().min(1),
+        gate: suiteGatePolicySchema.optional(),
+    })
+        .strict(),
+]);
+const manifestSchema = z
+    .object({
+    workspace: z.string().trim().min(1),
+    gate: suiteGatePolicySchema.optional(),
+    evals: z.array(evalEntrySchema).min(1),
+})
+    .strict();
+export function parseSuiteManifest(source, manifestPath) {
+    let document;
+    try {
+        document = parseYaml(source);
+    }
+    catch (error) {
+        throw new AutoevalError(`${manifestPath} is not valid YAML or JSON.`, {
+            kind: 'usage',
+            code: 'INVALID_SUITE_MANIFEST',
+            cause: error,
+        });
+    }
+    const parsed = manifestSchema.safeParse(document);
+    if (!parsed.success) {
+        const details = parsed.error.issues
+            .slice(0, 5)
+            .map((issue) => `${issue.path.map(String).join('.') || 'manifest'}: ${issue.message}`)
+            .join('; ');
+        throw new AutoevalError(`${manifestPath} is not a valid suite manifest: ${details}.`, {
+            kind: 'validation',
+            code: 'INVALID_SUITE_MANIFEST_SCHEMA',
+            cause: parsed.error,
+        });
+    }
+    const baseDirectory = dirname(resolve(manifestPath));
+    const toPath = (entry) => isAbsolute(entry) ? entry : resolve(baseDirectory, entry);
+    const entries = parsed.data.evals.map((entry) => typeof entry === 'string'
+        ? { file: toPath(entry), gate: mergePolicies(parsed.data.gate, undefined) }
+        : { file: toPath(entry.file), gate: mergePolicies(parsed.data.gate, entry.gate) });
+    return {
+        workspaceId: parsed.data.workspace,
+        entries,
+        evalFiles: entries.map((entry) => entry.file),
+    };
+}
+export async function readSuiteManifest(manifestPath) {
+    let source;
+    try {
+        source = await readFile(manifestPath, 'utf8');
+    }
+    catch (error) {
+        throw new AutoevalError(`Suite manifest was not found or could not be read: ${manifestPath}`, {
+            kind: 'usage',
+            code: 'SUITE_MANIFEST_READ_FAILED',
+            cause: error,
+        });
+    }
+    return parseSuiteManifest(source, manifestPath);
+}
+//# sourceMappingURL=manifest.js.map

package/dist/suite/manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/suite/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzD,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAwB,MAAM,mBAAmB,CAAC;AAE/F;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,CAAC;SACE,MAAM,CAAC;QACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9B,IAAI,EAAE,qBAAqB,CAAC,QAAQ,EAAE;KACvC,CAAC;SACD,MAAM,EAAE;CACZ,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC;KACrB,MAAM,CAAC;IACN,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,IAAI,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IACtC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,MAAM,EAAE,CAAC;AAgBZ,MAAM,UAAU,kBAAkB,CAAC,MAAc,EAAE,YAAoB;IACrE,IAAI,QAAiB,CAAC;IACtB,IAAI,CAAC;QACH,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAY,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAC,GAAG,YAAY,6BAA6B,EAAE;YACpE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,wBAAwB;YAC9B,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAChC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aACX,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,UAAU,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aACrF,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,aAAa,CAAC,GAAG,YAAY,mCAAmC,OAAO,GAAG,EAAE;YACpF,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,+BAA+B;YACrC,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,CAAC,KAAa,EAAU,EAAE,CACvC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAE5D,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAqB,CAAC,KAAK,EAAE,EAAE,CAClE,OAAO,KAAK,KAAK,QAAQ;QACvB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE;QAC3E,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CACpF,CAAC;IAEF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS;QAClC,OAAO;QACP,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,YAAoB;IAC1D,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAChD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAC,sDAAsD,YAAY,EAAE,EAAE;YAC5F,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,4BAA4B;YAClC,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IACD,OAAO,kBAAkB,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AAClD,CAAC"}

package/dist/suite/reporting.d.ts

@@ -0,0 +1,35 @@
+import type { SuiteSummary } from '../actions/suite.js';
+import type { SuiteGateReport } from '../gate/decision.js';
+/**
+ * Failure clustering.
+ *
+ * A blocked suite usually reports the same root cause many times: one upstream
+ * outage, one malformed artifact, one metric that regressed across every eval.
+ * Reading the per-eval list to rediscover that is noise, so this module groups
+ * failures by a normalized signature and reports each distinct cause once, with
+ * its count and one exemplar. It is pure: it reads only what the execution,
+ * results, and gate planes already produced.
+ */
+export type FailureClusterCategory = 'execution' | 'threshold' | 'inconclusive';
+export type FailureCluster = {
+    /** Stable machine identifier for the normalized cause. */
+    id: string;
+    category: FailureClusterCategory;
+    label: string;
+    count: number;
+    /** Eval labels (name or file) contributing to the cluster, in suite order. */
+    evals: string[];
+    /** One representative raw message, kept verbatim for debugging. */
+    exemplar: string;
+};
+/**
+ * Clusters everything that blocked a release: execution and result errors, and
+ * every failing or inconclusive gate check.
+ */
+export declare function clusterSuiteFailures(report: SuiteGateReport): FailureCluster[];
+/**
+ * Clusters execution and result-plane errors from a suite run, for the ungated
+ * `suite run` command where no thresholds are configured.
+ */
+export declare function clusterExecutionFailures(summary: SuiteSummary): FailureCluster[];
+//# sourceMappingURL=reporting.d.ts.map

package/dist/suite/reporting.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporting.d.ts","sourceRoot":"","sources":["../../src/suite/reporting.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,KAAK,EAAoB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE7E;;;;;;;;;GASG;AAEH,MAAM,MAAM,sBAAsB,GAAG,WAAW,GAAG,WAAW,GAAG,cAAc,CAAC;AAEhF,MAAM,MAAM,cAAc,GAAG;IAC3B,0DAA0D;IAC1D,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,sBAAsB,CAAC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAwFF;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,eAAe,GAAG,cAAc,EAAE,CAmC9E;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,YAAY,GAAG,cAAc,EAAE,CAehF"}