@agents-at-scale/ark 0.1.55 → 0.1.56

package/dist/commands/evaluation/index.d.ts

@@ -1,3 +0,0 @@
-import { Command } from 'commander';
-import type { ArkConfig } from '../../lib/config.js';
-export declare function createEvaluationCommand(_: ArkConfig): Command;

package/dist/commands/evaluation/index.js

@@ -1,60 +0,0 @@
-import { Command } from 'commander';
-import chalk from 'chalk';
-import { executeDirectEvaluation, executeQueryEvaluation, } from '../../lib/executeEvaluation.js';
-import { readStdin } from '../../lib/stdin.js';
-export function createEvaluationCommand(_) {
-    const evaluationCommand = new Command('evaluation');
-    evaluationCommand
-        .description('Execute evaluations against evaluators')
-        .argument('<evaluator-name>', 'Name of the evaluator to use')
-        .argument('[query-name]', 'Name of the query to evaluate (for query-based evaluation)')
-        .option('--input <input>', 'Input text for direct evaluation')
-        .option('--output <output>', 'Output text for direct evaluation')
-        .option('--response-target <target>', 'Response target for query evaluation (e.g., agent:my-agent)')
-        .option('--timeout <timeout>', 'Evaluation timeout (e.g., "30s", "5m")')
-        .option('--watch-timeout <timeout>', 'CLI watch timeout')
-        .action(async (evaluatorName, queryName, options) => {
-        if (options.input && options.output) {
-            await executeDirectEvaluation({
-                evaluatorName,
-                input: options.input,
-                output: options.output,
-                timeout: options.timeout,
-                watchTimeout: options.watchTimeout,
-            });
-        }
-        else if (queryName) {
-            await executeQueryEvaluation({
-                evaluatorName,
-                queryName,
-                responseTarget: options.responseTarget,
-                timeout: options.timeout,
-                watchTimeout: options.watchTimeout,
-            });
-        }
-        else {
-            const stdinQueryName = await readStdin();
-            if (stdinQueryName) {
-                await executeQueryEvaluation({
-                    evaluatorName,
-                    queryName: stdinQueryName,
-                    responseTarget: options.responseTarget,
-                    timeout: options.timeout,
-                    watchTimeout: options.watchTimeout,
-                });
-            }
-            else {
-                console.error(chalk.red('Error: Must provide either:'));
-                console.error('  - --input and --output for direct evaluation');
-                console.error('  - <query-name> for query-based evaluation');
-                console.error('  - Pipe query name from stdin');
-                console.error('\nExamples:');
-                console.error('  ark evaluation my-evaluator --input "test" --output "result"');
-                console.error('  ark evaluation my-evaluator my-query');
-                console.error('  echo "my-query" | ark evaluation my-evaluator');
-                process.exit(1);
-            }
-        }
-    });
-    return evaluationCommand;
-}

package/dist/lib/executeEvaluation.d.ts

@@ -1,16 +0,0 @@
-export interface DirectEvaluationOptions {
-    evaluatorName: string;
-    input: string;
-    output: string;
-    timeout?: string;
-    watchTimeout?: string;
-}
-export interface QueryEvaluationOptions {
-    evaluatorName: string;
-    queryName: string;
-    responseTarget?: string;
-    timeout?: string;
-    watchTimeout?: string;
-}
-export declare function executeDirectEvaluation(options: DirectEvaluationOptions): Promise<void>;
-export declare function executeQueryEvaluation(options: QueryEvaluationOptions): Promise<void>;

package/dist/lib/executeEvaluation.js

@@ -1,155 +0,0 @@
-import { execa } from 'execa';
-import ora from 'ora';
-import chalk from 'chalk';
-import output from './output.js';
-import { ExitCodes } from './errors.js';
-import { parseDuration } from './duration.js';
-async function waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay) {
-    const spinner = ora('Waiting for evaluation completion...').start();
-    try {
-        await execa('kubectl', [
-            'wait',
-            '--for=condition=Completed',
-            `evaluation/${evaluationName}`,
-            `--timeout=${Math.floor(watchTimeoutMs / 1000)}s`,
-        ], { timeout: watchTimeoutMs });
-    }
-    catch (error) {
-        spinner.stop();
-        if (error instanceof Error && error.message.includes('timed out waiting')) {
-            console.error(chalk.red(`Evaluation did not complete within ${watchTimeoutDisplay}`));
-            process.exit(ExitCodes.Timeout);
-        }
-        throw error;
-    }
-    spinner.stop();
-    const { stdout } = await execa('kubectl', ['get', 'evaluation', evaluationName, '-o', 'json'], { stdio: 'pipe' });
-    const evaluation = JSON.parse(stdout);
-    const status = evaluation.status;
-    if (status?.phase === 'done') {
-        console.log(chalk.green('\nEvaluation completed successfully:'));
-        if (status.score !== undefined) {
-            console.log(`Score: ${status.score}`);
-        }
-        if (status.passed !== undefined) {
-            console.log(`Result: ${status.passed ? chalk.green('PASSED') : chalk.red('FAILED')}`);
-        }
-        if (status.message) {
-            console.log(`Message: ${status.message}`);
-        }
-    }
-    else if (status?.phase === 'error') {
-        console.error(chalk.red(status.message || 'Evaluation failed with unknown error'));
-        process.exit(ExitCodes.OperationError);
-    }
-    else {
-        output.warning(`Unexpected evaluation phase: ${status?.phase}`);
-    }
-}
-export async function executeDirectEvaluation(options) {
-    const spinner = ora('Creating evaluation...').start();
-    const queryTimeoutMs = options.timeout
-        ? parseDuration(options.timeout)
-        : parseDuration('5m');
-    const watchTimeoutMs = options.watchTimeout
-        ? parseDuration(options.watchTimeout)
-        : queryTimeoutMs + 60000;
-    const timestamp = Date.now();
-    const evaluationName = `cli-eval-${timestamp}`;
-    const evaluationManifest = {
-        apiVersion: 'ark.mckinsey.com/v1alpha1',
-        kind: 'Evaluation',
-        metadata: {
-            name: evaluationName,
-        },
-        spec: {
-            type: 'direct',
-            evaluator: {
-                name: options.evaluatorName,
-            },
-            config: {
-                input: options.input,
-                output: options.output,
-            },
-            ...(options.timeout && { timeout: options.timeout }),
-            ttl: '1h',
-        },
-    };
-    try {
-        spinner.text = 'Submitting evaluation...';
-        await execa('kubectl', ['apply', '-f', '-'], {
-            input: JSON.stringify(evaluationManifest),
-            stdio: ['pipe', 'pipe', 'pipe'],
-        });
-        spinner.stop();
-        const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
-        await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
-    }
-    catch (error) {
-        spinner.stop();
-        console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
-        process.exit(ExitCodes.CliError);
-    }
-}
-export async function executeQueryEvaluation(options) {
-    const spinner = ora('Creating evaluation...').start();
-    const queryTimeoutMs = options.timeout
-        ? parseDuration(options.timeout)
-        : parseDuration('5m');
-    const watchTimeoutMs = options.watchTimeout
-        ? parseDuration(options.watchTimeout)
-        : queryTimeoutMs + 60000;
-    const timestamp = Date.now();
-    const evaluationName = `cli-eval-${timestamp}`;
-    let responseTarget;
-    if (options.responseTarget) {
-        const parts = options.responseTarget.split(':');
-        if (parts.length === 2) {
-            responseTarget = {
-                type: parts[0],
-                name: parts[1],
-            };
-        }
-        else {
-            spinner.stop();
-            console.error(chalk.red('Invalid response-target format. Use: type:name (e.g., agent:my-agent)'));
-            process.exit(ExitCodes.CliError);
-        }
-    }
-    const evaluationManifest = {
-        apiVersion: 'ark.mckinsey.com/v1alpha1',
-        kind: 'Evaluation',
-        metadata: {
-            name: evaluationName,
-        },
-        spec: {
-            type: 'query',
-            evaluator: {
-                name: options.evaluatorName,
-            },
-            config: {
-                queryRef: {
-                    name: options.queryName,
-                },
-                ...(responseTarget && { responseTarget }),
-            },
-            ...(options.timeout && { timeout: options.timeout }),
-            ttl: '1h',
-        },
-    };
-    try {
-        spinner.text = 'Submitting evaluation...';
-        await execa('kubectl', ['apply', '-f', '-'], {
-            input: JSON.stringify(evaluationManifest),
-            stdio: ['pipe', 'pipe', 'pipe'],
-        });
-        spinner.stop();
-        const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
-        await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
-    }
-    catch (error) {
-        spinner.stop();
-        console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
-        process.exit(ExitCodes.CliError);
-    }
-}