@agents-at-scale/ark 0.1.41 → 0.1.42

package/dist/lib/executeEvaluation.js

@@ -0,0 +1,155 @@
+import { execa } from 'execa';
+import ora from 'ora';
+import chalk from 'chalk';
+import output from './output.js';
+import { ExitCodes } from './errors.js';
+import { parseDuration } from './duration.js';
+async function waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay) {
+    const spinner = ora('Waiting for evaluation completion...').start();
+    try {
+        await execa('kubectl', [
+            'wait',
+            '--for=condition=Completed',
+            `evaluation/${evaluationName}`,
+            `--timeout=${Math.floor(watchTimeoutMs / 1000)}s`,
+        ], { timeout: watchTimeoutMs });
+    }
+    catch (error) {
+        spinner.stop();
+        if (error instanceof Error && error.message.includes('timed out waiting')) {
+            console.error(chalk.red(`Evaluation did not complete within ${watchTimeoutDisplay}`));
+            process.exit(ExitCodes.Timeout);
+        }
+        throw error;
+    }
+    spinner.stop();
+    const { stdout } = await execa('kubectl', ['get', 'evaluation', evaluationName, '-o', 'json'], { stdio: 'pipe' });
+    const evaluation = JSON.parse(stdout);
+    const status = evaluation.status;
+    if (status?.phase === 'done') {
+        console.log(chalk.green('\nEvaluation completed successfully:'));
+        if (status.score !== undefined) {
+            console.log(`Score: ${status.score}`);
+        }
+        if (status.passed !== undefined) {
+            console.log(`Result: ${status.passed ? chalk.green('PASSED') : chalk.red('FAILED')}`);
+        }
+        if (status.message) {
+            console.log(`Message: ${status.message}`);
+        }
+    }
+    else if (status?.phase === 'error') {
+        console.error(chalk.red(status.message || 'Evaluation failed with unknown error'));
+        process.exit(ExitCodes.OperationError);
+    }
+    else {
+        output.warning(`Unexpected evaluation phase: ${status?.phase}`);
+    }
+}
+export async function executeDirectEvaluation(options) {
+    const spinner = ora('Creating evaluation...').start();
+    const queryTimeoutMs = options.timeout
+        ? parseDuration(options.timeout)
+        : parseDuration('5m');
+    const watchTimeoutMs = options.watchTimeout
+        ? parseDuration(options.watchTimeout)
+        : queryTimeoutMs + 60000;
+    const timestamp = Date.now();
+    const evaluationName = `cli-eval-${timestamp}`;
+    const evaluationManifest = {
+        apiVersion: 'ark.mckinsey.com/v1alpha1',
+        kind: 'Evaluation',
+        metadata: {
+            name: evaluationName,
+        },
+        spec: {
+            type: 'direct',
+            evaluator: {
+                name: options.evaluatorName,
+            },
+            config: {
+                input: options.input,
+                output: options.output,
+            },
+            ...(options.timeout && { timeout: options.timeout }),
+            ttl: '1h',
+        },
+    };
+    try {
+        spinner.text = 'Submitting evaluation...';
+        await execa('kubectl', ['apply', '-f', '-'], {
+            input: JSON.stringify(evaluationManifest),
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        spinner.stop();
+        const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
+        await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
+    }
+    catch (error) {
+        spinner.stop();
+        console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
+        process.exit(ExitCodes.CliError);
+    }
+}
+export async function executeQueryEvaluation(options) {
+    const spinner = ora('Creating evaluation...').start();
+    const queryTimeoutMs = options.timeout
+        ? parseDuration(options.timeout)
+        : parseDuration('5m');
+    const watchTimeoutMs = options.watchTimeout
+        ? parseDuration(options.watchTimeout)
+        : queryTimeoutMs + 60000;
+    const timestamp = Date.now();
+    const evaluationName = `cli-eval-${timestamp}`;
+    let responseTarget;
+    if (options.responseTarget) {
+        const parts = options.responseTarget.split(':');
+        if (parts.length === 2) {
+            responseTarget = {
+                type: parts[0],
+                name: parts[1],
+            };
+        }
+        else {
+            spinner.stop();
+            console.error(chalk.red('Invalid response-target format. Use: type:name (e.g., agent:my-agent)'));
+            process.exit(ExitCodes.CliError);
+        }
+    }
+    const evaluationManifest = {
+        apiVersion: 'ark.mckinsey.com/v1alpha1',
+        kind: 'Evaluation',
+        metadata: {
+            name: evaluationName,
+        },
+        spec: {
+            type: 'query',
+            evaluator: {
+                name: options.evaluatorName,
+            },
+            config: {
+                queryRef: {
+                    name: options.queryName,
+                },
+                ...(responseTarget && { responseTarget }),
+            },
+            ...(options.timeout && { timeout: options.timeout }),
+            ttl: '1h',
+        },
+    };
+    try {
+        spinner.text = 'Submitting evaluation...';
+        await execa('kubectl', ['apply', '-f', '-'], {
+            input: JSON.stringify(evaluationManifest),
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        spinner.stop();
+        const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
+        await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
+    }
+    catch (error) {
+        spinner.stop();
+        console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
+        process.exit(ExitCodes.CliError);
+    }
+}

package/dist/lib/executeQuery.d.ts

@@ -9,6 +9,7 @@ export interface QueryOptions {
     timeout?: string;
     watchTimeout?: string;
     verbose?: boolean;
+    outputFormat?: string;
 }
 /**
  * Execute a query against any ARK target (model, agent, team)

package/dist/lib/executeQuery.js

@@ -7,12 +7,15 @@ import chalk from 'chalk';
 import output from './output.js';
 import { ExitCodes } from './errors.js';
 import { parseDuration } from './duration.js';
+import { getResource } from './kubectl.js';
 /**
  * Execute a query against any ARK target (model, agent, team)
  * This is the shared implementation used by all query commands
  */
 export async function executeQuery(options) {
-    const spinner = ora('Creating query...').start();
+    const spinner = options.outputFormat
+        ? null
+        : ora('Creating query...').start();
     const queryTimeoutMs = options.timeout
         ? parseDuration(options.timeout)
         : parseDuration('5m');
@@ -40,13 +43,15 @@ export async function executeQuery(options) {
     };
     try {
         // Apply the query
-        spinner.text = 'Submitting query...';
+        if (spinner)
+            spinner.text = 'Submitting query...';
         await execa('kubectl', ['apply', '-f', '-'], {
             input: JSON.stringify(queryManifest),
             stdio: ['pipe', 'pipe', 'pipe'],
         });
         // Watch for query completion using kubectl wait
-        spinner.text = 'Waiting for query completion...';
+        if (spinner)
+            spinner.text = 'Waiting for query completion...';
         try {
             await execa('kubectl', [
                 'wait',
@@ -56,7 +61,8 @@ export async function executeQuery(options) {
             ], { timeout: watchTimeoutMs });
         }
         catch (error) {
-            spinner.stop();
+            if (spinner)
+                spinner.stop();
             // Check if it's a timeout or other error
             if (error instanceof Error &&
                 error.message.includes('timed out waiting')) {
@@ -65,11 +71,35 @@ export async function executeQuery(options) {
             }
             // For other errors, fetch the query to check status
         }
-        spinner.stop();
+        if (spinner)
+            spinner.stop();
+        // If output format is specified, output the resource and return
+        if (options.outputFormat) {
+            try {
+                if (options.outputFormat === 'name') {
+                    console.log(queryName);
+                }
+                else if (options.outputFormat === 'json' ||
+                    options.outputFormat === 'yaml') {
+                    const { stdout } = await execa('kubectl', ['get', 'query', queryName, '-o', options.outputFormat], { stdio: 'pipe' });
+                    console.log(stdout);
+                }
+                else {
+                    console.error(chalk.red(`Invalid output format: ${options.outputFormat}. Use: yaml, json, or name`));
+                    process.exit(ExitCodes.CliError);
+                }
+                return;
+            }
+            catch (error) {
+                console.error(chalk.red(error instanceof Error
+                    ? error.message
+                    : 'Failed to fetch query resource'));
+                process.exit(ExitCodes.CliError);
+            }
+        }
         // Fetch final query state
         try {
-            const { stdout } = await execa('kubectl', ['get', 'query', queryName, '-o', 'json'], { stdio: 'pipe' });
-            const query = JSON.parse(stdout);
+            const query = await getResource('queries', queryName);
             const phase = query.status?.phase;
             // Check if query completed successfully or with error
             if (phase === 'done') {
@@ -88,7 +118,12 @@ export async function executeQuery(options) {
                 process.exit(ExitCodes.OperationError);
             }
             else if (phase === 'canceled') {
-                spinner.warn('Query canceled');
+                if (spinner) {
+                    spinner.warn('Query canceled');
+                }
+                else {
+                    output.warning('Query canceled');
+                }
                 if (query.status?.message) {
                     output.warning(query.status.message);
                 }
@@ -103,7 +138,8 @@ export async function executeQuery(options) {
         }
     }
     catch (error) {
-        spinner.stop();
+        if (spinner)
+            spinner.stop();
         console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
         process.exit(ExitCodes.CliError);
     }

package/dist/lib/executeQuery.spec.js

@@ -69,7 +69,7 @@ describe('executeQuery', () => {
                 if (args.includes('apply')) {
                     return { stdout: '', stderr: '', exitCode: 0 };
                 }
-                if (args.includes('get') && args.includes('query')) {
+                if (args.includes('get') && args.includes('queries')) {
                     return {
                         stdout: JSON.stringify(mockQueryResponse),
                         stderr: '',
@@ -98,7 +98,7 @@ describe('executeQuery', () => {
                 if (args.includes('apply')) {
                     return { stdout: '', stderr: '', exitCode: 0 };
                 }
-                if (args.includes('get') && args.includes('query')) {
+                if (args.includes('get') && args.includes('queries')) {
                     return {
                         stdout: JSON.stringify(mockQueryResponse),
                         stderr: '',
@@ -132,7 +132,7 @@ describe('executeQuery', () => {
                 if (args.includes('apply')) {
                     return { stdout: '', stderr: '', exitCode: 0 };
                 }
-                if (args.includes('get') && args.includes('query')) {
+                if (args.includes('get') && args.includes('queries')) {
                     return {
                         stdout: JSON.stringify(mockQueryResponse),
                         stderr: '',

package/dist/lib/kubectl.d.ts

@@ -0,0 +1,8 @@
+interface K8sResource {
+    metadata: {
+        name: string;
+        creationTimestamp?: string;
+    };
+}
+export declare function getResource<T extends K8sResource>(resourceType: string, name: string): Promise<T>;
+export {};

package/dist/lib/kubectl.js

@@ -0,0 +1,20 @@
+import { execa } from 'execa';
+export async function getResource(resourceType, name) {
+    if (name === '@latest') {
+        const result = await execa('kubectl', [
+            'get',
+            resourceType,
+            '--sort-by=.metadata.creationTimestamp',
+            '-o',
+            'json',
+        ], { stdio: 'pipe' });
+        const data = JSON.parse(result.stdout);
+        const resources = data.items || [];
+        if (resources.length === 0) {
+            throw new Error(`No ${resourceType} found`);
+        }
+        return resources[resources.length - 1];
+    }
+    const result = await execa('kubectl', ['get', resourceType, name, '-o', 'json'], { stdio: 'pipe' });
+    return JSON.parse(result.stdout);
+}

package/dist/lib/kubectl.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/lib/kubectl.spec.js

@@ -0,0 +1,88 @@
+import { jest } from '@jest/globals';
+const mockExeca = jest.fn();
+jest.unstable_mockModule('execa', () => ({
+    execa: mockExeca,
+}));
+const { getResource } = await import('./kubectl.js');
+describe('kubectl', () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+    });
+    describe('getResource', () => {
+        it('should get a specific resource by name', async () => {
+            const mockResource = {
+                metadata: {
+                    name: 'test-query',
+                    creationTimestamp: '2024-01-01T00:00:00Z',
+                },
+                spec: {
+                    value: 'test',
+                },
+            };
+            mockExeca.mockResolvedValue({
+                stdout: JSON.stringify(mockResource),
+            });
+            const result = await getResource('queries', 'test-query');
+            expect(result).toEqual(mockResource);
+            expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'queries', 'test-query', '-o', 'json'], { stdio: 'pipe' });
+        });
+        it('should get latest resource when name is @latest', async () => {
+            const mockResources = [
+                {
+                    metadata: {
+                        name: 'query-1',
+                        creationTimestamp: '2024-01-01T00:00:00Z',
+                    },
+                },
+                {
+                    metadata: {
+                        name: 'query-2',
+                        creationTimestamp: '2024-01-02T00:00:00Z',
+                    },
+                },
+                {
+                    metadata: {
+                        name: 'query-3',
+                        creationTimestamp: '2024-01-03T00:00:00Z',
+                    },
+                },
+            ];
+            mockExeca.mockResolvedValue({
+                stdout: JSON.stringify({ items: mockResources }),
+            });
+            const result = await getResource('queries', '@latest');
+            expect(result).toEqual(mockResources[2]);
+            expect(mockExeca).toHaveBeenCalledWith('kubectl', [
+                'get',
+                'queries',
+                '--sort-by=.metadata.creationTimestamp',
+                '-o',
+                'json',
+            ], { stdio: 'pipe' });
+        });
+        it('should throw error when @latest finds no resources', async () => {
+            mockExeca.mockResolvedValue({
+                stdout: JSON.stringify({ items: [] }),
+            });
+            await expect(getResource('queries', '@latest')).rejects.toThrow('No queries found');
+        });
+        it('should handle kubectl errors', async () => {
+            mockExeca.mockRejectedValue(new Error('kubectl error'));
+            await expect(getResource('queries', 'test-query')).rejects.toThrow('kubectl error');
+        });
+        it('should work with different resource types', async () => {
+            const mockAgent = {
+                metadata: {
+                    name: 'test-agent',
+                    creationTimestamp: '2024-01-01T00:00:00Z',
+                },
+            };
+            mockExeca.mockResolvedValue({
+                stdout: JSON.stringify(mockAgent),
+            });
+            const result = await getResource('agents', 'test-agent');
+            expect(result).toEqual(mockAgent);
+            expect(mockExeca).toHaveBeenCalledWith('kubectl', ['get', 'agents', 'test-agent', '-o', 'json'], { stdio: 'pipe' });
+        });
+    });
+});

package/dist/lib/stdin.d.ts

@@ -0,0 +1 @@
+export declare function readStdin(): Promise<string>;

package/dist/lib/stdin.js

@@ -0,0 +1,16 @@
+export async function readStdin() {
+    return new Promise((resolve) => {
+        if (process.stdin.isTTY) {
+            resolve('');
+            return;
+        }
+        let data = '';
+        process.stdin.setEncoding('utf8');
+        process.stdin.on('data', (chunk) => {
+            data += chunk;
+        });
+        process.stdin.on('end', () => {
+            resolve(data.trim());
+        });
+    });
+}

package/dist/lib/stdin.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/lib/stdin.spec.js

@@ -0,0 +1,82 @@
+import { describe, it, expect, beforeEach, afterEach, jest } from '@jest/globals';
+import { readStdin } from './stdin.js';
+import { Readable } from 'stream';
+describe('readStdin', () => {
+    let originalStdin;
+    let mockStdin;
+    beforeEach(() => {
+        originalStdin = process.stdin;
+        mockStdin = new Readable({
+            read() { },
+        });
+        Object.defineProperty(process, 'stdin', {
+            value: mockStdin,
+            writable: true,
+            configurable: true,
+        });
+    });
+    afterEach(() => {
+        Object.defineProperty(process, 'stdin', {
+            value: originalStdin,
+            writable: true,
+            configurable: true,
+        });
+        jest.clearAllMocks();
+    });
+    it('should return empty string when stdin is TTY', async () => {
+        mockStdin.isTTY = true;
+        const result = await readStdin();
+        expect(result).toBe('');
+    });
+    it('should read single chunk from stdin', async () => {
+        mockStdin.isTTY = false;
+        const promise = readStdin();
+        mockStdin.push('test-query');
+        mockStdin.push(null);
+        const result = await promise;
+        expect(result).toBe('test-query');
+    });
+    it('should read multiple chunks from stdin', async () => {
+        mockStdin.isTTY = false;
+        const promise = readStdin();
+        mockStdin.push('test-');
+        mockStdin.push('query-');
+        mockStdin.push('name');
+        mockStdin.push(null);
+        const result = await promise;
+        expect(result).toBe('test-query-name');
+    });
+    it('should trim whitespace from stdin input', async () => {
+        mockStdin.isTTY = false;
+        const promise = readStdin();
+        mockStdin.push('  test-query  \n');
+        mockStdin.push(null);
+        const result = await promise;
+        expect(result).toBe('test-query');
+    });
+    it('should handle empty stdin input', async () => {
+        mockStdin.isTTY = false;
+        const promise = readStdin();
+        mockStdin.push(null);
+        const result = await promise;
+        expect(result).toBe('');
+    });
+    it('should handle stdin with only whitespace', async () => {
+        mockStdin.isTTY = false;
+        const promise = readStdin();
+        mockStdin.push('   \n\n  ');
+        mockStdin.push(null);
+        const result = await promise;
+        expect(result).toBe('');
+    });
+    it('should handle multiline input', async () => {
+        mockStdin.isTTY = false;
+        const promise = readStdin();
+        mockStdin.push('line1\n');
+        mockStdin.push('line2\n');
+        mockStdin.push('line3');
+        mockStdin.push(null);
+        const result = await promise;
+        expect(result).toBe('line1\nline2\nline3');
+    });
+});

package/dist/lib/types.d.ts

@@ -52,6 +52,7 @@ export interface CommandVersionConfig {
 export interface K8sMetadata {
     name: string;
     namespace?: string;
+    creationTimestamp?: string;
 }
 export interface K8sCondition {
     type: string;
@@ -119,3 +120,41 @@ export interface ClusterInfo {
     user?: string;
     namespace?: string;
 }
+export interface EvaluationManifest {
+    apiVersion: string;
+    kind: 'Evaluation';
+    metadata: {
+        name: string;
+    };
+    spec: {
+        type: 'direct' | 'query';
+        evaluator: {
+            name: string;
+        };
+        config: {
+            input?: string;
+            output?: string;
+            queryRef?: {
+                name: string;
+            };
+            responseTarget?: {
+                type: string;
+                name: string;
+            };
+        };
+        timeout?: string;
+        ttl?: string;
+    };
+}
+export interface EvaluationStatus {
+    phase?: 'pending' | 'running' | 'done' | 'error';
+    score?: string;
+    passed?: boolean;
+    message?: string;
+}
+export interface Evaluation {
+    metadata: {
+        name: string;
+    };
+    status?: EvaluationStatus;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agents-at-scale/ark",
-  "version": "0.1.41",
+  "version": "0.1.42",
   "description": "Ark CLI - Interactive terminal interface for ARK agents",
   "main": "dist/index.js",
   "type": "module",
@@ -67,6 +67,7 @@
     "@types/debug": "^4.1.12",
     "@types/inquirer": "^9.0.7",
     "@types/jest": "^30.0.0",
+    "@types/marked-terminal": "^6.1.1",
     "@types/node": "^22.10.2",
     "@types/react": "^19.1.13",
     "@typescript-eslint/eslint-plugin": "^8.20.0",