@agents-at-scale/ark 0.1.41 → 0.1.42

package/dist/commands/completion/index.js

@@ -28,7 +28,7 @@ _ark_completion() {
   
   case \${COMP_CWORD} in
     1)
-      opts="agents chat cluster completion config dashboard docs generate install models query routes status targets teams tools uninstall help"
+      opts="agents chat cluster completion config dashboard docs generate install models queries query routes status targets teams tools uninstall help"
       COMPREPLY=( $(compgen -W "\${opts}" -- \${cur}) )
       return 0
       ;;
@@ -101,6 +101,11 @@ _ark_completion() {
           COMPREPLY=( $(compgen -W "\${targets}" -- \${cur}) )
           return 0
           ;;
+        queries)
+          opts="get"
+          COMPREPLY=( $(compgen -W "\${opts}" -- \${cur}) )
+          return 0
+          ;;
       esac
       ;;
   esac
@@ -138,6 +143,7 @@ _ark() {
         'generate[Generate ARK resources]' \\
         'install[Install ARK services]' \\
         'models[List available models]' \\
+        'queries[Manage query resources]' \\
         'query[Execute a single query]' \\
         'routes[List available routes]' \\
         'status[Check system status]' \\
@@ -216,6 +222,10 @@ _ark() {
           fi
           _values 'available targets' \${targets[@]}
           ;;
+        queries)
+          _values 'queries commands' \\
+            'get[Get a specific query]'
+          ;;
       esac
       ;;
   esac

package/dist/commands/evaluation/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+import type { ArkConfig } from '../../lib/config.js';
+export declare function createEvaluationCommand(_: ArkConfig): Command;

package/dist/commands/evaluation/index.js

@@ -0,0 +1,60 @@
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { executeDirectEvaluation, executeQueryEvaluation, } from '../../lib/executeEvaluation.js';
+import { readStdin } from '../../lib/stdin.js';
+export function createEvaluationCommand(_) {
+    const evaluationCommand = new Command('evaluation');
+    evaluationCommand
+        .description('Execute evaluations against evaluators')
+        .argument('<evaluator-name>', 'Name of the evaluator to use')
+        .argument('[query-name]', 'Name of the query to evaluate (for query-based evaluation)')
+        .option('--input <input>', 'Input text for direct evaluation')
+        .option('--output <output>', 'Output text for direct evaluation')
+        .option('--response-target <target>', 'Response target for query evaluation (e.g., agent:my-agent)')
+        .option('--timeout <timeout>', 'Evaluation timeout (e.g., "30s", "5m")')
+        .option('--watch-timeout <timeout>', 'CLI watch timeout')
+        .action(async (evaluatorName, queryName, options) => {
+        if (options.input && options.output) {
+            await executeDirectEvaluation({
+                evaluatorName,
+                input: options.input,
+                output: options.output,
+                timeout: options.timeout,
+                watchTimeout: options.watchTimeout,
+            });
+        }
+        else if (queryName) {
+            await executeQueryEvaluation({
+                evaluatorName,
+                queryName,
+                responseTarget: options.responseTarget,
+                timeout: options.timeout,
+                watchTimeout: options.watchTimeout,
+            });
+        }
+        else {
+            const stdinQueryName = await readStdin();
+            if (stdinQueryName) {
+                await executeQueryEvaluation({
+                    evaluatorName,
+                    queryName: stdinQueryName,
+                    responseTarget: options.responseTarget,
+                    timeout: options.timeout,
+                    watchTimeout: options.watchTimeout,
+                });
+            }
+            else {
+                console.error(chalk.red('Error: Must provide either:'));
+                console.error('  - --input and --output for direct evaluation');
+                console.error('  - <query-name> for query-based evaluation');
+                console.error('  - Pipe query name from stdin');
+                console.error('\nExamples:');
+                console.error('  ark evaluation my-evaluator --input "test" --output "result"');
+                console.error('  ark evaluation my-evaluator my-query');
+                console.error('  echo "my-query" | ark evaluation my-evaluator');
+                process.exit(1);
+            }
+        }
+    });
+    return evaluationCommand;
+}

package/dist/commands/evaluation/index.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/commands/evaluation/index.spec.js

@@ -0,0 +1,166 @@
+import { jest } from '@jest/globals';
+import { Command } from 'commander';
+const mockExecuteDirectEvaluation = jest.fn();
+const mockExecuteQueryEvaluation = jest.fn();
+jest.unstable_mockModule('../../lib/executeEvaluation.js', () => ({
+    executeDirectEvaluation: mockExecuteDirectEvaluation,
+    executeQueryEvaluation: mockExecuteQueryEvaluation,
+}));
+const { createEvaluationCommand } = await import('./index.js');
+describe('createEvaluationCommand', () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+    });
+    it('should create an evaluation command', () => {
+        const command = createEvaluationCommand({});
+        expect(command).toBeInstanceOf(Command);
+        expect(command.name()).toBe('evaluation');
+        expect(command.description()).toBe('Execute evaluations against evaluators');
+    });
+    describe('direct evaluation', () => {
+        it('should execute direct evaluation with required options', async () => {
+            mockExecuteDirectEvaluation.mockResolvedValue(undefined);
+            const command = createEvaluationCommand({});
+            await command.parseAsync([
+                'node',
+                'test',
+                'my-evaluator',
+                '--input',
+                'test-input',
+                '--output',
+                'test-output',
+            ]);
+            expect(mockExecuteDirectEvaluation).toHaveBeenCalledWith({
+                evaluatorName: 'my-evaluator',
+                input: 'test-input',
+                output: 'test-output',
+                timeout: undefined,
+                watchTimeout: undefined,
+            });
+        });
+        it('should execute direct evaluation with timeout options', async () => {
+            mockExecuteDirectEvaluation.mockResolvedValue(undefined);
+            const command = createEvaluationCommand({});
+            await command.parseAsync([
+                'node',
+                'test',
+                'my-evaluator',
+                '--input',
+                'test-input',
+                '--output',
+                'test-output',
+                '--timeout',
+                '10m',
+                '--watch-timeout',
+                '11m',
+            ]);
+            expect(mockExecuteDirectEvaluation).toHaveBeenCalledWith({
+                evaluatorName: 'my-evaluator',
+                input: 'test-input',
+                output: 'test-output',
+                timeout: '10m',
+                watchTimeout: '11m',
+            });
+        });
+    });
+    describe('query evaluation', () => {
+        it('should execute query evaluation with required arguments', async () => {
+            mockExecuteQueryEvaluation.mockResolvedValue(undefined);
+            const command = createEvaluationCommand({});
+            await command.parseAsync([
+                'node',
+                'test',
+                'my-evaluator',
+                'test-query',
+            ]);
+            expect(mockExecuteQueryEvaluation).toHaveBeenCalledWith({
+                evaluatorName: 'my-evaluator',
+                queryName: 'test-query',
+                responseTarget: undefined,
+                timeout: undefined,
+                watchTimeout: undefined,
+            });
+        });
+        it('should execute query evaluation from stdin', async () => {
+            mockExecuteQueryEvaluation.mockResolvedValue(undefined);
+            const command = createEvaluationCommand({});
+            const mockStdin = {
+                isTTY: false,
+                setEncoding: jest.fn(),
+                on: jest.fn((event, callback) => {
+                    if (event === 'data') {
+                        callback('piped-query-name');
+                    }
+                    else if (event === 'end') {
+                        callback();
+                    }
+                }),
+            };
+            const originalStdin = process.stdin;
+            Object.defineProperty(process, 'stdin', {
+                value: mockStdin,
+                writable: true,
+                configurable: true,
+            });
+            try {
+                await command.parseAsync(['node', 'test', 'my-evaluator']);
+                expect(mockExecuteQueryEvaluation).toHaveBeenCalledWith({
+                    evaluatorName: 'my-evaluator',
+                    queryName: 'piped-query-name',
+                    responseTarget: undefined,
+                    timeout: undefined,
+                    watchTimeout: undefined,
+                });
+            }
+            finally {
+                Object.defineProperty(process, 'stdin', {
+                    value: originalStdin,
+                    writable: true,
+                    configurable: true,
+                });
+            }
+        });
+        it('should execute query evaluation with response-target option', async () => {
+            mockExecuteQueryEvaluation.mockResolvedValue(undefined);
+            const command = createEvaluationCommand({});
+            await command.parseAsync([
+                'node',
+                'test',
+                'my-evaluator',
+                'test-query',
+                '--response-target',
+                'agent:my-agent',
+            ]);
+            expect(mockExecuteQueryEvaluation).toHaveBeenCalledWith({
+                evaluatorName: 'my-evaluator',
+                queryName: 'test-query',
+                responseTarget: 'agent:my-agent',
+                timeout: undefined,
+                watchTimeout: undefined,
+            });
+        });
+        it('should execute query evaluation with all options', async () => {
+            mockExecuteQueryEvaluation.mockResolvedValue(undefined);
+            const command = createEvaluationCommand({});
+            await command.parseAsync([
+                'node',
+                'test',
+                'my-evaluator',
+                'test-query',
+                '--response-target',
+                'agent:my-agent',
+                '--timeout',
+                '10m',
+                '--watch-timeout',
+                '11m',
+            ]);
+            expect(mockExecuteQueryEvaluation).toHaveBeenCalledWith({
+                evaluatorName: 'my-evaluator',
+                queryName: 'test-query',
+                responseTarget: 'agent:my-agent',
+                timeout: '10m',
+                watchTimeout: '11m',
+            });
+        });
+    });
+});

package/dist/commands/memory/index.d.ts

@@ -0,0 +1,15 @@
+import { Command } from 'commander';
+import type { ArkConfig } from '../../lib/config.js';
+export declare function listSessions(options: {
+    output?: string;
+}): Promise<void>;
+export declare function deleteSession(sessionId: string, options: {
+    output?: string;
+}): Promise<void>;
+export declare function deleteQuery(sessionId: string, queryId: string, options: {
+    output?: string;
+}): Promise<void>;
+export declare function deleteAll(options: {
+    output?: string;
+}): Promise<void>;
+export declare function createMemoryCommand(_: ArkConfig): Command;

package/dist/commands/memory/index.js

@@ -0,0 +1,130 @@
+import { Command } from 'commander';
+import output from '../../lib/output.js';
+import { ArkApiProxy } from '../../lib/arkApiProxy.js';
+export async function listSessions(options) {
+    try {
+        const proxy = new ArkApiProxy();
+        const arkApiClient = await proxy.start();
+        const sessions = await arkApiClient.getSessions();
+        if (options.output === 'json') {
+            output.info(JSON.stringify(sessions, null, 2));
+            return;
+        }
+        if (sessions.length === 0) {
+            output.info('No sessions found');
+            return;
+        }
+        output.info('Sessions:');
+        sessions.forEach((session) => {
+            output.info(`  ${session.sessionId} (memory: ${session.memoryName})`);
+        });
+        proxy.stop();
+    }
+    catch (error) {
+        output.error('Failed to list sessions:', error);
+        process.exit(1);
+    }
+}
+export async function deleteSession(sessionId, options) {
+    try {
+        const proxy = new ArkApiProxy();
+        const arkApiClient = await proxy.start();
+        const response = await arkApiClient.deleteSession(sessionId);
+        if (options.output === 'json') {
+            output.info(JSON.stringify(response, null, 2));
+            return;
+        }
+        output.success(`Session ${sessionId} deleted successfully`);
+        proxy.stop();
+    }
+    catch (error) {
+        output.error(`Failed to delete session ${sessionId}:`, error);
+        process.exit(1);
+    }
+}
+export async function deleteQuery(sessionId, queryId, options) {
+    try {
+        const proxy = new ArkApiProxy();
+        const arkApiClient = await proxy.start();
+        const response = await arkApiClient.deleteQueryMessages(sessionId, queryId);
+        if (options.output === 'json') {
+            output.info(JSON.stringify(response, null, 2));
+            return;
+        }
+        output.success(`Query ${queryId} messages deleted successfully from session ${sessionId}`);
+        proxy.stop();
+    }
+    catch (error) {
+        output.error(`Failed to delete query ${queryId} messages:`, error);
+        process.exit(1);
+    }
+}
+export async function deleteAll(options) {
+    try {
+        const proxy = new ArkApiProxy();
+        const arkApiClient = await proxy.start();
+        const response = await arkApiClient.deleteAllSessions();
+        if (options.output === 'json') {
+            output.info(JSON.stringify(response, null, 2));
+            return;
+        }
+        output.success('All sessions deleted successfully');
+        proxy.stop();
+    }
+    catch (error) {
+        output.error('Failed to delete all sessions:', error);
+        process.exit(1);
+    }
+}
+export function createMemoryCommand(_) {
+    const memoryCommand = new Command('memory');
+    memoryCommand.description('Manage memory sessions and queries').alias('mem');
+    // List sessions command
+    memoryCommand
+        .command('list')
+        .alias('ls')
+        .description('List all sessions')
+        .option('-o, --output <format>', 'output format (json or text)', 'text')
+        .action(async (options) => {
+        await listSessions(options);
+    });
+    // Delete command with subcommands (alias: reset for backward compatibility)
+    const deleteCommand = memoryCommand
+        .command('delete')
+        .alias('reset')
+        .description('Delete memory data')
+        .option('--all', 'Delete all sessions and their messages');
+    // Delete specific session
+    deleteCommand
+        .command('session')
+        .description('Delete a specific session')
+        .argument('<sessionId>', 'Session ID to delete')
+        .option('-o, --output <format>', 'output format (json or text)', 'text')
+        .action(async (sessionId, options) => {
+        await deleteSession(sessionId, options);
+    });
+    // Delete specific query
+    deleteCommand
+        .command('query')
+        .description('Delete messages for a specific query')
+        .argument('<sessionId>', 'Session ID')
+        .argument('<queryId>', 'Query ID to delete messages for')
+        .option('-o, --output <format>', 'output format (json or text)', 'text')
+        .action(async (sessionId, queryId, options) => {
+        await deleteQuery(sessionId, queryId, options);
+    });
+    // Handle --all on delete root
+    deleteCommand.action(async (options) => {
+        if (options.all) {
+            await deleteAll(options);
+            return;
+        }
+        // If no subcommand and no --all, show help
+        deleteCommand.help();
+    });
+    // Default action - list sessions
+    memoryCommand.action(async (options) => {
+        await listSessions(options);
+    });
+    return memoryCommand;
+}

package/dist/commands/memory/index.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/commands/memory/index.spec.js

@@ -0,0 +1,124 @@
+import { describe, it, expect, jest, beforeEach, afterEach, beforeAll, } from '@jest/globals';
+// ESM-safe mocking: declare variables to hold dynamically imported modules
+let createMemoryCommand;
+let deleteSession;
+let deleteQuery;
+let deleteAll;
+let ArkApiProxy;
+let output;
+// Mock dependencies
+jest.unstable_mockModule('../../lib/output.js', () => ({
+    default: {
+        info: jest.fn(),
+        success: jest.fn(),
+        error: jest.fn(),
+    },
+}));
+// Mock ArkApiProxy with a simpler approach
+jest.unstable_mockModule('../../lib/arkApiProxy.js', () => ({
+    __esModule: true,
+    ArkApiProxy: jest.fn().mockImplementation(() => ({
+        start: jest.fn(),
+        stop: jest.fn(),
+    })),
+}));
+beforeAll(async () => {
+    // After mocks are registered, dynamically import modules
+    ({ ArkApiProxy } = await import('../../lib/arkApiProxy.js'));
+    ({ default: output } = await import('../../lib/output.js'));
+    ({ createMemoryCommand, deleteSession, deleteQuery, deleteAll } = await import('./index.js'));
+});
+describe('Memory Command', () => {
+    let mockConfig;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        mockConfig = {};
+    });
+    afterEach(() => {
+        jest.restoreAllMocks();
+    });
+    describe('Command Structure', () => {
+        it('should create memory command with correct structure', () => {
+            const command = createMemoryCommand(mockConfig);
+            expect(command.name()).toBe('memory');
+            expect(command.alias()).toBe('mem');
+            expect(command.description()).toBe('Manage memory sessions and queries');
+        });
+        it('should have list subcommand', () => {
+            const command = createMemoryCommand(mockConfig);
+            const subcommands = command.commands.map((cmd) => cmd.name());
+            expect(subcommands).toContain('list');
+        });
+        it('should have delete subcommand with nested commands and flags', () => {
+            const command = createMemoryCommand(mockConfig);
+            const deleteCommand = command.commands.find((cmd) => cmd.name() === 'delete');
+            expect(deleteCommand).toBeDefined();
+            expect(deleteCommand?.description()).toBe('Delete memory data');
+            const deleteSubcommands = deleteCommand?.commands.map((cmd) => cmd.name()) || [];
+            expect(deleteSubcommands).toContain('session');
+            expect(deleteSubcommands).toContain('query');
+            // --all flag is supported on the delete root instead of an 'all' subcommand
+        });
+    });
+    describe('Command Creation', () => {
+        it('should create command without errors', () => {
+            expect(() => createMemoryCommand(mockConfig)).not.toThrow();
+        });
+        it('should return a command object', () => {
+            const command = createMemoryCommand(mockConfig);
+            expect(command).toBeDefined();
+            expect(typeof command.name).toBe('function');
+            expect(typeof command.description).toBe('function');
+        });
+    });
+    describe('Error Scenarios', () => {
+        let exitSpy;
+        beforeEach(async () => {
+            exitSpy = jest
+                .spyOn(process, 'exit')
+                .mockImplementation(((..._args) => undefined));
+        });
+        afterEach(() => {
+            exitSpy.mockRestore();
+        });
+        it('deleteSession handles 500 error', async () => {
+            const err = new Error('Internal Server Error');
+            const fakeClient = {
+                deleteSession: jest.fn().mockRejectedValue(err),
+            };
+            ArkApiProxy.mockImplementation(() => ({
+                start: jest.fn().mockResolvedValue(fakeClient),
+                stop: jest.fn(),
+            }));
+            await deleteSession('sess-1', { output: 'text' }).catch(() => { });
+            expect(output.error).toHaveBeenCalled();
+            expect(process.exit).toHaveBeenCalledWith(1);
+        });
+        it('deleteQuery handles network timeout', async () => {
+            const err = new Error('Network timeout');
+            const fakeClient = {
+                deleteQueryMessages: jest.fn().mockRejectedValue(err),
+            };
+            ArkApiProxy.mockImplementation(() => ({
+                start: jest.fn().mockResolvedValue(fakeClient),
+                stop: jest.fn(),
+            }));
+            await deleteQuery('sess-2', 'query-9', { output: 'text' }).catch(() => { });
+            expect(output.error).toHaveBeenCalled();
+            expect(process.exit).toHaveBeenCalledWith(1);
+        });
+        it('deleteAll handles no memory services reachable', async () => {
+            const err = new Error('No memory services reachable');
+            const fakeClient = {
+                deleteAllSessions: jest.fn().mockRejectedValue(err),
+            };
+            ArkApiProxy.mockImplementation(() => ({
+                start: jest.fn().mockResolvedValue(fakeClient),
+                stop: jest.fn(),
+            }));
+            await deleteAll({ output: 'text' }).catch(() => { });
+            expect(output.error).toHaveBeenCalled();
+            expect(process.exit).toHaveBeenCalledWith(1);
+        });
+    });
+});

package/dist/commands/models/create.d.ts

@@ -1,9 +1,8 @@
-export interface CreateModelOptions {
-    type?: string;
-    model?: string;
-    baseUrl?: string;
-    apiKey?: string;
-    apiVersion?: string;
+import { BaseCollectorOptions } from './providers/index.js';
+/**
+ * Common options for model creation.
+ */
+export interface CreateModelOptions extends BaseCollectorOptions {
     yes?: boolean;
 }
 export declare function createModel(modelName?: string, options?: CreateModelOptions): Promise<boolean>;