@agents-at-scale/ark 0.1.55 → 0.1.57

package/dist/commands/evaluation/index.d.ts

@@ -1,3 +0,0 @@
-import { Command } from 'commander';
-import type { ArkConfig } from '../../lib/config.js';
-export declare function createEvaluationCommand(_: ArkConfig): Command;

package/dist/commands/evaluation/index.js

@@ -1,60 +0,0 @@
-import { Command } from 'commander';
-import chalk from 'chalk';
-import { executeDirectEvaluation, executeQueryEvaluation, } from '../../lib/executeEvaluation.js';
-import { readStdin } from '../../lib/stdin.js';
-export function createEvaluationCommand(_) {
-    const evaluationCommand = new Command('evaluation');
-    evaluationCommand
-        .description('Execute evaluations against evaluators')
-        .argument('<evaluator-name>', 'Name of the evaluator to use')
-        .argument('[query-name]', 'Name of the query to evaluate (for query-based evaluation)')
-        .option('--input <input>', 'Input text for direct evaluation')
-        .option('--output <output>', 'Output text for direct evaluation')
-        .option('--response-target <target>', 'Response target for query evaluation (e.g., agent:my-agent)')
-        .option('--timeout <timeout>', 'Evaluation timeout (e.g., "30s", "5m")')
-        .option('--watch-timeout <timeout>', 'CLI watch timeout')
-        .action(async (evaluatorName, queryName, options) => {
-        if (options.input && options.output) {
-            await executeDirectEvaluation({
-                evaluatorName,
-                input: options.input,
-                output: options.output,
-                timeout: options.timeout,
-                watchTimeout: options.watchTimeout,
-            });
-        }
-        else if (queryName) {
-            await executeQueryEvaluation({
-                evaluatorName,
-                queryName,
-                responseTarget: options.responseTarget,
-                timeout: options.timeout,
-                watchTimeout: options.watchTimeout,
-            });
-        }
-        else {
-            const stdinQueryName = await readStdin();
-            if (stdinQueryName) {
-                await executeQueryEvaluation({
-                    evaluatorName,
-                    queryName: stdinQueryName,
-                    responseTarget: options.responseTarget,
-                    timeout: options.timeout,
-                    watchTimeout: options.watchTimeout,
-                });
-            }
-            else {
-                console.error(chalk.red('Error: Must provide either:'));
-                console.error('  - --input and --output for direct evaluation');
-                console.error('  - <query-name> for query-based evaluation');
-                console.error('  - Pipe query name from stdin');
-                console.error('\nExamples:');
-                console.error('  ark evaluation my-evaluator --input "test" --output "result"');
-                console.error('  ark evaluation my-evaluator my-query');
-                console.error('  echo "my-query" | ark evaluation my-evaluator');
-                process.exit(1);
-            }
-        }
-    });
-    return evaluationCommand;
-}

package/dist/lib/executeEvaluation.d.ts

@@ -1,16 +0,0 @@
-export interface DirectEvaluationOptions {
-    evaluatorName: string;
-    input: string;
-    output: string;
-    timeout?: string;
-    watchTimeout?: string;
-}
-export interface QueryEvaluationOptions {
-    evaluatorName: string;
-    queryName: string;
-    responseTarget?: string;
-    timeout?: string;
-    watchTimeout?: string;
-}
-export declare function executeDirectEvaluation(options: DirectEvaluationOptions): Promise<void>;
-export declare function executeQueryEvaluation(options: QueryEvaluationOptions): Promise<void>;

package/dist/lib/executeEvaluation.js

@@ -1,155 +0,0 @@
-import { execa } from 'execa';
-import ora from 'ora';
-import chalk from 'chalk';
-import output from './output.js';
-import { ExitCodes } from './errors.js';
-import { parseDuration } from './duration.js';
-async function waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay) {
-    const spinner = ora('Waiting for evaluation completion...').start();
-    try {
-        await execa('kubectl', [
-            'wait',
-            '--for=condition=Completed',
-            `evaluation/${evaluationName}`,
-            `--timeout=${Math.floor(watchTimeoutMs / 1000)}s`,
-        ], { timeout: watchTimeoutMs });
-    }
-    catch (error) {
-        spinner.stop();
-        if (error instanceof Error && error.message.includes('timed out waiting')) {
-            console.error(chalk.red(`Evaluation did not complete within ${watchTimeoutDisplay}`));
-            process.exit(ExitCodes.Timeout);
-        }
-        throw error;
-    }
-    spinner.stop();
-    const { stdout } = await execa('kubectl', ['get', 'evaluation', evaluationName, '-o', 'json'], { stdio: 'pipe' });
-    const evaluation = JSON.parse(stdout);
-    const status = evaluation.status;
-    if (status?.phase === 'done') {
-        console.log(chalk.green('\nEvaluation completed successfully:'));
-        if (status.score !== undefined) {
-            console.log(`Score: ${status.score}`);
-        }
-        if (status.passed !== undefined) {
-            console.log(`Result: ${status.passed ? chalk.green('PASSED') : chalk.red('FAILED')}`);
-        }
-        if (status.message) {
-            console.log(`Message: ${status.message}`);
-        }
-    }
-    else if (status?.phase === 'error') {
-        console.error(chalk.red(status.message || 'Evaluation failed with unknown error'));
-        process.exit(ExitCodes.OperationError);
-    }
-    else {
-        output.warning(`Unexpected evaluation phase: ${status?.phase}`);
-    }
-}
-export async function executeDirectEvaluation(options) {
-    const spinner = ora('Creating evaluation...').start();
-    const queryTimeoutMs = options.timeout
-        ? parseDuration(options.timeout)
-        : parseDuration('5m');
-    const watchTimeoutMs = options.watchTimeout
-        ? parseDuration(options.watchTimeout)
-        : queryTimeoutMs + 60000;
-    const timestamp = Date.now();
-    const evaluationName = `cli-eval-${timestamp}`;
-    const evaluationManifest = {
-        apiVersion: 'ark.mckinsey.com/v1alpha1',
-        kind: 'Evaluation',
-        metadata: {
-            name: evaluationName,
-        },
-        spec: {
-            type: 'direct',
-            evaluator: {
-                name: options.evaluatorName,
-            },
-            config: {
-                input: options.input,
-                output: options.output,
-            },
-            ...(options.timeout && { timeout: options.timeout }),
-            ttl: '1h',
-        },
-    };
-    try {
-        spinner.text = 'Submitting evaluation...';
-        await execa('kubectl', ['apply', '-f', '-'], {
-            input: JSON.stringify(evaluationManifest),
-            stdio: ['pipe', 'pipe', 'pipe'],
-        });
-        spinner.stop();
-        const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
-        await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
-    }
-    catch (error) {
-        spinner.stop();
-        console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
-        process.exit(ExitCodes.CliError);
-    }
-}
-export async function executeQueryEvaluation(options) {
-    const spinner = ora('Creating evaluation...').start();
-    const queryTimeoutMs = options.timeout
-        ? parseDuration(options.timeout)
-        : parseDuration('5m');
-    const watchTimeoutMs = options.watchTimeout
-        ? parseDuration(options.watchTimeout)
-        : queryTimeoutMs + 60000;
-    const timestamp = Date.now();
-    const evaluationName = `cli-eval-${timestamp}`;
-    let responseTarget;
-    if (options.responseTarget) {
-        const parts = options.responseTarget.split(':');
-        if (parts.length === 2) {
-            responseTarget = {
-                type: parts[0],
-                name: parts[1],
-            };
-        }
-        else {
-            spinner.stop();
-            console.error(chalk.red('Invalid response-target format. Use: type:name (e.g., agent:my-agent)'));
-            process.exit(ExitCodes.CliError);
-        }
-    }
-    const evaluationManifest = {
-        apiVersion: 'ark.mckinsey.com/v1alpha1',
-        kind: 'Evaluation',
-        metadata: {
-            name: evaluationName,
-        },
-        spec: {
-            type: 'query',
-            evaluator: {
-                name: options.evaluatorName,
-            },
-            config: {
-                queryRef: {
-                    name: options.queryName,
-                },
-                ...(responseTarget && { responseTarget }),
-            },
-            ...(options.timeout && { timeout: options.timeout }),
-            ttl: '1h',
-        },
-    };
-    try {
-        spinner.text = 'Submitting evaluation...';
-        await execa('kubectl', ['apply', '-f', '-'], {
-            input: JSON.stringify(evaluationManifest),
-            stdio: ['pipe', 'pipe', 'pipe'],
-        });
-        spinner.stop();
-        const watchTimeoutDisplay = options.watchTimeout ?? `${Math.floor(watchTimeoutMs / 1000)}s`;
-        await waitForEvaluationAndDisplayResults(evaluationName, watchTimeoutMs, watchTimeoutDisplay);
-    }
-    catch (error) {
-        spinner.stop();
-        console.error(chalk.red(error instanceof Error ? error.message : 'Unknown error'));
-        process.exit(ExitCodes.CliError);
-    }
-}

package/templates/marketplace/.editorconfig

@@ -1,24 +0,0 @@
-# EditorConfig is awesome: https://EditorConfig.org
-
-# top-most EditorConfig file
-root = true
-
-[*]
-indent_style = space
-indent_size = 2
-end_of_line = lf
-charset = utf-8
-trim_trailing_whitespace = true
-insert_final_newline = true
-
-[*.md]
-trim_trailing_whitespace = false
-
-[Makefile]
-indent_style = tab
-
-[*.py]
-indent_size = 4
-
-[*.go]
-indent_style = tab

package/templates/marketplace/.github/.keep

@@ -1,11 +0,0 @@
-# GitHub Configuration Directory
-
-GitHub-specific configuration files for the marketplace repository.
-
-Expected contents:
-- workflows/ - GitHub Actions workflows
-- ISSUE_TEMPLATE/ - Issue templates
-- PULL_REQUEST_TEMPLATE.md - PR template
-- CODEOWNERS - Code ownership rules
-
-These files help maintain quality and consistency in contributions.

package/templates/marketplace/.github/workflows/.keep

@@ -1,16 +0,0 @@
-# GitHub Workflows Directory
-
-GitHub Actions workflows for automating marketplace operations.
-
-Common workflows:
-- component-validation.yml - Validate contributed components
-- security-scan.yml - Security scanning of components
-- documentation-build.yml - Build and deploy documentation
-- release.yml - Automated releases
-- pr-checks.yml - Pull request validation
-
-Workflows should:
-- Be efficient and reliable
-- Provide clear feedback
-- Follow security best practices
-- Be well-documented

package/templates/marketplace/.helmignore

@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

package/templates/marketplace/.prettierrc.json

@@ -1,20 +0,0 @@
-{
-  "semi": true,
-  "trailingComma": "es5",
-  "singleQuote": true,
-  "printWidth": 80,
-  "tabWidth": 2,
-  "useTabs": false,
-  "arrowParens": "avoid",
-  "bracketSpacing": true,
-  "proseWrap": "preserve",
-  "quoteProps": "as-needed",
-  "overrides": [
-    {
-      "files": "*.yaml",
-      "options": {
-        "singleQuote": false
-      }
-    }
-  ]
-}

package/templates/marketplace/.yamllint.yml

@@ -1,53 +0,0 @@
-# Default rules for yamllint
-extends: default
-
-rules:
-  # 120 chars should be enough, but don't fail if a line is longer
-  line-length:
-    max: 120
-    level: warning
-
-  # accept both       key:value and key: value
-  colons:
-    max-spaces-before: 0
-    max-spaces-after: -1
-
-  # don't bother me with tabs vs spaces issues
-  indentation:
-    spaces: consistent
-    indent-sequences: whatever
-
-  # allow empty values like 'key:'
-  empty-values:
-    forbid-in-block-mappings: false
-    forbid-in-flow-mappings: false
-
-  # sometimes you have to use quotes
-  quoted-strings:
-    quote-type: any
-    required: false
-
-  # allow trailing spaces in comments
-  trailing-spaces:
-    level: warning
-
-  # allow truthy values
-  truthy:
-    allowed-values: ['true', 'false', 'on', 'off', 'yes', 'no']
-    check-keys: false
-
-  # allow ! in yaml
-  comments:
-    require-starting-space: false
-    ignore-shebangs: true
-    min-spaces-from-content: 1
-
-  # brackets
-  brackets:
-    max-spaces-inside: 1
-    max-spaces-inside-empty: 0
-
-  # braces
-  braces:
-    max-spaces-inside: 1
-    max-spaces-inside-empty: 0

package/templates/marketplace/agents/.keep

@@ -1,29 +0,0 @@
-# Agents Directory
-
-This directory contains reusable agent definitions that can be shared across projects.
-
-## Structure
-
-Each agent should be in its own file or subdirectory (complex agents) with:
-
-- agent.yaml (the main agent definition)
-- README.md (documentation)
-- examples/ (optional usage examples)
-
-## Naming Convention
-
-Use descriptive names that indicate the agent's purpose:
-
-- data-analyst/
-- code-reviewer/
-- security-scanner/
-- documentation-writer/
-
-## Contributing
-
-When contributing an agent:
-
-1. Ensure it follows the Ark agent specification
-2. Include comprehensive documentation
-3. Provide usage examples
-4. Test with multiple models when possible

package/templates/marketplace/docs/.keep

@@ -1,19 +0,0 @@
-# Documentation Directory
-
-Documentation for the marketplace and its components.
-
-## Structure
-
-- contributing.md - Guidelines for contributors
-- component-specs/ - Specifications for different component types
-- best-practices/ - Development and usage best practices
-- tutorials/ - Step-by-step guides
-- api/ - API documentation
-
-## Documentation Standards
-
-1. Use clear, concise language
-2. Include code examples
-3. Keep documentation up-to-date
-4. Follow markdown conventions
-5. Include diagrams where helpful

package/templates/marketplace/marketplace.json.example

@@ -1,59 +0,0 @@
-{
-  "version": "1.0.0",
-  "marketplace": "ARK Marketplace",
-  "items": [
-    {
-      "name": "phoenix",
-      "displayName": "Phoenix",
-      "description": "AI/ML observability and evaluation platform with OpenTelemetry integration",
-      "version": "0.1.5",
-      "author": "ARK Marketplace",
-      "homepage": "https://github.com/mckinsey/agents-at-scale-marketplace",
-      "repository": "https://github.com/mckinsey/agents-at-scale-marketplace",
-      "license": "Apache-2.0",
-      "tags": ["observability", "evaluation", "telemetry", "monitoring"],
-      "category": "observability",
-      "icon": "https://example.com/phoenix-icon.png",
-      "documentation": "https://mckinsey.github.io/agents-at-scale-marketplace/services/phoenix/",
-      "support": {
-        "url": "https://github.com/mckinsey/agents-at-scale-marketplace/issues"
-      },
-      "ark": {
-        "chartPath": "oci://ghcr.io/mckinsey/agents-at-scale-marketplace/charts/phoenix",
-        "namespace": "phoenix",
-        "helmReleaseName": "phoenix",
-        "installArgs": ["--create-namespace"],
-        "k8sServiceName": "phoenix",
-        "k8sServicePort": 6006,
-        "k8sDeploymentName": "phoenix"
-      }
-    },
-    {
-      "name": "langfuse",
-      "displayName": "Langfuse",
-      "description": "Open-source LLM observability and analytics platform with session tracking",
-      "version": "0.1.4",
-      "author": "ARK Marketplace",
-      "homepage": "https://github.com/mckinsey/agents-at-scale-marketplace",
-      "repository": "https://github.com/mckinsey/agents-at-scale-marketplace",
-      "license": "Apache-2.0",
-      "tags": ["observability", "analytics", "llm", "tracking"],
-      "category": "observability",
-      "icon": "https://example.com/langfuse-icon.png",
-      "documentation": "https://mckinsey.github.io/agents-at-scale-marketplace/services/langfuse/",
-      "support": {
-        "url": "https://github.com/mckinsey/agents-at-scale-marketplace/issues"
-      },
-      "ark": {
-        "chartPath": "oci://ghcr.io/mckinsey/agents-at-scale-marketplace/charts/langfuse",
-        "namespace": "telemetry",
-        "helmReleaseName": "langfuse",
-        "installArgs": ["--create-namespace"],
-        "k8sServiceName": "langfuse",
-        "k8sServicePort": 3000,
-        "k8sDeploymentName": "langfuse-web"
-      }
-    }
-  ]
-}
-

package/templates/marketplace/mcp-servers/.keep

@@ -1,32 +0,0 @@
-# MCP Servers Directory
-
-This directory contains Model Context Protocol (MCP) server configurations and implementations.
-
-## Structure
-
-Each MCP server should be in its own subdirectory with:
-
-- mcp-server.yaml (server configuration)
-- src/ (source code if custom implementation)
-- Dockerfile (if containerized)
-- README.md (documentation)
-- examples/ (usage examples)
-
-## Naming Convention
-
-Use descriptive names that indicate the server's purpose:
-
-- file-system-server/
-- database-server/
-- api-gateway-server/
-- knowledge-base-server/
-
-## Contributing
-
-When contributing an MCP server:
-
-1. Follow MCP protocol specifications
-2. Include security configurations
-3. Document all available tools/resources
-4. Provide integration examples
-5. Include monitoring and logging setup

package/templates/marketplace/models/.keep

@@ -1,23 +0,0 @@
-# Models Directory
-
-This directory contains model configurations for different AI providers and use cases.
-
-## Structure
-Each model configuration should be:
-- provider-specific directories (openai/, anthropic/, etc.)
-- model-specific YAML files within each directory
-- README.md documenting capabilities and limitations
-
-## Naming Convention
-Use provider and model names:
-- openai/gpt-4.yaml
-- anthropic/claude-3-sonnet.yaml
-- azure/gpt-4-32k.yaml
-- local/llama2-7b.yaml
-
-## Contributing
-When contributing a model configuration:
-1. Include provider-specific settings
-2. Document token limits and capabilities
-3. Specify recommended use cases
-4. Include cost considerations if applicable

package/templates/marketplace/projects/.keep

@@ -1,43 +0,0 @@
-# Projects Directory
-
-This directory contains complete, reusable Ark projects that demonstrate comprehensive use cases and multi-component solutions.
-
-## Purpose
-
-- **Full Project Templates**: Complete Ark project structures with multiple agents, teams, models, and tools working together
-- **End-to-End Solutions**: Real-world scenarios that showcase complex workflows and integrations
-- **Reference Implementations**: Best practices for structuring large-scale Ark deployments
-
-## Structure Guidelines
-
-Each project should include:
-
-- **Complete project structure**: All necessary YAML configurations
-- **README.md**: Detailed setup instructions, use case description, and architecture overview
-- **Documentation**: Architecture diagrams, workflow explanations, and deployment guides
-- **Examples**: Sample queries and expected outputs
-- **Dependencies**: Clear list of required models, tools, and external services
-
-## Naming Convention
-
-Use descriptive names that indicate the use case:
-
-- `customer-support-workflow/` - Multi-agent customer service system
-- `research-assistant-pipeline/` - Academic research and analysis workflow
-- `content-generation-studio/` - Creative content production system
-
-## Project Requirements
-
-- Each project must be self-contained and deployable
-- Include comprehensive documentation
-- Provide clear setup and configuration instructions
-- Include sample data or test cases where applicable
-- Follow Ark best practices for resource organization
-
-## Examples
-
-- Multi-agent workflows with specialized roles
-- Integration with external APIs and services
-- Complex data processing pipelines
-- Cross-functional team collaboration scenarios
-- Industry-specific solution templates

package/templates/marketplace/queries/.keep

@@ -1,25 +0,0 @@
-# Queries Directory
-
-This directory contains reusable query templates for common use cases.
-
-## Structure
-Each query should be in its own file or subdirectory:
-- query-name.yaml (simple queries)
-- complex-query/ (for queries with multiple files)
-  - query.yaml
-  - README.md
-  - examples/
-
-## Naming Convention
-Use descriptive names that indicate the query's purpose:
-- code-analysis.yaml
-- data-transformation.yaml
-- security-audit.yaml
-- document-summarization.yaml
-
-## Contributing
-When contributing a query:
-1. Make it parameterizable where possible
-2. Include clear documentation of expected inputs
-3. Provide example outputs
-4. Specify compatible agents/teams

package/templates/marketplace/teams/.keep

@@ -1,29 +0,0 @@
-# Teams Directory
-
-This directory contains reusable team configurations for multi-agent workflows.
-
-## Structure
-
-Each team should be in its own file (simple) or subdirectory (complex) with:
-
-- team.yaml (the main team definition)
-- README.md (documentation describing the workflow)
-- examples/ (optional usage examples)
-
-## Naming Convention
-
-Use descriptive names that indicate the team's workflow:
-
-- code-review-team/
-- data-pipeline-team/
-- content-creation-team/
-- security-audit-team/
-
-## Contributing
-
-When contributing a team:
-
-1. Document the team's workflow and strategy
-2. Explain when to use this team configuration
-3. Include examples of typical inputs and outputs
-4. Test the team configuration thoroughly