@llm-dev-ops/agentics-cli 1.4.27 → 1.4.29

package/dist/bundled-agents/test-bench-agents/package.json

@@ -1,113 +0,0 @@
-{
-  "name": "@llm-dev-ops/test-bench-sdk",
-  "version": "0.2.0",
-  "description": "TypeScript SDK for LLM Test Bench - A production-grade framework for testing, validation, and benchmarking of Large Language Models",
-  "main": "./dist/functions/index.js",
-  "types": "./dist/index.d.ts",
-  "type": "module",
-  "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "types": "./dist/index.d.ts"
-    },
-    "./functions": "./dist/functions/index.js"
-  },
-  "engines": {
-    "node": ">=18.0.0",
-    "npm": ">=9.0.0"
-  },
-  "scripts": {
-    "dev": "tsup --watch",
-    "build": "tsup",
-    "build:functions": "tsup --config tsup.functions.config.ts",
-    "build:all": "npm run build && npm run build:functions",
-    "build:clean": "rm -rf dist && npm run build:all",
-    "gcp-build": "npm run build:functions",
-    "typecheck": "tsc --noEmit",
-    "lint": "eslint src --ext .ts --max-warnings=0",
-    "lint:fix": "eslint src --ext .ts --fix",
-    "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\" \"*.{json,md}\"",
-    "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\" \"*.{json,md}\"",
-    "test": "vitest run",
-    "test:watch": "vitest watch",
-    "test:ui": "vitest --ui",
-    "test:coverage": "vitest run --coverage",
-    "test:coverage:check": "vitest run --coverage && node scripts/check-coverage.js",
-    "test:integration": "vitest run --config vitest.integration.config.ts",
-    "ci": "npm run lint && npm run typecheck && npm run test:coverage && npm run build:all",
-    "prepublishOnly": "npm run lint && npm run typecheck && npm run build",
-    "prepare": "husky install || true",
-    "clean": "rm -rf dist coverage test-results .tsbuildinfo node_modules/.cache",
-    "deploy": "npm run build:functions && gcloud functions deploy test-bench-agents --runtime nodejs20 --trigger-http --allow-unauthenticated --region us-central1 --project agentics-dev --entry-point api --memory 512MB --timeout 120s"
-  },
-  "keywords": [
-    "llm",
-    "testing",
-    "benchmark",
-    "validation",
-    "openai",
-    "anthropic",
-    "claude",
-    "gpt",
-    "cli",
-    "typescript"
-  ],
-  "author": "LLM Test Bench Contributors",
-  "license": "MIT",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/LLM-Dev-Ops/test-bench.git"
-  },
-  "bugs": {
-    "url": "https://github.com/LLM-Dev-Ops/test-bench/issues"
-  },
-  "homepage": "https://github.com/LLM-Dev-Ops/test-bench#readme",
-  "publishConfig": {
-    "access": "public"
-  },
-  "dependencies": {
-    "@google-cloud/functions-framework": "^3.5.1",
-    "ajv": "^8.12.0",
-    "chalk": "^5.3.0",
-    "claude-flow": "^2.7.47",
-    "express": "^4.22.1",
-    "glob": "^10.3.10",
-    "openai": "^4.20.1",
-    "p-limit": "^5.0.0",
-    "yaml": "^2.3.4",
-    "yargs": "^17.7.2",
-    "zod": "^3.22.4"
-  },
-  "devDependencies": {
-    "@types/express": "^4.17.25",
-    "@types/node": "^20.19.34",
-    "@types/yargs": "^17.0.32",
-    "@typescript-eslint/eslint-plugin": "^6.15.0",
-    "@typescript-eslint/parser": "^6.15.0",
-    "@vitest/coverage-v8": "^1.0.4",
-    "@vitest/ui": "^1.0.4",
-    "eslint": "^8.56.0",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-plugin-import": "^2.29.1",
-    "husky": "^8.0.3",
-    "lint-staged": "^15.2.0",
-    "prettier": "^3.1.1",
-    "tsup": "^8.0.1",
-    "typescript": "^5.9.3",
-    "vitest": "^1.0.4"
-  },
-  "lint-staged": {
-    "*.ts": [
-      "eslint --fix",
-      "prettier --write"
-    ],
-    "*.{json,md,yml,yaml}": [
-      "prettier --write"
-    ]
-  },
-  "files": [
-    "dist",
-    "README.md",
-    "LICENSE"
-  ]
-}

package/dist/bundled-agents/test-bench-agents/src/cli.ts

@@ -1,72 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * CLI entry point for LLM Test Bench TypeScript wrapper
- *
- * This provides a thin TypeScript wrapper around the Rust CLI,
- * allowing the tool to be used via npm while maintaining all
- * the performance and functionality of the Rust implementation.
- */
-
-import { spawn } from 'child_process';
-
-import { findCLIPath } from './utils/cli-executor.js';
-
-/**
- * Main CLI function
- */
-function main(): void {
-  // Find the CLI binary
-  const cliPath = findCLIPath();
-
-  if (!cliPath) {
-    console.error('Error: LLM Test Bench CLI not found.');
-    console.error('');
-    console.error('Please install the CLI using one of these methods:');
-    console.error('  1. cargo install llm-test-bench');
-    console.error('  2. Download from https://github.com/LLM-Dev-Ops/test-bench/releases');
-    console.error('');
-    console.error('Or set CARGO_HOME environment variable to point to your Rust installation.');
-    process.exit(1);
-  }
-
-  // Get arguments (remove 'node' and script name)
-  const args = process.argv.slice(2);
-
-  // Spawn the Rust CLI with all arguments
-  const child = spawn(cliPath, args, {
-    stdio: 'inherit', // Forward stdin, stdout, stderr
-    env: process.env,
-  });
-
-  // Handle process exit
-  child.on('exit', (code, signal) => {
-    if (signal) {
-      process.kill(process.pid, signal);
-    } else {
-      process.exit(code ?? 0);
-    }
-  });
-
-  // Handle errors
-  child.on('error', (err) => {
-    console.error(`Failed to start CLI: ${err.message}`);
-    process.exit(1);
-  });
-
-  // Handle signals to properly forward them to child process
-  const signals: NodeJS.Signals[] = ['SIGINT', 'SIGTERM', 'SIGHUP'];
-  signals.forEach((signal) => {
-    process.on(signal, () => {
-      child.kill(signal);
-    });
-  });
-}
-
-// Run the CLI
-try {
-  main();
-} catch (err) {
-  console.error('Unexpected error:', err);
-  process.exit(1);
-}

package/dist/bundled-agents/test-bench-agents/src/core/fleet-benchmark.ts

@@ -1,503 +0,0 @@
-/**
- * Fleet Benchmark API for TypeScript/JavaScript
- *
- * Provides programmatic access to fleet benchmark execution for simulators
- * and automation tools.
- */
-
-import { executeCLI, findCLIPath } from '../utils/cli-executor.js';
-import type { SDKConfig } from '../types/index.js';
-
-/**
- * Fleet configuration options
- */
-export interface FleetConfig {
-  /** Base directory for fleet results */
-  outputBaseDir: string;
-  /** Default concurrency for benchmark execution */
-  defaultConcurrency?: number;
-  /** Whether to save individual responses */
-  saveResponses?: boolean;
-  /** Whether to continue on individual test failures */
-  continueOnFailure?: boolean;
-  /** Optional request delay in milliseconds */
-  requestDelayMs?: number;
-  /** Path to configuration file */
-  configPath?: string;
-}
-
-/**
- * Fleet manifest structure
- */
-export interface FleetManifest {
-  /** Fleet identifier */
-  fleet_id: string;
-  /** Fleet description */
-  description?: string;
-  /** Repositories to benchmark */
-  repositories: RepositorySpec[];
-  /** Providers to benchmark against */
-  providers: string[];
-  /** Fleet-level configuration overrides */
-  config?: FleetManifestConfig;
-}
-
-/**
- * Repository specification in fleet manifest
- */
-export interface RepositorySpec {
-  /** Repository identifier */
-  id: string;
-  /** Repository name */
-  name: string;
-  /** Path to dataset file */
-  dataset_path: string;
-  /** Repository-specific metadata */
-  metadata?: Record<string, string>;
-}
-
-/**
- * Fleet manifest configuration
- */
-export interface FleetManifestConfig {
-  /** Concurrency override */
-  concurrency?: number;
-  /** Request delay override */
-  request_delay_ms?: number;
-  /** Configuration file override */
-  config_path?: string;
-}
-
-/**
- * Fleet execution handle returned immediately when executing a fleet benchmark
- */
-export interface FleetExecutionHandle {
-  /** Unique identifier for this fleet benchmark run */
-  runId: string;
-  /** Base directory where all artifacts will be stored */
-  artifactBaseDir: string;
-  /** Optional URL for monitoring execution status */
-  statusUrl?: string;
-  /** Metadata about the execution */
-  metadata: FleetExecutionMetadata;
-}
-
-/**
- * Metadata about a fleet execution
- */
-export interface FleetExecutionMetadata {
-  /** Fleet identifier from manifest */
-  fleetId: string;
-  /** Number of repositories in the fleet */
-  repositoryCount: number;
-  /** Providers being benchmarked */
-  providers: string[];
-  /** When execution was initiated */
-  startedAt: string;
-  /** Expected artifact paths */
-  expectedArtifacts: string[];
-}
-
-/**
- * Fleet benchmark results
- */
-export interface FleetBenchmarkResults {
-  /** Fleet identifier */
-  fleet_id: string;
-  /** Timestamp when fleet benchmark was executed */
-  timestamp: string;
-  /** Total number of repositories in the fleet */
-  total_repositories: number;
-  /** Results for each repository */
-  repository_results: RepositoryResults[];
-  /** Fleet-wide aggregated statistics */
-  fleet_summary: FleetSummary;
-  /** Per-provider statistics across the fleet */
-  provider_breakdown: Record<string, ProviderFleetStats>;
-  /** Per-category statistics across the fleet */
-  category_breakdown: Record<string, CategoryFleetStats>;
-  /** Fleet metadata */
-  metadata: FleetMetadata;
-}
-
-/**
- * Results for a single repository within the fleet
- */
-export interface RepositoryResults {
-  /** Repository identifier */
-  repository_id: string;
-  /** Repository name */
-  repository_name: string;
-  /** Provider used for this repository */
-  provider_name: string;
-  /** Benchmark results for this repository */
-  results: any; // BenchmarkResults
-  /** Repository-specific metadata */
-  repository_metadata: Record<string, string>;
-}
-
-/**
- * Fleet-wide summary statistics
- */
-export interface FleetSummary {
-  /** Total number of repositories */
-  total_repositories: number;
-  /** Total tests across all repositories */
-  total_tests: number;
-  /** Total successful tests */
-  total_succeeded: number;
-  /** Total failed tests */
-  total_failed: number;
-  /** Total timed out tests */
-  total_timeout: number;
-  /** Total skipped tests */
-  total_skipped: number;
-  /** Overall success rate (0.0 - 1.0) */
-  success_rate: number;
-  /** Average duration across all tests (milliseconds) */
-  avg_duration_ms: number;
-  /** Median latency across all tests (milliseconds) */
-  p50_duration_ms: number;
-  /** 95th percentile latency across all tests (milliseconds) */
-  p95_duration_ms: number;
-  /** 99th percentile latency across all tests (milliseconds) */
-  p99_duration_ms: number;
-  /** Minimum duration across all tests (milliseconds) */
-  min_duration_ms: number;
-  /** Maximum duration across all tests (milliseconds) */
-  max_duration_ms: number;
-  /** Total tokens used across all repositories */
-  total_tokens: number;
-  /** Average tokens per successful request */
-  avg_tokens_per_request: number;
-  /** Total cost across all repositories (USD) */
-  total_cost: number;
-  /** Average cost per repository (USD) */
-  avg_cost_per_repository: number;
-  /** Average number of tests per repository */
-  avg_tests_per_repository: number;
-  /** Total execution duration (milliseconds) */
-  total_duration_ms: number;
-}
-
-/**
- * Per-provider statistics across the fleet
- */
-export interface ProviderFleetStats {
-  /** Provider name */
-  provider_name: string;
-  /** Number of repositories using this provider */
-  repository_count: number;
-  /** Total tests for this provider */
-  total_tests: number;
-  /** Total successful tests */
-  total_succeeded: number;
-  /** Total failed tests */
-  total_failed: number;
-  /** Success rate for this provider */
-  success_rate: number;
-  /** Total tokens used by this provider */
-  total_tokens: number;
-  /** Total cost for this provider (USD) */
-  total_cost: number;
-  /** Average duration for this provider (milliseconds) */
-  avg_duration_ms: number;
-}
-
-/**
- * Per-category statistics across the fleet
- */
-export interface CategoryFleetStats {
-  /** Category name */
-  category_name: string;
-  /** Total tests in this category */
-  total_tests: number;
-  /** Total successful tests */
-  total_succeeded: number;
-  /** Total failed tests */
-  total_failed: number;
-  /** Success rate for this category */
-  success_rate: number;
-  /** Average duration for this category (milliseconds) */
-  avg_duration_ms: number;
-}
-
-/**
- * Fleet metadata
- */
-export interface FleetMetadata {
-  /** Total tests across the fleet */
-  total_tests: number;
-  /** Total duration across the fleet */
-  total_duration_ms: number;
-  /** When the fleet benchmark was executed */
-  execution_timestamp: string;
-  /** Aggregation algorithm version (for reproducibility) */
-  aggregation_version: string;
-  /** Custom metadata fields */
-  custom: Record<string, string>;
-}
-
-/**
- * Fleet Benchmark SDK
- *
- * Provides programmatic API for executing fleet benchmarks and retrieving results.
- * Designed to be consumed by simulators that need to trigger benchmarks programmatically.
- *
- * @example
- * ```typescript
- * import { FleetBenchmark } from 'llm-test-bench';
- *
- * const fleet = new FleetBenchmark({
- *   outputBaseDir: './fleet-results',
- *   defaultConcurrency: 10,
- * });
- *
- * // Execute fleet benchmark (returns immediately)
- * const handle = await fleet.executeFleet('./fleet-manifest.json');
- * console.log('Run ID:', handle.runId);
- * console.log('Artifacts:', handle.artifactBaseDir);
- *
- * // Wait for completion (optional)
- * const results = await fleet.waitForCompletion(handle.runId);
- * console.log('Fleet completed:', results.fleet_summary.success_rate);
- * ```
- */
-export class FleetBenchmark {
-  private cliPath: string;
-  private config: Required<FleetConfig>;
-
-  /**
-   * Create a new Fleet Benchmark SDK instance
-   *
-   * @param config - Fleet configuration
-   * @param sdkConfig - Optional SDK configuration (for CLI path, etc.)
-   * @throws {Error} If CLI binary cannot be found
-   */
-  constructor(config: FleetConfig, sdkConfig?: SDKConfig) {
-    const resolvedCliPath = sdkConfig?.cliPath ?? findCLIPath();
-    if (!resolvedCliPath) {
-      throw new Error(
-        'LLM Test Bench CLI not found. Please install it via cargo or npm, or provide cliPath in config.'
-      );
-    }
-
-    this.cliPath = resolvedCliPath;
-    this.config = {
-      outputBaseDir: config.outputBaseDir,
-      defaultConcurrency: config.defaultConcurrency ?? 5,
-      saveResponses: config.saveResponses ?? true,
-      continueOnFailure: config.continueOnFailure ?? true,
-      requestDelayMs: config.requestDelayMs,
-      configPath: config.configPath,
-    };
-  }
-
-  /**
-   * Execute a fleet benchmark and return handle immediately
-   *
-   * This method:
-   * 1. Validates the fleet manifest
-   * 2. Generates a deterministic run_id
-   * 3. Spawns async execution
-   * 4. Returns handle immediately with run_id and artifact locations
-   *
-   * @param manifestPath - Path to fleet manifest JSON file
-   * @param options - Optional execution options
-   * @returns Promise resolving to FleetExecutionHandle
-   * @throws {Error} If manifest is invalid or execution fails to start
-   *
-   * @example
-   * ```typescript
-   * const handle = await fleet.executeFleet('./fleet-manifest.json');
-   * console.log('Run ID:', handle.runId);
-   * console.log('Artifacts will be at:', handle.artifactBaseDir);
-   * ```
-   */
-  async executeFleet(
-    manifestPath: string,
-    options?: {
-      wait?: boolean;
-      format?: 'json' | 'summary' | 'quiet';
-    }
-  ): Promise<FleetExecutionHandle> {
-    const args = ['fleet', '--manifest', manifestPath, '--output', this.config.outputBaseDir];
-
-    args.push('--concurrency', this.config.defaultConcurrency.toString());
-
-    if (this.config.saveResponses) {
-      args.push('--save-responses');
-    }
-
-    if (this.config.requestDelayMs !== undefined) {
-      args.push('--delay', this.config.requestDelayMs.toString());
-    }
-
-    if (this.config.configPath) {
-      args.push('--config', this.config.configPath);
-    }
-
-    const format = options?.format ?? 'quiet';
-    args.push('--format', format);
-
-    if (!options?.wait) {
-      // Execute without waiting (async mode)
-      const result = await executeCLI<string>(this.cliPath, {
-        args,
-        cwd: process.cwd(),
-        timeout: 30000, // 30s timeout for startup
-        parseJson: false,
-      });
-
-      if (!result.success) {
-        throw new Error(`Failed to execute fleet benchmark: ${result.error}`);
-      }
-
-      // In quiet mode, stdout is just the run_id
-      const runId = result.stdout.trim();
-
-      // Parse metadata from stderr or construct expected paths
-      const handle: FleetExecutionHandle = {
-        runId,
-        artifactBaseDir: `${this.config.outputBaseDir}/${runId}`,
-        metadata: {
-          fleetId: runId.split('-')[0] || 'unknown',
-          repositoryCount: 0, // Unknown until execution completes
-          providers: [],
-          startedAt: new Date().toISOString(),
-          expectedArtifacts: [
-            `${this.config.outputBaseDir}/${runId}/fleet-results.json`,
-            `${this.config.outputBaseDir}/${runId}/fleet-summary.csv`,
-          ],
-        },
-      };
-
-      return handle;
-    } else {
-      // Execute and wait for completion
-      args.push('--wait');
-
-      const result = await executeCLI<FleetBenchmarkResults>(this.cliPath, {
-        args,
-        cwd: process.cwd(),
-        timeout: 600000, // 10 minute timeout for full execution
-        parseJson: format === 'json',
-      });
-
-      if (!result.success) {
-        throw new Error(`Fleet benchmark failed: ${result.error}`);
-      }
-
-      if (!result.data) {
-        throw new Error('No results returned from fleet benchmark');
-      }
-
-      const runId = result.data.fleet_id;
-
-      const handle: FleetExecutionHandle = {
-        runId,
-        artifactBaseDir: `${this.config.outputBaseDir}/${runId}`,
-        metadata: {
-          fleetId: result.data.fleet_id,
-          repositoryCount: result.data.total_repositories,
-          providers: Object.keys(result.data.provider_breakdown),
-          startedAt: result.data.timestamp,
-          expectedArtifacts: [
-            `${this.config.outputBaseDir}/${runId}/fleet-results.json`,
-            `${this.config.outputBaseDir}/${runId}/fleet-summary.csv`,
-          ],
-        },
-      };
-
-      return handle;
-    }
-  }
-
-  /**
-   * Get fleet results for a given run ID
-   *
-   * Loads the fleet results from disk for a previously executed fleet benchmark.
-   *
-   * @param runId - The run identifier to load results for
-   * @returns Promise resolving to FleetBenchmarkResults
-   * @throws {Error} If results cannot be found or loaded
-   *
-   * @example
-   * ```typescript
-   * const results = await fleet.getFleetResults('my-fleet-20231215-123456');
-   * console.log('Success rate:', results.fleet_summary.success_rate);
-   * ```
-   */
-  async getFleetResults(runId: string): Promise<FleetBenchmarkResults> {
-    const resultsPath = `${this.config.outputBaseDir}/${runId}/fleet-results.json`;
-
-    try {
-      const fs = await import('fs/promises');
-      const content = await fs.readFile(resultsPath, 'utf-8');
-      return JSON.parse(content) as FleetBenchmarkResults;
-    } catch (error) {
-      throw new Error(`Failed to load fleet results for ${runId}: ${error}`);
-    }
-  }
-
-  /**
-   * Wait for a fleet benchmark to complete
-   *
-   * Polls the results directory until the fleet-results.json file is available.
-   *
-   * @param runId - The run identifier to wait for
-   * @param options - Polling options
-   * @returns Promise resolving to FleetBenchmarkResults when complete
-   *
-   * @example
-   * ```typescript
-   * const handle = await fleet.executeFleet('./manifest.json');
-   * const results = await fleet.waitForCompletion(handle.runId);
-   * ```
-   */
-  async waitForCompletion(
-    runId: string,
-    options?: {
-      timeoutMs?: number;
-      pollIntervalMs?: number;
-    }
-  ): Promise<FleetBenchmarkResults> {
-    const timeoutMs = options?.timeoutMs ?? 600000; // 10 minutes default
-    const pollIntervalMs = options?.pollIntervalMs ?? 2000; // 2 seconds default
-
-    const startTime = Date.now();
-
-    while (Date.now() - startTime < timeoutMs) {
-      try {
-        return await this.getFleetResults(runId);
-      } catch (error) {
-        // Results not yet available, wait and retry
-        await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
-      }
-    }
-
-    throw new Error(`Timeout waiting for fleet benchmark ${runId} to complete`);
-  }
-
-  /**
-   * List all available fleet runs
-   *
-   * @returns Promise resolving to array of run IDs
-   *
-   * @example
-   * ```typescript
-   * const runs = await fleet.listRuns();
-   * console.log('Available runs:', runs);
-   * ```
-   */
-  async listRuns(): Promise<string[]> {
-    try {
-      const fs = await import('fs/promises');
-      const entries = await fs.readdir(this.config.outputBaseDir, { withFileTypes: true });
-      return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
-    } catch (error) {
-      // Directory doesn't exist yet
-      return [];
-    }
-  }
-}