@llm-dev-ops/agentics-cli 1.4.27 → 1.4.29

package/dist/bundled-agents/test-bench-agents/src/index.ts

@@ -1,100 +0,0 @@
-/**
- * LLM Test Bench - TypeScript SDK
- *
- * A comprehensive, production-ready framework for benchmarking, testing,
- * and evaluating Large Language Models.
- *
- * @packageDocumentation
- */
-
-// Export main SDK class
-export { LLMTestBench } from './core/llm-test-bench.js';
-
-// Export Fleet Benchmark API
-export { FleetBenchmark } from './core/fleet-benchmark.js';
-export type {
-  FleetConfig,
-  FleetManifest,
-  RepositorySpec,
-  FleetManifestConfig,
-  FleetExecutionHandle,
-  FleetExecutionMetadata,
-  FleetBenchmarkResults,
-  FleetSummary,
-  ProviderFleetStats,
-  CategoryFleetStats,
-} from './core/fleet-benchmark.js';
-
-// Export provider clients
-export {
-  ProviderClient,
-  OpenAIClient,
-  AnthropicClient,
-  GoogleClient,
-  ProviderClientFactory,
-} from './core/provider-client.js';
-
-// Export evaluator utilities
-export { Evaluator, createEvaluator } from './evaluators/index.js';
-
-// Export all types
-export * from './types/index.js';
-
-// Export execution span module (Agentics Foundational Execution Unit)
-export {
-  InstrumentedTestBench,
-  SpanManager,
-  executeWithSpans,
-  validateExecutionContext,
-  ExecutionContextError,
-  ExecutionInvariantError,
-  generateSpanId,
-  nowISO,
-  REPO_NAME,
-} from './execution/index.js';
-export type {
-  SpanType,
-  SpanStatus,
-  ExecutionContext,
-  ExecutionSpan,
-  ExecutionArtifact,
-  ExecutionResult,
-} from './execution/index.js';
-
-// Export policy gate (execution dependency enforcement)
-export {
-  PolicyGate,
-  PolicyGateError,
-  DEFAULT_EXECUTION_POLICY,
-} from './execution/index.js';
-export type {
-  SimulationIntent,
-  GateDisposition,
-  ExecutionPolicy,
-  ExemptionRecord,
-  ExemptionRequest,
-  PolicyCheckResult,
-} from './execution/index.js';
-
-// Export utilities
-export { executeCLI, findCLIPath } from './utils/cli-executor.js';
-export {
-  validateProviderConfig,
-  validateCompletionRequest,
-  validateBenchmarkConfig,
-  validateEvaluationConfig,
-  isValidModel,
-  isValidProvider,
-} from './utils/validators.js';
-
-/**
- * SDK version
- */
-export const VERSION = '0.1.2';
-
-/**
- * Default export for convenience
- */
-import { LLMTestBench } from './core/llm-test-bench.js';
-
-export default LLMTestBench;

package/dist/bundled-agents/test-bench-agents/src/types/benchmarks.ts

@@ -1,219 +0,0 @@
-/**
- * Benchmark types and interfaces
- *
- * This module defines TypeScript types for benchmarking LLM providers.
- */
-
-/**
- * Benchmark configuration
- */
-export interface BenchmarkConfig {
-  /** Number of concurrent requests */
-  concurrency?: number;
-
-  /** Whether to save individual responses */
-  saveResponses?: boolean;
-
-  /** Output file path for results */
-  outputPath?: string;
-
-  /** Maximum duration for the benchmark (milliseconds) */
-  maxDuration?: number;
-
-  /** Whether to show progress bar */
-  showProgress?: boolean;
-}
-
-/**
- * Latency metrics in milliseconds
- */
-export interface LatencyMetrics {
-  /** Median latency (P50) */
-  p50: number;
-
-  /** 95th percentile latency */
-  p95: number;
-
-  /** 99th percentile latency */
-  p99: number;
-
-  /** Average latency */
-  mean: number;
-
-  /** Minimum latency */
-  min: number;
-
-  /** Maximum latency */
-  max: number;
-
-  /** Standard deviation */
-  stdDev: number;
-}
-
-/**
- * Token usage metrics
- */
-export interface TokenMetrics {
-  /** Total prompt tokens across all requests */
-  totalPromptTokens: number;
-
-  /** Total completion tokens across all requests */
-  totalCompletionTokens: number;
-
-  /** Total tokens used */
-  totalTokens: number;
-
-  /** Average tokens per request */
-  averageTokensPerRequest: number;
-
-  /** Average tokens per second */
-  tokensPerSecond: number;
-}
-
-/**
- * Cost metrics in USD
- */
-export interface CostMetrics {
-  /** Total estimated cost */
-  totalCost: number;
-
-  /** Average cost per request */
-  averageCostPerRequest: number;
-
-  /** Input token cost */
-  inputCost: number;
-
-  /** Output token cost */
-  outputCost: number;
-}
-
-/**
- * Benchmark summary statistics
- */
-export interface BenchmarkSummary {
-  /** Total number of requests */
-  totalRequests: number;
-
-  /** Number of successful requests */
-  successfulRequests: number;
-
-  /** Number of failed requests */
-  failedRequests: number;
-
-  /** Success rate (0.0 - 1.0) */
-  successRate: number;
-
-  /** Latency metrics */
-  latency: LatencyMetrics;
-
-  /** Token usage metrics */
-  tokens: TokenMetrics;
-
-  /** Cost metrics (if available) */
-  cost?: CostMetrics;
-
-  /** Total duration in milliseconds */
-  totalDuration: number;
-
-  /** Requests per second */
-  requestsPerSecond: number;
-}
-
-/**
- * Individual benchmark result
- */
-export interface BenchmarkResult {
-  /** Request ID */
-  id: string;
-
-  /** Prompt used */
-  prompt: string;
-
-  /** Response content */
-  response?: string;
-
-  /** Whether the request succeeded */
-  success: boolean;
-
-  /** Error message (if failed) */
-  error?: string;
-
-  /** Latency in milliseconds */
-  latencyMs: number;
-
-  /** Token usage */
-  tokens?: {
-    prompt: number;
-    completion: number;
-    total: number;
-  };
-
-  /** Estimated cost in USD */
-  cost?: number;
-
-  /** Timestamp */
-  timestamp: string;
-}
-
-/**
- * Complete benchmark results
- */
-export interface BenchmarkResults {
-  /** Benchmark metadata */
-  metadata: {
-    /** Provider name */
-    provider: string;
-
-    /** Model name */
-    model: string;
-
-    /** Start timestamp */
-    startTime: string;
-
-    /** End timestamp */
-    endTime: string;
-
-    /** Configuration used */
-    config: BenchmarkConfig;
-  };
-
-  /** Summary statistics */
-  summary: BenchmarkSummary;
-
-  /** Individual results (if saveResponses was enabled) */
-  results?: BenchmarkResult[];
-}
-
-/**
- * Comparison result for multiple models
- */
-export interface ComparisonResult {
-  /** Comparison metadata */
-  metadata: {
-    /** Models compared */
-    models: Array<{
-      provider: string;
-      model: string;
-    }>;
-
-    /** Timestamp */
-    timestamp: string;
-
-    /** Number of test cases */
-    testCases: number;
-  };
-
-  /** Results for each model */
-  modelResults: Array<{
-    provider: string;
-    model: string;
-    summary: BenchmarkSummary;
-  }>;
-
-  /** Winner determination (if applicable) */
-  winner?: {
-    provider: string;
-    model: string;
-    reason: string;
-  };
-}

package/dist/bundled-agents/test-bench-agents/src/types/evaluators.ts

@@ -1,253 +0,0 @@
-/**
- * Evaluator types and interfaces
- *
- * This module defines TypeScript types for evaluating LLM responses.
- */
-
-/**
- * Evaluation metric types
- */
-export type EvaluatorType =
-  | 'perplexity'
-  | 'coherence'
-  | 'relevance'
-  | 'faithfulness'
-  | 'llm-as-judge'
-  | 'readability'
-  | 'sentiment'
-  | 'toxicity'
-  | 'pii-detection'
-  | 'custom';
-
-/**
- * Base evaluation result
- */
-export interface EvaluationResult {
-  /** Type of evaluator */
-  type: EvaluatorType;
-
-  /** Evaluation score (typically 0.0 - 1.0) */
-  score: number;
-
-  /** Detailed explanation of the score */
-  explanation?: string;
-
-  /** Additional metadata */
-  metadata?: Record<string, unknown>;
-
-  /** Timestamp */
-  timestamp: string;
-}
-
-/**
- * Perplexity evaluation result
- */
-export interface PerplexityResult extends EvaluationResult {
-  type: 'perplexity';
-
-  /** Perplexity value (lower is better) */
-  perplexity: number;
-
-  /** Log probability */
-  logProb: number;
-}
-
-/**
- * Coherence evaluation result
- */
-export interface CoherenceResult extends EvaluationResult {
-  type: 'coherence';
-
-  /** Coherence score (0.0 - 1.0) */
-  coherenceScore: number;
-
-  /** Semantic similarity scores between sentences */
-  sentenceSimilarities?: number[];
-
-  /** Discourse marker analysis */
-  discourseMarkers?: {
-    count: number;
-    types: string[];
-  };
-}
-
-/**
- * Relevance evaluation result
- */
-export interface RelevanceResult extends EvaluationResult {
-  type: 'relevance';
-
-  /** Relevance score (0.0 - 1.0) */
-  relevanceScore: number;
-
-  /** Similarity to reference/context */
-  similarity?: number;
-
-  /** Key topics covered */
-  topicsCovered?: string[];
-}
-
-/**
- * Faithfulness evaluation result
- */
-export interface FaithfulnessResult extends EvaluationResult {
-  type: 'faithfulness';
-
-  /** Faithfulness score (0.0 - 1.0) */
-  faithfulnessScore: number;
-
-  /** Claims that are supported by source */
-  supportedClaims?: number;
-
-  /** Total claims made */
-  totalClaims?: number;
-
-  /** Hallucination indicators */
-  hallucinations?: string[];
-}
-
-/**
- * LLM-as-Judge evaluation result
- */
-export interface LLMJudgeResult extends EvaluationResult {
-  type: 'llm-as-judge';
-
-  /** Judge's rating (typically 1-10) */
-  rating: number;
-
-  /** Judge's reasoning */
-  reasoning: string;
-
-  /** Criteria evaluated */
-  criteria: string[];
-
-  /** Judge model used */
-  judgeModel: string;
-}
-
-/**
- * Readability evaluation result
- */
-export interface ReadabilityResult extends EvaluationResult {
-  type: 'readability';
-
-  /** Flesch Reading Ease score */
-  fleschScore?: number;
-
-  /** Flesch-Kincaid Grade Level */
-  gradeLevel?: number;
-
-  /** Average sentence length */
-  avgSentenceLength?: number;
-
-  /** Average word length */
-  avgWordLength?: number;
-}
-
-/**
- * Sentiment evaluation result
- */
-export interface SentimentResult extends EvaluationResult {
-  type: 'sentiment';
-
-  /** Sentiment label */
-  sentiment: 'positive' | 'negative' | 'neutral';
-
-  /** Confidence score (0.0 - 1.0) */
-  confidence: number;
-
-  /** Detailed sentiment scores */
-  scores?: {
-    positive: number;
-    negative: number;
-    neutral: number;
-  };
-}
-
-/**
- * Toxicity evaluation result
- */
-export interface ToxicityResult extends EvaluationResult {
-  type: 'toxicity';
-
-  /** Toxicity score (0.0 - 1.0) */
-  toxicityScore: number;
-
-  /** Whether content is toxic */
-  isToxic: boolean;
-
-  /** Specific toxicity categories detected */
-  categories?: {
-    hate?: number;
-    harassment?: number;
-    violence?: number;
-    sexual?: number;
-    profanity?: number;
-  };
-}
-
-/**
- * PII detection result
- */
-export interface PIIDetectionResult extends EvaluationResult {
-  type: 'pii-detection';
-
-  /** Whether PII was detected */
-  piiDetected: boolean;
-
-  /** Types of PII found */
-  piiTypes?: Array<{
-    type: 'email' | 'phone' | 'ssn' | 'credit_card' | 'address' | 'name' | 'other';
-    value: string;
-    confidence: number;
-  }>;
-
-  /** Redacted text (if applicable) */
-  redactedText?: string;
-}
-
-/**
- * Evaluation configuration
- */
-export interface EvaluationConfig {
-  /** Evaluators to use */
-  evaluators: EvaluatorType[];
-
-  /** Reference text for comparison (optional) */
-  referenceText?: string;
-
-  /** Context or source documents (optional) */
-  context?: string[];
-
-  /** Custom evaluation criteria */
-  customCriteria?: string[];
-
-  /** Judge model to use for LLM-as-Judge */
-  judgeModel?: string;
-
-  /** Whether to run evaluations in parallel */
-  parallel?: boolean;
-
-  /** Cache evaluation results */
-  cacheResults?: boolean;
-}
-
-/**
- * Combined evaluation results
- */
-export interface CombinedEvaluationResults {
-  /** Text that was evaluated */
-  text: string;
-
-  /** Individual evaluation results */
-  results: EvaluationResult[];
-
-  /** Overall aggregate score */
-  overallScore: number;
-
-  /** Evaluation duration in milliseconds */
-  durationMs: number;
-
-  /** Timestamp */
-  timestamp: string;
-}

package/dist/bundled-agents/test-bench-agents/src/types/index.ts

@@ -1,55 +0,0 @@
-/**
- * Type definitions for LLM Test Bench SDK
- *
- * This module re-exports all type definitions for convenient importing.
- */
-
-export * from './providers.js';
-export * from './benchmarks.js';
-export * from './evaluators.js';
-
-/**
- * SDK Configuration
- */
-export interface SDKConfig {
-  /** Path to the LLM Test Bench CLI binary (optional, auto-detected if not provided) */
-  cliPath?: string;
-
-  /** Working directory for operations */
-  workingDir?: string;
-
-  /** Whether to enable verbose logging */
-  verbose?: boolean;
-
-  /** Environment variables to pass to CLI commands */
-  env?: Record<string, string>;
-
-  /** Default timeout for CLI operations in milliseconds */
-  timeout?: number;
-}
-
-/**
- * CLI execution result
- */
-export interface CLIResult<T = unknown> {
-  /** Whether the command succeeded */
-  success: boolean;
-
-  /** Parsed output data */
-  data?: T;
-
-  /** Error message (if failed) */
-  error?: string;
-
-  /** Exit code */
-  exitCode: number;
-
-  /** Raw stdout */
-  stdout: string;
-
-  /** Raw stderr */
-  stderr: string;
-
-  /** Execution duration in milliseconds */
-  durationMs: number;
-}