@llm-dev-ops/agentics-cli 1.4.27 → 1.4.29

package/dist/bundled-agents/test-bench-agents/src/evaluators/index.ts

@@ -1,137 +0,0 @@
-/**
- * Evaluator utilities and helpers
- *
- * Provides convenient functions for evaluating LLM responses.
- */
-
-import { LLMTestBench } from '../core/llm-test-bench.js';
-import type {
-  EvaluationConfig,
-  CombinedEvaluationResults,
-  EvaluatorType,
-} from '../types/index.js';
-
-/**
- * Evaluator helper class
- */
-export class Evaluator {
-  constructor(private ltb: LLMTestBench) {}
-
-  /**
-   * Evaluate text with all available evaluators
-   */
-  async evaluateAll(text: string, options?: {
-    referenceText?: string;
-    context?: string[];
-  }): Promise<CombinedEvaluationResults> {
-    const config: EvaluationConfig = {
-      evaluators: [
-        'perplexity',
-        'coherence',
-        'relevance',
-        'faithfulness',
-        'readability',
-        'sentiment',
-        'toxicity',
-        'pii-detection',
-      ],
-      ...options,
-      parallel: true,
-      cacheResults: true,
-    };
-
-    return this.ltb.evaluate(text, config);
-  }
-
-  /**
-   * Evaluate text quality (perplexity, coherence, readability)
-   */
-  async evaluateQuality(text: string): Promise<CombinedEvaluationResults> {
-    const config: EvaluationConfig = {
-      evaluators: ['perplexity', 'coherence', 'readability'],
-      parallel: true,
-    };
-
-    return this.ltb.evaluate(text, config);
-  }
-
-  /**
-   * Evaluate safety (toxicity, PII detection)
-   */
-  async evaluateSafety(text: string): Promise<CombinedEvaluationResults> {
-    const config: EvaluationConfig = {
-      evaluators: ['toxicity', 'pii-detection'],
-      parallel: true,
-    };
-
-    return this.ltb.evaluate(text, config);
-  }
-
-  /**
-   * Evaluate accuracy (relevance, faithfulness)
-   */
-  async evaluateAccuracy(
-    text: string,
-    options: {
-      referenceText?: string;
-      context?: string[];
-    }
-  ): Promise<CombinedEvaluationResults> {
-    const config: EvaluationConfig = {
-      evaluators: ['relevance', 'faithfulness'],
-      ...options,
-      parallel: true,
-    };
-
-    return this.ltb.evaluate(text, config);
-  }
-
-  /**
-   * Use LLM as judge to evaluate text
-   */
-  async llmAsJudge(
-    text: string,
-    options: {
-      criteria: string[];
-      judgeModel?: string;
-      referenceText?: string;
-    }
-  ): Promise<CombinedEvaluationResults> {
-    const config: EvaluationConfig = {
-      evaluators: ['llm-as-judge'],
-      customCriteria: options.criteria,
-      judgeModel: options.judgeModel ?? 'gpt-4',
-      referenceText: options.referenceText,
-    };
-
-    return this.ltb.evaluate(text, config);
-  }
-
-  /**
-   * Evaluate with specific evaluators
-   */
-  async evaluateWith(
-    text: string,
-    evaluators: EvaluatorType[],
-    options?: {
-      referenceText?: string;
-      context?: string[];
-      judgeModel?: string;
-    }
-  ): Promise<CombinedEvaluationResults> {
-    const config: EvaluationConfig = {
-      evaluators,
-      ...options,
-      parallel: true,
-    };
-
-    return this.ltb.evaluate(text, config);
-  }
-}
-
-/**
- * Create an evaluator instance
- */
-export function createEvaluator(ltb: LLMTestBench): Evaluator {
-  return new Evaluator(ltb);
-}

package/dist/bundled-agents/test-bench-agents/src/execution/default-policy.ts

@@ -1,98 +0,0 @@
-/**
- * Default execution policy for LLM-Test-Bench.
- *
- * This is the canonical policy enforced when no override is provided.
- * Benchmark, compare, evaluate, and all stress test intents are mandatory.
- * Complete and optimize are recommended but not hard-gated.
- */
-
-import type { ExecutionPolicy } from './policy.js';
-
-export const DEFAULT_EXECUTION_POLICY: ExecutionPolicy = {
-  policy_version: '1.0.0',
-  updated_at: '2026-02-12T00:00:00.000Z',
-  description:
-    'Default policy: benchmark, compare, evaluate, and all stress test intents ' +
-    'are mandatory execution dependencies. Complete and optimize are recommended.',
-  rules: [
-    // Core simulation intents - mandatory
-    {
-      intent: 'benchmark',
-      disposition: 'mandatory',
-      rationale: 'Benchmark results must appear in execution graph for auditability',
-    },
-    {
-      intent: 'compare',
-      disposition: 'mandatory',
-      rationale: 'Model comparisons are decision-critical and must be traced',
-    },
-    {
-      intent: 'evaluate',
-      disposition: 'mandatory',
-      rationale: 'Evaluations are quality gates and must be traced',
-    },
-    // Core intents - recommended
-    {
-      intent: 'complete',
-      disposition: 'recommended',
-      rationale: 'Completions should be traced but may be used for ad-hoc exploration',
-    },
-    {
-      intent: 'optimize',
-      disposition: 'recommended',
-      rationale: 'Optimization is advisory; should be traced but not hard-gated',
-    },
-    // Stress test intents - ALL mandatory
-    {
-      intent: 'load_ramp',
-      disposition: 'mandatory',
-      rationale: 'Load ramp results are critical for capacity planning',
-    },
-    {
-      intent: 'spike',
-      disposition: 'mandatory',
-      rationale: 'Spike test results inform resilience decisions',
-    },
-    {
-      intent: 'soak',
-      disposition: 'mandatory',
-      rationale: 'Soak test results inform stability decisions',
-    },
-    {
-      intent: 'extreme_input',
-      disposition: 'mandatory',
-      rationale: 'Extreme input handling must be auditable',
-    },
-    {
-      intent: 'adversarial',
-      disposition: 'mandatory',
-      rationale: 'Adversarial test results are security-critical',
-    },
-    {
-      intent: 'rate_limit_probe',
-      disposition: 'mandatory',
-      rationale: 'Rate limit probing affects operational parameters',
-    },
-    {
-      intent: 'timeout_boundary',
-      disposition: 'mandatory',
-      rationale: 'Timeout boundary results set SLA parameters',
-    },
-    {
-      intent: 'token_limit',
-      disposition: 'mandatory',
-      rationale: 'Token limit results inform context window policies',
-    },
-    {
-      intent: 'context_overflow',
-      disposition: 'mandatory',
-      rationale: 'Context overflow results are input validation critical',
-    },
-    {
-      intent: 'malformed_request',
-      disposition: 'mandatory',
-      rationale: 'Error handling behavior must be auditable',
-    },
-  ],
-  default_disposition: 'recommended',
-};

package/dist/bundled-agents/test-bench-agents/src/execution/index.ts

@@ -1,52 +0,0 @@
-/**
- * Execution span module for the Agentics Foundational Execution Unit.
- *
- * This module ensures this repository NEVER executes silently.
- * All externally-invoked operations must emit agent-level execution spans
- * and integrate into a hierarchical ExecutionGraph produced by a Core.
- *
- * @packageDocumentation
- */
-
-// Core types
-export type {
-  SpanType,
-  SpanStatus,
-  ExecutionContext,
-  ExecutionSpan,
-  ExecutionArtifact,
-  ExecutionResult,
-} from './types.js';
-export { generateSpanId, nowISO, REPO_NAME } from './types.js';
-
-// Span management
-export {
-  SpanManager,
-  validateExecutionContext,
-  ExecutionContextError,
-  ExecutionInvariantError,
-} from './span-manager.js';
-
-// Instrumented entry points
-export { InstrumentedTestBench, executeWithSpans } from './instrumented.js';
-
-// Policy gate
-export { PolicyGate, PolicyGateError } from './policy-gate.js';
-export type { PolicyCheckResult } from './policy-gate.js';
-export {
-  SimulationIntentSchema,
-  GateDispositionSchema,
-  PolicyRuleSchema,
-  ExecutionPolicySchema,
-  ExemptionRecordSchema,
-  ExemptionRequestSchema,
-} from './policy.js';
-export type {
-  SimulationIntent,
-  GateDisposition,
-  PolicyRule,
-  ExecutionPolicy,
-  ExemptionRecord,
-  ExemptionRequest,
-} from './policy.js';
-export { DEFAULT_EXECUTION_POLICY } from './default-policy.js';

package/dist/bundled-agents/test-bench-agents/src/execution/instrumented.ts

@@ -1,337 +0,0 @@
-/**
- * Instrumented wrapper for LLMTestBench.
- *
- * Wraps every externally-invoked operation so that it:
- * 1. Requires an ExecutionContext (rejects if parent_span_id is missing)
- * 2. Creates a repo-level span on entry
- * 3. Creates agent-level spans for each operation
- * 4. Attaches artifacts to agent spans
- * 5. Returns ExecutionResult with full span hierarchy
- *
- * This ensures the repository NEVER executes silently.
- */
-
-import type {
-  SDKConfig,
-  CompletionRequest,
-  CompletionResponse,
-  BenchmarkConfig,
-  BenchmarkResults,
-  ComparisonResult,
-  EvaluationConfig,
-  CombinedEvaluationResults,
-  ProviderName,
-  ModelInfo,
-} from '../types/index.js';
-import { LLMTestBench } from '../core/llm-test-bench.js';
-import type { ExecutionContext, ExecutionResult } from './types.js';
-import type { ExecutionPolicy } from './policy.js';
-import { SpanManager } from './span-manager.js';
-import { PolicyGate } from './policy-gate.js';
-
-/**
- * Instrumented LLM Test Bench that emits execution spans.
- *
- * This is the primary entry point when this repo operates as a
- * Foundational Execution Unit within the Agentics system.
- *
- * @example
- * ```typescript
- * const bench = new InstrumentedTestBench({ verbose: true });
- *
- * const result = await bench.benchmark(
- *   { execution_id: 'exec-123', parent_span_id: 'core-span-456' },
- *   { provider: 'openai', model: 'gpt-4', prompts: ['Hello'] }
- * );
- *
- * // result.repo_span - the repo-level span
- * // result.agent_spans - agent spans (benchmark-runner)
- * // result.artifacts - benchmark output artifacts
- * // result.data - BenchmarkResults
- * // result.valid - true if all invariants satisfied
- * ```
- */
-export class InstrumentedTestBench {
-  private readonly inner: LLMTestBench;
-  private readonly gate: PolicyGate;
-
-  constructor(config: SDKConfig = {}, policy?: ExecutionPolicy) {
-    this.inner = new LLMTestBench(config);
-    this.gate = new PolicyGate(policy);
-  }
-
-  /**
-   * Get version with execution span tracking.
-   */
-  async version(ctx: ExecutionContext): Promise<ExecutionResult<string>> {
-    const { mgr } = this.gate.enforce('benchmark', ctx);
-    const agent = mgr.startAgent('version-resolver');
-
-    try {
-      const version = await this.inner.version();
-      mgr.endAgent(agent.span_id, 'completed');
-      return mgr.finalize(version);
-    } catch (err) {
-      const reason = err instanceof Error ? err.message : String(err);
-      mgr.endAgent(agent.span_id, 'failed', [reason]);
-      return mgr.finalize<string>(undefined, 'failed');
-    }
-  }
-
-  /**
-   * List models with execution span tracking.
-   */
-  async listModels(
-    ctx: ExecutionContext,
-    provider?: ProviderName
-  ): Promise<ExecutionResult<ModelInfo[]>> {
-    const { mgr } = this.gate.enforce('benchmark', ctx);
-    const agent = mgr.startAgent('model-lister');
-
-    try {
-      const models = await this.inner.listModels(provider);
-      mgr.endAgent(agent.span_id, 'completed');
-      return mgr.finalize(models);
-    } catch (err) {
-      const reason = err instanceof Error ? err.message : String(err);
-      mgr.endAgent(agent.span_id, 'failed', [reason]);
-      return mgr.finalize<ModelInfo[]>(undefined, 'failed');
-    }
-  }
-
-  /**
-   * Run benchmark with execution span tracking.
-   * Emits agent spans for validation, execution, and result collection.
-   */
-  async benchmark(
-    ctx: ExecutionContext,
-    options: {
-      provider: ProviderName;
-      model: string;
-      prompts: string[];
-      config?: BenchmarkConfig;
-    }
-  ): Promise<ExecutionResult<BenchmarkResults>> {
-    const { mgr } = this.gate.enforce('benchmark', ctx);
-
-    // Agent 1: Input validation
-    const validator = mgr.startAgent('input-validator');
-    try {
-      if (!options.provider || !options.model || !options.prompts?.length) {
-        throw new Error('Missing required benchmark parameters');
-      }
-      mgr.endAgent(validator.span_id, 'completed');
-    } catch (err) {
-      const reason = err instanceof Error ? err.message : String(err);
-      mgr.endAgent(validator.span_id, 'failed', [reason]);
-      return mgr.finalize<BenchmarkResults>(undefined, 'failed');
-    }
-
-    // Agent 2: Benchmark execution
-    const runner = mgr.startAgent('benchmark-runner');
-    try {
-      const results = await this.inner.benchmark(options);
-
-      // Attach result artifact to the runner agent
-      mgr.addArtifact({
-        reference: `benchmark-${mgr.repoSpanId}-results.json`,
-        artifact_type: 'benchmark_results',
-        agent_span_id: runner.span_id,
-        metadata: {
-          provider: options.provider,
-          model: options.model,
-          prompt_count: options.prompts.length,
-        },
-      });
-
-      mgr.endAgent(runner.span_id, 'completed');
-      return mgr.finalize(results);
-    } catch (err) {
-      const reason = err instanceof Error ? err.message : String(err);
-      mgr.endAgent(runner.span_id, 'failed', [reason]);
-      return mgr.finalize<BenchmarkResults>(undefined, 'failed');
-    }
-  }
-
-  /**
-   * Compare models with execution span tracking.
-   */
-  async compare(
-    ctx: ExecutionContext,
-    options: {
-      models: Array<{ provider: ProviderName; model: string }>;
-      prompts: string[];
-      config?: BenchmarkConfig;
-    }
-  ): Promise<ExecutionResult<ComparisonResult>> {
-    const { mgr } = this.gate.enforce('compare', ctx);
-    const agent = mgr.startAgent('model-comparator');
-
-    try {
-      const result = await this.inner.compare(options);
-
-      mgr.addArtifact({
-        reference: `comparison-${mgr.repoSpanId}-results.json`,
-        artifact_type: 'comparison_results',
-        agent_span_id: agent.span_id,
-        metadata: {
-          model_count: options.models.length,
-          prompt_count: options.prompts.length,
-        },
-      });
-
-      mgr.endAgent(agent.span_id, 'completed');
-      return mgr.finalize(result);
-    } catch (err) {
-      const reason = err instanceof Error ? err.message : String(err);
-      mgr.endAgent(agent.span_id, 'failed', [reason]);
-      return mgr.finalize<ComparisonResult>(undefined, 'failed');
-    }
-  }
-
-  /**
-   * Evaluate text with execution span tracking.
-   */
-  async evaluate(
-    ctx: ExecutionContext,
-    text: string,
-    config: EvaluationConfig
-  ): Promise<ExecutionResult<CombinedEvaluationResults>> {
-    const { mgr } = this.gate.enforce('evaluate', ctx);
-    const agent = mgr.startAgent('evaluator');
-
-    try {
-      const result = await this.inner.evaluate(text, config);
-
-      mgr.addArtifact({
-        reference: `evaluation-${mgr.repoSpanId}-results.json`,
-        artifact_type: 'evaluation_results',
-        agent_span_id: agent.span_id,
-        metadata: {
-          evaluators: config.evaluators,
-          text_length: text.length,
-        },
-      });
-
-      mgr.endAgent(agent.span_id, 'completed');
-      return mgr.finalize(result);
-    } catch (err) {
-      const reason = err instanceof Error ? err.message : String(err);
-      mgr.endAgent(agent.span_id, 'failed', [reason]);
-      return mgr.finalize<CombinedEvaluationResults>(undefined, 'failed');
-    }
-  }
-
-  /**
-   * Get completion with execution span tracking.
-   */
-  async complete(
-    ctx: ExecutionContext,
-    request: CompletionRequest
-  ): Promise<ExecutionResult<CompletionResponse>> {
-    const { mgr } = this.gate.enforce('complete', ctx);
-    const agent = mgr.startAgent('completion-agent');
-
-    try {
-      const result = await this.inner.complete(request);
-
-      mgr.addArtifact({
-        reference: `completion-${mgr.repoSpanId}-response.json`,
-        artifact_type: 'completion_response',
-        agent_span_id: agent.span_id,
-        metadata: {
-          model: request.model,
-        },
-      });
-
-      mgr.endAgent(agent.span_id, 'completed');
-      return mgr.finalize(result);
-    } catch (err) {
-      const reason = err instanceof Error ? err.message : String(err);
-      mgr.endAgent(agent.span_id, 'failed', [reason]);
-      return mgr.finalize<CompletionResponse>(undefined, 'failed');
-    }
-  }
-
-  /**
-   * Optimize model selection with execution span tracking.
-   */
-  async optimize(
-    ctx: ExecutionContext,
-    options: {
-      prompts: string[];
-      metric: 'latency' | 'cost' | 'quality';
-      maxCost?: number;
-      minQuality?: number;
-    }
-  ): Promise<
-    ExecutionResult<{ provider: ProviderName; model: string; reason: string }>
-  > {
-    const { mgr } = this.gate.enforce('optimize', ctx);
-    const agent = mgr.startAgent('optimizer');
-
-    try {
-      const result = await this.inner.optimize(options);
-
-      mgr.addArtifact({
-        reference: `optimization-${mgr.repoSpanId}-recommendation.json`,
-        artifact_type: 'optimization_recommendation',
-        agent_span_id: agent.span_id,
-        metadata: {
-          metric: options.metric,
-          prompt_count: options.prompts.length,
-        },
-      });
-
-      mgr.endAgent(agent.span_id, 'completed');
-      return mgr.finalize(result);
-    } catch (err) {
-      const reason = err instanceof Error ? err.message : String(err);
-      mgr.endAgent(agent.span_id, 'failed', [reason]);
-      return mgr.finalize<{
-        provider: ProviderName;
-        model: string;
-        reason: string;
-      }>(undefined, 'failed');
-    }
-  }
-}
-
-/**
- * Execute an arbitrary async operation with full execution span instrumentation.
- *
- * Use this for ad-hoc operations that need to participate in the
- * execution graph without going through InstrumentedTestBench.
- *
- * @example
- * ```typescript
- * const result = await executeWithSpans(
- *   { execution_id: 'exec-1', parent_span_id: 'core-span-1' },
- *   async (mgr) => {
- *     const agent = mgr.startAgent('custom-agent');
- *     const data = await doWork();
- *     mgr.addArtifact({
- *       reference: 'output.json',
- *       artifact_type: 'custom',
- *       agent_span_id: agent.span_id,
- *     });
- *     mgr.endAgent(agent.span_id, 'completed');
- *     return data;
- *   }
- * );
- * ```
- */
-export async function executeWithSpans<T>(
-  ctx: ExecutionContext,
-  fn: (mgr: SpanManager) => Promise<T>
-): Promise<ExecutionResult<T>> {
-  const mgr = SpanManager.create(ctx);
-
-  try {
-    const data = await fn(mgr);
-    return mgr.finalize(data);
-  } catch (err) {
-    const reason = err instanceof Error ? err.message : String(err);
-    return mgr.finalizeFailure([reason]);
-  }
-}