@llm-dev-ops/agentics-cli 1.4.27 → 1.4.29

package/dist/bundled-agents/test-bench-agents/src/execution/policy-gate.ts

@@ -1,166 +0,0 @@
-/**
- * PolicyGate enforcer for the Agentics Foundational Execution Unit.
- *
- * Checks the execution policy before any simulation intent proceeds.
- * For mandatory intents, either an ExecutionContext must be present or
- * an ExemptionRequest must be provided. Exemptions are recorded as
- * first-class artifacts in the execution graph.
- */
-
-import type { ExecutionContext, ExecutionArtifact } from './types.js';
-import type {
-  SimulationIntent,
-  GateDisposition,
-  ExecutionPolicy,
-  ExemptionRequest,
-  ExemptionRecord,
-} from './policy.js';
-import { SpanManager } from './span-manager.js';
-import { DEFAULT_EXECUTION_POLICY } from './default-policy.js';
-import { nowISO } from './types.js';
-
-/**
- * Error thrown when a mandatory policy gate is not satisfied.
- */
-export class PolicyGateError extends Error {
-  public readonly intent: SimulationIntent;
-  public readonly disposition: GateDisposition;
-
-  constructor(
-    message: string,
-    intent: SimulationIntent,
-    disposition: GateDisposition,
-  ) {
-    super(message);
-    this.name = 'PolicyGateError';
-    this.intent = intent;
-    this.disposition = disposition;
-  }
-}
-
-/**
- * Result of a policy gate check.
- */
-export interface PolicyCheckResult {
-  allowed: boolean;
-  disposition: GateDisposition;
-  exemption?: ExemptionRecord;
-  error?: string;
-}
-
-/**
- * PolicyGate sits between callers and InstrumentedTestBench.
- *
- * It checks the intent against the policy BEFORE execution proceeds.
- * If the intent is mandatory and no ExecutionContext is provided,
- * callers must supply an ExemptionRequest -- otherwise the check fails.
- */
-export class PolicyGate {
-  private readonly policy: ExecutionPolicy;
-
-  constructor(policy?: ExecutionPolicy) {
-    this.policy = policy ?? DEFAULT_EXECUTION_POLICY;
-  }
-
-  /** Get the current policy (for inspection / serialization). */
-  getPolicy(): Readonly<ExecutionPolicy> {
-    return this.policy;
-  }
-
-  /**
-   * Check whether an intent may proceed without an ExecutionContext.
-   *
-   * @param intent - The simulation intent being requested
-   * @param ctx - The execution context (null if caller is trying to bypass)
-   * @param exemption - An optional exemption request if ctx is null
-   */
-  check(
-    intent: SimulationIntent,
-    ctx: ExecutionContext | null,
-    exemption?: ExemptionRequest,
-  ): PolicyCheckResult {
-    const rule = this.policy.rules.find((r: { intent: string }) => r.intent === intent);
-    const disposition = rule?.disposition ?? this.policy.default_disposition;
-
-    // If context is present, always allowed (properly instrumented path)
-    if (ctx !== null) {
-      return { allowed: true, disposition };
-    }
-
-    // No context -- evaluate disposition
-    switch (disposition) {
-      case 'optional':
-        return { allowed: true, disposition };
-
-      case 'recommended':
-        return { allowed: true, disposition };
-
-      case 'mandatory': {
-        if (!exemption) {
-          return {
-            allowed: false,
-            disposition,
-            error:
-              `Intent '${intent}' has mandatory policy gate. ` +
-              `Provide an ExecutionContext or an ExemptionRequest. ` +
-              `Rationale: ${rule?.rationale ?? 'policy default'}`,
-          };
-        }
-
-        const record: ExemptionRecord = {
-          intent,
-          reason: exemption.reason,
-          granted_by: exemption.granted_by,
-          granted_at: nowISO(),
-          execution_id: 'EXEMPTED',
-          parent_span_id: 'EXEMPTED',
-        };
-
-        return { allowed: true, disposition, exemption: record };
-      }
-
-      default:
-        return { allowed: true, disposition };
-    }
-  }
-
-  /**
-   * Enforce the policy gate for an instrumented execution.
-   *
-   * Called by InstrumentedTestBench before delegating to the inner
-   * LLMTestBench. Since ctx is always provided on this path,
-   * this validates and creates the SpanManager.
-   */
-  enforce(
-    intent: SimulationIntent,
-    ctx: ExecutionContext,
-  ): { mgr: SpanManager } {
-    const result = this.check(intent, ctx);
-    if (!result.allowed) {
-      throw new PolicyGateError(
-        result.error ?? 'Policy gate check failed',
-        intent,
-        result.disposition,
-      );
-    }
-    return { mgr: SpanManager.create(ctx) };
-  }
-
-  /**
-   * Record an exemption as a first-class artifact on a SpanManager.
-   */
-  recordExemption(
-    mgr: SpanManager,
-    agentSpanId: string,
-    exemption: ExemptionRecord,
-  ): void {
-    const artifact: ExecutionArtifact = {
-      reference: `policy-exemption-${exemption.intent}-${Date.now()}.json`,
-      artifact_type: 'policy_exemption',
-      agent_span_id: agentSpanId,
-      metadata: { ...exemption },
-    };
-    mgr.addArtifact(artifact);
-    mgr.addExemption(exemption);
-  }
-}

package/dist/bundled-agents/test-bench-agents/src/execution/policy.ts

@@ -1,130 +0,0 @@
-/**
- * Policy gate types for the Agentics Foundational Execution Unit.
- *
- * Defines the declarative policy that classifies simulation intents
- * as mandatory, recommended, or optional execution dependencies.
- * When a mandatory intent is bypassed, an ExemptionRecord must be
- * produced as a first-class artifact in the execution graph.
- */
-
-import { z } from 'zod';
-
-// =============================================================================
-// SIMULATION INTENT
-// =============================================================================
-
-/**
- * All recognized simulation intents.
- * Union of InstrumentedTestBench method intents and stress test types.
- */
-export const SimulationIntentSchema = z.enum([
-  // InstrumentedTestBench methods
-  'benchmark',
-  'compare',
-  'evaluate',
-  'complete',
-  'optimize',
-  // Stress test types
-  'load_ramp',
-  'spike',
-  'soak',
-  'extreme_input',
-  'adversarial',
-  'rate_limit_probe',
-  'timeout_boundary',
-  'token_limit',
-  'context_overflow',
-  'malformed_request',
-]);
-
-export type SimulationIntent = z.infer<typeof SimulationIntentSchema>;
-
-// =============================================================================
-// GATE DISPOSITION
-// =============================================================================
-
-/**
- * Gate disposition for a given intent.
- * - 'mandatory': MUST run through InstrumentedTestBench; bypass is an error
- *   unless an exemption is explicitly granted.
- * - 'recommended': Should run instrumented, but bypass does not fail.
- * - 'optional': No policy enforcement.
- */
-export const GateDispositionSchema = z.enum([
-  'mandatory',
-  'recommended',
-  'optional',
-]);
-
-export type GateDisposition = z.infer<typeof GateDispositionSchema>;
-
-// =============================================================================
-// POLICY RULES
-// =============================================================================
-
-/**
- * A single policy rule binding an intent to a gate disposition.
- */
-export const PolicyRuleSchema = z.object({
-  intent: SimulationIntentSchema,
-  disposition: GateDispositionSchema,
-  rationale: z.string().min(1),
-});
-
-export type PolicyRule = z.infer<typeof PolicyRuleSchema>;
-
-// =============================================================================
-// EXECUTION POLICY
-// =============================================================================
-
-/**
- * The complete execution policy. Declarative, JSON-serializable,
- * inspectable at rest.
- */
-export const ExecutionPolicySchema = z.object({
-  policy_version: z.string().regex(/^\d+\.\d+\.\d+$/),
-  updated_at: z.string().datetime(),
-  description: z.string(),
-  rules: z.array(PolicyRuleSchema).refine(
-    (rules: Array<{ intent: string; disposition: string; rationale: string }>) => {
-      const intents = rules.map((r: { intent: string }) => r.intent);
-      return new Set(intents).size === intents.length;
-    },
-    { message: 'Duplicate intent in policy rules' }
-  ),
-  default_disposition: GateDispositionSchema,
-});
-
-export type ExecutionPolicy = z.infer<typeof ExecutionPolicySchema>;
-
-// =============================================================================
-// EXEMPTION RECORDS
-// =============================================================================
-
-/**
- * An exemption record. Created when a mandatory intent is explicitly
- * waived. This is a FIRST-CLASS record in the execution graph,
- * attached as an artifact to the relevant span.
- */
-export const ExemptionRecordSchema = z.object({
-  intent: SimulationIntentSchema,
-  reason: z.string().min(1),
-  granted_by: z.string().min(1),
-  granted_at: z.string().datetime(),
-  expires_at: z.string().datetime().optional(),
-  execution_id: z.string(),
-  parent_span_id: z.string(),
-});
-
-export type ExemptionRecord = z.infer<typeof ExemptionRecordSchema>;
-
-/**
- * Exemption request passed by callers who need to bypass a mandatory gate.
- */
-export const ExemptionRequestSchema = z.object({
-  intent: SimulationIntentSchema,
-  reason: z.string().min(1),
-  granted_by: z.string().min(1),
-});
-
-export type ExemptionRequest = z.infer<typeof ExemptionRequestSchema>;

package/dist/bundled-agents/test-bench-agents/src/execution/span-manager.ts

@@ -1,292 +0,0 @@
-/**
- * Span manager for the Agentics Foundational Execution Unit.
- *
- * Manages the lifecycle of execution spans, enforces invariants,
- * and produces the final ExecutionResult.
- *
- * Invariants enforced:
- * - parent_span_id must be present and non-empty
- * - At least one agent span must be emitted for a valid execution
- * - Agents must not share spans
- * - Repo span is created on entry, closed on exit
- * - All spans are append-only and causally ordered
- */
-
-import type {
-  ExecutionContext,
-  ExecutionSpan,
-  ExecutionArtifact,
-  ExecutionResult,
-  SpanStatus,
-} from './types.js';
-import { generateSpanId, nowISO, REPO_NAME } from './types.js';
-import type { ExemptionRecord } from './policy.js';
-
-/**
- * Validates that an ExecutionContext is structurally valid.
- * Rejects execution if parent_span_id is missing.
- */
-export function validateExecutionContext(ctx: unknown): ctx is ExecutionContext {
-  if (!ctx || typeof ctx !== 'object') return false;
-  const c = ctx as Record<string, unknown>;
-  return (
-    typeof c['execution_id'] === 'string' &&
-    c['execution_id'].length > 0 &&
-    typeof c['parent_span_id'] === 'string' &&
-    c['parent_span_id'].length > 0
-  );
-}
-
-/**
- * Error thrown when execution context is missing or invalid.
- */
-export class ExecutionContextError extends Error {
-  constructor(message: string) {
-    super(message);
-    this.name = 'ExecutionContextError';
-  }
-}
-
-/**
- * Error thrown when execution invariants are violated.
- */
-export class ExecutionInvariantError extends Error {
-  constructor(message: string) {
-    super(message);
-    this.name = 'ExecutionInvariantError';
-  }
-}
-
-/**
- * Manages execution spans for a single repo-level invocation.
- *
- * Usage:
- *   const mgr = SpanManager.create(ctx);
- *   const agentSpan = mgr.startAgent('benchmark-runner');
- *   // ... do work ...
- *   mgr.addArtifact({ reference: 'results.json', artifact_type: 'benchmark', agent_span_id: agentSpan.span_id });
- *   mgr.endAgent(agentSpan.span_id, 'completed');
- *   const result = mgr.finalize(data);
- */
-export class SpanManager {
-  private readonly repoSpan: ExecutionSpan;
-  private readonly agentSpans: Map<string, ExecutionSpan> = new Map();
-  private readonly artifacts: ExecutionArtifact[] = [];
-  private readonly exemptions: ExemptionRecord[] = [];
-  private readonly activeAgents: Set<string> = new Set();
-  private finalized = false;
-
-  private constructor(ctx: ExecutionContext) {
-    this.repoSpan = {
-      type: 'repo',
-      repo_name: REPO_NAME,
-      span_id: generateSpanId(),
-      parent_span_id: ctx.parent_span_id,
-      start_time: nowISO(),
-      status: 'running',
-    };
-  }
-
-  /**
-   * Create a new SpanManager for an incoming execution.
-   * Rejects if parent_span_id is missing or invalid.
-   */
-  static create(ctx: unknown): SpanManager {
-    if (!validateExecutionContext(ctx)) {
-      throw new ExecutionContextError(
-        'Execution rejected: parent_span_id is missing or invalid. ' +
-          'Every externally-invoked operation must provide a valid ExecutionContext ' +
-          'with execution_id and parent_span_id.'
-      );
-    }
-    return new SpanManager(ctx);
-  }
-
-  /** Get the repo-level span ID */
-  get repoSpanId(): string {
-    return this.repoSpan.span_id;
-  }
-
-  /** Get the execution_id from the parent context */
-  get executionId(): string {
-    // Stored implicitly via the parent relationship
-    return this.repoSpan.parent_span_id;
-  }
-
-  /**
-   * Start a new agent-level span.
-   * Each agent gets its own unique span. Agents must not share spans.
-   */
-  startAgent(agentName: string): ExecutionSpan {
-    if (this.finalized) {
-      throw new ExecutionInvariantError('Cannot start agent after finalization');
-    }
-
-    const span: ExecutionSpan = {
-      type: 'agent',
-      agent_name: agentName,
-      repo_name: REPO_NAME,
-      span_id: generateSpanId(),
-      parent_span_id: this.repoSpan.span_id,
-      start_time: nowISO(),
-      status: 'running',
-    };
-
-    this.agentSpans.set(span.span_id, span);
-    this.activeAgents.add(span.span_id);
-    return { ...span };
-  }
-
-  /**
-   * End an agent-level span.
-   */
-  endAgent(
-    spanId: string,
-    status: 'completed' | 'failed',
-    failureReasons?: string[]
-  ): void {
-    const span = this.agentSpans.get(spanId);
-    if (!span) {
-      throw new ExecutionInvariantError(`Unknown agent span: ${spanId}`);
-    }
-    if (span.status !== 'running') {
-      throw new ExecutionInvariantError(
-        `Agent span ${spanId} already ended with status ${span.status}`
-      );
-    }
-
-    span.end_time = nowISO();
-    span.status = status;
-    if (failureReasons) {
-      span.failure_reasons = failureReasons;
-    }
-    this.activeAgents.delete(spanId);
-  }
-
-  /**
-   * Attach an artifact to an agent span.
-   * Artifacts must never be attached directly to the repo span.
-   */
-  addArtifact(artifact: ExecutionArtifact): void {
-    if (!this.agentSpans.has(artifact.agent_span_id)) {
-      throw new ExecutionInvariantError(
-        `Cannot attach artifact to unknown agent span: ${artifact.agent_span_id}. ` +
-          'Artifacts must be attached to an existing agent-level span.'
-      );
-    }
-    this.artifacts.push(artifact);
-  }
-
-  /**
-   * Record a policy exemption for this execution.
-   * Exemptions appear in the final ExecutionResult.exemptions array.
-   */
-  addExemption(record: ExemptionRecord): void {
-    this.exemptions.push(record);
-  }
-
-  /**
-   * Finalize the execution and produce the result.
-   *
-   * Enforces:
-   * - At least one agent span was emitted
-   * - No agents are still running
-   * - All invariants are satisfied
-   */
-  finalize<T>(data?: T, forceStatus?: SpanStatus): ExecutionResult<T> {
-    if (this.finalized) {
-      throw new ExecutionInvariantError('Already finalized');
-    }
-    this.finalized = true;
-
-    const validationErrors: string[] = [];
-
-    // Enforce: at least one agent span must exist
-    if (this.agentSpans.size === 0) {
-      validationErrors.push(
-        'No agent-level spans were emitted. Execution is INVALID.'
-      );
-    }
-
-    // Enforce: no running agents
-    if (this.activeAgents.size > 0) {
-      const running = Array.from(this.activeAgents)
-        .map((id) => this.agentSpans.get(id)?.agent_name ?? id)
-        .join(', ');
-      validationErrors.push(
-        `Agents still running at finalization: ${running}. ` +
-          'All agent spans must be ended before finalization.'
-      );
-      // Force-end them as failed
-      for (const id of this.activeAgents) {
-        this.endAgent(id, 'failed', [
-          'Agent was still running at finalization',
-        ]);
-      }
-    }
-
-    // Check if any agent executed without a span (structural check)
-    // This is enforced by the API: you cannot execute logic without calling startAgent
-
-    // Determine repo span status
-    const anyFailed = Array.from(this.agentSpans.values()).some(
-      (s) => s.status === 'failed'
-    );
-    const repoStatus: SpanStatus =
-      forceStatus ??
-      (validationErrors.length > 0
-        ? 'failed'
-        : anyFailed
-          ? 'failed'
-          : 'completed');
-
-    this.repoSpan.end_time = nowISO();
-    this.repoSpan.status = repoStatus;
-    if (repoStatus === 'failed') {
-      this.repoSpan.failure_reasons = validationErrors.length > 0
-        ? validationErrors
-        : Array.from(this.agentSpans.values())
-            .filter((s) => s.status === 'failed')
-            .flatMap((s) => s.failure_reasons ?? ['Agent failed']);
-    }
-
-    const valid = validationErrors.length === 0;
-
-    return {
-      repo_span: { ...this.repoSpan },
-      agent_spans: Array.from(this.agentSpans.values()).map((s) => ({ ...s })),
-      artifacts: [...this.artifacts],
-      data: valid ? data : undefined,
-      valid,
-      validation_errors: validationErrors.length > 0 ? validationErrors : undefined,
-      exemptions: this.exemptions.length > 0 ? [...this.exemptions] : undefined,
-    };
-  }
-
-  /**
-   * Finalize as failed with explicit reasons.
-   * Still returns all emitted spans per the contract.
-   */
-  finalizeFailure(reasons: string[]): ExecutionResult<never> {
-    if (!this.finalized) {
-      // End any running agents
-      for (const id of this.activeAgents) {
-        this.endAgent(id, 'failed', reasons);
-      }
-    }
-
-    this.finalized = true;
-    this.repoSpan.end_time = nowISO();
-    this.repoSpan.status = 'failed';
-    this.repoSpan.failure_reasons = reasons;
-
-    return {
-      repo_span: { ...this.repoSpan },
-      agent_spans: Array.from(this.agentSpans.values()).map((s) => ({ ...s })),
-      artifacts: [...this.artifacts],
-      valid: false,
-      validation_errors: reasons,
-      exemptions: this.exemptions.length > 0 ? [...this.exemptions] : undefined,
-    };
-  }
-}

package/dist/bundled-agents/test-bench-agents/src/execution/types.ts

@@ -1,125 +0,0 @@
-/**
- * Execution span types for the Agentics Foundational Execution Unit.
- *
- * This module defines the core types that enable this repository to emit
- * agent-level execution spans and integrate into a hierarchical
- * ExecutionGraph produced by a Core.
- *
- * Span hierarchy invariant:
- *   Core
- *     └─ Repo (this repo)
- *         └─ Agent (one or more)
- */
-
-import { randomUUID } from 'crypto';
-import type { ExemptionRecord } from './policy.js';
-
-/** Span type discriminator */
-export type SpanType = 'repo' | 'agent';
-
-/** Span status */
-export type SpanStatus = 'running' | 'completed' | 'failed';
-
-/**
- * Execution context provided by the calling Core.
- * Every externally-invoked operation MUST receive this.
- */
-export interface ExecutionContext {
-  /** Unique identifier for this execution graph */
-  execution_id: string;
-
-  /** Span ID from the Core that parents this repo's span */
-  parent_span_id: string;
-}
-
-/**
- * A single execution span within the graph.
- * JSON-serializable, append-only, causally ordered via parent_span_id.
- */
-export interface ExecutionSpan {
-  /** Discriminator: "repo" or "agent" */
-  type: SpanType;
-
-  /** Unique span identifier */
-  span_id: string;
-
-  /** Parent span (Core span for repo, repo span for agents) */
-  parent_span_id: string;
-
-  /** ISO-8601 start time */
-  start_time: string;
-
-  /** ISO-8601 end time (set on completion/failure) */
-  end_time?: string;
-
-  /** Current status */
-  status: SpanStatus;
-
-  /** Repo-level: name of this repository */
-  repo_name?: string;
-
-  /** Agent-level: name of the agent that executed */
-  agent_name?: string;
-
-  /** Failure reasons (populated when status is "failed") */
-  failure_reasons?: string[];
-}
-
-/**
- * An artifact produced during execution.
- * MUST be attached to the agent-level span that produced it.
- */
-export interface ExecutionArtifact {
-  /** Stable reference: ID, URI, hash, or filename */
-  reference: string;
-
-  /** Artifact type descriptor */
-  artifact_type: string;
-
-  /** Span ID of the agent that produced this artifact */
-  agent_span_id: string;
-
-  /** Optional metadata */
-  metadata?: Record<string, unknown>;
-}
-
-/**
- * Complete execution result returned by this repo.
- * Includes the repo-level span, nested agent-level spans,
- * and artifacts with evidence attached at correct levels.
- */
-export interface ExecutionResult<T = unknown> {
-  /** The repo-level span */
-  repo_span: ExecutionSpan;
-
-  /** Nested agent-level spans (one per agent that executed) */
-  agent_spans: ExecutionSpan[];
-
-  /** Artifacts produced, each attached to an agent span */
-  artifacts: ExecutionArtifact[];
-
-  /** The operation's return data (if successful) */
-  data?: T;
-
-  /** Whether the execution is valid per invariants */
-  valid: boolean;
-
-  /** Validation errors (if invalid) */
-  validation_errors?: string[];
-
-  /** Policy exemptions granted during this execution (if any) */
-  exemptions?: ExemptionRecord[];
-}
-
-/** Generate a new span ID */
-export function generateSpanId(): string {
-  return randomUUID();
-}
-
-/** Get current ISO-8601 timestamp */
-export function nowISO(): string {
-  return new Date().toISOString();
-}
-
-/** The canonical name for this repository */
-export const REPO_NAME = 'llm-test-bench';