@llm-dev-ops/agentics-cli 1.4.26 → 1.4.27

package/dist/bundled-agents/test-bench-agents/dist/execution/span-manager.js

@@ -0,0 +1,164 @@
+import { generateSpanId, nowISO, REPO_NAME } from './types.js';
+export function validateExecutionContext(ctx) {
+    if (!ctx || typeof ctx !== 'object')
+        return false;
+    const c = ctx;
+    return (typeof c['execution_id'] === 'string' &&
+        c['execution_id'].length > 0 &&
+        typeof c['parent_span_id'] === 'string' &&
+        c['parent_span_id'].length > 0);
+}
+export class ExecutionContextError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ExecutionContextError';
+    }
+}
+export class ExecutionInvariantError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ExecutionInvariantError';
+    }
+}
+export class SpanManager {
+    repoSpan;
+    agentSpans = new Map();
+    artifacts = [];
+    exemptions = [];
+    activeAgents = new Set();
+    finalized = false;
+    constructor(ctx) {
+        this.repoSpan = {
+            type: 'repo',
+            repo_name: REPO_NAME,
+            span_id: generateSpanId(),
+            parent_span_id: ctx.parent_span_id,
+            start_time: nowISO(),
+            status: 'running',
+        };
+    }
+    static create(ctx) {
+        if (!validateExecutionContext(ctx)) {
+            throw new ExecutionContextError('Execution rejected: parent_span_id is missing or invalid. ' +
+                'Every externally-invoked operation must provide a valid ExecutionContext ' +
+                'with execution_id and parent_span_id.');
+        }
+        return new SpanManager(ctx);
+    }
+    get repoSpanId() {
+        return this.repoSpan.span_id;
+    }
+    get executionId() {
+        return this.repoSpan.parent_span_id;
+    }
+    startAgent(agentName) {
+        if (this.finalized) {
+            throw new ExecutionInvariantError('Cannot start agent after finalization');
+        }
+        const span = {
+            type: 'agent',
+            agent_name: agentName,
+            repo_name: REPO_NAME,
+            span_id: generateSpanId(),
+            parent_span_id: this.repoSpan.span_id,
+            start_time: nowISO(),
+            status: 'running',
+        };
+        this.agentSpans.set(span.span_id, span);
+        this.activeAgents.add(span.span_id);
+        return { ...span };
+    }
+    endAgent(spanId, status, failureReasons) {
+        const span = this.agentSpans.get(spanId);
+        if (!span) {
+            throw new ExecutionInvariantError(`Unknown agent span: ${spanId}`);
+        }
+        if (span.status !== 'running') {
+            throw new ExecutionInvariantError(`Agent span ${spanId} already ended with status ${span.status}`);
+        }
+        span.end_time = nowISO();
+        span.status = status;
+        if (failureReasons) {
+            span.failure_reasons = failureReasons;
+        }
+        this.activeAgents.delete(spanId);
+    }
+    addArtifact(artifact) {
+        if (!this.agentSpans.has(artifact.agent_span_id)) {
+            throw new ExecutionInvariantError(`Cannot attach artifact to unknown agent span: ${artifact.agent_span_id}. ` +
+                'Artifacts must be attached to an existing agent-level span.');
+        }
+        this.artifacts.push(artifact);
+    }
+    addExemption(record) {
+        this.exemptions.push(record);
+    }
+    finalize(data, forceStatus) {
+        if (this.finalized) {
+            throw new ExecutionInvariantError('Already finalized');
+        }
+        this.finalized = true;
+        const validationErrors = [];
+        if (this.agentSpans.size === 0) {
+            validationErrors.push('No agent-level spans were emitted. Execution is INVALID.');
+        }
+        if (this.activeAgents.size > 0) {
+            const running = Array.from(this.activeAgents)
+                .map((id) => this.agentSpans.get(id)?.agent_name ?? id)
+                .join(', ');
+            validationErrors.push(`Agents still running at finalization: ${running}. ` +
+                'All agent spans must be ended before finalization.');
+            for (const id of this.activeAgents) {
+                this.endAgent(id, 'failed', [
+                    'Agent was still running at finalization',
+                ]);
+            }
+        }
+        const anyFailed = Array.from(this.agentSpans.values()).some((s) => s.status === 'failed');
+        const repoStatus = forceStatus ??
+            (validationErrors.length > 0
+                ? 'failed'
+                : anyFailed
+                    ? 'failed'
+                    : 'completed');
+        this.repoSpan.end_time = nowISO();
+        this.repoSpan.status = repoStatus;
+        if (repoStatus === 'failed') {
+            this.repoSpan.failure_reasons = validationErrors.length > 0
+                ? validationErrors
+                : Array.from(this.agentSpans.values())
+                    .filter((s) => s.status === 'failed')
+                    .flatMap((s) => s.failure_reasons ?? ['Agent failed']);
+        }
+        const valid = validationErrors.length === 0;
+        return {
+            repo_span: { ...this.repoSpan },
+            agent_spans: Array.from(this.agentSpans.values()).map((s) => ({ ...s })),
+            artifacts: [...this.artifacts],
+            data: valid ? data : undefined,
+            valid,
+            validation_errors: validationErrors.length > 0 ? validationErrors : undefined,
+            exemptions: this.exemptions.length > 0 ? [...this.exemptions] : undefined,
+        };
+    }
+    finalizeFailure(reasons) {
+        if (!this.finalized) {
+            for (const id of this.activeAgents) {
+                this.endAgent(id, 'failed', reasons);
+            }
+        }
+        this.finalized = true;
+        this.repoSpan.end_time = nowISO();
+        this.repoSpan.status = 'failed';
+        this.repoSpan.failure_reasons = reasons;
+        return {
+            repo_span: { ...this.repoSpan },
+            agent_spans: Array.from(this.agentSpans.values()).map((s) => ({ ...s })),
+            artifacts: [...this.artifacts],
+            valid: false,
+            validation_errors: reasons,
+            exemptions: this.exemptions.length > 0 ? [...this.exemptions] : undefined,
+        };
+    }
+}
+//# sourceMappingURL=span-manager.js.map

package/dist/bundled-agents/test-bench-agents/dist/execution/span-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"span-manager.js","sourceRoot":"","sources":["../../src/execution/span-manager.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAO/D,MAAM,UAAU,wBAAwB,CAAC,GAAY;IACnD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClD,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,OAAO,CACL,OAAO,CAAC,CAAC,cAAc,CAAC,KAAK,QAAQ;QACrC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC;QAC5B,OAAO,CAAC,CAAC,gBAAgB,CAAC,KAAK,QAAQ;QACvC,CAAC,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,CAAC,CAC/B,CAAC;AACJ,CAAC;AAKD,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IAC9C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAKD,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAChD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAaD,MAAM,OAAO,WAAW;IACL,QAAQ,CAAgB;IACxB,UAAU,GAA+B,IAAI,GAAG,EAAE,CAAC;IACnD,SAAS,GAAwB,EAAE,CAAC;IACpC,UAAU,GAAsB,EAAE,CAAC;IACnC,YAAY,GAAgB,IAAI,GAAG,EAAE,CAAC;IAC/C,SAAS,GAAG,KAAK,CAAC;IAE1B,YAAoB,GAAqB;QACvC,IAAI,CAAC,QAAQ,GAAG;YACd,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,cAAc,EAAE;YACzB,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,UAAU,EAAE,MAAM,EAAE;YACpB,MAAM,EAAE,SAAS;SAClB,CAAC;IACJ,CAAC;IAMD,MAAM,CAAC,MAAM,CAAC,GAAY;QACxB,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,qBAAqB,CAC7B,4DAA4D;gBAC1D,2EAA2E;gBAC3E,uCAAuC,CAC1C,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAGD,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/B,CAAC;IAGD,IAAI,WAAW;QAEb,OAAO,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;IACtC,CAAC;IAMD,UAAU,CAAC,SAAiB;QAC1B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,IAAI,uBAAuB,CAAC,uCAAuC,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,IAAI,GAAkB;YAC1B,IAAI,EAAE,OAAO;YACb,UAAU,EAAE,SAAS;YACrB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,cAAc,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;YACrC,UAAU,EAAE,MAAM,EAAE;YACpB,MAAM,EAAE,SAAS;SAClB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC;IACrB,CAAC;IAKD,QAAQ,CACN,MAAc,EACd,MAA8B,EAC9B,cAAyB;QAEzB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,uBAAuB,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,IAAI,uBAAuB,CAC/B,cAAc,MAAM,8BAA8B,IAAI,CAAC,MAAM,EAAE,CAChE,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,MAAM,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACxC,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAMD,WAAW,CAAC,QAA2B;QACrC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,uBAAuB,CAC/B,iDAAiD,QAAQ,CAAC,aAAa,IAAI;gBACzE,6DAA6D,CAChE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAMD,YAAY,CAAC,MAAuB;QAClC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAUD,QAAQ,CAAI,IAAQ,EAAE,WAAwB;QAC5C,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,IAAI,uBAAuB,CAAC,mBAAmB,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,MAAM,gBAAgB,GAAa,EAAE,CAAC;QAGtC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC/B,gBAAgB,CAAC,IAAI,CACnB,0DAA0D,CAC3D,CAAC;QACJ,CAAC;QAGD,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;iBAC1C,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,UAAU,IAAI,EAAE,CAAC;iBACtD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,gBAAgB,CAAC,IAAI,CACnB,yCAAyC,OAAO,IAAI;gBAClD,oDAAoD,CACvD,CAAC;YAEF,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE;oBAC1B,yCAAyC;iBAC1C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAMD,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CACzD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAC7B,CAAC;QACF,MAAM,UAAU,GACd,WAAW;YACX,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,SAAS;oBACT,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,WAAW,CAAC,CAAC;QAErB,IAAI,CAAC,QAAQ,CAAC,QAAQ,GAAG,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC;QAClC,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,eAAe,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC;gBACzD,CAAC,CAAC,gBAAgB;gBAClB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;qBACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC;qBACpC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,MAAM,KAAK,CAAC,CAAC;QAE5C,OAAO;YACL,SAAS,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE;YAC/B,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;YAC9B,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC9B,KAAK;YACL,iBAAiB,EAAE,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;YAC7E,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS;SAC1E,CAAC;IACJ,CAAC;IAMD,eAAe,CAAC,OAAiB;QAC/B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAEpB,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,QAAQ,GAAG,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,eAAe,GAAG,OAAO,CAAC;QAExC,OAAO;YACL,SAAS,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE;YAC/B,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;YAC9B,KAAK,EAAE,KAAK;YACZ,iBAAiB,EAAE,OAAO;YAC1B,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS;SAC1E,CAAC;IACJ,CAAC;CACF"}

package/dist/bundled-agents/test-bench-agents/dist/execution/types.d.ts

@@ -0,0 +1,37 @@
+import type { ExemptionRecord } from './policy.js';
+export type SpanType = 'repo' | 'agent';
+export type SpanStatus = 'running' | 'completed' | 'failed';
+export interface ExecutionContext {
+    execution_id: string;
+    parent_span_id: string;
+}
+export interface ExecutionSpan {
+    type: SpanType;
+    span_id: string;
+    parent_span_id: string;
+    start_time: string;
+    end_time?: string;
+    status: SpanStatus;
+    repo_name?: string;
+    agent_name?: string;
+    failure_reasons?: string[];
+}
+export interface ExecutionArtifact {
+    reference: string;
+    artifact_type: string;
+    agent_span_id: string;
+    metadata?: Record<string, unknown>;
+}
+export interface ExecutionResult<T = unknown> {
+    repo_span: ExecutionSpan;
+    agent_spans: ExecutionSpan[];
+    artifacts: ExecutionArtifact[];
+    data?: T;
+    valid: boolean;
+    validation_errors?: string[];
+    exemptions?: ExemptionRecord[];
+}
+export declare function generateSpanId(): string;
+export declare function nowISO(): string;
+export declare const REPO_NAME = "llm-test-bench";
+//# sourceMappingURL=types.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/execution/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/execution/types.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAGnD,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;AAGxC,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;AAM5D,MAAM,WAAW,gBAAgB;IAE/B,YAAY,EAAE,MAAM,CAAC;IAGrB,cAAc,EAAE,MAAM,CAAC;CACxB;AAMD,MAAM,WAAW,aAAa;IAE5B,IAAI,EAAE,QAAQ,CAAC;IAGf,OAAO,EAAE,MAAM,CAAC;IAGhB,cAAc,EAAE,MAAM,CAAC;IAGvB,UAAU,EAAE,MAAM,CAAC;IAGnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,MAAM,EAAE,UAAU,CAAC;IAGnB,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAMD,MAAM,WAAW,iBAAiB;IAEhC,SAAS,EAAE,MAAM,CAAC;IAGlB,aAAa,EAAE,MAAM,CAAC;IAGtB,aAAa,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAOD,MAAM,WAAW,eAAe,CAAC,CAAC,GAAG,OAAO;IAE1C,SAAS,EAAE,aAAa,CAAC;IAGzB,WAAW,EAAE,aAAa,EAAE,CAAC;IAG7B,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAG/B,IAAI,CAAC,EAAE,CAAC,CAAC;IAGT,KAAK,EAAE,OAAO,CAAC;IAGf,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAG7B,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;CAChC;AAGD,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAGD,wBAAgB,MAAM,IAAI,MAAM,CAE/B;AAGD,eAAO,MAAM,SAAS,mBAAmB,CAAC"}

package/dist/bundled-agents/test-bench-agents/dist/execution/types.js

@@ -0,0 +1,9 @@
+import { randomUUID } from 'crypto';
+export function generateSpanId() {
+    return randomUUID();
+}
+export function nowISO() {
+    return new Date().toISOString();
+}
+export const REPO_NAME = 'llm-test-bench';
+//# sourceMappingURL=types.js.map

package/dist/bundled-agents/test-bench-agents/dist/execution/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/execution/types.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAqGpC,MAAM,UAAU,cAAc;IAC5B,OAAO,UAAU,EAAE,CAAC;AACtB,CAAC;AAGD,MAAM,UAAU,MAAM;IACpB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAGD,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAC"}

package/dist/bundled-agents/test-bench-agents/dist/index.d.ts

@@ -0,0 +1,16 @@
+export { LLMTestBench } from './core/llm-test-bench.js';
+export { FleetBenchmark } from './core/fleet-benchmark.js';
+export type { FleetConfig, FleetManifest, RepositorySpec, FleetManifestConfig, FleetExecutionHandle, FleetExecutionMetadata, FleetBenchmarkResults, FleetSummary, ProviderFleetStats, CategoryFleetStats, } from './core/fleet-benchmark.js';
+export { ProviderClient, OpenAIClient, AnthropicClient, GoogleClient, ProviderClientFactory, } from './core/provider-client.js';
+export { Evaluator, createEvaluator } from './evaluators/index.js';
+export * from './types/index.js';
+export { InstrumentedTestBench, SpanManager, executeWithSpans, validateExecutionContext, ExecutionContextError, ExecutionInvariantError, generateSpanId, nowISO, REPO_NAME, } from './execution/index.js';
+export type { SpanType, SpanStatus, ExecutionContext, ExecutionSpan, ExecutionArtifact, ExecutionResult, } from './execution/index.js';
+export { PolicyGate, PolicyGateError, DEFAULT_EXECUTION_POLICY, } from './execution/index.js';
+export type { SimulationIntent, GateDisposition, ExecutionPolicy, ExemptionRecord, ExemptionRequest, PolicyCheckResult, } from './execution/index.js';
+export { executeCLI, findCLIPath } from './utils/cli-executor.js';
+export { validateProviderConfig, validateCompletionRequest, validateBenchmarkConfig, validateEvaluationConfig, isValidModel, isValidProvider, } from './utils/validators.js';
+export declare const VERSION = "0.1.2";
+import { LLMTestBench } from './core/llm-test-bench.js';
+export default LLMTestBench;
+//# sourceMappingURL=index.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,YAAY,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,oBAAoB,EACpB,sBAAsB,EACtB,qBAAqB,EACrB,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAGnE,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EACL,qBAAqB,EACrB,WAAW,EACX,gBAAgB,EAChB,wBAAwB,EACxB,qBAAqB,EACrB,uBAAuB,EACvB,cAAc,EACd,MAAM,EACN,SAAS,GACV,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,UAAU,EACV,eAAe,EACf,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EACL,sBAAsB,EACtB,yBAAyB,EACzB,uBAAuB,EACvB,wBAAwB,EACxB,YAAY,EACZ,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAK/B,eAAO,MAAM,OAAO,UAAU,CAAC;AAK/B,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAExD,eAAe,YAAY,CAAC"}

package/dist/bundled-agents/test-bench-agents/dist/index.js

@@ -0,0 +1,13 @@
+export { LLMTestBench } from './core/llm-test-bench.js';
+export { FleetBenchmark } from './core/fleet-benchmark.js';
+export { ProviderClient, OpenAIClient, AnthropicClient, GoogleClient, ProviderClientFactory, } from './core/provider-client.js';
+export { Evaluator, createEvaluator } from './evaluators/index.js';
+export * from './types/index.js';
+export { InstrumentedTestBench, SpanManager, executeWithSpans, validateExecutionContext, ExecutionContextError, ExecutionInvariantError, generateSpanId, nowISO, REPO_NAME, } from './execution/index.js';
+export { PolicyGate, PolicyGateError, DEFAULT_EXECUTION_POLICY, } from './execution/index.js';
+export { executeCLI, findCLIPath } from './utils/cli-executor.js';
+export { validateProviderConfig, validateCompletionRequest, validateBenchmarkConfig, validateEvaluationConfig, isValidModel, isValidProvider, } from './utils/validators.js';
+export const VERSION = '0.1.2';
+import { LLMTestBench } from './core/llm-test-bench.js';
+export default LLMTestBench;
+//# sourceMappingURL=index.js.map

package/dist/bundled-agents/test-bench-agents/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAe3D,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAGnE,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EACL,qBAAqB,EACrB,WAAW,EACX,gBAAgB,EAChB,wBAAwB,EACxB,qBAAqB,EACrB,uBAAuB,EACvB,cAAc,EACd,MAAM,EACN,SAAS,GACV,MAAM,sBAAsB,CAAC;AAW9B,OAAO,EACL,UAAU,EACV,eAAe,EACf,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAW9B,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EACL,sBAAsB,EACtB,yBAAyB,EACzB,uBAAuB,EACvB,wBAAwB,EACxB,YAAY,EACZ,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAK/B,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAK/B,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAExD,eAAe,YAAY,CAAC"}

package/dist/bundled-agents/test-bench-agents/dist/types/benchmarks.d.ts

@@ -0,0 +1,87 @@
+export interface BenchmarkConfig {
+    concurrency?: number;
+    saveResponses?: boolean;
+    outputPath?: string;
+    maxDuration?: number;
+    showProgress?: boolean;
+}
+export interface LatencyMetrics {
+    p50: number;
+    p95: number;
+    p99: number;
+    mean: number;
+    min: number;
+    max: number;
+    stdDev: number;
+}
+export interface TokenMetrics {
+    totalPromptTokens: number;
+    totalCompletionTokens: number;
+    totalTokens: number;
+    averageTokensPerRequest: number;
+    tokensPerSecond: number;
+}
+export interface CostMetrics {
+    totalCost: number;
+    averageCostPerRequest: number;
+    inputCost: number;
+    outputCost: number;
+}
+export interface BenchmarkSummary {
+    totalRequests: number;
+    successfulRequests: number;
+    failedRequests: number;
+    successRate: number;
+    latency: LatencyMetrics;
+    tokens: TokenMetrics;
+    cost?: CostMetrics;
+    totalDuration: number;
+    requestsPerSecond: number;
+}
+export interface BenchmarkResult {
+    id: string;
+    prompt: string;
+    response?: string;
+    success: boolean;
+    error?: string;
+    latencyMs: number;
+    tokens?: {
+        prompt: number;
+        completion: number;
+        total: number;
+    };
+    cost?: number;
+    timestamp: string;
+}
+export interface BenchmarkResults {
+    metadata: {
+        provider: string;
+        model: string;
+        startTime: string;
+        endTime: string;
+        config: BenchmarkConfig;
+    };
+    summary: BenchmarkSummary;
+    results?: BenchmarkResult[];
+}
+export interface ComparisonResult {
+    metadata: {
+        models: Array<{
+            provider: string;
+            model: string;
+        }>;
+        timestamp: string;
+        testCases: number;
+    };
+    modelResults: Array<{
+        provider: string;
+        model: string;
+        summary: BenchmarkSummary;
+    }>;
+    winner?: {
+        provider: string;
+        model: string;
+        reason: string;
+    };
+}
+//# sourceMappingURL=benchmarks.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/types/benchmarks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmarks.d.ts","sourceRoot":"","sources":["../../src/types/benchmarks.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAKD,MAAM,WAAW,cAAc;IAE7B,GAAG,EAAE,MAAM,CAAC;IAGZ,GAAG,EAAE,MAAM,CAAC;IAGZ,GAAG,EAAE,MAAM,CAAC;IAGZ,IAAI,EAAE,MAAM,CAAC;IAGb,GAAG,EAAE,MAAM,CAAC;IAGZ,GAAG,EAAE,MAAM,CAAC;IAGZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAKD,MAAM,WAAW,YAAY;IAE3B,iBAAiB,EAAE,MAAM,CAAC;IAG1B,qBAAqB,EAAE,MAAM,CAAC;IAG9B,WAAW,EAAE,MAAM,CAAC;IAGpB,uBAAuB,EAAE,MAAM,CAAC;IAGhC,eAAe,EAAE,MAAM,CAAC;CACzB;AAKD,MAAM,WAAW,WAAW;IAE1B,SAAS,EAAE,MAAM,CAAC;IAGlB,qBAAqB,EAAE,MAAM,CAAC;IAG9B,SAAS,EAAE,MAAM,CAAC;IAGlB,UAAU,EAAE,MAAM,CAAC;CACpB;AAKD,MAAM,WAAW,gBAAgB;IAE/B,aAAa,EAAE,MAAM,CAAC;IAGtB,kBAAkB,EAAE,MAAM,CAAC;IAG3B,cAAc,EAAE,MAAM,CAAC;IAGvB,WAAW,EAAE,MAAM,CAAC;IAGpB,OAAO,EAAE,cAAc,CAAC;IAGxB,MAAM,EAAE,YAAY,CAAC;IAGrB,IAAI,CAAC,EAAE,WAAW,CAAC;IAGnB,aAAa,EAAE,MAAM,CAAC;IAGtB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAKD,MAAM,WAAW,eAAe;IAE9B,EAAE,EAAE,MAAM,CAAC;IAGX,MAAM,EAAE,MAAM,CAAC;IAGf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,OAAO,EAAE,OAAO,CAAC;IAGjB,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,SAAS,EAAE,MAAM,CAAC;IAGlB,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IAGF,IAAI,CAAC,EAAE,MAAM,CAAC;IAGd,SAAS,EAAE,MAAM,CAAC;CACnB;AAKD,MAAM,WAAW,gBAAgB;IAE/B,QAAQ,EAAE;QAER,QAAQ,EAAE,MAAM,CAAC;QAGjB,KAAK,EAAE,MAAM,CAAC;QAGd,SAAS,EAAE,MAAM,CAAC;QAGlB,OAAO,EAAE,MAAM,CAAC;QAGhB,MAAM,EAAE,eAAe,CAAC;KACzB,CAAC;IAGF,OAAO,EAAE,gBAAgB,CAAC;IAG1B,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;CAC7B;AAKD,MAAM,WAAW,gBAAgB;IAE/B,QAAQ,EAAE;QAER,MAAM,EAAE,KAAK,CAAC;YACZ,QAAQ,EAAE,MAAM,CAAC;YACjB,KAAK,EAAE,MAAM,CAAC;SACf,CAAC,CAAC;QAGH,SAAS,EAAE,MAAM,CAAC;QAGlB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IAGF,YAAY,EAAE,KAAK,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,gBAAgB,CAAC;KAC3B,CAAC,CAAC;IAGH,MAAM,CAAC,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH"}

package/dist/bundled-agents/test-bench-agents/dist/types/benchmarks.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=benchmarks.js.map

package/dist/bundled-agents/test-bench-agents/dist/types/benchmarks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmarks.js","sourceRoot":"","sources":["../../src/types/benchmarks.ts"],"names":[],"mappings":""}

package/dist/bundled-agents/test-bench-agents/dist/types/evaluators.d.ts

@@ -0,0 +1,98 @@
+export type EvaluatorType = 'perplexity' | 'coherence' | 'relevance' | 'faithfulness' | 'llm-as-judge' | 'readability' | 'sentiment' | 'toxicity' | 'pii-detection' | 'custom';
+export interface EvaluationResult {
+    type: EvaluatorType;
+    score: number;
+    explanation?: string;
+    metadata?: Record<string, unknown>;
+    timestamp: string;
+}
+export interface PerplexityResult extends EvaluationResult {
+    type: 'perplexity';
+    perplexity: number;
+    logProb: number;
+}
+export interface CoherenceResult extends EvaluationResult {
+    type: 'coherence';
+    coherenceScore: number;
+    sentenceSimilarities?: number[];
+    discourseMarkers?: {
+        count: number;
+        types: string[];
+    };
+}
+export interface RelevanceResult extends EvaluationResult {
+    type: 'relevance';
+    relevanceScore: number;
+    similarity?: number;
+    topicsCovered?: string[];
+}
+export interface FaithfulnessResult extends EvaluationResult {
+    type: 'faithfulness';
+    faithfulnessScore: number;
+    supportedClaims?: number;
+    totalClaims?: number;
+    hallucinations?: string[];
+}
+export interface LLMJudgeResult extends EvaluationResult {
+    type: 'llm-as-judge';
+    rating: number;
+    reasoning: string;
+    criteria: string[];
+    judgeModel: string;
+}
+export interface ReadabilityResult extends EvaluationResult {
+    type: 'readability';
+    fleschScore?: number;
+    gradeLevel?: number;
+    avgSentenceLength?: number;
+    avgWordLength?: number;
+}
+export interface SentimentResult extends EvaluationResult {
+    type: 'sentiment';
+    sentiment: 'positive' | 'negative' | 'neutral';
+    confidence: number;
+    scores?: {
+        positive: number;
+        negative: number;
+        neutral: number;
+    };
+}
+export interface ToxicityResult extends EvaluationResult {
+    type: 'toxicity';
+    toxicityScore: number;
+    isToxic: boolean;
+    categories?: {
+        hate?: number;
+        harassment?: number;
+        violence?: number;
+        sexual?: number;
+        profanity?: number;
+    };
+}
+export interface PIIDetectionResult extends EvaluationResult {
+    type: 'pii-detection';
+    piiDetected: boolean;
+    piiTypes?: Array<{
+        type: 'email' | 'phone' | 'ssn' | 'credit_card' | 'address' | 'name' | 'other';
+        value: string;
+        confidence: number;
+    }>;
+    redactedText?: string;
+}
+export interface EvaluationConfig {
+    evaluators: EvaluatorType[];
+    referenceText?: string;
+    context?: string[];
+    customCriteria?: string[];
+    judgeModel?: string;
+    parallel?: boolean;
+    cacheResults?: boolean;
+}
+export interface CombinedEvaluationResults {
+    text: string;
+    results: EvaluationResult[];
+    overallScore: number;
+    durationMs: number;
+    timestamp: string;
+}
+//# sourceMappingURL=evaluators.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/types/evaluators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.d.ts","sourceRoot":"","sources":["../../src/types/evaluators.ts"],"names":[],"mappings":"AASA,MAAM,MAAM,aAAa,GACrB,YAAY,GACZ,WAAW,GACX,WAAW,GACX,cAAc,GACd,cAAc,GACd,aAAa,GACb,WAAW,GACX,UAAU,GACV,eAAe,GACf,QAAQ,CAAC;AAKb,MAAM,WAAW,gBAAgB;IAE/B,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,EAAE,MAAM,CAAC;IAGd,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAGnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAKD,MAAM,WAAW,gBAAiB,SAAQ,gBAAgB;IACxD,IAAI,EAAE,YAAY,CAAC;IAGnB,UAAU,EAAE,MAAM,CAAC;IAGnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAKD,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACvD,IAAI,EAAE,WAAW,CAAC;IAGlB,cAAc,EAAE,MAAM,CAAC;IAGvB,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAGhC,gBAAgB,CAAC,EAAE;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;CACH;AAKD,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACvD,IAAI,EAAE,WAAW,CAAC;IAGlB,cAAc,EAAE,MAAM,CAAC;IAGvB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAKD,MAAM,WAAW,kBAAmB,SAAQ,gBAAgB;IAC1D,IAAI,EAAE,cAAc,CAAC;IAGrB,iBAAiB,EAAE,MAAM,CAAC;IAG1B,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAKD,MAAM,WAAW,cAAe,SAAQ,gBAAgB;IACtD,IAAI,EAAE,cAAc,CAAC;IAGrB,MAAM,EAAE,MAAM,CAAC;IAGf,SAAS,EAAE,MAAM,CAAC;IAGlB,QAAQ,EAAE,MAAM,EAAE,CAAC;IAGnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAKD,MAAM,WAAW,iBAAkB,SAAQ,gBAAgB;IACzD,IAAI,EAAE,aAAa,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAKD,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACvD,IAAI,EAAE,WAAW,CAAC;IAGlB,SAAS,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;IAG/C,UAAU,EAAE,MAAM,CAAC;IAGnB,MAAM,CAAC,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAKD,MAAM,WAAW,cAAe,SAAQ,gBAAgB;IACtD,IAAI,EAAE,UAAU,CAAC;IAGjB,aAAa,EAAE,MAAM,CAAC;IAGtB,OAAO,EAAE,OAAO,CAAC;IAGjB,UAAU,CAAC,EAAE;QACX,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAKD,MAAM,WAAW,kBAAmB,SAAQ,gBAAgB;IAC1D,IAAI,EAAE,eAAe,CAAC;IAGtB,WAAW,EAAE,OAAO,CAAC;IAGrB,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,IAAI,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,GAAG,aAAa,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,CAAC;QAC/E,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IAGH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAKD,MAAM,WAAW,gBAAgB;IAE/B,UAAU,EAAE,aAAa,EAAE,CAAC;IAG5B,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAGnB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAG1B,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAGnB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAKD,MAAM,WAAW,yBAAyB;IAExC,IAAI,EAAE,MAAM,CAAC;IAGb,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAG5B,YAAY,EAAE,MAAM,CAAC;IAGrB,UAAU,EAAE,MAAM,CAAC;IAGnB,SAAS,EAAE,MAAM,CAAC;CACnB"}

package/dist/bundled-agents/test-bench-agents/dist/types/evaluators.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=evaluators.js.map

package/dist/bundled-agents/test-bench-agents/dist/types/evaluators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.js","sourceRoot":"","sources":["../../src/types/evaluators.ts"],"names":[],"mappings":""}

package/dist/bundled-agents/test-bench-agents/dist/types/index.d.ts

@@ -0,0 +1,20 @@
+export * from './providers.js';
+export * from './benchmarks.js';
+export * from './evaluators.js';
+export interface SDKConfig {
+    cliPath?: string;
+    workingDir?: string;
+    verbose?: boolean;
+    env?: Record<string, string>;
+    timeout?: number;
+}
+export interface CLIResult<T = unknown> {
+    success: boolean;
+    data?: T;
+    error?: string;
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+    durationMs: number;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAMA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAKhC,MAAM,WAAW,SAAS;IAExB,OAAO,CAAC,EAAE,MAAM,CAAC;IAGjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAGlB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAG7B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAKD,MAAM,WAAW,SAAS,CAAC,CAAC,GAAG,OAAO;IAEpC,OAAO,EAAE,OAAO,CAAC;IAGjB,IAAI,CAAC,EAAE,CAAC,CAAC;IAGT,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,QAAQ,EAAE,MAAM,CAAC;IAGjB,MAAM,EAAE,MAAM,CAAC;IAGf,MAAM,EAAE,MAAM,CAAC;IAGf,UAAU,EAAE,MAAM,CAAC;CACpB"}

package/dist/bundled-agents/test-bench-agents/dist/types/index.js

@@ -0,0 +1,4 @@
+export * from './providers.js';
+export * from './benchmarks.js';
+export * from './evaluators.js';
+//# sourceMappingURL=index.js.map

package/dist/bundled-agents/test-bench-agents/dist/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAMA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC"}

package/dist/bundled-agents/test-bench-agents/dist/types/providers.d.ts

@@ -0,0 +1,64 @@
+export type ProviderName = 'openai' | 'anthropic' | 'google' | 'mistral' | 'azure-openai' | 'bedrock' | 'cohere' | 'groq' | 'huggingface' | 'ollama' | 'perplexity' | 'replicate' | 'together';
+export interface ProviderConfig {
+    name: ProviderName;
+    apiKeyEnv: string;
+    baseUrl: string;
+    defaultModel: string;
+    timeoutSeconds?: number;
+    maxRetries?: number;
+    enabled?: boolean;
+}
+export interface CompletionRequest {
+    model: string;
+    prompt: string;
+    maxTokens?: number;
+    temperature?: number;
+    topP?: number;
+    stop?: string[];
+    stream?: boolean;
+}
+export type FinishReason = 'stop' | 'length' | 'content_filter' | 'tool_calls' | 'error';
+export interface TokenUsage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+}
+export interface CompletionResponse {
+    content: string;
+    model: string;
+    finishReason: FinishReason;
+    usage: TokenUsage;
+    timestamp: string;
+    latencyMs: number;
+    estimatedCost?: number;
+}
+export interface CompletionChunk {
+    content: string;
+    done: boolean;
+    finishReason?: FinishReason;
+    usage?: TokenUsage;
+}
+export interface ModelInfo {
+    id: string;
+    provider: ProviderName;
+    name: string;
+    description?: string;
+    contextWindow?: number;
+    maxOutputTokens?: number;
+    inputCostPerMillion?: number;
+    outputCostPerMillion?: number;
+    capabilities?: {
+        streaming?: boolean;
+        vision?: boolean;
+        audio?: boolean;
+        functionCalling?: boolean;
+        json?: boolean;
+    };
+}
+export interface ProviderError {
+    type: 'rate_limit' | 'authentication' | 'network' | 'invalid_request' | 'provider_error' | 'timeout';
+    message: string;
+    statusCode?: number;
+    details?: unknown;
+}
+//# sourceMappingURL=providers.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/types/providers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../../src/types/providers.ts"],"names":[],"mappings":"AAUA,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,SAAS,GACT,cAAc,GACd,SAAS,GACT,QAAQ,GACR,MAAM,GACN,aAAa,GACb,QAAQ,GACR,YAAY,GACZ,WAAW,GACX,UAAU,CAAC;AAKf,MAAM,WAAW,cAAc;IAE7B,IAAI,EAAE,YAAY,CAAC;IAGnB,SAAS,EAAE,MAAM,CAAC;IAGlB,OAAO,EAAE,MAAM,CAAC;IAGhB,YAAY,EAAE,MAAM,CAAC;IAGrB,cAAc,CAAC,EAAE,MAAM,CAAC;IAGxB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAKD,MAAM,WAAW,iBAAiB;IAEhC,KAAK,EAAE,MAAM,CAAC;IAGd,MAAM,EAAE,MAAM,CAAC;IAGf,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,MAAM,CAAC;IAGd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAGhB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAKD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,QAAQ,GAAG,gBAAgB,GAAG,YAAY,GAAG,OAAO,CAAC;AAKzF,MAAM,WAAW,UAAU;IAEzB,YAAY,EAAE,MAAM,CAAC;IAGrB,gBAAgB,EAAE,MAAM,CAAC;IAGzB,WAAW,EAAE,MAAM,CAAC;CACrB;AAKD,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,CAAC;IAGhB,KAAK,EAAE,MAAM,CAAC;IAGd,YAAY,EAAE,YAAY,CAAC;IAG3B,KAAK,EAAE,UAAU,CAAC;IAGlB,SAAS,EAAE,MAAM,CAAC;IAGlB,SAAS,EAAE,MAAM,CAAC;IAGlB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAKD,MAAM,WAAW,eAAe;IAE9B,OAAO,EAAE,MAAM,CAAC;IAGhB,IAAI,EAAE,OAAO,CAAC;IAGd,YAAY,CAAC,EAAE,YAAY,CAAC;IAG5B,KAAK,CAAC,EAAE,UAAU,CAAC;CACpB;AAKD,MAAM,WAAW,SAAS;IAExB,EAAE,EAAE,MAAM,CAAC;IAGX,QAAQ,EAAE,YAAY,CAAC;IAGvB,IAAI,EAAE,MAAM,CAAC;IAGb,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAG9B,YAAY,CAAC,EAAE;QACb,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,IAAI,CAAC,EAAE,OAAO,CAAC;KAChB,CAAC;CACH;AAKD,MAAM,WAAW,aAAa;IAE5B,IAAI,EAAE,YAAY,GAAG,gBAAgB,GAAG,SAAS,GAAG,iBAAiB,GAAG,gBAAgB,GAAG,SAAS,CAAC;IAGrG,OAAO,EAAE,MAAM,CAAC;IAGhB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB"}

package/dist/bundled-agents/test-bench-agents/dist/types/providers.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=providers.js.map

package/dist/bundled-agents/test-bench-agents/dist/types/providers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/types/providers.ts"],"names":[],"mappings":""}

package/dist/bundled-agents/test-bench-agents/dist/utils/cli-executor.d.ts

@@ -0,0 +1,11 @@
+import { CLIResult } from '../types/index.js';
+export interface ExecuteOptions {
+    args: string[];
+    cwd?: string;
+    env?: Record<string, string>;
+    timeout?: number;
+    parseJson?: boolean;
+}
+export declare function executeCLI<T = unknown>(cliPath: string, options: ExecuteOptions): Promise<CLIResult<T>>;
+export declare function findCLIPath(): string | undefined;
+//# sourceMappingURL=cli-executor.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/utils/cli-executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-executor.d.ts","sourceRoot":"","sources":["../../src/utils/cli-executor.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAK9C,MAAM,WAAW,cAAc;IAE7B,IAAI,EAAE,MAAM,EAAE,CAAC;IAGf,GAAG,CAAC,EAAE,MAAM,CAAC;IAGb,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAG7B,OAAO,CAAC,EAAE,MAAM,CAAC;IAGjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AASD,wBAAsB,UAAU,CAAC,CAAC,GAAG,OAAO,EAC1C,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAmGvB;AAOD,wBAAgB,WAAW,IAAI,MAAM,GAAG,SAAS,CAuBhD"}