@assemblyline-agents/cli 0.1.0

package/dist/eval-types.d.ts

@@ -0,0 +1,271 @@
+import type { JsonObject, JsonValue } from "@assemblyline-agents/core";
+import type { RunStatus, RunTimeline } from "@assemblyline-agents/runtime";
+export type EvalOutcome = "passed" | "failed" | "errored";
+export type EvalTrajectoryMode = "contains" | "ordered" | "exact" | "forbid";
+export type EvalValueMatchMode = "partial" | "exact";
+export interface EvalTrajectoryStep {
+    tool: string;
+    arguments?: unknown;
+    argumentsMatch?: EvalValueMatchMode;
+    status?: string;
+    result?: unknown;
+    resultMatch?: EvalValueMatchMode;
+}
+export interface EvalCustomEvaluatorExpectation {
+    name: string;
+    config?: JsonObject;
+    /** Metric key used by minScore when the evaluator returns several metrics. */
+    metric?: string;
+    /** Require at least one numeric metric at or above this value. */
+    minScore?: number;
+    /** Defaults to true; fail when any returned metric explicitly has pass=false. */
+    requirePass?: boolean;
+}
+export interface EvalOutputExpectation {
+    contains?: string[];
+    matches?: string;
+    equals?: string;
+    json?: {
+        valid?: boolean;
+        jsonSchema?: unknown;
+        fields?: Record<string, unknown>;
+    };
+}
+/** Assertions that need no model call; usable per case and per conversation turn. */
+export interface EvalDeterministicExpectations {
+    status?: RunStatus;
+    output?: EvalOutputExpectation;
+    toolsCalled?: string[];
+    toolsNotCalled?: string[];
+    trajectory?: {
+        mode: EvalTrajectoryMode;
+        steps: EvalTrajectoryStep[];
+    };
+    maxCostUsd?: number;
+}
+export interface EvalTurn {
+    message: string;
+    approve?: boolean;
+    tool?: string;
+    toolInput?: unknown;
+    /** Deterministic assertions against this turn alone; failures are prefixed `turn N:`. */
+    expect?: EvalDeterministicExpectations;
+}
+/** Exactly one of `result`, `results` (consumed per call, last repeats), or `error`. */
+export interface EvalToolMock {
+    result?: JsonValue;
+    results?: JsonValue[];
+    error?: string;
+}
+export interface EvalMocks {
+    tools: Record<string, EvalToolMock>;
+}
+export interface EvalFixtures {
+    /** Sandbox files seeded before the first turn; relative paths land under /workspace. */
+    sandbox?: Record<string, string>;
+    /** Memory documents (path starting with /memory/) seeded at agent scope. */
+    memory?: Record<string, string>;
+}
+export interface EvalCase {
+    name: string;
+    description?: string;
+    tags?: string[];
+    /** Attempts for this case (1..100); wins over the --repetitions suite flag. */
+    repetitions?: number;
+    input: {
+        /** Single-turn message; mutually exclusive with `turns`. */
+        message?: string;
+        /** Scripted conversation; mutually exclusive with `message`. */
+        turns?: EvalTurn[];
+        channel?: string;
+        tool?: string;
+        toolInput?: unknown;
+        approve?: boolean;
+        recentHistory?: string[];
+    };
+    mocks?: EvalMocks;
+    fixtures?: EvalFixtures;
+    expect: EvalDeterministicExpectations & {
+        evaluators?: EvalCustomEvaluatorExpectation[];
+        judge?: {
+            criteria: string;
+            model?: string;
+            threshold?: number;
+        };
+    };
+}
+export interface DiscoveredEvalCase {
+    path: string;
+    name: string;
+    evalCase?: EvalCase;
+    error?: string;
+}
+export interface EvalTurnRecord {
+    message: string;
+    status: RunStatus;
+    response: string;
+    output?: JsonValue;
+    toolsCalled: string[];
+    costUsd: number;
+    runId: string;
+}
+export interface EvalActual {
+    status: RunStatus;
+    response: string;
+    output?: JsonValue;
+    toolsCalled: string[];
+    toolCalls?: RunTimeline["toolCalls"];
+    /** False when any model transaction lacks a provider-reported/reconciled cash receipt. */
+    costComplete: boolean;
+    costUsd: number;
+    /** Per-turn records for conversation cases; status/response/output reflect the final turn. */
+    turns?: EvalTurnRecord[];
+}
+export interface EvalMetric {
+    key: string;
+    score?: number;
+    value?: JsonValue;
+    pass?: boolean;
+    comment?: string;
+    evaluator?: string;
+}
+export type EvalModelComplete = (input: {
+    prompt: string;
+    systemPrompt?: string;
+    model?: string;
+}) => Promise<{
+    text: string;
+    costUsd: number;
+}>;
+export interface EvalEvaluatorContext {
+    evalCase: EvalCase;
+    actual: EvalActual;
+    /** Final-turn timeline; `timelines` has one entry per conversation turn. */
+    timeline: RunTimeline;
+    timelines: RunTimeline[];
+    config: JsonObject;
+    repetition: number;
+    attempt: number;
+    /**
+     * Model completer bound to the suite judge configuration (defaults to the
+     * judge model). Cost is accounted into the case's judgeCostUsd.
+     */
+    complete: EvalModelComplete;
+}
+export interface EvalEvaluator {
+    name: string;
+    description?: string;
+    evaluate(context: EvalEvaluatorContext): EvalMetric | EvalMetric[] | Promise<EvalMetric | EvalMetric[]>;
+}
+export declare function defineEvaluator<T extends EvalEvaluator>(evaluator: T): T;
+export interface EvalJudgeResult {
+    score: number;
+    pass: boolean;
+    reasoning: string;
+    costUsd: number;
+    model: string;
+}
+export interface EvalCaseResult {
+    caseId: string;
+    name: string;
+    path: string;
+    tags: string[];
+    outcome: EvalOutcome;
+    repetition: number;
+    /** Planned repetitions for this case (case field or suite flag) — the pass-distribution denominator. */
+    repetitionsPlanned?: number;
+    attempts: number;
+    durationMs: number;
+    runCostUsd: number;
+    judgeCostUsd: number;
+    failures: string[];
+    error?: string;
+    runId?: string;
+    status?: RunStatus;
+    response?: string;
+    output?: JsonValue;
+    turns?: EvalTurnRecord[];
+    judge?: EvalJudgeResult;
+    metrics?: EvalMetric[];
+}
+export interface EvalExecutionConfig {
+    filter?: string;
+    tag?: string;
+    judgeModel: string;
+    concurrency: number;
+    timeoutMs: number;
+    approve: boolean;
+    failFast: boolean;
+    repetitions: number;
+    retryAttempts: number;
+    retryBackoffMs: number;
+}
+export interface EvalExperimentManifest {
+    schemaVersion: 1;
+    id: string;
+    startedAt: string;
+    frameworkVersion: string;
+    agentRevision: string;
+    suiteFingerprint: string;
+    configFingerprint: string;
+    caseCount: number;
+    executionCount: number;
+    config: EvalExecutionConfig;
+}
+export interface EvalRegression {
+    caseId: string;
+    name: string;
+    baselinePassRate: number;
+    currentPassRate: number;
+}
+export interface EvalComparison {
+    baselineExperimentId: string;
+    regressions: EvalRegression[];
+    improvements: EvalRegression[];
+    addedCaseIds: string[];
+    removedCaseIds: string[];
+    passRateDelta: number;
+    costDeltaUsd: number;
+    costDeltaPercent?: number;
+    /** actual=null represents an unbounded percentage (zero-cost baseline to positive cost). */
+    gates: Array<{
+        name: string;
+        passed: boolean;
+        actual: number | null;
+        limit: number;
+    }>;
+    passed: boolean;
+}
+export interface EvalReport {
+    schemaVersion: 1;
+    experiment: EvalExperimentManifest;
+    startedAt: string;
+    durationMs: number;
+    agentRevision: string;
+    results: EvalCaseResult[];
+    comparison?: EvalComparison;
+    artifacts?: {
+        root: string;
+        experimentPath: string;
+        resultsPath: string;
+    };
+    summary: {
+        total: number;
+        uniqueCases: number;
+        repetitions: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        passRate: number;
+        runCostUsd: number;
+        judgeCostUsd: number;
+        totalCostUsd: number;
+        byTag: Record<string, {
+            total: number;
+            passed: number;
+            failed: number;
+            errored: number;
+        }>;
+    };
+}
+//# sourceMappingURL=eval-types.d.ts.map

package/dist/eval-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-types.d.ts","sourceRoot":"","sources":["../src/eval-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAE3E,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAC1D,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;AAC7E,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,OAAO,CAAC;AAErD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,kBAAkB,CAAC;CAClC;AAED,MAAM,WAAW,8BAA8B;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iFAAiF;IACjF,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE;QACL,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KAClC,CAAC;CACH;AAED,qFAAqF;AACrF,MAAM,WAAW,6BAA6B;IAC5C,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE;QAAE,IAAI,EAAE,kBAAkB,CAAC;QAAC,KAAK,EAAE,kBAAkB,EAAE,CAAA;KAAE,CAAC;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,yFAAyF;IACzF,MAAM,CAAC,EAAE,6BAA6B,CAAC;CACxC;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,YAAY;IAC3B,wFAAwF;IACxF,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,4EAA4E;IAC5E,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE;QACL,4DAA4D;QAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,gEAAgE;QAChE,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;KAC1B,CAAC;IACF,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,MAAM,EAAE,6BAA6B,GAAG;QACtC,UAAU,CAAC,EAAE,8BAA8B,EAAE,CAAC;QAC9C,KAAK,CAAC,EAAE;YAAE,QAAQ,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAC;YAAC,SAAS,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAClE,CAAC;CACH;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;IACrC,0FAA0F;IAC1F,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,8FAA8F;IAC9F,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAEjD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,4EAA4E;IAC5E,QAAQ,EAAE,WAAW,CAAC;IACtB,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,oBAAoB,GAAG,UAAU,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,UAAU,GAAG,UAAU,EAAE,CAAC,CAAC;CACzG;AAED,wBAAgB,eAAe,CAAC,CAAC,SAAS,aAAa,EAAE,SAAS,EAAE,CAAC,GAAG,CAAC,CAExE;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,wGAAwG;IACxG,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,CAAC,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,mBAAmB,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4FAA4F;IAC5F,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtF,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,CAAC,CAAC;IACjB,UAAU,EAAE,sBAAsB,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1E,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAC3F,CAAC;CACH"}

package/dist/eval-types.js

@@ -0,0 +1,4 @@
+export function defineEvaluator(evaluator) {
+    return evaluator;
+}
+//# sourceMappingURL=eval-types.js.map

package/dist/eval-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-types.js","sourceRoot":"","sources":["../src/eval-types.ts"],"names":[],"mappings":"AAuKA,MAAM,UAAU,eAAe,CAA0B,SAAY;IACnE,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/eval.d.ts

@@ -0,0 +1,26 @@
+import { type ParsedArgs } from "./args.js";
+import { type EvalCaseRuntimeDependencies } from "./eval-case-runner.js";
+import type { EvalEvaluator, EvalReport } from "./eval-types.js";
+export { evaluateCase, evaluateExpectations } from "./eval-assertions.js";
+export { discoverCases } from "./eval-cases.js";
+export { compareEvalReports, loadBaselineReport } from "./eval-comparison.js";
+export { EvalEvaluatorLoader, evaluateCustomExpectations } from "./eval-custom.js";
+export { seedEvalFixtures, validateFixtures } from "./eval-fixtures.js";
+export { createToolStubs, validateMocks } from "./eval-mocks.js";
+export { conversationIdForCase, EVAL_CONVERSATION_ID, normalizeTurns, runConversation, type EvalRuntimeHost } from "./eval-turns.js";
+export { createGatewayEvalHost, remoteEvalIncompatibilities } from "./eval-gateway-host.js";
+export { createExperimentArtifacts, createExperimentManifest, fingerprintEvalSuite, fingerprintValue, writeExperimentManifest, writeExperimentReport } from "./eval-experiment.js";
+export { createDefaultJudgeCompleter, EvalJudgeResponseError, judgeCase, robustParseJudge } from "./eval-judge.js";
+export { createEvalReport, formatEvalReport } from "./eval-report.js";
+export { isTransientProviderError, retryTransient } from "./eval-retry.js";
+export { evaluateTrajectory } from "./eval-trajectory.js";
+export { defineEvaluator } from "./eval-types.js";
+export type * from "./eval-types.js";
+export interface EvalCommandDependencies extends EvalCaseRuntimeDependencies {
+    evaluators?: Record<string, EvalEvaluator>;
+    env?: Record<string, string | undefined>;
+    now?: () => Date;
+    log?: (line: string) => void;
+}
+export declare function runEvalCommand(agentRoot: string, parsed: ParsedArgs, dependencies?: EvalCommandDependencies): Promise<EvalReport>;
+//# sourceMappingURL=eval.d.ts.map

package/dist/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAEA,OAAO,EAA2B,KAAK,UAAU,EAAE,MAAM,WAAW,CAAC;AACrE,OAAO,EAA0B,KAAK,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AAejG,OAAO,KAAK,EAGV,aAAa,EAEb,UAAU,EACX,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,cAAc,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACrI,OAAO,EAAE,qBAAqB,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAC5F,OAAO,EACL,yBAAyB,EACzB,wBAAwB,EACxB,oBAAoB,EACpB,gBAAgB,EAChB,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,mBAAmB,iBAAiB,CAAC;AAErC,MAAM,WAAW,uBAAwB,SAAQ,2BAA2B;IAC1E,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC3C,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACzC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAC9B;AAQD,wBAAsB,cAAc,CAClC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,UAAU,EAClB,YAAY,GAAE,uBAA4B,GACzC,OAAO,CAAC,UAAU,CAAC,CAuHrB"}

package/dist/eval.js

@@ -0,0 +1,218 @@
+import { relative, resolve } from "node:path";
+import { buildAgent } from "@assemblyline-agents/compiler";
+import { booleanFlag, stringFlag } from "./args.js";
+import { runEvalCaseWithRetries } from "./eval-case-runner.js";
+import { discoverCases } from "./eval-cases.js";
+import { createGatewayEvalHost, remoteEvalIncompatibilities } from "./eval-gateway-host.js";
+import { compareEvalReports, loadBaselineReport } from "./eval-comparison.js";
+import { EvalEvaluatorLoader } from "./eval-custom.js";
+import { createExperimentArtifacts, createExperimentManifest, writeExperimentManifest, writeExperimentReport } from "./eval-experiment.js";
+import { createDefaultJudgeCompleter } from "./eval-judge.js";
+import { createEvalReport, formatEvalReport } from "./eval-report.js";
+export { evaluateCase, evaluateExpectations } from "./eval-assertions.js";
+export { discoverCases } from "./eval-cases.js";
+export { compareEvalReports, loadBaselineReport } from "./eval-comparison.js";
+export { EvalEvaluatorLoader, evaluateCustomExpectations } from "./eval-custom.js";
+export { seedEvalFixtures, validateFixtures } from "./eval-fixtures.js";
+export { createToolStubs, validateMocks } from "./eval-mocks.js";
+export { conversationIdForCase, EVAL_CONVERSATION_ID, normalizeTurns, runConversation } from "./eval-turns.js";
+export { createGatewayEvalHost, remoteEvalIncompatibilities } from "./eval-gateway-host.js";
+export { createExperimentArtifacts, createExperimentManifest, fingerprintEvalSuite, fingerprintValue, writeExperimentManifest, writeExperimentReport } from "./eval-experiment.js";
+export { createDefaultJudgeCompleter, EvalJudgeResponseError, judgeCase, robustParseJudge } from "./eval-judge.js";
+export { createEvalReport, formatEvalReport } from "./eval-report.js";
+export { isTransientProviderError, retryTransient } from "./eval-retry.js";
+export { evaluateTrajectory } from "./eval-trajectory.js";
+export { defineEvaluator } from "./eval-types.js";
+export async function runEvalCommand(agentRoot, parsed, dependencies = {}) {
+    const env = dependencies.env ?? process.env;
+    const now = dependencies.now ?? (() => new Date());
+    const started = now();
+    const filter = stringFlag(parsed, "filter");
+    const tag = stringFlag(parsed, "tag");
+    const discovered = discoverCases(agentRoot, {
+        ...(filter ? { filter } : {}),
+        ...(tag ? { tag } : {})
+    });
+    const concurrency = positiveIntegerFlag(parsed, "concurrency", 1);
+    const timeoutMs = positiveIntegerFlag(parsed, "timeout", 120) * 1_000;
+    const repetitions = positiveIntegerFlag(parsed, "repetitions", 1);
+    const retryAttempts = nonnegativeIntegerFlag(parsed, "retry-attempts", 2);
+    const retryBackoffMs = nonnegativeIntegerFlag(parsed, "retry-backoff-ms", 250);
+    const approve = booleanFlag(parsed, "approve");
+    const failFast = booleanFlag(parsed, "fail-fast");
+    const outDir = stringFlag(parsed, "out");
+    const artifact = buildAgent({ root: agentRoot, ...(outDir ? { outDir } : {}) });
+    const judgeModel = stringFlag(parsed, "judge-model") ?? env.OPENEVE_EVAL_JUDGE_MODEL ?? artifact.manifest.agent.model;
+    const config = {
+        ...(filter ? { filter } : {}),
+        ...(tag ? { tag } : {}),
+        judgeModel,
+        concurrency,
+        timeoutMs,
+        approve,
+        failFast,
+        repetitions,
+        retryAttempts,
+        retryBackoffMs
+    };
+    const experiment = createExperimentManifest({
+        startedAt: started.toISOString(),
+        agentRevision: artifact.manifest.agentRevision,
+        agentRoot,
+        cases: discovered,
+        config
+    });
+    const artifacts = createExperimentArtifacts(artifact.root, experiment);
+    writeExperimentManifest(experiment, artifacts);
+    const evaluatorLoader = new EvalEvaluatorLoader(agentRoot, resolve(artifacts.root, "evaluator-cache"), dependencies.evaluators);
+    const executions = expandExecutions(discovered, repetitions);
+    const judgeComplete = dependencies.judgeComplete ?? createDefaultJudgeCompleter(env);
+    // --url targets a deployed gateway instead of the in-process runtime. Cases
+    // that depend on local adapters (fixtures, mocks) or run() inputs the HTTP
+    // surface lacks (multi-turn, seeded history) are refused upfront rather than
+    // silently graded against a runtime that never saw them.
+    const gatewayUrl = stringFlag(parsed, "url");
+    let createRuntime = dependencies.createRuntime;
+    if (gatewayUrl && !createRuntime) {
+        const token = stringFlag(parsed, "token") ?? env.OPENEVE_ADMIN_TOKEN;
+        if (!token)
+            throw new Error("Remote eval targets require --token or ASSEMBLY_LINE_ADMIN_TOKEN.");
+        const host = createGatewayEvalHost({ url: gatewayUrl, token });
+        const capabilities = await host.capabilities().catch(() => []);
+        const incompatible = discovered.flatMap((item) => {
+            const reasons = item.evalCase ? remoteEvalIncompatibilities(item.evalCase, capabilities) : [];
+            return reasons.length > 0 ? [`${item.name} (${reasons.join(", ")})`] : [];
+        });
+        if (incompatible.length > 0) {
+            throw new Error(`${incompatible.length} eval case(s) need the in-process runtime and cannot run against --url:\n- ${incompatible.join("\n- ")}\nRun them without --url, or exclude them with --tag/--filter.`);
+        }
+        createRuntime = () => host;
+    }
+    const results = await executePool(executions, concurrency, failFast, async (execution, index) => {
+        const { item, repetition, repetitionsPlanned } = execution;
+        const caseId = caseIdentifier(agentRoot, item.path);
+        if (!item.evalCase)
+            return malformedResult(item, caseId);
+        const caseRoot = resolve(artifacts.root, "cases", `${String(index + 1).padStart(4, "0")}-${safeName(item.name)}`, `repetition-${String(repetition).padStart(2, "0")}`);
+        return runEvalCaseWithRetries({
+            item: { ...item, evalCase: item.evalCase },
+            caseId,
+            repetition,
+            repetitionsPlanned,
+            caseRoot,
+            artifactRoot: artifact.root,
+            evaluatorLoader,
+            approve,
+            timeoutMs,
+            judgeModel,
+            retryAttempts,
+            retryBackoffMs,
+            env,
+            dependencies: {
+                ...(createRuntime ? { createRuntime } : {}),
+                judgeComplete,
+                ...(dependencies.sleep ? { sleep: dependencies.sleep } : {})
+            }
+        });
+    });
+    const durationMs = Math.max(0, now().getTime() - started.getTime());
+    const reportArtifacts = {
+        root: artifacts.root,
+        experimentPath: artifacts.experimentPath,
+        resultsPath: artifacts.resultsPath
+    };
+    let report = createEvalReport(experiment, durationMs, results, { artifacts: reportArtifacts });
+    const baselinePath = stringFlag(parsed, "baseline");
+    if (baselinePath) {
+        const baseline = loadBaselineReport(baselinePath);
+        const comparison = compareEvalReports(baseline, report, {
+            maxRegressions: nonnegativeIntegerFlag(parsed, "max-regressions", 0),
+            ...(parsed.flags["max-cost-increase-percent"] === undefined
+                ? {}
+                : { maxCostIncreasePercent: nonnegativeNumberFlag(parsed, "max-cost-increase-percent") })
+        });
+        report = createEvalReport(experiment, durationMs, results, { artifacts: reportArtifacts, comparison });
+    }
+    writeExperimentReport(report, artifacts);
+    (dependencies.log ?? console.log)(booleanFlag(parsed, "json") ? JSON.stringify(report, null, 2) : formatEvalReport(report));
+    if (report.summary.failed + report.summary.errored > 0 || report.comparison?.passed === false)
+        process.exitCode = 1;
+    return report;
+}
+async function executePool(cases, concurrency, failFast, execute) {
+    const results = new Array(cases.length);
+    let next = 0;
+    let stopped = false;
+    const workers = Array.from({ length: Math.min(concurrency, Math.max(1, cases.length)) }, async () => {
+        while (!stopped) {
+            const index = next;
+            next += 1;
+            const item = cases[index];
+            if (!item)
+                return;
+            const result = await execute(item, index);
+            results[index] = result;
+            if (failFast && result.outcome !== "passed")
+                stopped = true;
+        }
+    });
+    await Promise.all(workers);
+    return results.filter((result) => result !== undefined);
+}
+function expandExecutions(cases, repetitions) {
+    return cases.flatMap((item) => {
+        if (!item.evalCase)
+            return [{ item, repetition: 1, repetitionsPlanned: 1 }];
+        const planned = item.evalCase.repetitions ?? repetitions;
+        return Array.from({ length: planned }, (_unused, index) => ({ item, repetition: index + 1, repetitionsPlanned: planned }));
+    });
+}
+function malformedResult(item, caseId) {
+    return {
+        caseId,
+        name: item.name,
+        path: item.path,
+        tags: [],
+        outcome: "errored",
+        repetition: 1,
+        repetitionsPlanned: 1,
+        attempts: 1,
+        durationMs: 0,
+        runCostUsd: 0,
+        judgeCostUsd: 0,
+        failures: [],
+        error: `Invalid eval case: ${item.error ?? "unknown error"}`
+    };
+}
+function positiveIntegerFlag(parsed, name, fallback) {
+    const raw = parsed.flags[name];
+    if (raw === undefined)
+        return fallback;
+    const value = typeof raw === "string" ? Number(raw) : Number.NaN;
+    if (!Number.isInteger(value) || value <= 0)
+        throw new Error(`--${name} must be a positive integer.`);
+    return value;
+}
+function nonnegativeIntegerFlag(parsed, name, fallback) {
+    const raw = parsed.flags[name];
+    if (raw === undefined)
+        return fallback;
+    const value = typeof raw === "string" ? Number(raw) : Number.NaN;
+    if (!Number.isInteger(value) || value < 0)
+        throw new Error(`--${name} must be a non-negative integer.`);
+    return value;
+}
+function nonnegativeNumberFlag(parsed, name) {
+    const raw = parsed.flags[name];
+    const value = typeof raw === "string" ? Number(raw) : Number.NaN;
+    if (!Number.isFinite(value) || value < 0)
+        throw new Error(`--${name} must be a non-negative number.`);
+    return value;
+}
+function caseIdentifier(agentRoot, path) {
+    return relative(agentRoot, path).replaceAll("\\", "/");
+}
+function safeName(value) {
+    return value.toLocaleLowerCase().replace(/[^a-z0-9._-]+/gu, "-").replace(/^-+|-+$/gu, "").slice(0, 80) || "case";
+}
+//# sourceMappingURL=eval.js.map

package/dist/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,UAAU,EAAmB,MAAM,WAAW,CAAC;AACrE,OAAO,EAAE,sBAAsB,EAAoC,MAAM,uBAAuB,CAAC;AACjG,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAC5F,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EACL,yBAAyB,EACzB,wBAAwB,EAGxB,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAStE,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,cAAc,EAAE,eAAe,EAAwB,MAAM,iBAAiB,CAAC;AACrI,OAAO,EAAE,qBAAqB,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAC5F,OAAO,EACL,yBAAyB,EACzB,wBAAwB,EACxB,oBAAoB,EACpB,gBAAgB,EAChB,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAgBlD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,MAAkB,EAClB,eAAwC,EAAE;IAE1C,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IAC5C,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,aAAa,CAAC,SAAS,EAAE;QAC1C,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxB,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC;IACtE,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,sBAAsB,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,cAAc,GAAG,sBAAsB,CAAC,MAAM,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAChF,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,GAAG,CAAC,wBAAwB,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;IACtH,MAAM,MAAM,GAAwB;QAClC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvB,UAAU;QACV,WAAW;QACX,SAAS;QACT,OAAO;QACP,QAAQ;QACR,WAAW;QACX,aAAa;QACb,cAAc;KACf,CAAC;IACF,MAAM,UAAU,GAAG,wBAAwB,CAAC;QAC1C,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE;QAChC,aAAa,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa;QAC9C,SAAS;QACT,KAAK,EAAE,UAAU;QACjB,MAAM;KACP,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,yBAAyB,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACvE,uBAAuB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,IAAI,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,iBAAiB,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;IAChI,MAAM,UAAU,GAAG,gBAAgB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAC7D,MAAM,aAAa,GAAG,YAAY,CAAC,aAAa,IAAI,2BAA2B,CAAC,GAAG,CAAC,CAAC;IACrF,4EAA4E;IAC5E,2EAA2E;IAC3E,6EAA6E;IAC7E,yDAAyD;IACzD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7C,IAAI,aAAa,GAAG,YAAY,CAAC,aAAa,CAAC;IAC/C,IAAI,UAAU,IAAI,CAAC,aAAa,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,mBAAmB,CAAC;QACrE,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACjG,MAAM,IAAI,GAAG,qBAAqB,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC;QAC3E,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9F,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,CAAC,CAAC,CAAC;QACH,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,GAAG,YAAY,CAAC,MAAM,8EAA8E,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,gEAAgE,CAC9L,CAAC;QACJ,CAAC;QACD,aAAa,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC;IAC7B,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;QAC9F,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,kBAAkB,EAAE,GAAG,SAAS,CAAC;QAC3D,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,OAAO,CACtB,SAAS,CAAC,IAAI,EACd,OAAO,EACP,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAC9D,cAAc,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACpD,CAAC;QACF,OAAO,sBAAsB,CAAC;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YAC1C,MAAM;YACN,UAAU;YACV,kBAAkB;YAClB,QAAQ;YACR,YAAY,EAAE,QAAQ,CAAC,IAAI;YAC3B,eAAe;YACf,OAAO;YACP,SAAS;YACT,UAAU;YACV,aAAa;YACb,cAAc;YACd,GAAG;YACH,YAAY,EAAE;gBACZ,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3C,aAAa;gBACb,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7D;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG;QACtB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,cAAc,EAAE,SAAS,CAAC,cAAc;QACxC,WAAW,EAAE,SAAS,CAAC,WAAW;KACnC,CAAC;IACF,IAAI,MAAM,GAAG,gBAAgB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC,CAAC;IAC/F,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACpD,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,QAAQ,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE;YACtD,cAAc,EAAE,sBAAsB,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACpE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,KAAK,SAAS;gBACzD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,sBAAsB,EAAE,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,CAAC,EAAE,CAAC;SAC5F,CAAC,CAAC;QACH,MAAM,GAAG,gBAAgB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC,CAAC;IACzG,CAAC;IACD,qBAAqB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC,YAAY,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5H,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,MAAM,KAAK,KAAK;QAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACpH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,KAAsB,EACtB,WAAmB,EACnB,QAAiB,EACjB,OAAwE;IAExE,MAAM,OAAO,GAAsC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAClG,OAAO,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,IAAI,IAAI,CAAC,CAAC;YACV,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC1C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;YACxB,IAAI,QAAQ,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ;gBAAE,OAAO,GAAG,IAAI,CAAC;QAC9D,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAA4B,EAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AACpF,CAAC;AAED,SAAS,gBAAgB,CAAC,KAA2B,EAAE,WAAmB;IACxE,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC5B,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,WAAW,CAAC;QACzD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,GAAG,CAAC,EAAE,kBAAkB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IAC7H,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,IAAwB,EAAE,MAAc;IAC/D,OAAO;QACL,MAAM;QACN,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,SAAS;QAClB,UAAU,EAAE,CAAC;QACb,kBAAkB,EAAE,CAAC;QACrB,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;QACf,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,sBAAsB,IAAI,CAAC,KAAK,IAAI,eAAe,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAkB,EAAE,IAAY,EAAE,QAAgB;IAC7E,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,IAAI,8BAA8B,CAAC,CAAC;IACrG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAkB,EAAE,IAAY,EAAE,QAAgB;IAChF,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,IAAI,kCAAkC,CAAC,CAAC;IACxG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAkB,EAAE,IAAY;IAC7D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,IAAI,iCAAiC,CAAC,CAAC;IACtG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,IAAY;IACrD,OAAO,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,iBAAiB,EAAE,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC;AACnH,CAAC"}

package/dist/help.d.ts

@@ -0,0 +1,12 @@
+export declare function printHelp(command?: string): void;
+export declare function printCommandHelp(command: string): void;
+export declare function isKnownCommand(command: string): boolean;
+export declare function printIssues(issues: {
+    level: string;
+    code: string;
+    message: string;
+    path?: string;
+}[], options?: {
+    agentRoot?: string;
+}): void;
+//# sourceMappingURL=help.d.ts.map

package/dist/help.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"help.d.ts","sourceRoot":"","sources":["../src/help.ts"],"names":[],"mappings":"AAoQA,wBAAgB,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAuBhD;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAsBtD;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAEvD;AAED,wBAAgB,WAAW,CACzB,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,EACzE,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GACnC,IAAI,CAQN"}