@tangle-network/agent-eval 0.95.0 → 0.96.0

package/dist/control-Doncu-B_.d.ts

@@ -0,0 +1,259 @@
+import { b as FeedbackLabel, p as ProposedSideEffect } from './feedback-trajectory-BxY0cKfs.js';
+import { C as ControlEvalResult, a as ControlRunResult, h as ControlRuntimeConfig } from './control-runtime-Acf9CGhw.js';
+import { T as TraceEmitter } from './emitter-C2rqGH_l.js';
+import { F as FailureClass } from './schema-m0gsnbt3.js';
+import { T as TraceStore } from './store-BcFXE6LG.js';
+import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from './run-record-CP2ObebC.js';
+
+interface ActionExecutionPolicy {
+    allowedTypes?: string[];
+    blockedTypes?: string[];
+    alwaysRequireApprovalTypes?: string[];
+    autoApproveTypes?: string[];
+    requireApprovalForExternalSideEffects?: boolean;
+    requireApprovalAboveCostUsd?: number;
+    maxActionCostUsd?: number;
+    remainingBudgetUsd?: number;
+    expectedOutcomeRequired?: boolean;
+    killCriteriaRequired?: boolean;
+}
+interface ActionPolicyDecision {
+    allowed: boolean;
+    blocked: boolean;
+    requiresApproval: boolean;
+    reasons: string[];
+    label?: FeedbackLabel;
+}
+declare function evaluateActionPolicy(action: ProposedSideEffect, policy?: ActionExecutionPolicy, options?: {
+    createdAt?: string;
+}): ActionPolicyDecision;
+
+interface RunEvidenceMetadata {
+    experimentId: string;
+    candidateId: string;
+    seed: number;
+    model: string;
+    promptHash: string;
+    configHash: string;
+    commitSha: string;
+    splitTag: RunSplitTag;
+    tokenUsage: RunTokenUsage;
+    queueMs?: number;
+    judgeMetadata?: RunRecord['judgeMetadata'];
+    raw?: Record<string, number>;
+}
+interface ControlRunToRunRecordOptions extends RunEvidenceMetadata {
+    runId?: string;
+    score?: number;
+    failureMode?: string;
+}
+/**
+ * Project a completed control-loop run into the strict RunRecord shape used by
+ * release gates, optimizer tables, and research reports.
+ *
+ * The control loop owns live execution evidence. The caller still supplies the
+ * experiment-cell metadata because prompt/config hashes, split assignment,
+ * model snapshot, and commit SHA are product/harness concerns.
+ */
+declare function controlRunToRunRecord<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult>(run: ControlRunResult<TState, TAction, TActionResult, TEval>, options: ControlRunToRunRecordOptions): RunRecord;
+declare function scoreFromEvals(evals: readonly ControlEvalResult[]): number | undefined;
+
+/**
+ * Propose / Verify / Review — the core multi-shot primitive.
+ *
+ *   shot N:  propose(state, priorReview) → new state
+ *            verify(state)               → pass/fail, optional layers
+ *            review(state, verification, memory) → observations + next-shot
+ *                                                   instruction + shouldContinue
+ *            memory.append(entry)
+ *
+ * Roles are strictly separated:
+ *
+ *   - The WORKER is whatever the caller wraps in `propose`. It is
+ *     stateful — caller owns its resume/session mechanism.
+ *   - The VERIFIER grades the state. It produces the ground truth.
+ *     The reviewer cannot overturn or downgrade a verification layer.
+ *   - The REVIEWER is stateless per call. Its continuity is the
+ *     `ReviewMemoryStore` — durable JSONL by default, or any store
+ *     implementing the interface. It reads memory + trace summary +
+ *     verification and directs the NEXT proposer shot.
+ *
+ * This shape is load-bearing. The reviewer never grades; the verifier
+ * never directs. Two processes, two prompts, two concerns — which is
+ * what keeps the loop from confirmation-biasing itself into "all
+ * passed" when it didn't.
+ *
+ * Short-circuits and soft-fails are both first-class:
+ *   - verify.pass === true  → reviewer LLM call is skipped, memory
+ *     records a success entry, loop exits.
+ *   - review throws         → the shot still counts; the loop uses the
+ *     last-known instruction (or `fallbackInstruction`) for the next
+ *     propose call. A transient reviewer failure must NEVER abort a
+ *     valid arc.
+ *
+ * Composable: `propose` itself can be another `runProposeReview` call.
+ * That's the dogfooding path — a harness built on this primitive is in
+ * turn evaluable by it.
+ */
+
+interface Verification {
+    pass: boolean;
+    score?: number;
+    failingLayers?: string[];
+    details?: unknown;
+}
+interface Review {
+    observations: string;
+    diagnosis: string;
+    nextShotInstruction: string;
+    shouldContinue: boolean;
+    confidence: number;
+}
+interface ReviewMemoryEntry extends Review {
+    shot: number;
+    timestamp: number;
+    verification: {
+        pass: boolean;
+        score?: number;
+        failingLayers?: string[];
+    };
+}
+interface ProposeInput<State> {
+    shot: number;
+    goal: string;
+    state: State;
+    priorReview: Review | null;
+    abortSignal: AbortSignal;
+    emitter?: TraceEmitter;
+}
+interface ProposeOutput<State, Summary = unknown> {
+    state: State;
+    traceSummary?: Summary;
+}
+interface ReviewInput<State, Summary = unknown> {
+    shot: number;
+    goal: string;
+    state: State;
+    verification: Verification;
+    traceSummary: Summary | undefined;
+    memory: ReviewMemoryEntry[];
+}
+type ProposeFn<State, Summary = unknown> = (input: ProposeInput<State>) => Promise<ProposeOutput<State, Summary>>;
+type VerifyFn<State> = (state: State) => Promise<Verification>;
+type ReviewFn<State, Summary = unknown> = (input: ReviewInput<State, Summary>) => Promise<Review>;
+interface ReviewMemoryStore {
+    load(): Promise<ReviewMemoryEntry[]>;
+    append(entry: ReviewMemoryEntry): Promise<void>;
+}
+interface ProposeReviewConfig<State, Summary = unknown> {
+    goal: string;
+    initialState: State;
+    propose: ProposeFn<State, Summary>;
+    verify: VerifyFn<State>;
+    review: ReviewFn<State, Summary>;
+    /** Hard shot cap. Default 10. */
+    maxShots?: number;
+    /** Wall-clock cap in ms. Default 10 min. */
+    maxWallMs?: number;
+    /**
+     * If the reviewer returns confidence ≤ floor on `confidenceFloorWindow`
+     * consecutive shots, terminate early. Default floor 0.3, window 2.
+     * Set window to 0 or floor to <0 to disable.
+     */
+    confidenceFloor?: number;
+    confidenceFloorWindow?: number;
+    /** Defaults to an in-memory store if omitted. */
+    memory?: ReviewMemoryStore;
+    /** If provided, emit a Run + per-shot spans. */
+    store?: TraceStore;
+    scenarioId?: string;
+    projectId?: string;
+    variantId?: string;
+    /**
+     * Used when the reviewer soft-fails on shot 1 (no prior instruction to
+     * fall back to). Default is a generic "inspect failures and fix".
+     */
+    fallbackInstruction?: string;
+}
+interface ProposeReviewShot<State, Summary = unknown> {
+    shot: number;
+    state: State;
+    verification: Verification;
+    traceSummary: Summary | undefined;
+    review: Review;
+    reviewAvailable: boolean;
+    reviewError?: string;
+    durationMs: number;
+}
+interface ProposeReviewReport<State, Summary = unknown> {
+    runId: string | null;
+    completed: boolean;
+    shots: ProposeReviewShot<State, Summary>[];
+    finalState: State;
+    finalVerification: Verification;
+    failureClass?: FailureClass;
+    wallMs: number;
+    score: number;
+}
+declare function inMemoryReviewStore(initial?: ReviewMemoryEntry[]): ReviewMemoryStore;
+declare function jsonlReviewStore(path: string): ReviewMemoryStore;
+declare function runProposeReview<State, Summary = unknown>(config: ProposeReviewConfig<State, Summary>): Promise<ProposeReviewReport<State, Summary>>;
+type LlmJsonCall = (req: {
+    system: string;
+    user: string;
+}) => Promise<unknown>;
+interface LlmReviewerConfig<State, Summary = unknown> {
+    callJson: LlmJsonCall;
+    renderState?: (state: State) => string;
+    renderTraceSummary?: (summary: Summary | undefined) => string;
+    /** Appended to the default system prompt. */
+    systemPromptAddendum?: string;
+}
+declare function createLlmReviewer<State, Summary = unknown>(cfg: LlmReviewerConfig<State, Summary>): ReviewFn<State, Summary>;
+
+interface ProposeReviewControlState<State, Summary = unknown> {
+    shot: number;
+    state: State;
+    priorReview: Review | null;
+    verification: Verification;
+    traceSummary?: Summary;
+    memory: ReviewMemoryEntry[];
+    completed: boolean;
+    reviewAvailable: boolean;
+    reviewError?: string;
+}
+interface ProposeReviewControlAction {
+    type: 'propose-review-shot';
+    shot: number;
+}
+interface ProposeReviewControlResult<State, Summary = unknown> {
+    state: State;
+    verification: Verification;
+    traceSummary?: Summary;
+    review: Review | null;
+    reviewAvailable: boolean;
+    reviewError?: string;
+}
+interface ProposeReviewControlConfig<State, Summary = unknown> {
+    goal: string;
+    initialState: State;
+    propose: ProposeFn<State, Summary>;
+    verify: VerifyFn<State>;
+    review: ReviewFn<State, Summary>;
+    maxShots?: number;
+    maxWallMs?: number;
+    memory?: ReviewMemoryStore;
+    store?: TraceStore;
+    scenarioId?: string;
+    projectId?: string;
+    variantId?: string;
+    fallbackInstruction?: string;
+    confidenceFloor?: number;
+    confidenceFloorWindow?: number;
+    failureClassFromVerification?: (verification: Verification) => FailureClass | undefined;
+    actionFailure?: ControlRuntimeConfig<ProposeReviewControlState<State, Summary>, ProposeReviewControlAction, ProposeReviewControlResult<State, Summary>>['actionFailure'];
+}
+declare function runProposeReviewAsControlLoop<State, Summary = unknown>(config: ProposeReviewControlConfig<State, Summary>): Promise<ControlRunResult<ProposeReviewControlState<State, Summary>, ProposeReviewControlAction, ProposeReviewControlResult<State, Summary>>>;
+declare function controlFailureClassFromVerification(verification: Verification): FailureClass | undefined;
+
+export { type ActionExecutionPolicy as A, type ControlRunToRunRecordOptions as C, type LlmJsonCall as L, type ProposeFn as P, type Review as R, type Verification as V, type ActionPolicyDecision as a, type LlmReviewerConfig as b, type ProposeInput as c, type ProposeOutput as d, type ProposeReviewConfig as e, type ProposeReviewControlAction as f, type ProposeReviewControlConfig as g, type ProposeReviewControlResult as h, type ProposeReviewControlState as i, type ProposeReviewReport as j, type ProposeReviewShot as k, type ReviewFn as l, type ReviewInput as m, type ReviewMemoryEntry as n, type ReviewMemoryStore as o, type RunEvidenceMetadata as p, type VerifyFn as q, controlFailureClassFromVerification as r, controlRunToRunRecord as s, createLlmReviewer as t, evaluateActionPolicy as u, inMemoryReviewStore as v, jsonlReviewStore as w, runProposeReview as x, runProposeReviewAsControlLoop as y, scoreFromEvals as z };

package/dist/{control-runtime.d.ts → control-runtime-Acf9CGhw.d.ts}

@@ -1,3 +1,7 @@
+import { T as TraceEmitter } from './emitter-C2rqGH_l.js';
+import { F as FailureClass } from './schema-m0gsnbt3.js';
+import { T as TraceStore } from './store-BcFXE6LG.js';
+
 /**
  * Policy-based agent control runtime.
  *
@@ -9,12 +13,10 @@
  * critic/revise, driver intervention, specialist calls, and human escalation
  * are all just actions chosen by the control policy.
  */
-import { TraceEmitter } from './trace/emitter';
-import type { FailureClass } from './trace/schema';
-import type { TraceStore } from './trace/store';
-export type ControlSeverity = 'info' | 'warning' | 'error' | 'critical';
-export type ControlActionFailureMode = 'continue' | 'stop';
-export interface ControlEvalResult {
+
+type ControlSeverity = 'info' | 'warning' | 'error' | 'critical';
+type ControlActionFailureMode = 'continue' | 'stop';
+interface ControlEvalResult {
     /** Stable validator or judge id. */
     id: string;
     /** Whether this check passed. */
@@ -32,12 +34,12 @@ export interface ControlEvalResult {
     /** Structured details for downstream control policies and reports. */
     metadata?: Record<string, unknown>;
 }
-export interface ControlBudget {
+interface ControlBudget {
     maxSteps: number;
     maxWallMs?: number;
     maxCostUsd?: number;
 }
-export interface ControlStopPolicies<TState, TAction> {
+interface ControlStopPolicies<TState, TAction> {
     /**
      * Stop after N consecutive steps with no state fingerprint change and
      * less than `minScoreDelta` score movement. Disabled when omitted.
@@ -55,7 +57,7 @@ export interface ControlStopPolicies<TState, TAction> {
     /** Override the default JSON/string fingerprint for repeated-action checks. */
     actionFingerprint?: (action: TAction) => string;
 }
-export interface ControlContext<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
+interface ControlContext<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
     intent: string;
     state: TState;
     evals: TEval[];
@@ -68,7 +70,7 @@ export interface ControlContext<TState, TAction, TActionResult, TEval extends Co
     abortSignal: AbortSignal;
     emitter?: TraceEmitter;
 }
-export type ControlDecision<TAction> = {
+type ControlDecision<TAction> = {
     type: 'continue';
     action: TAction;
     reason?: string;
@@ -78,26 +80,26 @@ export type ControlDecision<TAction> = {
     pass?: boolean;
     score?: number;
 };
-export interface StopDecision {
+interface StopDecision {
     stop: boolean;
     pass: boolean;
     reason: string;
     score?: number;
     failureClass?: FailureClass;
 }
-export interface ControlActionOutcome<TActionResult> {
+interface ControlActionOutcome<TActionResult> {
     ok: boolean;
     result?: TActionResult;
     error?: string;
     costUsd?: number;
     durationMs: number;
 }
-export interface ControlRuntimeError {
+interface ControlRuntimeError {
     phase: 'observe' | 'validate' | 'decide' | 'act' | 'stop-policy' | 'on-step' | 'trace';
     stepIndex: number;
     message: string;
 }
-export interface ControlStep<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
+interface ControlStep<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
     index: number;
     decision: ControlDecision<TAction>;
     beforeState: TState;
@@ -108,7 +110,7 @@ export interface ControlStep<TState, TAction, TActionResult, TEval extends Contr
     startedAt: string;
     endedAt: string;
 }
-export interface ControlRunResult<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
+interface ControlRunResult<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
     intent: string;
     pass: boolean;
     completed: boolean;
@@ -125,7 +127,7 @@ export interface ControlRunResult<TState, TAction, TActionResult, TEval extends
     runtimeErrors: ControlRuntimeError[];
     stoppedBy: 'policy' | 'stop-policy' | 'budget' | 'abort' | 'runtime-error';
 }
-export interface ControlRuntimeConfig<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
+interface ControlRuntimeConfig<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
     intent: string;
     budget?: Partial<ControlBudget>;
     signal?: AbortSignal;
@@ -170,10 +172,11 @@ export interface ControlRuntimeConfig<TState, TAction, TActionResult, TEval exte
     projectId?: string;
     variantId?: string;
 }
-export declare function runAgentControlLoop<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult>(config: ControlRuntimeConfig<TState, TAction, TActionResult, TEval>): Promise<ControlRunResult<TState, TAction, TActionResult, TEval>>;
-export declare function stopOnNoProgress<TState, TAction>(maxNoProgressSteps: number, options?: Omit<ControlStopPolicies<TState, TAction>, 'maxNoProgressSteps'>): ControlStopPolicies<TState, TAction>;
-export declare function stopOnRepeatedAction<TState, TAction>(maxRepeatedActions: number, options?: Omit<ControlStopPolicies<TState, TAction>, 'maxRepeatedActions'>): ControlStopPolicies<TState, TAction>;
-export declare function objectiveEval(input: Omit<ControlEvalResult, 'objective'>): ControlEvalResult;
-export declare function subjectiveEval(input: Omit<ControlEvalResult, 'objective'>): ControlEvalResult;
-export declare function allCriticalPassed(evals: ControlEvalResult[]): boolean;
-//# sourceMappingURL=control-runtime.d.ts.map
+declare function runAgentControlLoop<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult>(config: ControlRuntimeConfig<TState, TAction, TActionResult, TEval>): Promise<ControlRunResult<TState, TAction, TActionResult, TEval>>;
+declare function stopOnNoProgress<TState, TAction>(maxNoProgressSteps: number, options?: Omit<ControlStopPolicies<TState, TAction>, 'maxNoProgressSteps'>): ControlStopPolicies<TState, TAction>;
+declare function stopOnRepeatedAction<TState, TAction>(maxRepeatedActions: number, options?: Omit<ControlStopPolicies<TState, TAction>, 'maxRepeatedActions'>): ControlStopPolicies<TState, TAction>;
+declare function objectiveEval(input: Omit<ControlEvalResult, 'objective'>): ControlEvalResult;
+declare function subjectiveEval(input: Omit<ControlEvalResult, 'objective'>): ControlEvalResult;
+declare function allCriticalPassed(evals: ControlEvalResult[]): boolean;
+
+export { type ControlEvalResult as C, type StopDecision as S, type ControlRunResult as a, type ControlStep as b, type ControlActionFailureMode as c, type ControlActionOutcome as d, type ControlBudget as e, type ControlContext as f, type ControlDecision as g, type ControlRuntimeConfig as h, type ControlRuntimeError as i, type ControlSeverity as j, type ControlStopPolicies as k, allCriticalPassed as l, stopOnRepeatedAction as m, subjectiveEval as n, objectiveEval as o, runAgentControlLoop as r, stopOnNoProgress as s };

package/dist/control.d.ts

@@ -1,11 +1,10 @@
-export type { ActionExecutionPolicy, ActionPolicyDecision, } from './action-policy';
-export { evaluateActionPolicy } from './action-policy';
-export type { ControlActionFailureMode, ControlActionOutcome, ControlBudget, ControlContext, ControlDecision, ControlEvalResult, ControlRunResult, ControlRuntimeConfig, ControlRuntimeError, ControlSeverity, ControlStep, ControlStopPolicies, StopDecision, } from './control-runtime';
-export { allCriticalPassed, objectiveEval, runAgentControlLoop, stopOnNoProgress, stopOnRepeatedAction, subjectiveEval, } from './control-runtime';
-export type { ProposeReviewConfig, ProposeReviewReport, } from './propose-review';
-export { runProposeReview } from './propose-review';
-export type { ProposeReviewControlAction, ProposeReviewControlConfig, ProposeReviewControlResult, ProposeReviewControlState, } from './propose-review-control';
-export { runProposeReviewAsControlLoop } from './propose-review-control';
-export type { ControlRunToRunRecordOptions, RunEvidenceMetadata, } from './run-evidence';
-export { controlRunToRunRecord, scoreFromEvals, } from './run-evidence';
-//# sourceMappingURL=control.d.ts.map
+export { A as ActionExecutionPolicy, a as ActionPolicyDecision, C as ControlRunToRunRecordOptions, e as ProposeReviewConfig, f as ProposeReviewControlAction, g as ProposeReviewControlConfig, h as ProposeReviewControlResult, i as ProposeReviewControlState, j as ProposeReviewReport, p as RunEvidenceMetadata, s as controlRunToRunRecord, u as evaluateActionPolicy, x as runProposeReview, y as runProposeReviewAsControlLoop, z as scoreFromEvals } from './control-Doncu-B_.js';
+export { c as ControlActionFailureMode, d as ControlActionOutcome, e as ControlBudget, f as ControlContext, g as ControlDecision, C as ControlEvalResult, a as ControlRunResult, h as ControlRuntimeConfig, i as ControlRuntimeError, j as ControlSeverity, b as ControlStep, k as ControlStopPolicies, S as StopDecision, l as allCriticalPassed, o as objectiveEval, r as runAgentControlLoop, s as stopOnNoProgress, m as stopOnRepeatedAction, n as subjectiveEval } from './control-runtime-Acf9CGhw.js';
+import './feedback-trajectory-BxY0cKfs.js';
+import './dataset-BbGkaN2I.js';
+import './errors-CzMUYo7b.js';
+import './emitter-C2rqGH_l.js';
+import './schema-m0gsnbt3.js';
+import './store-BcFXE6LG.js';
+import './run-record-CP2ObebC.js';
+import '@tangle-network/agent-interface';