@tangle-network/agent-eval 0.23.1 → 0.24.0

package/dist/feedback-trajectory-c43WGtTX.d.ts

@@ -1,346 +0,0 @@
-import { D as DatasetSplit, a as DatasetScenario } from './dataset-B9qvlm_o.js';
-import { T as TraceEmitter } from './emitter-B2XqDKFU.js';
-import { F as FailureClass, T as TraceStore } from './store-u47QaJ9G.js';
-
-/**
- * Policy-based agent control runtime.
- *
- * This is the minimal reusable loop behind driver-agent patterns:
- *
- *   observe state -> validate -> decide next action -> act -> observe -> ...
- *
- * It deliberately does not model named "topologies". Direct execution,
- * critic/revise, driver intervention, specialist calls, and human escalation
- * are all just actions chosen by the control policy.
- */
-
-type ControlSeverity = 'info' | 'warning' | 'error' | 'critical';
-type ControlActionFailureMode = 'continue' | 'stop';
-interface ControlEvalResult {
-    /** Stable validator or judge id. */
-    id: string;
-    /** Whether this check passed. */
-    passed: boolean;
-    /** Optional normalized score. 1 = best, 0 = worst. */
-    score?: number;
-    /** Objective validators should usually be "error" or "critical" when failed. */
-    severity?: ControlSeverity;
-    /** Human-readable result. */
-    detail?: string;
-    /** Small evidence string or pointer. Avoid large payloads. */
-    evidence?: string;
-    /** True when the result came from deterministic state, not LLM judgment. */
-    objective?: boolean;
-    /** Structured details for downstream control policies and reports. */
-    metadata?: Record<string, unknown>;
-}
-interface ControlBudget {
-    maxSteps: number;
-    maxWallMs?: number;
-    maxCostUsd?: number;
-}
-interface ControlStopPolicies<TState, TAction> {
-    /**
-     * Stop after N consecutive steps with no state fingerprint change and
-     * less than `minScoreDelta` score movement. Disabled when omitted.
-     */
-    maxNoProgressSteps?: number;
-    /**
-     * Stop after the same action fingerprint is selected N consecutive
-     * times. Disabled when omitted.
-     */
-    maxRepeatedActions?: number;
-    /** Minimum score movement that counts as progress. Default 0.001. */
-    minScoreDelta?: number;
-    /** Override the default JSON/string fingerprint for state comparisons. */
-    stateFingerprint?: (state: TState) => string;
-    /** Override the default JSON/string fingerprint for repeated-action checks. */
-    actionFingerprint?: (action: TAction) => string;
-}
-interface ControlContext<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
-    intent: string;
-    state: TState;
-    evals: TEval[];
-    history: ControlStep<TState, TAction, TActionResult, TEval>[];
-    budget: ControlBudget;
-    stepIndex: number;
-    wallMs: number;
-    spentCostUsd: number;
-    remainingCostUsd?: number;
-    abortSignal: AbortSignal;
-    emitter?: TraceEmitter;
-}
-type ControlDecision<TAction> = {
-    type: 'continue';
-    action: TAction;
-    reason?: string;
-} | {
-    type: 'stop';
-    reason: string;
-    pass?: boolean;
-    score?: number;
-};
-interface StopDecision {
-    stop: boolean;
-    pass: boolean;
-    reason: string;
-    score?: number;
-    failureClass?: FailureClass;
-}
-interface ControlActionOutcome<TActionResult> {
-    ok: boolean;
-    result?: TActionResult;
-    error?: string;
-    costUsd?: number;
-    durationMs: number;
-}
-interface ControlRuntimeError {
-    phase: 'observe' | 'validate' | 'decide' | 'act' | 'stop-policy' | 'on-step' | 'trace';
-    stepIndex: number;
-    message: string;
-}
-interface ControlStep<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
-    index: number;
-    decision: ControlDecision<TAction>;
-    beforeState: TState;
-    afterState: TState;
-    evalsBefore: TEval[];
-    evalsAfter: TEval[];
-    actionOutcome?: ControlActionOutcome<TActionResult>;
-    startedAt: string;
-    endedAt: string;
-}
-interface ControlRunResult<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
-    intent: string;
-    pass: boolean;
-    completed: boolean;
-    reason: string;
-    score?: number;
-    steps: ControlStep<TState, TAction, TActionResult, TEval>[];
-    finalState: TState | undefined;
-    finalEvals: TEval[];
-    wallMs: number;
-    spentCostUsd: number;
-    runId: string | null;
-    failureClass?: FailureClass;
-    runtimeErrors: ControlRuntimeError[];
-    stoppedBy: 'policy' | 'stop-policy' | 'budget' | 'abort' | 'runtime-error';
-}
-interface ControlRuntimeConfig<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult> {
-    intent: string;
-    budget?: Partial<ControlBudget>;
-    signal?: AbortSignal;
-    /** Defaults to `continue`: action failures are recorded, then the policy gets another chance. */
-    actionFailure?: ControlActionFailureMode;
-    /**
-     * Extract cost from an action result. Used for `maxCostUsd` budget
-     * enforcement and trace budget ledger emission.
-     */
-    getActionCostUsd?: (ctx: {
-        action: TAction;
-        result: TActionResult;
-        state: TState;
-        evals: TEval[];
-        history: ControlStep<TState, TAction, TActionResult, TEval>[];
-    }) => number | undefined;
-    /** Read typed task/product state. Prefer structured state over transcript-only context. */
-    observe: (ctx: {
-        history: ControlStep<TState, TAction, TActionResult, TEval>[];
-        abortSignal: AbortSignal;
-    }) => Promise<TState> | TState;
-    /** Objective validators first, subjective judges only where objective state is insufficient. */
-    validate: (ctx: {
-        intent: string;
-        state: TState;
-        history: ControlStep<TState, TAction, TActionResult, TEval>[];
-        abortSignal: AbortSignal;
-    }) => Promise<TEval[]> | TEval[];
-    /** Choose the next control action. Can call a worker, ask user, run critic, inspect state, or stop. */
-    decide: (ctx: ControlContext<TState, TAction, TActionResult, TEval>) => Promise<ControlDecision<TAction>> | ControlDecision<TAction>;
-    /** Execute the action selected by the policy. */
-    act: (action: TAction, ctx: ControlContext<TState, TAction, TActionResult, TEval>) => Promise<TActionResult> | TActionResult;
-    /** Final stopping policy. Called before decide and after each action. */
-    shouldStop?: (ctx: ControlContext<TState, TAction, TActionResult, TEval>) => Promise<StopDecision> | StopDecision;
-    /** Optional hook for tracing or live progress updates. */
-    onStep?: (step: ControlStep<TState, TAction, TActionResult, TEval>) => Promise<void> | void;
-    /** Optional generic stuck-loop policies. Custom `shouldStop` still runs first. */
-    stopPolicies?: ControlStopPolicies<TState, TAction>;
-    /** Optional trace sink. Emits one run plus one span per control step. */
-    store?: TraceStore;
-    scenarioId?: string;
-    projectId?: string;
-    variantId?: string;
-}
-declare function runAgentControlLoop<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult>(config: ControlRuntimeConfig<TState, TAction, TActionResult, TEval>): Promise<ControlRunResult<TState, TAction, TActionResult, TEval>>;
-declare function stopOnNoProgress<TState, TAction>(maxNoProgressSteps: number, options?: Omit<ControlStopPolicies<TState, TAction>, 'maxNoProgressSteps'>): ControlStopPolicies<TState, TAction>;
-declare function stopOnRepeatedAction<TState, TAction>(maxRepeatedActions: number, options?: Omit<ControlStopPolicies<TState, TAction>, 'maxRepeatedActions'>): ControlStopPolicies<TState, TAction>;
-declare function objectiveEval(input: Omit<ControlEvalResult, 'objective'>): ControlEvalResult;
-declare function subjectiveEval(input: Omit<ControlEvalResult, 'objective'>): ControlEvalResult;
-declare function allCriticalPassed(evals: ControlEvalResult[]): boolean;
-
-type FeedbackArtifactType = 'text' | 'code' | 'plan' | 'research' | 'action' | 'ui' | 'decision' | 'data' | 'other';
-type FeedbackLabelSource = 'user' | 'judge' | 'environment' | 'metric' | 'policy' | 'system';
-type FeedbackLabelKind = 'approve' | 'reject' | 'select' | 'edit' | 'rank' | 'rate' | 'comment' | 'metric_outcome' | 'policy_block' | 'revision_request';
-type FeedbackSeverity = 'info' | 'warning' | 'error' | 'critical';
-interface FeedbackTask {
-    intent: string;
-    context?: unknown;
-}
-interface ProposedSideEffect {
-    type: string;
-    risk?: 'low' | 'medium' | 'high';
-    costUsd?: number;
-    externalSideEffect?: boolean;
-    requiresApproval?: boolean;
-    metadata?: Record<string, unknown>;
-}
-interface FeedbackLabel {
-    id?: string;
-    source: FeedbackLabelSource;
-    kind: FeedbackLabelKind;
-    value: unknown;
-    reason?: string;
-    severity?: FeedbackSeverity;
-    createdAt: string;
-    metadata?: Record<string, unknown>;
-}
-interface FeedbackAttempt {
-    id: string;
-    stepIndex: number;
-    artifactType: FeedbackArtifactType;
-    artifact: unknown;
-    options?: unknown[];
-    proposedAction?: ProposedSideEffect;
-    evals?: ControlEvalResult[];
-    feedback?: FeedbackLabel[];
-    createdAt: string;
-    metadata?: Record<string, unknown>;
-}
-interface FeedbackOutcome {
-    success?: boolean;
-    score?: number;
-    metrics?: Record<string, number>;
-    costUsd?: number;
-    detail?: string;
-    observedAt?: string;
-    metadata?: Record<string, unknown>;
-}
-interface FeedbackTrajectory {
-    id: string;
-    projectId?: string;
-    scenarioId?: string;
-    task: FeedbackTask;
-    attempts: FeedbackAttempt[];
-    labels: FeedbackLabel[];
-    outcome?: FeedbackOutcome;
-    split?: DatasetSplit;
-    tags?: Record<string, string>;
-    createdAt: string;
-    updatedAt?: string;
-    metadata?: Record<string, unknown>;
-}
-interface FeedbackTrajectoryStore {
-    save(trajectory: FeedbackTrajectory): Promise<void>;
-    get(id: string): Promise<FeedbackTrajectory | null>;
-    list(filter?: FeedbackTrajectoryFilter): Promise<FeedbackTrajectory[]>;
-    appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory>;
-    appendLabel(id: string, label: FeedbackLabel, attemptId?: string): Promise<FeedbackTrajectory>;
-}
-interface FeedbackTrajectoryFilter {
-    projectId?: string;
-    scenarioId?: string;
-    split?: DatasetSplit;
-    tag?: [string, string];
-}
-interface FeedbackSplitPolicy {
-    trainPct?: number;
-    devPct?: number;
-    testPct?: number;
-    holdoutPct?: number;
-}
-interface PreferenceMemoryEntry {
-    instruction: string;
-    rationale: string;
-    weight: number;
-    sourceTrajectoryId: string;
-    sourceLabelId?: string;
-    category?: string;
-}
-interface FeedbackOptimizerRow {
-    scenarioId: string;
-    trajectoryId: string;
-    labelKinds: FeedbackLabelKind[];
-    score?: number;
-    metadata?: Record<string, unknown>;
-}
-interface FeedbackReplayResult {
-    trajectoryId: string;
-    pass: boolean;
-    score?: number;
-    labels: FeedbackLabel[];
-    outcome?: FeedbackOutcome;
-    metadata?: Record<string, unknown>;
-}
-interface FeedbackReplayAdapter {
-    replay(trajectory: FeedbackTrajectory): Promise<Omit<FeedbackReplayResult, 'trajectoryId'>> | Omit<FeedbackReplayResult, 'trajectoryId'>;
-}
-declare class InMemoryFeedbackTrajectoryStore implements FeedbackTrajectoryStore {
-    private readonly trajectories;
-    save(trajectory: FeedbackTrajectory): Promise<void>;
-    get(id: string): Promise<FeedbackTrajectory | null>;
-    list(filter?: FeedbackTrajectoryFilter): Promise<FeedbackTrajectory[]>;
-    appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory>;
-    appendLabel(id: string, label: FeedbackLabel, attemptId?: string): Promise<FeedbackTrajectory>;
-}
-declare class FileSystemFeedbackTrajectoryStore implements FeedbackTrajectoryStore {
-    private readonly dir;
-    private readonly memory;
-    private loaded;
-    constructor(options: {
-        dir: string;
-    });
-    save(trajectory: FeedbackTrajectory): Promise<void>;
-    get(id: string): Promise<FeedbackTrajectory | null>;
-    list(filter?: FeedbackTrajectoryFilter): Promise<FeedbackTrajectory[]>;
-    appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory>;
-    appendLabel(id: string, label: FeedbackLabel, attemptId?: string): Promise<FeedbackTrajectory>;
-    private append;
-    private load;
-}
-declare function createFeedbackTrajectory(input: {
-    id?: string;
-    projectId?: string;
-    scenarioId?: string;
-    task: FeedbackTask;
-    attempts?: FeedbackAttempt[];
-    labels?: FeedbackLabel[];
-    outcome?: FeedbackOutcome;
-    split?: DatasetSplit;
-    tags?: Record<string, string>;
-    createdAt?: string;
-    metadata?: Record<string, unknown>;
-}): FeedbackTrajectory;
-declare function assignFeedbackSplit(trajectory: Pick<FeedbackTrajectory, 'id' | 'projectId' | 'scenarioId' | 'task'>, policy?: FeedbackSplitPolicy): DatasetSplit;
-declare function withAssignedFeedbackSplit(trajectory: FeedbackTrajectory, policy?: FeedbackSplitPolicy): FeedbackTrajectory;
-declare function feedbackTrajectoryToDatasetScenario(trajectory: FeedbackTrajectory): DatasetScenario;
-declare function feedbackTrajectoriesToDatasetScenarios(trajectories: FeedbackTrajectory[]): DatasetScenario[];
-declare function feedbackTrajectoryToOptimizerRow(trajectory: FeedbackTrajectory): FeedbackOptimizerRow;
-declare function feedbackTrajectoriesToOptimizerRows(trajectories: FeedbackTrajectory[]): FeedbackOptimizerRow[];
-declare function replayFeedbackTrajectory(trajectory: FeedbackTrajectory, adapter: FeedbackReplayAdapter): Promise<FeedbackReplayResult>;
-declare function replayFeedbackTrajectories(trajectories: FeedbackTrajectory[], adapter: FeedbackReplayAdapter): Promise<FeedbackReplayResult[]>;
-declare function summarizePreferenceMemory(trajectories: FeedbackTrajectory[], options?: {
-    maxEntries?: number;
-}): PreferenceMemoryEntry[];
-declare function renderPreferenceMemoryMarkdown(entries: PreferenceMemoryEntry[]): string;
-declare function serializeFeedbackTrajectoriesJsonl(trajectories: FeedbackTrajectory[]): string;
-declare function parseFeedbackTrajectoriesJsonl(jsonl: string): FeedbackTrajectory[];
-declare function controlRunToFeedbackTrajectory<TState, TAction, TActionResult>(run: ControlRunResult<TState, TAction, TActionResult>, options?: {
-    projectId?: string;
-    scenarioId?: string;
-    artifactType?: FeedbackArtifactType;
-    artifactFromStep?: (step: ControlStep<TState, TAction, TActionResult>) => unknown;
-    proposedActionFromStep?: (step: ControlStep<TState, TAction, TActionResult>) => ProposedSideEffect | undefined;
-    createdAt?: string;
-}): FeedbackTrajectory;
-
-export { replayFeedbackTrajectory as A, serializeFeedbackTrajectoriesJsonl as B, summarizePreferenceMemory as C, withAssignedFeedbackSplit as D, type ControlSeverity as E, type FeedbackArtifactType as F, type ControlEvalResult as G, type ControlActionFailureMode as H, InMemoryFeedbackTrajectoryStore as I, type ControlActionOutcome as J, type ControlBudget as K, type ControlContext as L, type ControlDecision as M, type ControlRunResult as N, type ControlRuntimeConfig as O, type PreferenceMemoryEntry as P, type ControlRuntimeError as Q, type ControlStep as R, type ControlStopPolicies as S, type StopDecision as T, allCriticalPassed as U, objectiveEval as V, runAgentControlLoop as W, stopOnNoProgress as X, stopOnRepeatedAction as Y, subjectiveEval as Z, type FeedbackAttempt as a, type FeedbackLabel as b, type FeedbackLabelKind as c, type FeedbackLabelSource as d, type FeedbackOptimizerRow as e, type FeedbackOutcome as f, type FeedbackReplayAdapter as g, type FeedbackReplayResult as h, type FeedbackSeverity as i, type FeedbackSplitPolicy as j, type FeedbackTask as k, type FeedbackTrajectory as l, type FeedbackTrajectoryFilter as m, type FeedbackTrajectoryStore as n, FileSystemFeedbackTrajectoryStore as o, type ProposedSideEffect as p, assignFeedbackSplit as q, controlRunToFeedbackTrajectory as r, createFeedbackTrajectory as s, feedbackTrajectoriesToDatasetScenarios as t, feedbackTrajectoriesToOptimizerRows as u, feedbackTrajectoryToDatasetScenario as v, feedbackTrajectoryToOptimizerRow as w, parseFeedbackTrajectoriesJsonl as x, renderPreferenceMemoryMarkdown as y, replayFeedbackTrajectories as z };