@dsh-enhanced/assistant-evaluation 0.1.23 → 0.1.30

package/lib/benchmark/strategy-evidence.d.ts

@@ -0,0 +1,161 @@
+import type { TaskAcceptanceContract, TaskVerificationReceipt } from '@dsh-enhanced/task-acceptance-contract';
+import type { StrategyBenchmarkMeterSnapshot } from './strategy-meter.js';
+import type { BenchmarkBudget, BenchmarkCase, BenchmarkCell, BenchmarkPlan, BenchmarkVariant, BenchmarkVersions } from './types.js';
+import type { StrategyBenchmarkPlan } from './strategy-plan.js';
+export declare const strategyEvidenceProtocol = "dsh-native-goal-strategy-evidence/v1";
+export interface StrategyEvidenceRequest {
+    planId: string;
+    dataset: BenchmarkPlan['dataset'];
+    cell: BenchmarkCell;
+    task: BenchmarkCase;
+    variant: BenchmarkVariant;
+    budget: BenchmarkBudget;
+}
+export interface StrategyEvidenceNative {
+    parent: {
+        sessionId: string;
+        goalId: string;
+        nativeGoalId: string;
+        definitionVersion: number;
+        definitionDigest: string;
+        lifecycle: 'completed' | 'failed' | 'cancelled' | 'unknown';
+        quiescent: boolean;
+    };
+    runs: readonly {
+        runId: string;
+        executionStatus: 'succeeded' | 'unknown';
+        quiescent: boolean;
+    }[];
+    strategies: readonly {
+        strategyId: string;
+        parentRunId: string;
+        kind: 'investigate' | 'review' | 'compare';
+        outcome: 'advice' | 'execution-failed' | 'cancelled' | 'unknown';
+        children: readonly {
+            sessionId: string;
+            stopReason: string;
+            quiescent: boolean;
+        }[];
+    }[];
+    outcomeAssessments: readonly {
+        contract: TaskAcceptanceContract;
+        triggerRunId: string | null;
+        dispatchedAt: number | null;
+        execution: {
+            status: 'succeeded' | 'unknown';
+            quiescent: boolean;
+            completedAt: number;
+        } | null;
+    }[];
+    selectedOutcomeContractId: string | null;
+    receipts: readonly {
+        runId: string;
+        taskKind: 'goal-step' | 'goal-outcome';
+        contract: TaskAcceptanceContract;
+        receipt: TaskVerificationReceipt;
+        quiescent: boolean;
+    }[];
+}
+export interface StrategyEvidenceWrite {
+    protocol: typeof strategyEvidenceProtocol;
+    version: 1;
+    plan: StrategyBenchmarkPlan;
+    request: StrategyEvidenceRequest;
+    input: {
+        digest: string;
+    };
+    acceptance: {
+        digest: string;
+        verdict: 'achieved' | 'not-achieved' | 'unknown';
+    };
+    versions: BenchmarkVersions;
+    meter: StrategyBenchmarkMeterSnapshot;
+    native: StrategyEvidenceNative;
+    outcome: {
+        status: 'completed' | 'unknown';
+        verdict: 'achieved' | 'not-achieved' | 'unknown';
+        quiescent: boolean;
+    };
+}
+export interface StrategyEvidenceObject extends Omit<StrategyEvidenceWrite, 'plan'> {
+    planDigest: string;
+}
+export interface StrategyEvidenceStoreOptions {
+    stateDirectory: string;
+    candidateWorkspace: string;
+    createDirectories?: boolean;
+}
+export declare const strategyFailureProtocol = "dsh-native-goal-strategy-failure/v1";
+export interface StrategyFailureSnapshot {
+    runtimeRoot: string | null;
+    stage: 'setup' | 'executing' | 'cleanup' | 'evidence' | 'closed';
+    cleanup: 'pending' | 'succeeded' | 'unknown';
+    meter: StrategyBenchmarkMeterSnapshot | null;
+    /** Current-authority Host observation, or null when no authorized snapshot is available. */
+    goalSnapshot: unknown | null;
+    lastGoalObservation: {
+        observedAt: number;
+        value: Readonly<Record<string, unknown>>;
+    } | null;
+    failureStage: 'setup' | 'executing' | 'cleanup' | 'evidence' | null;
+}
+export interface StrategyFailureWrite {
+    protocol: typeof strategyFailureProtocol;
+    version: 1;
+    plan: StrategyBenchmarkPlan;
+    request: StrategyEvidenceRequest;
+    reason: 'adapter-error' | 'interrupted' | 'timeout' | 'invalid-observation' | 'budget-exceeded' | 'not-quiescent';
+    observedAt: number;
+    snapshot: StrategyFailureSnapshot;
+}
+export interface StrategyFailureObject extends Omit<StrategyFailureWrite, 'plan'> {
+    planDigest: string;
+}
+export declare const strategyCellProtocol = "dsh-native-goal-strategy-cell/v1";
+export interface StrategyCellWrite {
+    protocol: typeof strategyCellProtocol;
+    version: 1;
+    plan: StrategyBenchmarkPlan;
+    request: StrategyEvidenceRequest;
+    nativeEvidenceDigest: string;
+    capabilities: {
+        sourceIdentity: unknown;
+        runtimeCommitment: {
+            source: string;
+            sourceIdentityDigest: string;
+            image: string;
+            dockerPath: string;
+            stepMaxDurationMs: number;
+            stopTimeoutMs: number;
+        };
+        requests: readonly {
+            sessionId: string;
+            kind: 'parent' | 'child';
+            observation: unknown | null;
+        }[];
+    };
+}
+export interface StrategyCellObject extends Omit<StrategyCellWrite, 'plan'> {
+    planDigest: string;
+}
+export declare class StrategyEvidenceStore {
+    #private;
+    constructor(input: StrategyEvidenceStoreOptions);
+    write(input: StrategyEvidenceWrite): Readonly<{
+        digest: string;
+        path: string;
+    }>;
+    read(planInput: StrategyBenchmarkPlan, cell: BenchmarkCell, digest: string): Readonly<StrategyEvidenceObject>;
+    /** A failure observation is terminal evidence of uncertainty, never an acceptance receipt. */
+    writeFailure(input: StrategyFailureWrite): Readonly<{
+        digest: string;
+        path: string;
+    }>;
+    readFailure(planInput: StrategyBenchmarkPlan, cell: BenchmarkCell, digest: string): Readonly<StrategyFailureObject>;
+    writeCell(input: StrategyCellWrite): Readonly<{
+        digest: string;
+        path: string;
+    }>;
+    readCell(planInput: StrategyBenchmarkPlan, cell: BenchmarkCell, digest: string): Readonly<StrategyCellObject>;
+}
+//# sourceMappingURL=strategy-evidence.d.ts.map

package/lib/benchmark/strategy-evidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"strategy-evidence.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-evidence.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAA;AAG7G,OAAO,KAAK,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAA;AACzE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAEnI,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAE/D,eAAO,MAAM,wBAAwB,yCAAyC,CAAA;AAC9E,MAAM,WAAW,uBAAuB;IAAG,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAAC,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,gBAAgB,CAAC;IAAC,MAAM,EAAE,eAAe,CAAA;CAAE;AAC5L,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,iBAAiB,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,CAAA;IACzM,IAAI,EAAE,SAAS;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,WAAW,GAAG,SAAS,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,EAAE,CAAA;IAChG,UAAU,EAAE,SAAS;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,aAAa,GAAG,QAAQ,GAAG,SAAS,CAAC;QAAC,OAAO,EAAE,QAAQ,GAAG,kBAAkB,GAAG,WAAW,GAAG,SAAS,CAAC;QAAC,QAAQ,EAAE,SAAS;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,OAAO,CAAA;SAAE,EAAE,CAAA;KAAE,EAAE,CAAA;IACpQ,kBAAkB,EAAE,SAAS;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE;YAAE,MAAM,EAAE,WAAW,GAAG,SAAS,CAAC;YAAC,SAAS,EAAE,OAAO,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,GAAG,IAAI,CAAA;KAAE,EAAE,CAAA;IAC7N,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAA;IACxC,QAAQ,EAAE,SAAS;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,WAAW,GAAG,cAAc,CAAC;QAAC,QAAQ,EAAE,sBAAsB,CAAC;QAAC,OAAO,EAAE,uBAAuB,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,EAAE,CAAA;CACvK;AACD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,OAAO,wBAAwB,CAAC;IAAC,OAAO,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,uBAAuB,CAAA;IACpH,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAAC,UAAU,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,SAAS,CAAA;KAAE,CAAC;IAAC,QAAQ,EAAE,iBAAiB,CAAA;IACxI,KAAK,EAAE,8BAA8B,CAAC;IAAC,MAAM,EAAE,sBAAsB,CAAA;IACrE,OAAO,EAAE;QAAE,MAAM,EAAE,WAAW,GAAG,SAAS,CAAC;QAAC,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,SAAS,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,CAAA;CACnH;AACD,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,qBAAqB,EAAE,MAAM,CAAC;IAAG,UAAU,EAAE,MAAM,CAAA;CAAE;AAC1G,MAAM,WAAW,4BAA4B;IAAG,cAAc,EAAE,MAAM,CAAC;IAAC,kBAAkB,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAAE;AAEjI,eAAO,MAAM,uBAAuB,wCAAwC,CAAA;AAC5E,MAAM,WAAW,uBAAuB;IACtC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,KAAK,EAAE,OAAO,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;IAChE,OAAO,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,CAAA;IAC5C,KAAK,EAAE,8BAA8B,GAAG,IAAI,CAAA;IAC5C,4FAA4F;IAC5F,YAAY,EAAE,OAAO,GAAG,IAAI,CAAA;IAC5B,mBAAmB,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;KAAE,GAAG,IAAI,CAAA;IAC5F,YAAY,EAAE,OAAO,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,GAAG,IAAI,CAAA;CACpE;AACD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,OAAO,uBAAuB,CAAC;IAAC,OAAO,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,uBAAuB,CAAA;IACnH,MAAM,EAAE,eAAe,GAAG,aAAa,GAAG,SAAS,GAAG,qBAAqB,GAAG,iBAAiB,GAAG,eAAe,CAAA;IACjH,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,uBAAuB,CAAA;CAClC;AACD,MAAM,WAAW,qBAAsB,SAAQ,IAAI,CAAC,oBAAoB,EAAE,MAAM,CAAC;IAAG,UAAU,EAAE,MAAM,CAAA;CAAE;AACxG,eAAO,MAAM,oBAAoB,qCAAqC,CAAA;AACtE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,OAAO,oBAAoB,CAAC;IAAC,OAAO,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,uBAAuB,CAAA;IAChH,oBAAoB,EAAE,MAAM,CAAA;IAC5B,YAAY,EAAE;QACZ,cAAc,EAAE,OAAO,CAAA;QACvB,iBAAiB,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,oBAAoB,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAC;YAAC,iBAAiB,EAAE,MAAM,CAAC;YAAC,aAAa,EAAE,MAAM,CAAA;SAAE,CAAA;QACxJ,QAAQ,EAAE,SAAS;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;YAAC,WAAW,EAAE,OAAO,GAAG,IAAI,CAAA;SAAE,EAAE,CAAA;KAClG,CAAA;CACF;AACD,MAAM,WAAW,kBAAmB,SAAQ,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAAG,UAAU,EAAE,MAAM,CAAA;CAAE;AAkKlG,qBAAa,qBAAqB;;gBAEpB,KAAK,EAAE,4BAA4B;IAQ/C,KAAK,CAAC,KAAK,EAAE,qBAAqB,GAAG,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAmB/E,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,sBAAsB,CAAC;IAgB7G,8FAA8F;IAC9F,YAAY,CAAC,KAAK,EAAE,oBAAoB,GAAG,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAMrF,WAAW,CAAC,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,qBAAqB,CAAC;IAGnH,SAAS,CAAC,KAAK,EAAE,iBAAiB,GAAG,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAM/E,QAAQ,CAAC,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,kBAAkB,CAAC;CA6C9G"}