engineering-behavior-observatory 0.2.1

package/dist/src/semantic-judge.d.ts

@@ -0,0 +1,194 @@
+import { query as claudeQuery, type EffortLevel } from "@anthropic-ai/claude-agent-sdk";
+import { type BehaviorAssertion } from "./behavior-assertions.js";
+import { type CodexReasoningEffort } from "./codex.js";
+import { type StructuralObservationSet } from "./structural-observations.js";
+import type { NativeEvidenceReference, NormalizationInput, UniformEvent } from "./uniform-events.js";
+type DigestString = `sha256:${string}`;
+type JsonRecord = Record<string, unknown>;
+export declare const SEMANTIC_JUDGE_PROMPT_VERSION = "1.0.0";
+export declare const CLAUDE_SEMANTIC_JUDGE_BACKEND_ID = "claude-agent-sdk";
+export type SemanticJudgeRequest = {
+    schemaVersion: "ebo.semantic-judge-request/v1";
+    id: string;
+    behavior: {
+        vocabularyVersion: string;
+        categoryId: string;
+        dimensionId: string;
+    };
+    rubric: {
+        id: string;
+        version: string;
+        instructions: string;
+    };
+    evaluator: {
+        provider: "anthropic" | "openai";
+        model: string;
+        effort: EffortLevel | CodexReasoningEffort;
+        backend?: "claude-agent-sdk" | "codex-app-server";
+        executable?: string;
+    };
+    selection: {
+        eventIds: readonly string[];
+        structuralObservationIds: readonly string[];
+        includeOutcomeObservations: boolean;
+    };
+    limits: {
+        maxEvidenceItems: number;
+        maxRecordChars: number;
+        maxInputChars: number;
+        maxOutputChars: number;
+        maxCitations: number;
+        maxWallClockMs: number;
+        maxTurns: number;
+        maxBudgetUsd?: number;
+    };
+    blinding: {
+        evaluatedModelIdentity: "redact" | "retain";
+    };
+};
+export type SemanticJudgeEvidenceItem = {
+    kind: "event" | "structural-observation";
+    id: string;
+    citation?: {
+        eventId: string;
+        nativeReference: NativeEvidenceReference;
+    };
+    content: string;
+    truncated: boolean;
+};
+export type SemanticJudgeInput = {
+    schemaVersion: "ebo.semantic-judge-input/v1";
+    promptVersion: typeof SEMANTIC_JUDGE_PROMPT_VERSION;
+    runId: string;
+    attemptId: string;
+    dataset: {
+        schemaVersion: "ebo.normalized-dataset/v1";
+        digest: DigestString;
+    };
+    behavior: SemanticJudgeRequest["behavior"];
+    rubric: SemanticJudgeRequest["rubric"];
+    selection: {
+        requestedEventIds: readonly string[];
+        requestedStructuralObservationIds: readonly string[];
+        includedEventIds: readonly string[];
+        includedStructuralObservationIds: readonly string[];
+        omitted: readonly string[];
+        truncated: readonly string[];
+    };
+    blinding: {
+        evaluatedModelIdentity: "redacted" | "retained";
+        redactionCount: number;
+        sourceIdentityPreserved: true;
+        residualClues: readonly string[];
+    };
+    limits: Pick<SemanticJudgeRequest["limits"], "maxEvidenceItems" | "maxRecordChars" | "maxInputChars" | "maxCitations">;
+    evidence: readonly SemanticJudgeEvidenceItem[];
+};
+export type SemanticJudgeUsage = {
+    totalCostUsd: number;
+    numTurns: number;
+    mainLoop: unknown;
+    byModel: unknown;
+};
+export type SemanticJudgeBackendResult = {
+    status: "completed";
+    response: unknown;
+    raw: unknown;
+    rawModelResponse?: unknown;
+    timing?: {
+        durationMs: number;
+        durationApiMs: number;
+    };
+    usage?: SemanticJudgeUsage;
+} | {
+    status: "failed";
+    kind: "provider" | "timeout" | "interrupted";
+    message: string;
+    raw?: unknown;
+    rawModelResponse?: unknown;
+    timing?: {
+        durationMs: number;
+        durationApiMs: number;
+    };
+    usage?: SemanticJudgeUsage;
+};
+export type SemanticJudgeBackend = {
+    id: "claude-agent-sdk" | "codex-app-server";
+    version: string;
+    run: (prompt: string, request: SemanticJudgeRequest, signal?: AbortSignal) => Promise<SemanticJudgeBackendResult>;
+};
+type RecordReference = {
+    path: string;
+    digest: DigestString;
+    sharingClass: "restricted";
+    truncated?: boolean;
+};
+type Availability<Value> = {
+    status: "available";
+    value: Value;
+} | {
+    status: "unavailable";
+    reason: string;
+};
+export type SemanticJudgmentRecord = {
+    schemaVersion: "ebo.semantic-judgment/v1";
+    id: string;
+    runId: string;
+    attemptId: string;
+    createdAt: string;
+    status: "proposed" | "failed";
+    input: RecordReference & {
+        digest: DigestString;
+    };
+    evaluator: {
+        provider: "anthropic" | "openai";
+        model: string;
+        effort: EffortLevel | CodexReasoningEffort;
+        backend: {
+            id: "claude-agent-sdk" | "codex-app-server";
+            version: string;
+        };
+        environment: {
+            parentPreserved: boolean;
+            modelEffortOverrides: "removed";
+            ambientTelemetry: "removed";
+            removedKeys: readonly string[];
+            mode?: "replace";
+            allowedKeys?: readonly string[];
+            authentication?: "existing-auth-json-only";
+        };
+        limits: SemanticJudgeRequest["limits"];
+    };
+    rawResponse?: RecordReference;
+    rawModelResponse?: RecordReference;
+    parse: {
+        status: "valid";
+    } | {
+        status: "failed";
+        kind: string;
+        message: string;
+    };
+    timing: Availability<{
+        durationMs: number;
+        durationApiMs: number;
+    }>;
+    usage: Availability<SemanticJudgeUsage>;
+    assertion?: RecordReference & {
+        id: string;
+    };
+};
+export type RunAgentSdkSemanticJudgeOptions = {
+    bundleRoot: string;
+    observations: StructuralObservationSet;
+    request: SemanticJudgeRequest;
+    outputRoot: string;
+    backend?: SemanticJudgeBackend;
+    now?: () => string;
+    signal?: AbortSignal;
+};
+export declare function runAgentSdkSemanticJudge(options: RunAgentSdkSemanticJudgeOptions): Promise<SemanticJudgmentRecord>;
+export declare function runClaudeAgentSdkSemanticJudge(prompt: string, request: SemanticJudgeRequest, queryFunction?: typeof claudeQuery, signal?: AbortSignal): Promise<SemanticJudgeBackendResult>;
+export declare function packageSemanticJudgeInput(events: readonly UniformEvent[], capture: NormalizationInput<unknown>, observations: StructuralObservationSet, request: SemanticJudgeRequest, datasetDigest: DigestString, evaluatedModelId: string): SemanticJudgeInput;
+export declare function parseSemanticJudgeResponse(value: unknown, request: SemanticJudgeRequest, input: SemanticJudgeInput, evaluatorVersion?: string): BehaviorAssertion;
+export declare function semanticJudgeResponseSchema(maxCitations: number): JsonRecord;
+export {};