@attest-ai/core 0.6.0 → 1.0.0-rc.1

package/dist/cjs/proto/index.d.cts

@@ -5,6 +5,16 @@ declare const STEP_AGENT_CALL: "agent_call";
 declare const STATUS_PASS: "pass";
 declare const STATUS_SOFT_FAIL: "soft_fail";
 declare const STATUS_HARD_FAIL: "hard_fail";
+declare const STATUS_NOT_EVALUATED: "not_evaluated";
+declare const REASON_PROVIDER_UNAVAILABLE: "provider_unavailable";
+declare const REASON_PROVIDER_NOT_CONFIGURED: "provider_not_configured";
+declare const REASON_SIMULATION: "simulation";
+declare const REASON_MISSING_EVIDENCE: "missing_evidence";
+declare const REASON_SHORT_CIRCUITED: "short_circuited";
+declare const OUTCOME_PASS: "pass";
+declare const OUTCOME_FAIL: "fail";
+declare const OUTCOME_SKIP: "skip";
+type Outcome = typeof OUTCOME_PASS | typeof OUTCOME_FAIL | typeof OUTCOME_SKIP;
 declare const TYPE_SCHEMA: "schema";
 declare const TYPE_CONSTRAINT: "constraint";
 declare const TYPE_TRACE: "trace";
@@ -125,6 +135,46 @@ interface Assertion {
     readonly spec: Record<string, unknown>;
     readonly request_id?: string;
 }
+/**
+ * Audit trail for an LLM-judge evaluation. Mirrors
+ * engine/pkg/types/assertion.go::JudgeMetadata so report v2 payloads
+ * round-trip without losing the model / rubric / variance evidence
+ * reviewers need to triage a judge failure.
+ */
+interface JudgeMetadata {
+    readonly model?: string;
+    readonly rubric_name?: string;
+    readonly rubric_version?: string;
+    readonly prompt_hash?: string;
+    readonly sample_scores?: readonly number[];
+    readonly score_mean?: number;
+    readonly score_stddev?: number;
+    readonly high_variance?: boolean;
+    readonly bias_probes?: readonly BiasProbe[];
+    readonly calibration?: JudgeAgreement;
+}
+/**
+ * One judge-bias probe result. Score is the score the judge produced on
+ * the mutated input; delta is `score - baseline`. A well-calibrated
+ * judge has |delta| close to zero. Mirrors
+ * engine/pkg/types/assertion.go::BiasProbe.
+ */
+interface BiasProbe {
+    readonly name: string;
+    readonly score: number;
+    readonly delta: number;
+}
+/**
+ * Calibration agreement summary keyed by
+ * (rubric_name, rubric_version, prompt_hash). Mirrors
+ * engine/pkg/types/assertion.go::JudgeAgreement.
+ */
+interface JudgeAgreement {
+    readonly label_count: number;
+    readonly agreement: number;
+    readonly cohen_kappa: number;
+    readonly roc_auc?: number;
+}
 interface AssertionResult {
     readonly assertion_id: string;
     readonly status: string;
@@ -133,7 +183,44 @@ interface AssertionResult {
     readonly cost?: number;
     readonly duration_ms?: number;
     readonly request_id?: string;
+    /** Set only when status is "not_evaluated" (REASON_* constants). */
+    readonly reason?: string;
+    readonly threshold_source?: string;
+    readonly layer?: number;
+    readonly type?: string;
+    readonly trace_node_path?: string;
+    readonly expected?: string;
+    readonly actual?: string;
+    readonly suggested_action?: string;
+    readonly judge_metadata?: JudgeMetadata;
+    /**
+     * Heuristic root-cause label set by the engine
+     * (broken_code / flaky_judge / bad_rubric / missing_trace_data /
+     * stochastic_variance). Empty/undefined for passes and any result the
+     * engine declined to label. Mirrors
+     * engine/pkg/types/failure_class.go.
+     */
+    readonly failure_class?: string;
 }
+/**
+ * FailureClass values mirror engine/pkg/types/failure_class.go. Use
+ * these constants when matching on AssertionResult.failure_class so a
+ * typo in either codebase fails fast.
+ */
+declare const FailureClass: {
+    readonly BrokenCode: "broken_code";
+    readonly FlakyJudge: "flaky_judge";
+    readonly BadRubric: "bad_rubric";
+    readonly MissingTraceData: "missing_trace_data";
+    readonly StochasticVariance: "stochastic_variance";
+};
+type FailureClass = (typeof FailureClass)[keyof typeof FailureClass];
+/**
+ * Map an assertion-type string to its pipeline layer (1–8). Returns 0
+ * when the type is unknown so callers can route uncategorised results
+ * into their own bucket.
+ */
+declare function layerForType(assertionType: string | undefined): number;
 interface ErrorData {
     readonly error_type: string;
     readonly retryable: boolean;
@@ -182,6 +269,13 @@ interface EvaluateBatchResult {
     readonly results: readonly AssertionResult[];
     readonly total_cost?: number;
     readonly total_duration_ms?: number;
+    /**
+     * True when this result was produced by an SDK simulation shim rather
+     * than a real engine evaluation. Engine-produced results always carry
+     * simulated=false. CI gates can fail builds that accidentally evaluate
+     * against the simulator by checking this flag.
+     */
+    readonly simulated?: boolean;
 }
 interface ShutdownResult {
     readonly sessions_completed: number;
@@ -235,6 +329,36 @@ interface GenerateUserMessageResult {
     readonly message: string;
 }
 
+/**
+ * Protocol diagnostic types for client-side observability.
+ *
+ * The reader loop emits diagnostics whenever it cannot route a response
+ * line: malformed JSON, missing id, unknown id, or non-routable protocol
+ * errors. Diagnostics are buffered in a bounded ring (32 entries) and
+ * trigger a desync error when the rate exceeds the configured threshold.
+ */
+type ProtocolDiagnosticKind = "malformed_json" | "non_object_response" | "invalid_jsonrpc_version" | "missing_id" | "unknown_id" | "non_routable_error";
+interface ProtocolDiagnostic {
+    readonly kind: ProtocolDiagnosticKind;
+    readonly message: string;
+    readonly rawLine: string;
+    readonly timestampMs: number;
+}
+interface ProtocolLogger {
+    warn(message: string, ...args: unknown[]): void;
+    error(message: string, ...args: unknown[]): void;
+}
+declare class ProtocolDiagnosticBuffer {
+    private readonly capacity;
+    private readonly entries;
+    constructor(capacity?: number);
+    push(diagnostic: ProtocolDiagnostic): void;
+    snapshot(): readonly ProtocolDiagnostic[];
+    countWithin(windowMs: number, nowMs: number): number;
+    clear(): void;
+}
+declare function previewLine(line: string): string;
+
 declare class ProtocolError extends Error {
     readonly code: number;
     readonly errorMessage: string;
@@ -246,6 +370,16 @@ declare class EngineTimeoutError extends Error {
     readonly timeoutMs: number;
     constructor(method: string, timeoutMs: number);
 }
+/**
+ * Raised when the reader loop detects that engine output has lost
+ * framing or contains repeated unparseable lines. All in-flight requests
+ * are rejected with this error so callers see a visible failure rather
+ * than a hung promise.
+ */
+declare class ProtocolDesyncError extends Error {
+    readonly diagnostics: readonly ProtocolDiagnostic[];
+    constructor(message: string, diagnostics: readonly ProtocolDiagnostic[]);
+}
 
 /**
  * Encode a JSON-RPC 2.0 request as an NDJSON line.
@@ -266,4 +400,4 @@ declare function extractResult(response: RPCResponse): unknown;
  */
 declare function extractId(response: RPCResponse): number;
 
-export { type AgentId, type Assertion, type AssertionId, type AssertionResult, type AssertionSpec, type ConstraintSpec, type ContentSpec, type ConversationMessage, type DriftReport, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, type EmbeddingSpec, EngineTimeoutError, type ErrorData, type EvaluateBatchParams, type EvaluateBatchResult, type GenerateUserMessageParams, type GenerateUserMessageResult, type InitializeParams, type InitializeResult, type LlmJudgeSpec, type PluginSpec, ProtocolError, type QueryDriftParams, type QueryDriftResult, type RPCError, type RPCRequest, type RPCResponse, STATUS_HARD_FAIL, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, type SchemaSpec, type ShutdownResult, type SimulateFaultConfig, type SimulatePersona, type Step, type SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, type Trace, type TraceId, type TraceMetadata, type TraceSpec, type TraceTreeSpec, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, traceId };
+export { type AgentId, type Assertion, type AssertionId, type AssertionResult, type AssertionSpec, type BiasProbe, type ConstraintSpec, type ContentSpec, type ConversationMessage, type DriftReport, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, type EmbeddingSpec, EngineTimeoutError, type ErrorData, type EvaluateBatchParams, type EvaluateBatchResult, FailureClass, type GenerateUserMessageParams, type GenerateUserMessageResult, type InitializeParams, type InitializeResult, type JudgeAgreement, type JudgeMetadata, type LlmJudgeSpec, OUTCOME_FAIL, OUTCOME_PASS, OUTCOME_SKIP, type Outcome, type PluginSpec, ProtocolDesyncError, type ProtocolDiagnostic, ProtocolDiagnosticBuffer, type ProtocolDiagnosticKind, ProtocolError, type ProtocolLogger, type QueryDriftParams, type QueryDriftResult, REASON_MISSING_EVIDENCE, REASON_PROVIDER_NOT_CONFIGURED, REASON_PROVIDER_UNAVAILABLE, REASON_SHORT_CIRCUITED, REASON_SIMULATION, type RPCError, type RPCRequest, type RPCResponse, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, type SchemaSpec, type ShutdownResult, type SimulateFaultConfig, type SimulatePersona, type Step, type SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, type Trace, type TraceId, type TraceMetadata, type TraceSpec, type TraceTreeSpec, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId };

package/dist/esm/cli.d.ts

@@ -1,9 +1,24 @@
 #!/usr/bin/env node
 /**
- * CLI entry point for Attest: npx attest or attest (after global install).
+ * The `attest` command line (`npx attest`).
  *
- * Ported from Python sdks/python/src/attest/__main__.py
+ * `attest init` runs in the SDK. Every other command (`doctor`, `baseline`, `policy`,
+ * `cache`, `calibrate`, `version` and any command a later engine adds) runs in
+ * `attest-engine`: the CLI resolves a version-checked engine with `resolveEngine`, checks
+ * that the engine's `help` lists the command, and forwards the arguments, stdio and exit
+ * code unchanged. A missing, mismatched or too-old engine exits 1 before anything runs, so
+ * a gate such as `attest policy evaluate` never exits 0 without evaluating.
  */
-declare function main(argv?: string[]): void;
+/**
+ * `{command: summary}` from `<engine> help`, or null if it lists none.
+ *
+ * stdin is closed and the probe times out after 10 s, so an engine that predates the
+ * command registry (and starts the JSON-RPC server for `help`) cannot hang the CLI.
+ */
+declare function engineCommands(engine: string, timeoutMs?: number): Map<string, string> | null;
+/** Map a child's exit to a shell exit status (signal N → 128 + N). */
+declare function exitStatus(status: number | null, signal: NodeJS.Signals | null): number;
+/** Dispatch `attest` arguments and resolve to the process exit code. */
+declare function main(argv?: string[]): Promise<number>;
 
-export { main };
+export { engineCommands, exitStatus, main };