@attest-ai/core 0.9.0 → 1.0.0-rc.2

package/dist/cjs/proto/index.d.cts

@@ -5,6 +5,18 @@ declare const STEP_AGENT_CALL: "agent_call";
 declare const STATUS_PASS: "pass";
 declare const STATUS_SOFT_FAIL: "soft_fail";
 declare const STATUS_HARD_FAIL: "hard_fail";
+declare const STATUS_NOT_EVALUATED: "not_evaluated";
+declare const REASON_PROVIDER_UNAVAILABLE: "provider_unavailable";
+declare const REASON_PROVIDER_NOT_CONFIGURED: "provider_not_configured";
+declare const REASON_SIMULATION: "simulation";
+declare const REASON_MISSING_EVIDENCE: "missing_evidence";
+declare const REASON_SHORT_CIRCUITED: "short_circuited";
+declare const REASON_BUDGET_EXHAUSTED: "budget_exhausted";
+declare const REASON_JUDGE_ABSTAINED: "judge_abstained";
+declare const OUTCOME_PASS: "pass";
+declare const OUTCOME_FAIL: "fail";
+declare const OUTCOME_SKIP: "skip";
+type Outcome = typeof OUTCOME_PASS | typeof OUTCOME_FAIL | typeof OUTCOME_SKIP;
 declare const TYPE_SCHEMA: "schema";
 declare const TYPE_CONSTRAINT: "constraint";
 declare const TYPE_TRACE: "trace";
@@ -97,6 +109,30 @@ interface TraceMetadata {
     readonly model?: string;
     readonly timestamp?: string;
 }
+/** Trace v2: a step whose span ended in error. */
+interface StepError {
+    readonly type: string;
+    readonly message?: string;
+}
+/** Trace v2: a recorded model request/response pair. */
+interface Exchange {
+    readonly request: Record<string, unknown>;
+    readonly response: Record<string, unknown>;
+}
+/** Trace v2: one conversation turn and the indices of its steps. */
+interface Turn {
+    readonly index: number;
+    readonly span_id?: string;
+    readonly input?: unknown;
+    readonly output?: unknown;
+    readonly step_indices: readonly number[];
+}
+/** Trace v2: control passing from one agent to another at an agent_call step. */
+interface Handoff {
+    readonly from_agent: string;
+    readonly to_agent: string;
+    readonly step_index: number;
+}
 interface Step {
     readonly type: string;
     readonly name: string;
@@ -108,6 +144,12 @@ interface Step {
     readonly ended_at_ms?: number;
     readonly agent_id?: string;
     readonly agent_role?: string;
+    readonly span_id?: string;
+    readonly parent_span_id?: string;
+    readonly tool_call_id?: string;
+    readonly retry_of?: number;
+    readonly error?: StepError;
+    readonly exchange?: Exchange;
 }
 interface Trace {
     readonly trace_id: string;
@@ -118,6 +160,9 @@ interface Trace {
     readonly steps: readonly Step[];
     readonly metadata?: TraceMetadata;
     readonly parent_trace_id?: string;
+    readonly conversation_id?: string;
+    readonly turns?: readonly Turn[];
+    readonly handoffs?: readonly Handoff[];
 }
 interface Assertion {
     readonly assertion_id: string;
@@ -132,6 +177,7 @@ interface Assertion {
  * reviewers need to triage a judge failure.
  */
 interface JudgeMetadata {
+    readonly provider?: string;
     readonly model?: string;
     readonly rubric_name?: string;
     readonly rubric_version?: string;
@@ -173,6 +219,8 @@ interface AssertionResult {
     readonly cost?: number;
     readonly duration_ms?: number;
     readonly request_id?: string;
+    /** Set only when status is "not_evaluated" (REASON_* constants). */
+    readonly reason?: string;
     readonly threshold_source?: string;
     readonly layer?: number;
     readonly type?: string;
@@ -181,7 +229,28 @@ interface AssertionResult {
     readonly actual?: string;
     readonly suggested_action?: string;
     readonly judge_metadata?: JudgeMetadata;
+    /**
+     * Heuristic root-cause label set by the engine
+     * (broken_code / flaky_judge / bad_rubric / missing_trace_data /
+     * stochastic_variance). Empty/undefined for passes and any result the
+     * engine declined to label. Mirrors
+     * engine/pkg/types/failure_class.go.
+     */
+    readonly failure_class?: string;
 }
+/**
+ * FailureClass values mirror engine/pkg/types/failure_class.go. Use
+ * these constants when matching on AssertionResult.failure_class so a
+ * typo in either codebase fails fast.
+ */
+declare const FailureClass: {
+    readonly BrokenCode: "broken_code";
+    readonly FlakyJudge: "flaky_judge";
+    readonly BadRubric: "bad_rubric";
+    readonly MissingTraceData: "missing_trace_data";
+    readonly StochasticVariance: "stochastic_variance";
+};
+type FailureClass = (typeof FailureClass)[keyof typeof FailureClass];
 /**
  * Map an assertion-type string to its pipeline layer (1–8). Returns 0
  * when the type is unknown so callers can route uncategorised results
@@ -228,14 +297,44 @@ interface InitializeResult {
     readonly max_trace_size_bytes?: number;
     readonly max_steps_per_trace?: number;
 }
+/**
+ * Limits for one evaluate_batch call on top of the session budget. A judge or
+ * embedding call that cannot reserve against them is not sent and its
+ * assertion is not_evaluated with reason "budget_exhausted".
+ */
+interface BatchBudget {
+    readonly max_judge_calls?: number;
+    readonly max_cost_usd?: number;
+}
 interface EvaluateBatchParams {
     readonly trace: Trace;
     readonly assertions: readonly Assertion[];
+    /** Omitted when no per-batch budget applies. */
+    readonly budget?: BatchBudget;
+}
+/**
+ * Attest-initiated model usage (embeddings, judges, judge repeats, bias
+ * probes, simulated-user messages). Agent calls not routed through attest
+ * are not counted.
+ */
+interface Usage {
+    readonly model_calls: number;
+    /** Judge-provider requests, the part of model_calls that max_judge_calls limits. */
+    readonly judge_calls: number;
+    readonly input_tokens: number;
+    readonly output_tokens: number;
+    readonly cached_tokens: number;
+    readonly cost_usd: number;
+    readonly retry_cost_usd: number;
+    readonly unpriced_calls: number;
+    readonly budget_refused: number;
 }
 interface EvaluateBatchResult {
     readonly results: readonly AssertionResult[];
     readonly total_cost?: number;
     readonly total_duration_ms?: number;
+    /** Always sent by the engine; validated by AttestClient.evaluateBatch. */
+    readonly usage: Usage;
     /**
      * True when this result was produced by an SDK simulation shim rather
      * than a real engine evaluation. Engine-produced results always carry
@@ -347,6 +446,13 @@ declare class ProtocolDesyncError extends Error {
     readonly diagnostics: readonly ProtocolDiagnostic[];
     constructor(message: string, diagnostics: readonly ProtocolDiagnostic[]);
 }
+/**
+ * The engine rejected an OTLP/JSON export (`attest-engine trace convert`). The message is
+ * the engine's error, which names the offending span or field.
+ */
+declare class OTLPConversionError extends Error {
+    constructor(message: string);
+}
 
 /**
  * Encode a JSON-RPC 2.0 request as an NDJSON line.
@@ -367,4 +473,4 @@ declare function extractResult(response: RPCResponse): unknown;
  */
 declare function extractId(response: RPCResponse): number;
 
-export { type AgentId, type Assertion, type AssertionId, type AssertionResult, type AssertionSpec, type BiasProbe, type ConstraintSpec, type ContentSpec, type ConversationMessage, type DriftReport, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, type EmbeddingSpec, EngineTimeoutError, type ErrorData, type EvaluateBatchParams, type EvaluateBatchResult, type GenerateUserMessageParams, type GenerateUserMessageResult, type InitializeParams, type InitializeResult, type JudgeAgreement, type JudgeMetadata, type LlmJudgeSpec, type PluginSpec, ProtocolDesyncError, type ProtocolDiagnostic, ProtocolDiagnosticBuffer, type ProtocolDiagnosticKind, ProtocolError, type ProtocolLogger, type QueryDriftParams, type QueryDriftResult, type RPCError, type RPCRequest, type RPCResponse, STATUS_HARD_FAIL, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, type SchemaSpec, type ShutdownResult, type SimulateFaultConfig, type SimulatePersona, type Step, type SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, type Trace, type TraceId, type TraceMetadata, type TraceSpec, type TraceTreeSpec, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId };
+export { type AgentId, type Assertion, type AssertionId, type AssertionResult, type AssertionSpec, type BatchBudget, type BiasProbe, type ConstraintSpec, type ContentSpec, type ConversationMessage, type DriftReport, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, type EmbeddingSpec, EngineTimeoutError, type ErrorData, type EvaluateBatchParams, type EvaluateBatchResult, type Exchange, FailureClass, type GenerateUserMessageParams, type GenerateUserMessageResult, type Handoff, type InitializeParams, type InitializeResult, type JudgeAgreement, type JudgeMetadata, type LlmJudgeSpec, OTLPConversionError, OUTCOME_FAIL, OUTCOME_PASS, OUTCOME_SKIP, type Outcome, type PluginSpec, ProtocolDesyncError, type ProtocolDiagnostic, ProtocolDiagnosticBuffer, type ProtocolDiagnosticKind, ProtocolError, type ProtocolLogger, type QueryDriftParams, type QueryDriftResult, REASON_BUDGET_EXHAUSTED, REASON_JUDGE_ABSTAINED, REASON_MISSING_EVIDENCE, REASON_PROVIDER_NOT_CONFIGURED, REASON_PROVIDER_UNAVAILABLE, REASON_SHORT_CIRCUITED, REASON_SIMULATION, type RPCError, type RPCRequest, type RPCResponse, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, type SchemaSpec, type ShutdownResult, type SimulateFaultConfig, type SimulatePersona, type Step, type StepError, type SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, type Trace, type TraceId, type TraceMetadata, type TraceSpec, type TraceTreeSpec, type Turn, type Usage, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId };

package/dist/esm/cli.d.ts

@@ -1,9 +1,24 @@
 #!/usr/bin/env node
 /**
- * CLI entry point for Attest: npx attest or attest (after global install).
+ * The `attest` command line (`npx attest`).
  *
- * Ported from Python sdks/python/src/attest/__main__.py
+ * `attest init` runs in the SDK. Every other command (`doctor`, `baseline`, `policy`,
+ * `cache`, `calibrate`, `version` and any command a later engine adds) runs in
+ * `attest-engine`: the CLI resolves a version-checked engine with `resolveEngine`, checks
+ * that the engine's `help` lists the command, and forwards the arguments, stdio and exit
+ * code unchanged. A missing, mismatched or too-old engine exits 1 before anything runs, so
+ * a gate such as `attest policy evaluate` never exits 0 without evaluating.
  */
-declare function main(argv?: string[]): void;
+/**
+ * `{command: summary}` from `<engine> help`, or null if it lists none.
+ *
+ * stdin is closed and the probe times out after 10 s, so an engine that predates the
+ * command registry (and starts the JSON-RPC server for `help`) cannot hang the CLI.
+ */
+declare function engineCommands(engine: string, timeoutMs?: number): Map<string, string> | null;
+/** Map a child's exit to a shell exit status (signal N → 128 + N). */
+declare function exitStatus(status: number | null, signal: NodeJS.Signals | null): number;
+/** Dispatch `attest` arguments and resolve to the process exit code. */
+declare function main(argv?: string[]): Promise<number>;
 
-export { main };
+export { engineCommands, exitStatus, main };