@attest-ai/core 1.0.1 → 1.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,12 @@
1
1
  import { ProtocolLogger, ProtocolDiagnostic, Trace, Assertion, BatchBudget, BatchContext, EvaluateBatchResult, DriftReport, SimulatePersona, ConversationMessage, SimulateFaultConfig, InitializeResult, Usage, Step, AssertionResult, Outcome } from './proto/index.js';
2
- export { AgentId, AssertionId, AssertionSpec, BiasProbe, ConstraintSpec, ContentSpec, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, EffectSpec, EmbeddingSpec, EngineTimeoutError, ErrorData, EvaluateBatchParams, Exchange, FailureClass, GenerateUserMessageParams, GenerateUserMessageResult, Handoff, InitializeParams, JudgeAgreement, JudgeMetadata, LlmJudgeSpec, OTLPConversionError, OUTCOME_FAIL, OUTCOME_PASS, OUTCOME_SKIP, PluginSpec, ProtocolDesyncError, ProtocolDiagnosticBuffer, ProtocolDiagnosticKind, ProtocolError, QueryDriftParams, QueryDriftResult, REASON_BUDGET_EXHAUSTED, REASON_JUDGE_ABSTAINED, REASON_MISSING_EVIDENCE, REASON_PROVIDER_NOT_CONFIGURED, REASON_PROVIDER_UNAVAILABLE, REASON_REPLAY_MISS, REASON_SHORT_CIRCUITED, REASON_SIMULATION, RPCError, RPCRequest, RPCResponse, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, SchemaSpec, ShutdownResult, StepError, SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EFFECT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, TraceId, TraceMetadata, TraceSpec, TraceTreeSpec, Turn, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId } from './proto/index.js';
2
+ export { AgentId, AssertionId, AssertionSpec, BiasProbe, ConstraintSpec, ContentSpec, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, EffectSpec, EmbeddingSpec, EngineTimeoutError, ErrorData, EvaluateBatchParams, Exchange, FailureClass, GenerateUserMessageParams, GenerateUserMessageResult, Handoff, InitializeParams, JudgeAgreement, JudgeMetadata, LlmJudgeSpec, OTLPConversionError, OUTCOME_FAIL, OUTCOME_PASS, OUTCOME_SKIP, PluginSpec, ProtocolDesyncError, ProtocolDiagnosticBuffer, ProtocolDiagnosticKind, ProtocolError, QueryDriftParams, QueryDriftResult, REASON_BUDGET_EXHAUSTED, REASON_JUDGE_ABSTAINED, REASON_JUDGE_MALFORMED, REASON_MISSING_EVIDENCE, REASON_PROVIDER_AUTH, REASON_PROVIDER_NOT_CONFIGURED, REASON_PROVIDER_UNAVAILABLE, REASON_REPLAY_MISS, REASON_SHORT_CIRCUITED, REASON_SIMULATION, RPCError, RPCRequest, RPCResponse, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, SchemaSpec, ShutdownResult, StepError, SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EFFECT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, TraceId, TraceMetadata, TraceSpec, TraceTreeSpec, Turn, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId } from './proto/index.js';
3
3
  import { ChildProcess } from 'node:child_process';
4
4
  import { Interface } from 'node:readline';
5
5
  import { AsyncLocalStorage } from 'node:async_hooks';
6
6
  import { EventEmitter } from 'node:events';
7
7
 
8
- declare const VERSION = "1.0.1";
9
- declare const ENGINE_VERSION = "1.0.1";
8
+ declare const VERSION = "1.1.0-rc.1";
9
+ declare const ENGINE_VERSION = "1.1.0-rc.1";
10
10
 
11
11
  /** Budget a timed-out request may have consumed; null means the whole remaining budget. */
12
12
  interface BudgetReservation {
@@ -93,6 +93,7 @@ declare class AttestClient {
93
93
  }): Promise<boolean>;
94
94
  queryDrift(assertionId: string, windowSize?: number, options?: {
95
95
  timeout?: number;
96
+ staticThreshold?: number;
96
97
  }): Promise<DriftReport>;
97
98
  generateUserMessage(persona: SimulatePersona, conversationHistory: readonly ConversationMessage[], faultConfig?: SimulateFaultConfig, options?: {
98
99
  timeout?: number;
@@ -357,11 +358,23 @@ declare class WorldChecks<C> {
357
358
  where?: Where;
358
359
  }): C;
359
360
  }
361
+ type ChainObserver = (chain: ExpectChain) => void;
362
+ /**
363
+ * Register the function called with every new {@link ExpectChain}. The Vitest integration uses
364
+ * it to fail a test that builds a chain it never evaluates. One observer at a time; pass
365
+ * `undefined` to remove it.
366
+ */
367
+ declare function setChainObserver(observer: ChainObserver | undefined): void;
360
368
  declare class ExpectChain {
361
369
  private readonly _result;
362
370
  private readonly _assertions;
363
371
  private _worldBytes;
372
+ private _evaluated;
364
373
  constructor(result: AgentResult);
374
+ /** True once the chain was handed to an evaluation. */
375
+ get evaluated(): boolean;
376
+ /** Record that the chain was handed to an evaluation. */
377
+ markEvaluated(): void;
365
378
  get assertions(): Assertion[];
366
379
  get trace(): Trace;
367
380
  private add;
@@ -1071,6 +1084,35 @@ declare class ExpectChain {
1071
1084
  */
1072
1085
  declare function attestExpect(result: AgentResult | Trace): ExpectChain;
1073
1086
 
1087
+ /**
1088
+ * Assertion ids, id scheme 2 (contract C3). Normative text:
1089
+ * proto/attest/v1/protocol-spec.md, "Assertion ids (id scheme 2)"; executable form:
1090
+ * protocol-tests/ids/assertion-ids.json.
1091
+ */
1092
+ declare const ID_SCHEME = 2;
1093
+ /** What an id is computed for; names the assertion in rejection errors. */
1094
+ interface AssertionIdentity {
1095
+ readonly scopeId: string;
1096
+ readonly type: string;
1097
+ readonly spec: unknown;
1098
+ }
1099
+ /** RFC 8785 (JCS) serialization of an assertion spec; no spec is `{}`. */
1100
+ declare function canonicalSpec(spec: unknown, identity?: AssertionIdentity): string;
1101
+ /** The exact bytes (as text) that are hashed: the JCS array `[scope_id, type, spec, occurrence_index]`. */
1102
+ declare function canonicalIdInput(scopeId: string, type: string, spec: unknown, occurrenceIndex: number): string;
1103
+ /** `a2_` plus the first 32 hex characters of SHA-256 over the UTF-8 bytes of the id input. */
1104
+ declare function assertionIdOf(scopeId: string, type: string, spec: unknown, occurrenceIndex: number): string;
1105
+ /**
1106
+ * Assigns ids inside one scope. The occurrence index of an assertion is the number of earlier
1107
+ * assertions given an id here with the same (type, canonical spec), in call order.
1108
+ */
1109
+ declare class AssertionIdAssigner {
1110
+ readonly scopeId: string;
1111
+ private readonly seen;
1112
+ constructor(scopeId: string);
1113
+ next(type: string, spec: unknown): string;
1114
+ }
1115
+
1074
1116
  /**
1075
1117
  * Capture: turn a run into a verified fixture, and fixtures into tests.
1076
1118
  *
@@ -1081,7 +1123,9 @@ declare function attestExpect(result: AgentResult | Trace): ExpectChain;
1081
1123
  * - `record()` captures the code in a callback in this process.
1082
1124
  * - `loadFixture()` reads a fixture (`proto/attest/v1/fixture-v1.schema.json`).
1083
1125
  * - `emitVitest()` writes the fixture's drafted contracts as a Vitest file, using the
1084
- * `attestExpect()` DSL. `attest capture` and `record()` call it.
1126
+ * `attestExpect()` DSL. `attest capture` and `record()` call it. For a fixture captured
1127
+ * from a command, the test re-runs that command under strict replay (`replayRun()`) and
1128
+ * evaluates the contracts on the new run's trace.
1085
1129
  */
1086
1130
 
1087
1131
  /** A capture did not produce a fixture, or produced one that is not verified. */
@@ -1133,6 +1177,15 @@ interface Fixture {
1133
1177
  declare function loadFixture(file: string | URL): Fixture;
1134
1178
  /** The `ExpectChain` method call that emits `contract`'s spec; throws when there is none. */
1135
1179
  declare function dslCall(contract: Contract): string;
1180
+ /**
1181
+ * Where a drafted replay test re-runs the fixture's source command: `cwd` is the directory
1182
+ * the capture ran in (the command's relative paths resolve there), `world` the world file
1183
+ * the capture hosted, if any.
1184
+ */
1185
+ interface Rerun {
1186
+ cwd: string;
1187
+ world?: string;
1188
+ }
1136
1189
  /**
1137
1190
  * Where the drafted test for a fixture goes: `tests/attest/<name>.test.ts` for
1138
1191
  * `tests/attest/fixtures/<name>.attest.json`.
@@ -1141,18 +1194,56 @@ declare function testPathFor(fixturePath: string): string;
1141
1194
  /**
1142
1195
  * Render a fixture's drafted contracts as a Vitest module.
1143
1196
  *
1144
- * A verified fixture becomes a test that enforces every contract with `evaluate`. An
1145
- * incident reproduction (`expect_fail`) becomes a test that checks, with `inspect`, that
1146
- * the expected failures still fail and the other contracts still pass; it stays green
1147
- * while it reproduces the incident.
1197
+ * With `rerun` and a fixture captured from a command, the test re-runs that command under
1198
+ * strict replay and evaluates every contract on the new run's trace (an incident
1199
+ * reproduction included: it passes once the agent no longer reproduces the incident).
1200
+ * Otherwise a verified fixture becomes a test that enforces every contract on the recorded
1201
+ * trace with `evaluate`, and an incident reproduction (`expect_fail`) a test that checks,
1202
+ * with `inspect`, that the expected failures still fail and the other contracts still
1203
+ * pass; it stays green while it reproduces the incident.
1148
1204
  */
1149
- declare function renderVitest(fixture: Fixture, fixturePath: string, testPath: string): string;
1205
+ declare function renderVitest(fixture: Fixture, fixturePath: string, testPath: string, rerun?: Rerun): string;
1150
1206
  /**
1151
1207
  * Write the drafted Vitest file for a fixture unless it already exists. Returns the path
1152
1208
  * written, or null when a test file is already there (re-capturing never overwrites an
1153
- * edited test).
1209
+ * edited test). `rerun` makes the test of a command fixture re-run the command under strict
1210
+ * replay (see `renderVitest`).
1211
+ */
1212
+ declare function emitVitest(fixturePath: string, rerun?: Rerun): string | null;
1213
+ interface ReplayRunOptions {
1214
+ /** The argv to run, written in the reviewed test; must equal the fixture's `source.command`. */
1215
+ command: readonly string[];
1216
+ /** Directory to run the command in (the one the capture ran in). */
1217
+ cwd?: string | URL;
1218
+ /** World file the capture hosted, when the fixture has a `world` section. */
1219
+ world?: string | URL;
1220
+ /** Engine binary; default is the SDK's discovery order. */
1221
+ enginePath?: string;
1222
+ /** Milliseconds before the run is abandoned (default 300000). */
1223
+ timeoutMs?: number;
1224
+ }
1225
+ /**
1226
+ * Re-run a fixture's source command under strict replay and return the new run's trace.
1227
+ *
1228
+ * Runs `attest-engine replay --mode strict --no-live --trace-out ... -- <command>`: the
1229
+ * OpenAI- and Anthropic-compatible calls the command makes are answered from the fixture's
1230
+ * recorded exchanges, and a call that matches no recording fails. Other providers are not
1231
+ * blocked: the replay proxy redirects only `OPENAI_BASE_URL` and `ANTHROPIC_BASE_URL`, and
1232
+ * `ATTEST_EGRESS=block` only affects processes that install attest's guard.
1233
+ * `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` and `GEMINI_API_KEY` are removed from the
1234
+ * environment and `OPENAI_API_KEY` is a placeholder; run without other providers'
1235
+ * credentials. The new run's trace is written unredacted to a private temporary directory
1236
+ * removed afterwards.
1237
+ *
1238
+ * The fixture file can be changed by a pull request, so its `source.command` is never run:
1239
+ * `options.command` is, and the call rejects when the fixture's differs, or when it has
1240
+ * redacted or templated (`{{id_N}}`) arguments.
1241
+ *
1242
+ * Rejects with `CaptureError` when the fixture has no replayable source command, a request
1243
+ * matched no recording, the command exited differently than when recorded, or no trace came
1244
+ * out.
1154
1245
  */
1155
- declare function emitVitest(fixturePath: string): string | null;
1246
+ declare function replayRun(fixture: string | URL, options: ReplayRunOptions): Promise<Trace>;
1156
1247
  /** A running in-process capture. */
1157
1248
  interface Capture {
1158
1249
  name: string;
@@ -1411,6 +1502,17 @@ declare class PluginRegistry {
1411
1502
  listPlugins(pluginType?: string): string[];
1412
1503
  }
1413
1504
  declare function executePluginAssertion(plugin: AttestPlugin, trace: Trace, spec: Record<string, unknown>, client: AttestClient, traceId: string, assertionId: string, timeoutMs?: number): Promise<PluginResult>;
1505
+ /**
1506
+ * Run the registered plugin of every `plugin` assertion and submit its result to the engine
1507
+ * (`submit_plugin_result`) so the following `evaluate_batch` finds it. Order matters: submit
1508
+ * before evaluate. A plugin assertion whose plugin is not registered is left without a
1509
+ * submission, so the engine reports it `not_evaluated` (`missing_evidence`) instead of a failure.
1510
+ */
1511
+ declare function submitPluginResults(registry: PluginRegistry, client: AttestClient, trace: Trace, assertions: readonly {
1512
+ assertion_id: string;
1513
+ type: string;
1514
+ spec: Record<string, unknown>;
1515
+ }[]): Promise<void>;
1414
1516
  /** Explicitly register a plugin instance into the registry. */
1415
1517
  declare function registerPlugin(registry: PluginRegistry, name: string, plugin: AttestPlugin): void;
1416
1518
  /**
@@ -1431,16 +1533,26 @@ declare class Sampler {
1431
1533
  constructor(rate: number);
1432
1534
  shouldSample(): boolean;
1433
1535
  }
1536
+ type AlertStatus = "drift_detected" | "hard_fail";
1537
+ /** Webhook JSON body shared by both SDK runners. Every key is always present. */
1434
1538
  interface AlertPayload {
1435
- drift_type: string;
1436
- score: number | string;
1437
- trace_id: string;
1438
- [key: string]: unknown;
1539
+ runner: string;
1540
+ assertion_id: string;
1541
+ /** Alert trigger: `hard_fail` wins over `drift_detected` when both hold. */
1542
+ status: AlertStatus;
1543
+ /** Status of the evaluated assertion result. */
1544
+ result_status: "pass" | "soft_fail" | "hard_fail";
1545
+ score: number | null;
1546
+ baseline_mean: number | null;
1547
+ baseline_count: number | null;
1548
+ static_threshold: number;
1549
+ trace_id: string | null;
1439
1550
  }
1440
1551
  declare class AlertDispatcher extends EventEmitter {
1441
1552
  private readonly webhookUrl;
1442
1553
  private readonly slackUrl;
1443
- constructor(webhookUrl?: string, slackUrl?: string);
1554
+ private readonly timeoutMs;
1555
+ constructor(webhookUrl?: string, slackUrl?: string, timeoutMs?: number);
1444
1556
  dispatch(alert: AlertPayload): Promise<void>;
1445
1557
  private postJson;
1446
1558
  }
@@ -1451,22 +1563,53 @@ declare class ContinuousEvalRunner {
1451
1563
  private readonly dispatcher;
1452
1564
  private readonly queue;
1453
1565
  private readonly maxSize;
1566
+ private readonly name;
1567
+ private readonly drainTimeoutMs;
1568
+ private readonly onError;
1569
+ private readonly inFlight;
1570
+ private chain;
1454
1571
  private running;
1455
1572
  private loopTimer;
1456
- constructor(client: AttestClient, assertions: readonly Assertion[], options?: {
1573
+ constructor(client: AttestClient, assertions: readonly Assertion[], options: {
1574
+ /** Id scope of the assertions (C3): their ids derive from this name, so keep it stable. */
1575
+ name: string;
1457
1576
  sampleRate?: number;
1458
1577
  alertWebhook?: string;
1459
1578
  alertSlackUrl?: string;
1460
1579
  maxSize?: number;
1580
+ /** Called with every evaluation, drift-read and dispatch error (also logged and emitted as `alertError`). */
1581
+ onError?: (error: unknown) => void;
1582
+ /** Bound on one alert POST (default 10000). */
1583
+ dispatchTimeoutMs?: number;
1584
+ /** Bound on how long `stop()` waits for in-flight work (default 15000). */
1585
+ drainTimeoutMs?: number;
1461
1586
  });
1462
1587
  get alertEmitter(): AlertDispatcher;
1588
+ /** Runs `fn` after every previously queued evaluation, so evaluate → query_drift pairs never interleave. */
1589
+ private serialize;
1590
+ /** Surfaces an error to the log and `onError` without ever throwing or leaving a rejected promise. */
1591
+ private reportError;
1592
+ private logObserverFailure;
1593
+ private emitError;
1463
1594
  evaluateTrace(trace: Trace): Promise<EvaluateBatchResult | null>;
1595
+ private evaluateAndAlert;
1596
+ private staticThresholdFor;
1597
+ private alertIfNeeded;
1464
1598
  submit(trace: Trace): boolean;
1465
1599
  start(): void;
1466
- stop(): void;
1600
+ /**
1601
+ * Stops the loop, then processes the traces already queued and waits for in-flight evaluations
1602
+ * and dispatches, all bounded by `drainTimeoutMs`. At the bound the whole remaining queue is
1603
+ * discarded (nothing is evaluated after the timeout and nothing is left for a restart), the
1604
+ * evaluation already in flight is left to finish, and the timeout with the number of discarded
1605
+ * traces is reported through `onError`.
1606
+ */
1607
+ stop(): Promise<void>;
1608
+ private drainQueue;
1467
1609
  get queueLength(): number;
1468
1610
  get isRunning(): boolean;
1469
1611
  private scheduleNext;
1612
+ private tick;
1470
1613
  }
1471
1614
 
1472
1615
  /**
@@ -1660,34 +1803,81 @@ declare class ManualAdapter {
1660
1803
  createBuilder(): TraceBuilder;
1661
1804
  }
1662
1805
 
1663
- interface OpenAIUsage {
1806
+ /** Options every single-call provider adapter accepts. */
1807
+ interface ProviderCallOptions {
1808
+ inputMessages?: Record<string, unknown>[];
1809
+ costUsd?: number;
1810
+ /** Explicit latency; wins over the span between `startedAtMs` and `endedAtMs`. */
1811
+ latencyMs?: number;
1812
+ /** Wall-clock ms when the request was sent. */
1813
+ startedAtMs?: number;
1814
+ /** Wall-clock ms when the response was received. */
1815
+ endedAtMs?: number;
1816
+ }
1817
+
1818
+ interface ChatUsage {
1819
+ prompt_tokens?: number;
1820
+ completion_tokens?: number;
1664
1821
  total_tokens?: number;
1665
1822
  }
1666
- interface OpenAIToolCall {
1823
+ interface ChatToolCall {
1667
1824
  function: {
1668
1825
  name: string;
1669
1826
  arguments: string;
1670
1827
  };
1671
1828
  }
1672
- interface OpenAIMessage {
1829
+ interface ChatMessage {
1673
1830
  content: string | null;
1674
- tool_calls?: OpenAIToolCall[];
1675
- }
1676
- interface OpenAIChoice {
1677
- message: OpenAIMessage;
1831
+ refusal?: string | null;
1832
+ tool_calls?: ChatToolCall[];
1678
1833
  }
1834
+ /** An OpenAI Chat Completions response. */
1679
1835
  interface OpenAIChatCompletion {
1836
+ object?: string;
1680
1837
  model?: string;
1681
- usage?: OpenAIUsage;
1682
- choices: OpenAIChoice[];
1838
+ usage?: ChatUsage;
1839
+ choices: {
1840
+ message: ChatMessage;
1841
+ }[];
1683
1842
  }
1843
+ type ResponsesContentPart = {
1844
+ type: "output_text";
1845
+ text: string;
1846
+ } | {
1847
+ type: "refusal";
1848
+ refusal: string;
1849
+ };
1850
+ type ResponsesOutputItem = {
1851
+ type: "message";
1852
+ content: ResponsesContentPart[];
1853
+ } | {
1854
+ type: "function_call";
1855
+ name: string;
1856
+ arguments: string;
1857
+ } | {
1858
+ type: string;
1859
+ };
1860
+ /** An OpenAI Responses API response. */
1861
+ interface OpenAIResponse {
1862
+ object: "response";
1863
+ model?: string;
1864
+ usage?: {
1865
+ input_tokens?: number;
1866
+ output_tokens?: number;
1867
+ total_tokens?: number;
1868
+ };
1869
+ output: ResponsesOutputItem[];
1870
+ }
1871
+ /**
1872
+ * Captures OpenAI Chat Completions and Responses API calls into Attest traces.
1873
+ *
1874
+ * A model refusal is the response's message: it becomes the completion and the trace
1875
+ * output, and is also recorded as `refusal` on the llm_call step.
1876
+ */
1684
1877
  declare class OpenAIAdapter {
1685
1878
  private readonly agentId;
1686
1879
  constructor(agentId?: string);
1687
- traceFromResponse(response: OpenAIChatCompletion, options?: {
1688
- inputMessages?: Record<string, unknown>[];
1689
- costUsd?: number;
1690
- latencyMs?: number;
1880
+ traceFromResponse(response: OpenAIChatCompletion | OpenAIResponse, options?: ProviderCallOptions & {
1691
1881
  structuredOutput?: Record<string, unknown>;
1692
1882
  }): Trace;
1693
1883
  }
@@ -1696,29 +1886,29 @@ interface AnthropicTextBlock {
1696
1886
  type: "text";
1697
1887
  text: string;
1698
1888
  }
1889
+ /** A client-executed (`tool_use`) or Anthropic-executed (`server_tool_use`) tool call. */
1699
1890
  interface AnthropicToolUseBlock {
1700
- type: "tool_use";
1891
+ type: "tool_use" | "server_tool_use";
1701
1892
  name: string;
1702
1893
  input: Record<string, unknown>;
1703
1894
  }
1704
- type AnthropicContentBlock = AnthropicTextBlock | AnthropicToolUseBlock;
1705
- interface AnthropicUsage {
1706
- input_tokens: number;
1707
- output_tokens: number;
1895
+ /** Any other block (tool results, thinking, ...): carries no text or call of its own. */
1896
+ interface AnthropicOtherBlock {
1897
+ type: string;
1708
1898
  }
1899
+ type AnthropicContentBlock = AnthropicTextBlock | AnthropicToolUseBlock | AnthropicOtherBlock;
1709
1900
  interface AnthropicMessage {
1710
1901
  model?: string;
1711
- usage?: AnthropicUsage;
1902
+ usage?: {
1903
+ input_tokens: number;
1904
+ output_tokens: number;
1905
+ };
1712
1906
  content: AnthropicContentBlock[];
1713
1907
  }
1714
1908
  declare class AnthropicAdapter {
1715
1909
  private readonly agentId;
1716
1910
  constructor(agentId?: string);
1717
- traceFromResponse(response: AnthropicMessage, options?: {
1718
- inputMessages?: Record<string, unknown>[];
1719
- costUsd?: number;
1720
- latencyMs?: number;
1721
- }): Trace;
1911
+ traceFromResponse(response: AnthropicMessage, options?: ProviderCallOptions): Trace;
1722
1912
  }
1723
1913
 
1724
1914
  interface GeminiFunctionCall {
@@ -1727,6 +1917,8 @@ interface GeminiFunctionCall {
1727
1917
  }
1728
1918
  interface GeminiPart {
1729
1919
  text?: string;
1920
+ /** Thought-summary parts are the model's reasoning, not the answer. */
1921
+ thought?: boolean;
1730
1922
  functionCall?: GeminiFunctionCall;
1731
1923
  }
1732
1924
  interface GeminiCandidate {
@@ -1734,25 +1926,39 @@ interface GeminiCandidate {
1734
1926
  parts: GeminiPart[];
1735
1927
  };
1736
1928
  }
1929
+ interface GeminiUsage {
1930
+ promptTokenCount?: number;
1931
+ candidatesTokenCount?: number;
1932
+ totalTokenCount?: number;
1933
+ }
1737
1934
  interface GeminiResponse {
1738
1935
  text?: string;
1936
+ modelVersion?: string;
1937
+ usageMetadata?: GeminiUsage;
1739
1938
  candidates?: GeminiCandidate[];
1740
1939
  }
1940
+ interface GeminiOptions extends ProviderCallOptions {
1941
+ /** A plain-text prompt; `inputMessages` is the standard way to pass the input. */
1942
+ inputText?: string;
1943
+ /** Overrides the response's `modelVersion`. */
1944
+ model?: string;
1945
+ }
1741
1946
  declare class GeminiAdapter {
1742
1947
  private readonly agentId;
1743
1948
  constructor(agentId?: string);
1744
- traceFromResponse(response: GeminiResponse, options?: {
1745
- inputText?: string;
1746
- costUsd?: number;
1747
- latencyMs?: number;
1748
- model?: string;
1749
- }): Trace;
1949
+ traceFromResponse(response: GeminiResponse, options?: GeminiOptions): Trace;
1750
1950
  }
1751
1951
 
1752
1952
  interface OllamaResponse {
1753
1953
  model?: string;
1754
1954
  message?: {
1755
1955
  content?: string;
1956
+ tool_calls?: {
1957
+ function?: {
1958
+ name?: string;
1959
+ arguments?: Record<string, unknown>;
1960
+ };
1961
+ }[];
1756
1962
  };
1757
1963
  eval_count?: number;
1758
1964
  prompt_eval_count?: number;
@@ -1760,10 +1966,7 @@ interface OllamaResponse {
1760
1966
  declare class OllamaAdapter {
1761
1967
  private readonly agentId;
1762
1968
  constructor(agentId?: string);
1763
- traceFromResponse(response: OllamaResponse, options?: {
1764
- inputMessages?: Record<string, unknown>[];
1765
- latencyMs?: number;
1766
- }): Trace;
1969
+ traceFromResponse(response: OllamaResponse, options?: Omit<ProviderCallOptions, "costUsd">): Trace;
1767
1970
  }
1768
1971
 
1769
1972
  type AttributeValue = string | number | boolean | readonly (string | number | boolean | null | undefined)[];
@@ -1814,58 +2017,78 @@ declare class OTelAdapter {
1814
2017
  }
1815
2018
 
1816
2019
  /**
1817
- * Minimal type definitions for `@langchain/core` callback handler.
1818
- * These avoid a hard dependency on `@langchain/core` — the actual types
1819
- * are checked at runtime when the user provides real LangChain objects.
2020
+ * The part of `@langchain/core`'s `BaseCallbackHandler` this adapter relies on. It is
2021
+ * declared locally so the published typings do not import an optional peer dependency.
2022
+ */
2023
+ interface CallbackHandlerBase {
2024
+ name: string;
2025
+ awaitHandlers: boolean;
2026
+ raiseError: boolean;
2027
+ }
2028
+ type CallbackHandlerConstructor = new () => CallbackHandlerBase;
2029
+ declare const BaseCallbackHandler: CallbackHandlerConstructor;
2030
+ type Values = Record<string, unknown>;
2031
+ interface Message {
2032
+ content?: unknown;
2033
+ usage_metadata?: {
2034
+ input_tokens?: number;
2035
+ output_tokens?: number;
2036
+ total_tokens?: number;
2037
+ };
2038
+ }
2039
+ interface LLMResult {
2040
+ generations: {
2041
+ text?: string;
2042
+ message?: Message;
2043
+ }[][];
2044
+ llmOutput?: Values;
2045
+ }
2046
+ /**
2047
+ * Captures a LangChain.js run into an Attest trace.
2048
+ *
2049
+ * A `BaseCallbackHandler` from `@langchain/core`: pass it in `callbacks` and call
2050
+ * `buildTrace()` once the run resolves. Model and tool runs are keyed by `runId`, so
2051
+ * interleaved runs are attributed correctly; steps are recorded in completion order.
2052
+ *
2053
+ * ```ts
2054
+ * const adapter = new LangChainAdapter("my-agent");
2055
+ * await agent.invoke(input, { callbacks: [adapter] });
2056
+ * const trace = adapter.buildTrace();
2057
+ * ```
1820
2058
  */
1821
- interface LangChainLLMResult {
1822
- generations: Array<Array<{
1823
- text: string;
1824
- [key: string]: unknown;
1825
- }>>;
1826
- llmOutput?: Record<string, unknown>;
1827
- }
1828
- interface LangChainToolEnd {
1829
- name?: string;
1830
- output: string;
1831
- [key: string]: unknown;
1832
- }
1833
- interface LangChainChainValues {
1834
- [key: string]: unknown;
1835
- }
1836
- declare class LangChainAdapter {
2059
+ declare class LangChainAdapter extends BaseCallbackHandler {
2060
+ name: string;
1837
2061
  private readonly agentId;
1838
- private readonly llmCalls;
1839
- private readonly toolCalls;
1840
- private chainOutput;
1841
- private startTime;
2062
+ private input;
2063
+ private output;
2064
+ private steps;
2065
+ private llmStarts;
2066
+ private toolStarts;
2067
+ private totalTokens;
2068
+ private model;
2069
+ private startedAtMs;
2070
+ private endedAtMs;
1842
2071
  constructor(agentId?: string);
1843
- /**
1844
- * Reset accumulated state for a new trace capture session.
1845
- */
2072
+ /** Handlers are shared by the callback manager, never cloned: a clone would lose the capture. */
2073
+ copy(): this;
2074
+ /** Reset accumulated state for a new capture session. */
1846
2075
  reset(): void;
1847
- /**
1848
- * Call this when a chain starts (maps to `handleChainStart`).
1849
- */
1850
- handleChainStart(): void;
1851
- /**
1852
- * Call this when an LLM call completes (maps to `handleLLMEnd`).
1853
- */
1854
- handleLLMEnd(output: LangChainLLMResult): void;
1855
- /**
1856
- * Call this when a tool call completes (maps to `handleToolEnd`).
1857
- */
1858
- handleToolEnd(output: string, metadata?: LangChainToolEnd): void;
1859
- /**
1860
- * Call this when a chain completes (maps to `handleChainEnd`).
1861
- */
1862
- handleChainEnd(output: LangChainChainValues): void;
1863
- /**
1864
- * Build a Trace from accumulated callback events.
1865
- *
1866
- * @param options.costUsd - Total cost override.
2076
+ private now;
2077
+ handleChainStart(_chain: unknown, inputs: Values, _runId: string, parentRunId?: string): void;
2078
+ handleChainEnd(outputs: Values, _runId: string, parentRunId?: string): void;
2079
+ handleChatModelStart(_llm: unknown, _messages: unknown[][], runId: string, _parentRunId?: string, extraParams?: Values): void;
2080
+ handleLLMStart(_llm: unknown, _prompts: string[], runId: string, _parentRunId?: string, extraParams?: Values): void;
2081
+ handleLLMEnd(output: LLMResult, runId: string): void;
2082
+ handleToolStart(tool: unknown, input: string, runId: string, _parentRunId?: string, _tags?: string[], _metadata?: Values, runName?: string): void;
2083
+ handleToolEnd(output: unknown, runId: string): void;
2084
+ handleToolError(error: unknown, runId: string): void;
2085
+ private finishTool;
2086
+ private step;
2087
+ /**
2088
+ * Build a Trace from the captured callback events.
2089
+ *
2090
+ * @param options.costUsd - Total cost.
1867
2091
  * @param options.model - Model name override.
1868
- * @returns A finalized Trace object.
1869
2092
  */
1870
2093
  buildTrace(options?: {
1871
2094
  costUsd?: number;
@@ -1877,5 +2100,5 @@ interface TraceAdapter {
1877
2100
  traceFromResponse(response: unknown, options?: Record<string, unknown>): Trace;
1878
2101
  }
1879
2102
 
1880
- export { ADVERSARIAL_USER, Agent, AgentResult, AlertDispatcher, AnthropicAdapter, Assertion, AssertionResult, AttestClient, BatchBudget, BatchContext, CONFUSED_USER, COOPERATIVE_USER, CaptureError, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EgressBlockedError, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, InitializeResult, LangChainAdapter, ManualAdapter, MockToolRegistry, OTelAdapter, OllamaAdapter, OpenAIAdapter, Outcome, PluginRegistry, ProtocolDiagnostic, ProtocolLogger, RepeatResult, ReplayError, ReplayMissError, ReplayNotEvaluatedError, Sampler, ScenarioResult, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, TraceBuilder, TraceTree, Usage, VERSION, WorldChecks, WorldError, activeBuilder, addUsage, agent, attestExpect, computeAgreement, config, delegate, dslCall, egressBlockRequested, emitVitest, executePluginAssertion, faultInject, firstMiss, formatUsage, getSampleRate, installEgressGuard, isProviderHost, isSimulationMode, liveLine, loadFixture, loadLabelsCSV, loadLabelsJSONL, loadPackagePlugins, mockTool, parseUsage, promptHash, record, registerPlugin, renderAssertionFailure, renderDiagnostics, renderSummary, renderVerdictDiagnostics, renderVitest, repeat, requireSimulationCapability, resetConfig, resolveMode, scenario, simulationEnvEnabled, startReplay, startWorld, sumUsage, testPathFor, tier, uninstallEgressGuard, usageSummaryLine, zeroUsage };
1881
- export type { AgreementResult, AlertPayload, AttestClientOptions, AttestPlugin, Capture, Contract, ContractResult, FaultInjectOptions, Fixture, LabelPair, LabeledRecord, MockTool, MockToolFn, PluginResult, RecordOptions, RecordResult, RenderOptions, ReplayMode, ReplayModeSetting, ReplayOptions, ReplayRecord, ReplaySession, ReplaySummary, ScenarioConfig, ScenarioContext, ScenarioOptions, TraceAdapter, Verdict, WorldOptions, WorldSession, WorldSnapshot };
2103
+ export { ADVERSARIAL_USER, Agent, AgentResult, AlertDispatcher, AnthropicAdapter, Assertion, AssertionIdAssigner, AssertionResult, AttestClient, BatchBudget, BatchContext, CONFUSED_USER, COOPERATIVE_USER, CaptureError, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EgressBlockedError, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, ID_SCHEME, InitializeResult, LangChainAdapter, ManualAdapter, MockToolRegistry, OTelAdapter, OllamaAdapter, OpenAIAdapter, Outcome, PluginRegistry, ProtocolDiagnostic, ProtocolLogger, RepeatResult, ReplayError, ReplayMissError, ReplayNotEvaluatedError, Sampler, ScenarioResult, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, TraceBuilder, TraceTree, Usage, VERSION, WorldChecks, WorldError, activeBuilder, addUsage, agent, assertionIdOf, attestExpect, canonicalIdInput, canonicalSpec, computeAgreement, config, delegate, dslCall, egressBlockRequested, emitVitest, executePluginAssertion, faultInject, firstMiss, formatUsage, getSampleRate, installEgressGuard, isProviderHost, isSimulationMode, liveLine, loadFixture, loadLabelsCSV, loadLabelsJSONL, loadPackagePlugins, mockTool, parseUsage, promptHash, record, registerPlugin, renderAssertionFailure, renderDiagnostics, renderSummary, renderVerdictDiagnostics, renderVitest, repeat, replayRun, requireSimulationCapability, resetConfig, resolveMode, scenario, setChainObserver, simulationEnvEnabled, startReplay, startWorld, submitPluginResults, sumUsage, testPathFor, tier, uninstallEgressGuard, usageSummaryLine, zeroUsage };
2104
+ export type { AgreementResult, AlertPayload, AttestClientOptions, AttestPlugin, Capture, Contract, ContractResult, FaultInjectOptions, Fixture, LabelPair, LabeledRecord, MockTool, MockToolFn, PluginResult, RecordOptions, RecordResult, RenderOptions, ReplayMode, ReplayModeSetting, ReplayOptions, ReplayRecord, ReplayRunOptions, ReplaySession, ReplaySummary, Rerun, ScenarioConfig, ScenarioContext, ScenarioOptions, TraceAdapter, Verdict, WorldOptions, WorldSession, WorldSnapshot };