@atlanai/sdk 0.2.1 → 0.2.3

package/README.md

@@ -39,10 +39,41 @@ Pass a default workspace once instead of repeating it on every call:
 const client = new AtlanClient({ gatewayOrigin, bearerToken, workspace: "workspace_01example" });
 ```
 
-## Start an eval run
+## Run an eval
+
+`Eval()` executes the task and scorers, creates the Registry experiment, emits
+one trace per case, uploads trace-linked results, and finalizes the run. Registry
+derives the durable score summary during that finalization:
+
+```typescript
+import { Eval } from "@atlanai/sdk";
+
+const result = await Eval("Anthropic Evaluation", {
+  data: () => [
+    { input: "What is 2+2?", expected: "4" },
+    { input: "What is the capital of France?", expected: "Paris" },
+  ],
+  task: async (input) => callModel(input),
+  scores: [{
+    name: "accuracy",
+    scorer: ({ output, expected }) => output === expected ? 1 : 0,
+  }],
+});
+
+console.log(result.experimentId);
+```
+
+Set `ATLAN_API_KEY` and `ATLAN_WORKSPACE_ID`. Replace inline `data` with
+`dataset: "dataset_..."` or one exact Registry dataset name. The runner refuses
+to proceed when tracing is disabled because trace-less eval rows break the
+evidence chain. After flushing, it reads every case trace back through the
+experiment filter before it uploads results; a missing trace marks the
+experiment failed.
+
+## Control an eval lifecycle directly
 
 Resolve an existing dataset by artifact ID or exact name, then create the
-Registry experiment before the runner emits any traces:
+Registry experiment before another runner emits any traces:
 
 ```typescript
 import { createContextManifest, startExperiment } from "@atlanai/sdk";
@@ -67,31 +98,51 @@ await propagateAttributes(run.traceOptions, () => existingRunner());
 
 `run.experiment` is the generated create response and `run.id` is its
 `experimentId`. The helper pins the resolved dataset version and context
-manifest in the immutable experiment config. Execution, result upload,
-summarization, and terminal status remain the existing `client.experiments`
-operations.
+manifest in the immutable experiment config. Use it when another harness owns
+execution, result upload, and finalization.
 
 ## Tracing
 
 ```typescript
-import * as atlan from "@atlanai/sdk/tracing";
+import { initLogger, wrapTraced } from "@atlanai/sdk/tracing";
 
-const tracer = atlan.init({ apiKey: "<your-tracing-key>", workspaceId: "workspace_01example" });
+const logger = initLogger({ projectName: "support-agent" });
+const handleRequest = wrapTraced(async (input: string) => callModel(input));
 
-await tracer.startAsCurrentSpan("handle-request", { asType: "task" }, async (span) => {
-  span.update({ model: "claude-sonnet-5", usage: { inputTokens: 120, outputTokens: 40 } });
-});
+await logger.traced(async (span) => {
+  const output = await handleRequest("hello");
+  span.log({ input: "hello", output });
+}, { name: "handle-request", type: "task" });
+
+await logger.flush();
 ```
 
-Tracing has its own API key and lifecycle — it does not reuse the management
-client's bearer token or transport. It ships as a separate subpath
+Tracing has its own API key and lifecycle. It does not reuse the management
+client's transport. It ships as a separate subpath
 (`@atlanai/sdk/tracing`) so importing it doesn't pull OpenTelemetry into a
-management-only bundle.
+management-only bundle. npm still installs the OpenTelemetry dependencies with
+the package because npm cannot make dependencies conditional on an imported
+subpath; splitting tracing into a second package would trade that install size
+for a second version and release lifecycle.
+
+Vercel AI SDK v3-v6 uses its native OpenTelemetry path:
+
+```typescript
+import * as ai from "ai";
+import { initLogger, wrapAISDK } from "@atlanai/sdk/tracing";
+
+initLogger({ projectName: "support-agent" });
+const { generateText, streamText } = wrapAISDK(ai);
+```
+
+See [evals and tracing](../docs/evals-and-tracing.md) for framework setup,
+context manifests, serverless flushing, and the tested integration matrix.
 
 ## Errors
 
 Every non-2xx response rejects with `AtlanAPIError`, with `.status`, `.code`,
-and (where the gateway includes one) `.traceId`:
+and (where the gateway includes one) `.traceId`. A failure before any response
+is also normalized as `AtlanAPIError` with `status === 0`:
 
 ```typescript
 import { AtlanAPIError } from "@atlanai/sdk";

package/dist/client.d.ts

@@ -1,5 +1,6 @@
 import { type Operation } from "./operations";
 import { type Resource } from "./resources";
+import type { ResourceMap, ResourceRoots } from "./resource-types";
 type TokenProvider = () => string | Promise<string>;
 type FetchImplementation = typeof fetch;
 type GeneratedConfiguration = {
@@ -45,11 +46,13 @@ export declare class ServiceNamespace {
 }
 /** Build a namespace that also answers its operations by method name. */
 export declare function createService(name: string, module: GeneratedModule, configuration: GeneratedConfiguration, serviceOperations: readonly Omit<Operation, "service">[]): Service;
+export interface AtlanClient extends ResourceRoots {
+}
 export declare class AtlanClient {
     /** Per-service generated clients, for an operation the map has not surfaced. */
     readonly raw: Readonly<Record<string, Service>>;
     /** Every resource, keyed by its dotted path. */
-    readonly resources: Readonly<Record<string, Resource>>;
+    readonly resources: ResourceMap & Readonly<Record<string, Resource>>;
     readonly workspace: string | undefined;
     constructor(options: AtlanClientOptions);
     /**

package/dist/client.js

@@ -154,6 +154,9 @@ function wrapGeneratedAPI(api, operationMethods) {
                     if (response instanceof Response) {
                         throw await errors_1.AtlanAPIError.fromResponse(response);
                     }
+                    const cause = error?.cause;
+                    if (cause instanceof errors_1.AtlanAPIError)
+                        throw cause;
                     throw error;
                 }
             };
@@ -205,7 +208,7 @@ class AtlanClient {
         // Resources, not services. A service is a base path; a resource is what a
         // caller is actually working with, so `client.sessions.messages` reads the
         // way the API does.
-        const resources = (0, resources_1.buildResources)(resources_1.resourceMap, namespaces);
+        const resources = (0, resources_1.buildResources)(resources_1.resourceMap, namespaces, workspace);
         this.resources = Object.freeze(resources);
         Object.assign(this, resources);
         this.workspace = workspace;
@@ -264,10 +267,19 @@ function withTimeout(baseFetch, timeoutMs) {
         const callerSignal = init?.signal;
         const forwardAbort = () => controller.abort(callerSignal?.reason);
         callerSignal?.addEventListener("abort", forwardAbort, { once: true });
-        const timer = setTimeout(() => controller.abort(new Error("request timed out")), timeoutMs);
+        let timedOut = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            controller.abort(new Error("request timed out"));
+        }, timeoutMs);
         try {
             return await baseFetch(input, { ...init, signal: controller.signal });
         }
+        catch (error) {
+            if (callerSignal?.aborted || error instanceof errors_1.AtlanAPIError)
+                throw error;
+            throw errors_1.AtlanAPIError.fromConnectionError(timedOut);
+        }
         finally {
             clearTimeout(timer);
             callerSignal?.removeEventListener("abort", forwardAbort);

package/dist/errors.d.ts

@@ -12,4 +12,5 @@ export declare class AtlanAPIError extends Error {
         traceId?: string;
     });
     static fromResponse(response: Response): Promise<AtlanAPIError>;
+    static fromConnectionError(timedOut?: boolean): AtlanAPIError;
 }

package/dist/errors.js

@@ -9,7 +9,10 @@ class AtlanAPIError extends Error {
     traceId;
     constructor(options) {
         const suffix = options.code ? ` (${options.code})` : "";
-        super(`Atlan API request failed with status ${options.status}${suffix}`);
+        const message = options.status === 0
+            ? `${options.title ?? "Could not reach the Atlan Gateway"}${options.detail ? `; ${options.detail}` : ""}`
+            : `Atlan API request failed with status ${options.status}${suffix}`;
+        super(message);
         this.name = "AtlanAPIError";
         this.status = options.status;
         this.code = options.code ?? "";
@@ -39,5 +42,13 @@ class AtlanAPIError extends Error {
             traceId: typeof problem.trace_id === "string" ? problem.trace_id : undefined,
         });
     }
+    static fromConnectionError(timedOut = false) {
+        return new AtlanAPIError({
+            status: 0,
+            code: timedOut ? "request_timeout" : "connection_error",
+            title: timedOut ? "Atlan Gateway request timed out" : "Could not reach the Atlan Gateway",
+            detail: "Check gatewayOrigin, DNS, VPN, and network access",
+        });
+    }
 }
 exports.AtlanAPIError = AtlanAPIError;

package/dist/evals.d.ts

@@ -1,4 +1,4 @@
-/** Small lifecycle helpers for evaluation runners. */
+/** Evaluation lifecycle helpers and the declarative `Eval()` runner. */
 import { AtlanClient } from "./client";
 export declare const CONTEXT_MANIFEST_SCHEMA = "atlan.eval.context-manifest/v1";
 export interface ContextItem {
@@ -6,6 +6,7 @@ export interface ContextItem {
     name: string;
     digest: string;
     version: string;
+    artifactKind?: string;
     artifactId?: string;
     versionOrdinal?: number;
 }
@@ -14,6 +15,7 @@ export interface ContextManifestItem {
     name: string;
     digest: string;
     version: string;
+    artifact_kind?: string;
     artifact_id?: string;
     version_ordinal?: number;
 }
@@ -44,3 +46,136 @@ export declare class EvalRun {
 }
 /** Create a running experiment over a dataset ID or exact dataset name. */
 export declare function startExperiment(client: AtlanClient, dataset: string, body: Readonly<Record<string, unknown>>, options?: StartExperimentOptions): Promise<EvalRun>;
+export interface EvalCase<Input, Expected = unknown, Metadata = Record<string, unknown>> {
+    input: Input;
+    expected?: Expected;
+    metadata?: Metadata;
+    tags?: readonly string[];
+    /** Stable case checkpoint ID. Dataset-backed runs use the Registry record ID. */
+    id?: string;
+}
+export interface EvalHooks<Expected, Metadata> {
+    expected: Expected | undefined;
+    metadata: Metadata;
+    span: EvalTraceSpan;
+    tags: readonly string[] | undefined;
+    trialIndex: number;
+}
+export interface EvalScorerArgs<Input, Output, Expected, Metadata> extends EvalCase<Input, Expected, Metadata> {
+    output: Output;
+}
+export interface ScoreValue {
+    name?: string;
+    score: number | boolean | null;
+    metadata?: Record<string, unknown>;
+}
+export type EvalScorerReturn = number | boolean | null | ScoreValue | readonly ScoreValue[];
+export interface EvalScorer<Input, Output, Expected, Metadata> {
+    name: string;
+    scorer(args: EvalScorerArgs<Input, Output, Expected, Metadata>): EvalScorerReturn | Promise<EvalScorerReturn>;
+    /** Cite an existing Registry scorer instead of resolving/registering by name. */
+    scorerId?: string;
+    scorerVersion?: number;
+    /** Override the generated Registry artifact name. */
+    registryName?: string;
+    scorerKind?: "code" | "llm_judge" | "human";
+    scope?: "result" | "session" | "trace";
+    spec?: Record<string, unknown>;
+    outputs?: Record<string, unknown>;
+}
+export type EvalData<Input, Expected, Metadata> = readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | Promise<readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>>> | (() => readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | Promise<readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>>>);
+export interface Evaluator<Input, Output, Expected, Metadata> {
+    /** Inline cases. Omit when `dataset` names a Registry dataset. */
+    data?: EvalData<Input, Expected, Metadata>;
+    /** Dataset artifact ID or exact name; never fuzzy matched. */
+    dataset?: string;
+    task(input: Input, hooks: EvalHooks<Expected, Metadata>): Output | Promise<Output>;
+    scores: readonly EvalScorer<Input, Output, Expected, Metadata>[];
+    experimentName?: string;
+    description?: string;
+}
+export interface EvalOptions {
+    /** Reuse a configured management client; otherwise ATLAN_* env is used. */
+    client?: AtlanClient;
+    /** Reuse a configured tracing logger; otherwise `initLogger()` is called. */
+    logger?: EvalTraceLogger;
+    gatewayOrigin?: string;
+    apiKey?: string;
+    workspaceId?: string;
+    projectId?: string;
+    subjectKind?: "agent" | "harness" | "skill";
+    subjectId?: string;
+    baselineExperimentId?: string;
+    config?: Record<string, unknown>;
+    contextManifest?: ContextManifest;
+    /** Keep case outputs in the returned object. Defaults to true. */
+    returnResults?: boolean;
+    /** Read traces back before committing results. Defaults to true. */
+    verifyTraces?: boolean;
+    /** Maximum wait for trace-store consistency. Defaults to 30 seconds. */
+    traceVerificationTimeoutMs?: number;
+    /** Continue a running experiment, skipping case IDs already in Registry. */
+    resumeExperimentId?: string;
+    onStart?: (metadata: {
+        experimentId: string;
+        experiment: unknown;
+    }) => void;
+}
+export interface EvalCaseResult<Input, Output, Expected, Metadata> extends EvalCase<Input, Expected, Metadata> {
+    caseId: string;
+    output?: Output;
+    error?: string;
+    scores: Record<string, number | null>;
+    traceId: string;
+    durationMs: number;
+}
+export interface EvalSummary {
+    scores: Record<string, {
+        mean: number | null;
+        count: number;
+    }>;
+    metrics: {
+        cases: number;
+        successfulCases: number;
+        failedCases: number;
+        durationMs: number;
+    };
+}
+export interface EvalResultWithSummary<Input, Output, Expected, Metadata> {
+    experimentId: string;
+    experiment: unknown;
+    dataset: unknown;
+    summary: Record<string, unknown>;
+    results: EvalCaseResult<Input, Output, Expected, Metadata>[];
+}
+/** Structural trace surface keeps the management bundle decoupled at build time. */
+export interface EvalTraceSpan {
+    readonly traceId: string;
+    readonly spanId?: string;
+    update(fields: Record<string, unknown>): EvalTraceSpan;
+    score(name: string, value: unknown, options: {
+        scorerId: string;
+        scorerVersion: number;
+        comment?: string;
+    }): void;
+    recordFailure(error: unknown): void;
+    markEvidenceComplete?(): EvalTraceSpan;
+}
+interface EvalTraceClient {
+    readonly isEnabled: boolean;
+    startAsCurrentSpan<T>(name: string, options: Record<string, unknown>, callback: (span: EvalTraceSpan) => T): T;
+}
+export interface EvalTraceLogger {
+    readonly client: EvalTraceClient;
+    flush(): Promise<void>;
+}
+/**
+ * Run a complete Registry evaluation from a compact Braintrust-shaped definition.
+ *
+ * Every case is a root OTel trace. Task/provider spans and scorer spans nest
+ * beneath it, verdicts cite immutable Registry scorer versions, result rows
+ * carry the root trace ID, traces flush before summary, and the experiment is
+ * explicitly finalized.
+ */
+export declare function Eval<Input, Output, Expected = unknown, Metadata extends Record<string, unknown> = Record<string, unknown>>(name: string, evaluator: Evaluator<Input, Output, Expected, Metadata>, options?: EvalOptions): Promise<EvalResultWithSummary<Input, Output, Expected, Metadata>>;
+export {};