@atlanai/sdk 0.2.1 → 0.2.2

package/README.md

@@ -39,10 +39,41 @@ Pass a default workspace once instead of repeating it on every call:
 const client = new AtlanClient({ gatewayOrigin, bearerToken, workspace: "workspace_01example" });
 ```
 
-## Start an eval run
+## Run an eval
+
+`Eval()` executes the task and scorers, creates the Registry experiment, emits
+one trace per case, uploads trace-linked results, and finalizes the run. Registry
+derives the durable score summary during that finalization:
+
+```typescript
+import { Eval } from "@atlanai/sdk";
+
+const result = await Eval("Anthropic Evaluation", {
+  data: () => [
+    { input: "What is 2+2?", expected: "4" },
+    { input: "What is the capital of France?", expected: "Paris" },
+  ],
+  task: async (input) => callModel(input),
+  scores: [{
+    name: "accuracy",
+    scorer: ({ output, expected }) => output === expected ? 1 : 0,
+  }],
+});
+
+console.log(result.experimentId);
+```
+
+Set `ATLAN_API_KEY` and `ATLAN_WORKSPACE_ID`. Replace inline `data` with
+`dataset: "dataset_..."` or one exact Registry dataset name. The runner refuses
+to proceed when tracing is disabled because trace-less eval rows break the
+evidence chain. After flushing, it reads every case trace back through the
+experiment filter before it uploads results; a missing trace marks the
+experiment failed.
+
+## Control an eval lifecycle directly
 
 Resolve an existing dataset by artifact ID or exact name, then create the
-Registry experiment before the runner emits any traces:
+Registry experiment before another runner emits any traces:
 
 ```typescript
 import { createContextManifest, startExperiment } from "@atlanai/sdk";
@@ -67,27 +98,43 @@ await propagateAttributes(run.traceOptions, () => existingRunner());
 
 `run.experiment` is the generated create response and `run.id` is its
 `experimentId`. The helper pins the resolved dataset version and context
-manifest in the immutable experiment config. Execution, result upload,
-summarization, and terminal status remain the existing `client.experiments`
-operations.
+manifest in the immutable experiment config. Use it when another harness owns
+execution, result upload, and finalization.
 
 ## Tracing
 
 ```typescript
-import * as atlan from "@atlanai/sdk/tracing";
+import { initLogger, wrapTraced } from "@atlanai/sdk/tracing";
 
-const tracer = atlan.init({ apiKey: "<your-tracing-key>", workspaceId: "workspace_01example" });
+const logger = initLogger({ projectName: "support-agent" });
+const handleRequest = wrapTraced(async (input: string) => callModel(input));
 
-await tracer.startAsCurrentSpan("handle-request", { asType: "task" }, async (span) => {
-  span.update({ model: "claude-sonnet-5", usage: { inputTokens: 120, outputTokens: 40 } });
-});
+await logger.traced(async (span) => {
+  const output = await handleRequest("hello");
+  span.log({ input: "hello", output });
+}, { name: "handle-request", type: "task" });
+
+await logger.flush();
 ```
 
-Tracing has its own API key and lifecycle — it does not reuse the management
-client's bearer token or transport. It ships as a separate subpath
+Tracing has its own API key and lifecycle. It does not reuse the management
+client's transport. It ships as a separate subpath
 (`@atlanai/sdk/tracing`) so importing it doesn't pull OpenTelemetry into a
 management-only bundle.
 
+Vercel AI SDK v3-v6 uses its native OpenTelemetry path:
+
+```typescript
+import * as ai from "ai";
+import { initLogger, wrapAISDK } from "@atlanai/sdk/tracing";
+
+initLogger({ projectName: "support-agent" });
+const { generateText, streamText } = wrapAISDK(ai);
+```
+
+See [evals and tracing](../docs/evals-and-tracing.md) for framework setup,
+context manifests, serverless flushing, and the tested integration matrix.
+
 ## Errors
 
 Every non-2xx response rejects with `AtlanAPIError`, with `.status`, `.code`,

package/dist/client.js

@@ -205,7 +205,7 @@ class AtlanClient {
         // Resources, not services. A service is a base path; a resource is what a
         // caller is actually working with, so `client.sessions.messages` reads the
         // way the API does.
-        const resources = (0, resources_1.buildResources)(resources_1.resourceMap, namespaces);
+        const resources = (0, resources_1.buildResources)(resources_1.resourceMap, namespaces, workspace);
         this.resources = Object.freeze(resources);
         Object.assign(this, resources);
         this.workspace = workspace;

package/dist/evals.d.ts

@@ -1,4 +1,4 @@
-/** Small lifecycle helpers for evaluation runners. */
+/** Evaluation lifecycle helpers and the declarative `Eval()` runner. */
 import { AtlanClient } from "./client";
 export declare const CONTEXT_MANIFEST_SCHEMA = "atlan.eval.context-manifest/v1";
 export interface ContextItem {
@@ -44,3 +44,134 @@ export declare class EvalRun {
 }
 /** Create a running experiment over a dataset ID or exact dataset name. */
 export declare function startExperiment(client: AtlanClient, dataset: string, body: Readonly<Record<string, unknown>>, options?: StartExperimentOptions): Promise<EvalRun>;
+export interface EvalCase<Input, Expected = unknown, Metadata = Record<string, unknown>> {
+    input: Input;
+    expected?: Expected;
+    metadata?: Metadata;
+    tags?: readonly string[];
+    /** Registry dataset-record join; normally populated by `dataset`. */
+    id?: string;
+}
+export interface EvalHooks<Expected, Metadata> {
+    expected: Expected | undefined;
+    metadata: Metadata;
+    span: EvalTraceSpan;
+    tags: readonly string[] | undefined;
+    trialIndex: number;
+}
+export interface EvalScorerArgs<Input, Output, Expected, Metadata> extends EvalCase<Input, Expected, Metadata> {
+    output: Output;
+}
+export interface ScoreValue {
+    name?: string;
+    score: number | boolean | null;
+    metadata?: Record<string, unknown>;
+}
+export type EvalScorerReturn = number | boolean | null | ScoreValue | readonly ScoreValue[];
+export interface EvalScorer<Input, Output, Expected, Metadata> {
+    name: string;
+    scorer(args: EvalScorerArgs<Input, Output, Expected, Metadata>): EvalScorerReturn | Promise<EvalScorerReturn>;
+    /** Cite an existing Registry scorer instead of resolving/registering by name. */
+    scorerId?: string;
+    scorerVersion?: number;
+    /** Override the generated Registry artifact name. */
+    registryName?: string;
+    scorerKind?: "code" | "llm_judge" | "human";
+    scope?: "result" | "session" | "trace";
+    spec?: Record<string, unknown>;
+    outputs?: Record<string, unknown>;
+}
+export type EvalData<Input, Expected, Metadata> = readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | Promise<readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>>> | (() => readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | Promise<readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>>>);
+export interface Evaluator<Input, Output, Expected, Metadata> {
+    /** Inline cases. Omit when `dataset` names a Registry dataset. */
+    data?: EvalData<Input, Expected, Metadata>;
+    /** Dataset artifact ID or exact name; never fuzzy matched. */
+    dataset?: string;
+    task(input: Input, hooks: EvalHooks<Expected, Metadata>): Output | Promise<Output>;
+    scores: readonly EvalScorer<Input, Output, Expected, Metadata>[];
+    experimentName?: string;
+    description?: string;
+}
+export interface EvalOptions {
+    /** Reuse a configured management client; otherwise ATLAN_* env is used. */
+    client?: AtlanClient;
+    /** Reuse a configured tracing logger; otherwise `initLogger()` is called. */
+    logger?: EvalTraceLogger;
+    gatewayOrigin?: string;
+    apiKey?: string;
+    workspaceId?: string;
+    projectId?: string;
+    subjectKind?: "agent" | "harness" | "skill";
+    subjectId?: string;
+    baselineExperimentId?: string;
+    config?: Record<string, unknown>;
+    contextManifest?: ContextManifest;
+    /** Keep case outputs in the returned object. Defaults to true. */
+    returnResults?: boolean;
+    /** Read traces back before committing results. Defaults to true. */
+    verifyTraces?: boolean;
+    /** Maximum wait for trace-store consistency. Defaults to 30 seconds. */
+    traceVerificationTimeoutMs?: number;
+    /** Continue a running experiment, skipping case IDs already in Registry. */
+    resumeExperimentId?: string;
+    onStart?: (metadata: {
+        experimentId: string;
+        experiment: unknown;
+    }) => void;
+}
+export interface EvalCaseResult<Input, Output, Expected, Metadata> extends EvalCase<Input, Expected, Metadata> {
+    caseId: string;
+    output?: Output;
+    error?: string;
+    scores: Record<string, number | null>;
+    traceId: string;
+    durationMs: number;
+}
+export interface EvalSummary {
+    scores: Record<string, {
+        mean: number | null;
+        count: number;
+    }>;
+    metrics: {
+        cases: number;
+        successfulCases: number;
+        failedCases: number;
+        durationMs: number;
+    };
+}
+export interface EvalResultWithSummary<Input, Output, Expected, Metadata> {
+    experimentId: string;
+    experiment: unknown;
+    dataset: unknown;
+    summary: Record<string, unknown>;
+    results: EvalCaseResult<Input, Output, Expected, Metadata>[];
+}
+/** Structural trace surface keeps the management bundle decoupled at build time. */
+export interface EvalTraceSpan {
+    readonly traceId: string;
+    update(fields: Record<string, unknown>): EvalTraceSpan;
+    score(name: string, value: unknown, options: {
+        scorerId: string;
+        scorerVersion: number;
+        comment?: string;
+    }): void;
+    recordFailure(error: unknown): void;
+}
+interface EvalTraceClient {
+    readonly isEnabled: boolean;
+    startAsCurrentSpan<T>(name: string, options: Record<string, unknown>, callback: (span: EvalTraceSpan) => T): T;
+}
+export interface EvalTraceLogger {
+    readonly client: EvalTraceClient;
+    flush(): Promise<void>;
+}
+/**
+ * Run a complete Registry evaluation from a compact Braintrust-shaped definition.
+ *
+ * Every case is a root OTel trace. Task/provider spans and scorer spans nest
+ * beneath it, verdicts cite immutable Registry scorer versions, result rows
+ * carry the root trace ID, traces flush before summary, and the experiment is
+ * explicitly finalized.
+ */
+export declare function Eval<Input, Output, Expected = unknown, Metadata extends Record<string, unknown> = Record<string, unknown>>(name: string, evaluator: Evaluator<Input, Output, Expected, Metadata>, options?: EvalOptions): Promise<EvalResultWithSummary<Input, Output, Expected, Metadata>>;
+export {};