@atlanai/sdk 0.1.1 → 0.2.2

package/README.md

@@ -39,23 +39,102 @@ Pass a default workspace once instead of repeating it on every call:
 const client = new AtlanClient({ gatewayOrigin, bearerToken, workspace: "workspace_01example" });
 ```
 
+## Run an eval
+
+`Eval()` executes the task and scorers, creates the Registry experiment, emits
+one trace per case, uploads trace-linked results, and finalizes the run. Registry
+derives the durable score summary during that finalization:
+
+```typescript
+import { Eval } from "@atlanai/sdk";
+
+const result = await Eval("Anthropic Evaluation", {
+  data: () => [
+    { input: "What is 2+2?", expected: "4" },
+    { input: "What is the capital of France?", expected: "Paris" },
+  ],
+  task: async (input) => callModel(input),
+  scores: [{
+    name: "accuracy",
+    scorer: ({ output, expected }) => output === expected ? 1 : 0,
+  }],
+});
+
+console.log(result.experimentId);
+```
+
+Set `ATLAN_API_KEY` and `ATLAN_WORKSPACE_ID`. Replace inline `data` with
+`dataset: "dataset_..."` or one exact Registry dataset name. The runner refuses
+to proceed when tracing is disabled because trace-less eval rows break the
+evidence chain. After flushing, it reads every case trace back through the
+experiment filter before it uploads results; a missing trace marks the
+experiment failed.
+
+## Control an eval lifecycle directly
+
+Resolve an existing dataset by artifact ID or exact name, then create the
+Registry experiment before another runner emits any traces:
+
+```typescript
+import { createContextManifest, startExperiment } from "@atlanai/sdk";
+import { propagateAttributes } from "@atlanai/sdk/tracing";
+
+const contextManifest = await createContextManifest([{
+  kind: "file",
+  name: "CLAUDE.md",
+  version: "git:0123456789abcdef0123456789abcdef01234567",
+  digest: `sha256:${"0".repeat(64)}`,
+}]);
+
+const run = await startExperiment(
+  client,
+  "conversational-studio-daily", // exact name, or dataset_... ID
+  { name: "candidate-run", config: { model: "example-model" } },
+  { contextManifest },
+);
+
+await propagateAttributes(run.traceOptions, () => existingRunner());
+```
+
+`run.experiment` is the generated create response and `run.id` is its
+`experimentId`. The helper pins the resolved dataset version and context
+manifest in the immutable experiment config. Use it when another harness owns
+execution, result upload, and finalization.
+
 ## Tracing
 
 ```typescript
-import * as atlan from "@atlanai/sdk/tracing";
+import { initLogger, wrapTraced } from "@atlanai/sdk/tracing";
 
-const tracer = atlan.init({ apiKey: "<your-tracing-key>", workspaceId: "workspace_01example" });
+const logger = initLogger({ projectName: "support-agent" });
+const handleRequest = wrapTraced(async (input: string) => callModel(input));
 
-await tracer.startAsCurrentSpan("handle-request", { asType: "task" }, async (span) => {
-  span.update({ model: "claude-sonnet-5", usage: { inputTokens: 120, outputTokens: 40 } });
-});
+await logger.traced(async (span) => {
+  const output = await handleRequest("hello");
+  span.log({ input: "hello", output });
+}, { name: "handle-request", type: "task" });
+
+await logger.flush();
 ```
 
-Tracing has its own API key and lifecycle — it does not reuse the management
-client's bearer token or transport. It ships as a separate subpath
+Tracing has its own API key and lifecycle. It does not reuse the management
+client's transport. It ships as a separate subpath
 (`@atlanai/sdk/tracing`) so importing it doesn't pull OpenTelemetry into a
 management-only bundle.
 
+Vercel AI SDK v3-v6 uses its native OpenTelemetry path:
+
+```typescript
+import * as ai from "ai";
+import { initLogger, wrapAISDK } from "@atlanai/sdk/tracing";
+
+initLogger({ projectName: "support-agent" });
+const { generateText, streamText } = wrapAISDK(ai);
+```
+
+See [evals and tracing](../docs/evals-and-tracing.md) for framework setup,
+context manifests, serverless flushing, and the tested integration matrix.
+
 ## Errors
 
 Every non-2xx response rejects with `AtlanAPIError`, with `.status`, `.code`,

package/dist/client.js

@@ -37,6 +37,7 @@ exports.AtlanClient = exports.ServiceNamespace = exports.WORKSPACE_HEADER = void
 exports.createService = createService;
 const agentRaw = __importStar(require("./raw/agent/index"));
 const apiRaw = __importStar(require("./raw/api/index"));
+const evalRaw = __importStar(require("./raw/eval/index"));
 const fileRaw = __importStar(require("./raw/file/index"));
 const mcpRaw = __importStar(require("./raw/mcp/index"));
 const modelRaw = __importStar(require("./raw/model/index"));
@@ -111,7 +112,7 @@ function forwardOperations(namespace, operationMethods) {
     // operation reads in the contract and how a caller would guess it —
     // `client.agents.createAgent(...)`. Both spellings resolve, because the long
     // one is what every existing caller and published example uses, and across
-    // all 189 operations no short name collides with another, with an OpenAPI
+    // all 221 operations no short name collides with another, with an OpenAPI
     // tag, or with a property of the namespace.
     const prefixed = (property) => namespace.name + property.charAt(0).toUpperCase() + property.slice(1);
     return new Proxy(namespace, {
@@ -187,6 +188,7 @@ class AtlanClient {
             model: modelRaw,
             api: apiRaw,
             otel: otelRaw,
+            eval: evalRaw,
         };
         const namespaces = {};
         for (const service of operations_1.operationManifest.services) {
@@ -203,7 +205,7 @@ class AtlanClient {
         // Resources, not services. A service is a base path; a resource is what a
         // caller is actually working with, so `client.sessions.messages` reads the
         // way the API does.
-        const resources = (0, resources_1.buildResources)(resources_1.resourceMap, namespaces);
+        const resources = (0, resources_1.buildResources)(resources_1.resourceMap, namespaces, workspace);
         this.resources = Object.freeze(resources);
         Object.assign(this, resources);
         this.workspace = workspace;

package/dist/evals.d.ts

@@ -0,0 +1,177 @@
+/** Evaluation lifecycle helpers and the declarative `Eval()` runner. */
+import { AtlanClient } from "./client";
+export declare const CONTEXT_MANIFEST_SCHEMA = "atlan.eval.context-manifest/v1";
+export interface ContextItem {
+    kind: string;
+    name: string;
+    digest: string;
+    version: string;
+    artifactId?: string;
+    versionOrdinal?: number;
+}
+export interface ContextManifestItem {
+    kind: string;
+    name: string;
+    digest: string;
+    version: string;
+    artifact_id?: string;
+    version_ordinal?: number;
+}
+export interface ContextManifest {
+    schema_version: typeof CONTEXT_MANIFEST_SCHEMA;
+    items: readonly ContextManifestItem[];
+    digest: string;
+}
+/** Build the same canonical, order-independent context manifest as the Python SDK. */
+export declare function createContextManifest(input: readonly ContextItem[]): Promise<ContextManifest>;
+/** Resolve one dataset by artifact ID or exact name, never fuzzy matching. */
+export declare function resolveDataset(client: AtlanClient, idOrExactName: string, workspaceId?: string): Promise<unknown>;
+export interface StartExperimentOptions {
+    contextManifest?: ContextManifest;
+}
+export declare class EvalRun {
+    readonly experiment: unknown;
+    readonly dataset: unknown;
+    readonly contextManifest?: ContextManifest | undefined;
+    constructor(experiment: unknown, dataset: unknown, contextManifest?: ContextManifest | undefined);
+    get id(): string;
+    get experimentId(): string;
+    /** Options for `propagateAttributes` from `@atlanai/sdk/tracing`. */
+    get traceOptions(): {
+        experimentId: string;
+        metadata?: Record<string, unknown>;
+    };
+}
+/** Create a running experiment over a dataset ID or exact dataset name. */
+export declare function startExperiment(client: AtlanClient, dataset: string, body: Readonly<Record<string, unknown>>, options?: StartExperimentOptions): Promise<EvalRun>;
+export interface EvalCase<Input, Expected = unknown, Metadata = Record<string, unknown>> {
+    input: Input;
+    expected?: Expected;
+    metadata?: Metadata;
+    tags?: readonly string[];
+    /** Registry dataset-record join; normally populated by `dataset`. */
+    id?: string;
+}
+export interface EvalHooks<Expected, Metadata> {
+    expected: Expected | undefined;
+    metadata: Metadata;
+    span: EvalTraceSpan;
+    tags: readonly string[] | undefined;
+    trialIndex: number;
+}
+export interface EvalScorerArgs<Input, Output, Expected, Metadata> extends EvalCase<Input, Expected, Metadata> {
+    output: Output;
+}
+export interface ScoreValue {
+    name?: string;
+    score: number | boolean | null;
+    metadata?: Record<string, unknown>;
+}
+export type EvalScorerReturn = number | boolean | null | ScoreValue | readonly ScoreValue[];
+export interface EvalScorer<Input, Output, Expected, Metadata> {
+    name: string;
+    scorer(args: EvalScorerArgs<Input, Output, Expected, Metadata>): EvalScorerReturn | Promise<EvalScorerReturn>;
+    /** Cite an existing Registry scorer instead of resolving/registering by name. */
+    scorerId?: string;
+    scorerVersion?: number;
+    /** Override the generated Registry artifact name. */
+    registryName?: string;
+    scorerKind?: "code" | "llm_judge" | "human";
+    scope?: "result" | "session" | "trace";
+    spec?: Record<string, unknown>;
+    outputs?: Record<string, unknown>;
+}
+export type EvalData<Input, Expected, Metadata> = readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | Promise<readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>>> | (() => readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | Promise<readonly EvalCase<Input, Expected, Metadata>[] | Iterable<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>>>);
+export interface Evaluator<Input, Output, Expected, Metadata> {
+    /** Inline cases. Omit when `dataset` names a Registry dataset. */
+    data?: EvalData<Input, Expected, Metadata>;
+    /** Dataset artifact ID or exact name; never fuzzy matched. */
+    dataset?: string;
+    task(input: Input, hooks: EvalHooks<Expected, Metadata>): Output | Promise<Output>;
+    scores: readonly EvalScorer<Input, Output, Expected, Metadata>[];
+    experimentName?: string;
+    description?: string;
+}
+export interface EvalOptions {
+    /** Reuse a configured management client; otherwise ATLAN_* env is used. */
+    client?: AtlanClient;
+    /** Reuse a configured tracing logger; otherwise `initLogger()` is called. */
+    logger?: EvalTraceLogger;
+    gatewayOrigin?: string;
+    apiKey?: string;
+    workspaceId?: string;
+    projectId?: string;
+    subjectKind?: "agent" | "harness" | "skill";
+    subjectId?: string;
+    baselineExperimentId?: string;
+    config?: Record<string, unknown>;
+    contextManifest?: ContextManifest;
+    /** Keep case outputs in the returned object. Defaults to true. */
+    returnResults?: boolean;
+    /** Read traces back before committing results. Defaults to true. */
+    verifyTraces?: boolean;
+    /** Maximum wait for trace-store consistency. Defaults to 30 seconds. */
+    traceVerificationTimeoutMs?: number;
+    /** Continue a running experiment, skipping case IDs already in Registry. */
+    resumeExperimentId?: string;
+    onStart?: (metadata: {
+        experimentId: string;
+        experiment: unknown;
+    }) => void;
+}
+export interface EvalCaseResult<Input, Output, Expected, Metadata> extends EvalCase<Input, Expected, Metadata> {
+    caseId: string;
+    output?: Output;
+    error?: string;
+    scores: Record<string, number | null>;
+    traceId: string;
+    durationMs: number;
+}
+export interface EvalSummary {
+    scores: Record<string, {
+        mean: number | null;
+        count: number;
+    }>;
+    metrics: {
+        cases: number;
+        successfulCases: number;
+        failedCases: number;
+        durationMs: number;
+    };
+}
+export interface EvalResultWithSummary<Input, Output, Expected, Metadata> {
+    experimentId: string;
+    experiment: unknown;
+    dataset: unknown;
+    summary: Record<string, unknown>;
+    results: EvalCaseResult<Input, Output, Expected, Metadata>[];
+}
+/** Structural trace surface keeps the management bundle decoupled at build time. */
+export interface EvalTraceSpan {
+    readonly traceId: string;
+    update(fields: Record<string, unknown>): EvalTraceSpan;
+    score(name: string, value: unknown, options: {
+        scorerId: string;
+        scorerVersion: number;
+        comment?: string;
+    }): void;
+    recordFailure(error: unknown): void;
+}
+interface EvalTraceClient {
+    readonly isEnabled: boolean;
+    startAsCurrentSpan<T>(name: string, options: Record<string, unknown>, callback: (span: EvalTraceSpan) => T): T;
+}
+export interface EvalTraceLogger {
+    readonly client: EvalTraceClient;
+    flush(): Promise<void>;
+}
+/**
+ * Run a complete Registry evaluation from a compact Braintrust-shaped definition.
+ *
+ * Every case is a root OTel trace. Task/provider spans and scorer spans nest
+ * beneath it, verdicts cite immutable Registry scorer versions, result rows
+ * carry the root trace ID, traces flush before summary, and the experiment is
+ * explicitly finalized.
+ */
+export declare function Eval<Input, Output, Expected = unknown, Metadata extends Record<string, unknown> = Record<string, unknown>>(name: string, evaluator: Evaluator<Input, Output, Expected, Metadata>, options?: EvalOptions): Promise<EvalResultWithSummary<Input, Output, Expected, Metadata>>;
+export {};