@hue-run/sdk 0.1.2

package/EVALUATIONS.md

@@ -0,0 +1,141 @@
+# Local evaluations
+
+The SDK executes targets and scorers on your machine. Hue stores pinned definitions, experiment progress and results. It does not execute uploaded source code. Follow the [installation guide](https://docs.hue.run/installation) to add `@hue-run/sdk` to your application.
+
+```ts
+import { randomUUID } from "node:crypto";
+import { createHue } from "@hue-run/sdk";
+import { builtins, createEvaluationClient, runExperiment, rescore } from "@hue-run/sdk/evals";
+
+const connection = { apiKey: process.env.HUE_API_KEY! };
+const client = createEvaluationClient(connection);
+const hue = createHue({ ...connection, serviceName: "evaluation-demo", captureContent: false });
+
+// Registry writes use optimistic revisions; they do not have automatic retries.
+const dataset = await client.createDataset({ name: "Greetings", slug: "greetings" });
+let draft = dataset.versions[0];
+({ version: draft } = await client.addCase(draft.id, {
+  expectedRevision: draft.revision,
+  externalKey: "hello",
+  inputs: { message: "hello" },
+  expected: "Hello!",
+}));
+const frozen = await client.freezeDatasetVersion(draft.id, draft.revision);
+const scorer = await client.createScorer({ name: "Exact", slug: "exact" });
+const exact = await client.publishScorerVersion(scorer.id, builtins.exactMatch());
+
+// Persist each creation key/returned ID in your application before retrying creation.
+const experiment = await client.createExperiment({
+  idempotencyKey: randomUUID(),
+  name: "Greeting configuration A",
+  datasetVersionId: frozen.id,
+  scorerVersionIds: [exact.id],
+  config: { greeting: "Hello!" },
+});
+try {
+  const report = await runExperiment({
+    client,
+    hue,
+    experimentId: experiment.id,
+    checkpointDirectory: `.hue-checkpoints/${experiment.id}`,
+    persistResultContent: true,
+    traceEvidence: { mode: "required" },
+    concurrency: 2,
+    // Replace this deterministic example with your real agent invocation.
+    target: async (_inputs, { config }) => (config as { greeting: string }).greeting,
+  });
+  const historical = await client.createEvaluationRun({
+    idempotencyKey: randomUUID(),
+    name: "Exact rescore",
+    subjectIds: report.subjectIds,
+    scorerVersionIds: [exact.id],
+  });
+  await rescore({
+    client,
+    runId: historical.id,
+    checkpointDirectory: `.hue-checkpoints/${historical.id}`,
+    persistResultContent: true,
+  });
+} finally {
+  await hue.shutdown();
+}
+```
+
+Create another experiment with the same frozen version and different `config` to compare configurations. The runner reads the exact experiment case/version and scorer definitions; it never resolves a mutable latest version. `rescore` accepts an existing evaluation-run ID and has no target callback. Subject IDs refer to immutable saved outputs and trace evidence.
+
+## Content and result states
+
+Both choices are required and independent:
+
+- `captureContent` configures telemetry helpers and Vercel telemetry callbacks.
+- `persistResultContent` configures completion output, target error messages, scorer evidence and arbitrary scorer explanations, including local checkpoint files. When false, the runner keeps these only in memory while computing scores and uploads generic explanations. Declared metric values are always uploaded, including custom text metrics; do not put sensitive output in a metric unless that is intended.
+
+Frozen dataset inputs/references already exist on Hue. Their presence is independent of these switches. No output is inferred from telemetry. JavaScript `undefined` means unavailable output; JSON `null`, false, zero and empty string remain present values. Historical subjects with unavailable output produce skipped results without executing a scorer callback. A failed quality metric remains `state:"scored"` with `passed:false`. Target errors and scorer errors remain separate. Target error types are generalized to `TargetError`; when content is enabled, bounded messages may be stored. Never put credentials in error messages, output, custom metric values, names or metadata.
+
+`traceEvidence:{mode:"required"}` waits for trace and log export acknowledgement after the root span ends. The explicit alternative `{mode:"omit",reason:"..."}` stores the omission reason and declared trace ID without a fake snapshot. It can complete despite an export failure; export diagnostics remain available on the Hue client. There is no automatic fallback to omission.
+
+## Local scorers
+
+Built-ins execute exact typed JSON equality, string inclusion (with pinned case sensitivity), and JSON Schema draft 2020-12 via pinned Ajv. Exact/includes skip absent references; includes skips non-string operands. Object key order does not affect exact match; array order and scalar types do.
+
+JSON Schema compilation and validation run in an isolated worker with a default 2-second deadline (`schemaTimeoutMillis:100..60000`), terminated before returning a timeout error. No remote schema loading, custom formats, coercion or default insertion is enabled. Compilation errors are errors, not failed quality scores. Worker startup time counts toward the deadline. This is an execution bound, not a general security sandbox. Schema registration also enforces the server's supported schema subset. Ajv documents [draft 2020-12 support](https://ajv.js.org/json-schema.html) and [schema/regular-expression security considerations](https://ajv.js.org/security.html).
+
+```ts
+import { readFile } from "node:fs/promises";
+import { defineLocalScorer } from "@hue-run/sdk/evals";
+import { score } from "./my-scorer.js";
+
+const local = defineLocalScorer({
+  source: await readFile(new URL("./my-scorer.js", import.meta.url)),
+  entrypoint: "score",
+  metrics: [{ name: "quality", type: "number", min: 0, max: 1 }],
+  score,
+});
+const identity = await client.createScorer({ name: "Quality", slug: "quality" });
+const published = await client.publishScorerVersion(identity.id, local.definition);
+// Pin published.id in the experiment and pass scorers:[local] to the runner.
+```
+
+A callback receives `{inputs,hasOutput,output?,hasExpected,expected?,metadata,executionState}` and returns one of:
+
+- `{state:"scored",metrics:[{name,value,passed?}],explanation?,evidence?}` (explanation or evidence required).
+- `{state:"error",error:{type,message?}}`.
+- `{state:"skipped",explanation:"reason"}`.
+
+All declared metrics must appear exactly once and satisfy pinned types, bounds and categories. The binding must match language, entrypoint, SHA-256 source digest and metric definitions. The digest is an authenticated caller declaration; it does not attest closures, dependency versions or actual execution. Callback source is never downloaded or evaluated. Callbacks are trusted local code; they have **no execution timeout or side-effect cancellation**. Concurrency limits active cases to 1–16 (default 1), with scorers evaluated sequentially within each case.
+
+### Hosted and manual scorer pins
+
+The local runner leaves `llm_judge` and `manual` pins pending and reports their IDs in `deferredScorerVersionIds`. It does not upload a synthetic skipped result that would occupy their immutable result slot. Manual results require a human session. Hosted dispatch is an explicit separate API operation: inspect `getJudgeBudget()`, then call `createJudgeJobs(runId,{idempotencyKey,jobs:[{evaluationItemId,scorerVersionId}]})`. `listJudgeJobs`, `getJudgeJob` and `cancelJudgeJob` expose job progress and cancellation requests. These methods never claim that local execution has hosted provenance. Hosted job endpoints are covered by HTTP contract tests here; live hosted model execution is a separate platform acceptance phase. `listResults` and `getResult` read recorded local or hosted results.
+
+When present, the budget's `authentication` reports credential resolution only. An
+`available` status or `configured: true` does not prove that a provider accepted the
+credential, has funds, or permits the selected model. The project's `enabled`,
+allocation and `blocked` fields are separate admission controls. Treat absent
+authentication details as unknown.
+
+Job reads preserve `originalChargeState` and the original provider `receipt`.
+`chargeState` and `actualMicroUsd` reflect a separately verified reconciliation
+when one exists; `reconciliation` is otherwise `null`. Its evidence reference,
+reason and timestamp explain that settlement. A settled charge does not change
+an interrupted job's execution state or rerun the model.
+
+## Checkpoints and failures
+
+Use one dedicated mode-0700 directory per experiment/rescore run. Files are mode 0600, written through fsync and atomic rename, and protected against accidental corruption by a digest. This is local storage, not encryption. Do not check it into Git. The manifest binds project, origin, frozen versions/configuration, scorer pins and content choices. Keep the directory until you no longer need upload recovery.
+
+An exclusive `.lock` prevents two processes from invoking targets through the same checkpoint. A process crash can leave the lock behind. Confirm the recorded process has stopped before explicitly removing that lock; the SDK never guesses ownership from elapsed time. Removing a lock does not authorize another target call.
+
+The runner saves a starting marker before `start`, and a running marker before invoking a target. If an outcome is not durably saved, resume throws `UncertainExecutionError` and never reruns the target. Replaying an original start key only recovers its execution ID. The low-level `startExecution` API requires an explicit `previousExecutionId`; replacing a still-started attempt additionally requires `allowUncertainRetry:true`. The convenience runner does not automatically adopt externally created attempts. Create a fresh experiment for a fresh target run after investigating side effects.
+
+After a target completes and local scoring finishes, the runner saves the allowed completion/result payloads before uploading. Network/API errors leave those payloads and stable keys available for another call to `runExperiment` with the same directory. Saved receipt IDs prevent duplicate result writes. Serialization failure raises `OutcomeSerializationError` with the execution ID; it does not relabel the target as failed or invoke it again. Scoring/upload failures never change target state.
+
+A crash between target completion and saving its permitted result still leaves an uncertain outcome; metadata-only mode intentionally cannot reconstruct discarded output. Export acknowledgement is saved separately. If required telemetry export was not acknowledged, a fresh empty exporter is not evidence of prior receipt: automatic completion is refused. Inspect/export the original trace or use the low-level completion API with an explicit omission policy; do not rerun a known completed target to manufacture telemetry. API completion/results retain their own idempotency guarantees for explicit recovery.
+
+Concurrent flush calls each perform a fresh serialized drain. Because OTLP partial responses do not identify rejected records, any export failure during a case prevents the runner from acknowledging that case's required evidence, including a failure already surfaced by another concurrent flush. This deliberately favors explicit recovery over accepting possibly incomplete evidence.
+
+The runner stops scheduling more cases after an operational failure and waits for already active cases before releasing the lock. It does not undo target side effects. A historical rescore can repeat local scoring after a crash before its checkpoint was saved; its result uploads use saved immutable payloads and stable keys once prepared.
+
+## Verification boundaries
+
+`scripts/verify-package.mjs` installs a real packed tarball outside the monorepo and runs HTTP contract tests against a synthetic service plus actual OpenTelemetry exporters. It checks two configurations, rescoring without target invocation, absent/null output, upload resume, uncertain execution, exclusive checkpoints, source/metric contracts, content policy and terminating schema workers. `scripts/verify-evaluation-api.mjs` is a separate opt-in acceptance against a real Hue receiver/API; it creates synthetic datasets/scorers/experiments in the project associated with the supplied development key.

package/LICENSE

@@ -0,0 +1,18 @@
+MIT License
+
+Copyright (c) 2026 Hue contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,196 @@
+# Hue TypeScript SDK
+
+A Node 24 / Bun 1.3.9 client for Hue's standard OTLP HTTP endpoints. It uses the
+OpenTelemetry JavaScript SDK and official OTLP protobuf exporter components for
+traces and correlated logs. The package is named `@hue-run/sdk`.
+
+## Install
+
+```bash
+npm install @hue-run/sdk
+```
+
+Or with Bun:
+
+```bash
+bun add @hue-run/sdk
+```
+
+Run the command in your application's server package. See the [compatibility guide](https://docs.hue.run/sdks/compatibility) before adding Hue to an application with existing OpenTelemetry or AI SDK dependencies.
+
+## Start
+
+```ts
+import { createHue, HueExportError } from "@hue-run/sdk";
+
+const hue = createHue({
+  apiKey: process.env.HUE_API_KEY!, // a project service key, on the server only
+  serviceName: "my-agent",
+  captureContent: false, // required: explicitly choose true or false
+  onExportIssue: (issue) => console.error(issue), // sanitized counts, never server bodies
+});
+
+await hue.checkConnection(); // GET /api/v1/projects/current
+await hue.withSpan(
+  "chat",
+  async (span) => {
+    const output = await hue.tool("uppercase", "hello", () => "hello".toUpperCase());
+    span.setOutput(output);
+    hue.recordMessages({ output: [{ role: "assistant", content: output }] });
+  },
+  { sessionId: "session-123", userId: "user-123", input: "hello" },
+);
+
+try {
+  await hue.flush(); // await both traces and logs; don't discard this promise
+} catch (error) {
+  if (error instanceof HueExportError) console.error(error.issues, error.report);
+}
+await hue.shutdown(); // flushes and releases providers owned by this client
+```
+
+The default destination is `https://app.hue.run`. Data goes to
+`/api/v1/otlp/v1/traces` and `/api/v1/otlp/v1/logs` with a Bearer project key.
+Set `baseUrl` only for another Hue deployment. It must be an origin without an API path; a trailing slash is accepted.
+HTTPS is required except for loopback HTTP. Redirects are refused for both
+project checks and exports. There is no proprietary tracing protocol, lab API
+wrapper, database dependency, or dependency on the Hue application workspace.
+
+## Vercel AI SDK 7
+
+Compatible optional peers are `ai@^7.0.99` and `@ai-sdk/otel@^1.0.99`, alongside
+`@opentelemetry/api@1.9.1`. For an app without global AI SDK telemetry integrations,
+configure telemetry on each agent or generation call:
+
+```ts
+import { ToolLoopAgent } from "ai";
+import { hueTelemetry } from "@hue-run/sdk/ai-sdk";
+
+const agent = new ToolLoopAgent({
+  model: process.env.AI_MODEL!, // actual configured AI Gateway provider/model
+  telemetry: hueTelemetry(hue),
+});
+await hue.withSpan(
+  "chat",
+  async (span) => {
+    const result = await agent.stream({ prompt: "Hello" });
+    // Consume the stream inside the span's callback so completion/error timing is correct.
+    for await (const text of result.textStream) process.stdout.write(text);
+    span.setOutput(await result.text);
+  },
+  { sessionId: "session-123" },
+);
+await hue.flush();
+```
+
+`hueTelemetry(hue)` supplies per-call integrations, which AI SDK 7 uses **instead of
+globally registered integrations for that call**. Global registration remains intact,
+but those integrations do not receive that call's events. To keep an existing
+OpenTelemetry exporter, attach Hue's transport to the same provider using the
+[existing-provider recipe](#existing-opentelemetry-providers).
+
+Reuse the client across server requests. Await stream completion before flushing;
+returning a streaming `Response` does not mean its stream has finished. The
+[Next.js streaming recipe](https://docs.hue.run/integrations/opentelemetry#flush-streamed-responses-in-next-js)
+shows how to keep completion and flushing within the request's background lifetime.
+For a standalone script, put the operation in `try` and call `await hue.shutdown()`
+in `finally`. Shut down a shared server client only when the application stops.
+
+The integration creates real Vercel provider, streaming and tool spans and passes
+`recordInputs` and `recordOutputs` explicitly. The installed-package verification
+tests the matching 7.0.99/1.0.99 and
+7.0.100/1.0.100 pairs. Compatible-major ranges do not mean every later release
+has been verified. Other OTel
+instrumentations can use `hue.tracer` directly or explicitly attach the processors
+below. Instrumentations that only use a global provider need your application's
+normal OTel setup; Hue does not silently replace it.
+
+## Privacy and content
+
+`captureContent: false` disables manual input/output/messages/tool content and
+removes recognized GenAI, Vercel, OpenInference and OpenLLMetry content attributes,
+legacy GenAI content events, log bodies, status messages and exception text before
+export. Model/provider/token metadata remains available. Generic custom attribute
+names cannot be classified automatically; use them deliberately.
+
+`captureContent: true` captures supplied content. Accepted content is stored by Hue;
+there is no SDK retention timer or automatic content expiry. To redact strings
+before export, supply `redact(value, path)`; it applies to supported strings in
+attributes, resources, event/link attributes and log bodies. Return a string.
+A throwing callback or invalid/oversized content fails closed: that record is
+counted as failed and the flush reports it. Shared resources are redacted once per
+export batch. Do not put user content or secrets in span names or scope names.
+
+Manual helpers encode JSON values without converting null into absence. Unknown
+outputs and usage remain absent. This SDK does not estimate tokens or cost. Error
+helpers mark span status and record an exception; thrown application errors remain
+errors and are rethrown unchanged. `withSpan` ends its span in `finally`.
+
+## Existing OpenTelemetry providers
+
+Attach processors while constructing your providers. Hue uses local async context
+for its own helpers and never registers/replaces the global tracer, logger, or
+context manager.
+
+```ts
+import { TracerProvider } from "@opentelemetry/sdk-trace";
+import { LoggerProvider } from "@opentelemetry/sdk-logs";
+import { createHue, createHueTransport } from "@hue-run/sdk";
+
+const transport = createHueTransport({
+  apiKey: process.env.HUE_API_KEY!,
+  serviceName: "existing-app",
+  captureContent: false,
+});
+const tracerProvider = new TracerProvider({ spanProcessors: [transport.spanProcessor] });
+const loggerProvider = new LoggerProvider({ processors: [transport.logRecordProcessor] });
+const hue = createHue({ transport, tracerProvider, loggerProvider });
+// Your application owns and configures these providers and their resources.
+await hue.shutdown(); // flushes; does not shut down these externally owned providers
+// During application shutdown, shut down your providers, then await transport.shutdown().
+```
+
+For external parent context pass `parentContext` to `withSpan`, or use standard
+OTel context propagation in your application. `getContext()` exposes the helper's
+current context for APIs taking an explicit context. Session/user identifiers are
+inherited within a client callback. Separate requests require separate callbacks.
+
+## Delivery behavior
+
+Exports use official OTel retry handling for temporary HTTP/network failures. Each
+request is limited to 1 MiB before gzip (with space reserved for gzip overhead) and each content value to 256 KiB. Batches
+split at record boundaries. Each signal queues at most 2,048 records, including
+exports in flight; overflow is reported through the callback, counters and next
+flush. This is an in-memory queue, not durable storage.
+
+`flush()` waits for the current trace and log export work. A partial rejection,
+invalid acknowledgement, queue drop or failure throws `HueExportError`; its
+`report` contains cumulative accepted/rejected/failed/pending counts. Accepted
+means the collector acknowledged receipt, not that a complete trace has arrived.
+A malformed response reports uncertain acceptance as failure. Partial successes
+are not retried. Warning-only acknowledgements with zero rejected records remain
+successful; the callback and issue history expose a sanitized warning. The next
+non-overlapping flush reports new failures; overlapping callers also observe failures
+from their shared in-flight work. Counters remain cumulative and `transport.getIssues()`
+keeps the latest 128 sanitized issues. Each concurrent caller receives a fresh serialized
+drain, including records emitted before its call. Stop request production
+before shutdown so late spans cannot race it. A client does not own instrumented
+operations still running in the application.
+
+## Package verification
+
+From the repository root with Node 24 and Bun 1.3.9 on PATH:
+
+```sh
+node packages/sdk-typescript/scripts/verify-package.mjs
+```
+
+This copies the SDK to a temporary directory, performs a frozen installation,
+builds and packs it, installs the tarball into a separate consumer, runs the real
+HTTP exporter suite against that installed package, and installs/builds the
+standalone reference chatbot. It prints the artifact paths. No package is
+published. The chatbot README describes running that external installation.
+
+# Local evaluation workflows
+
+The optional `@hue-run/sdk/evals` entry point supports dataset/scorer registration, frozen-version experiments, local built-in/custom scoring, upload resume, and historical rescoring. See the [evaluation guide](https://docs.hue.run/evaluations/first-evaluation) for the complete journey, content policy and checkpoint recovery contract.

package/dist/ai-sdk.d.ts

@@ -0,0 +1,4 @@
+import type { TelemetryOptions } from "ai";
+import type { HueClient } from "./client.js";
+/** Use as the call's telemetry option; it does not change global AI SDK integrations. */
+export declare function hueTelemetry(hue: HueClient): TelemetryOptions;

package/dist/ai-sdk.js

@@ -0,0 +1,10 @@
+import { OpenTelemetry } from "@ai-sdk/otel";
+/** Use as the call's telemetry option; it does not change global AI SDK integrations. */
+export function hueTelemetry(hue) {
+    return {
+        isEnabled: true,
+        recordInputs: hue.captureContent,
+        recordOutputs: hue.captureContent,
+        integrations: [new OpenTelemetry({ tracer: hue.tracer, usage: true })],
+    };
+}

package/dist/client.d.ts

@@ -0,0 +1,40 @@
+import { type Context, type Span, type Tracer } from "@opentelemetry/api";
+import { HueTransport } from "./transport.js";
+import type { ExportReport, FlushableLoggerProvider, FlushableTracerProvider, HueOptions, HueSpan, JsonValue, ProjectConnection, SpanOptions } from "./types.js";
+export interface ExistingHueProviders {
+    transport: HueTransport;
+    tracerProvider: FlushableTracerProvider;
+    loggerProvider: FlushableLoggerProvider;
+}
+export declare class HueConnectionError extends Error {
+    readonly status?: number | undefined;
+    constructor(message: string, status?: number | undefined);
+}
+export declare class HueClient {
+    readonly transport: HueTransport;
+    readonly tracer: Tracer;
+    readonly captureContent: boolean;
+    private logger;
+    private storage;
+    private tracerProvider;
+    private loggerProvider;
+    private ownedProviders?;
+    private closed;
+    private shutdownPromise?;
+    private flushPromise?;
+    constructor(options: HueOptions | ExistingHueProviders);
+    getContext(): Context;
+    withSpan<T>(name: string, callback: (span: HueSpan) => Promise<T> | T, options?: SpanOptions): Promise<T>;
+    tool<T extends JsonValue | undefined>(name: string, input: JsonValue, execute: () => Promise<T> | T): Promise<T>;
+    recordError(span: Span, error: unknown): void;
+    recordMessages(messages: {
+        input?: JsonValue;
+        output?: JsonValue;
+    }, explicitContext?: Context): void;
+    private setContent;
+    checkConnection(): Promise<ProjectConnection>;
+    flush(): Promise<ExportReport>;
+    private flushOnce;
+    shutdown(): Promise<ExportReport>;
+}
+export declare function createHue(options: HueOptions | ExistingHueProviders): HueClient;