@hue-run/sdk 0.1.2

package/dist/client.js

@@ -0,0 +1,309 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { context, SpanKind, SpanStatusCode, trace, } from "@opentelemetry/api";
+import { SeverityNumber } from "@opentelemetry/api-logs";
+import { LoggerProvider } from "@opentelemetry/sdk-logs";
+import { TracerProvider } from "@opentelemetry/sdk-trace";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { MAX_CONTENT_BYTES } from "./config.js";
+import { createHueTransport, HueExportError } from "./transport.js";
+export class HueConnectionError extends Error {
+    status;
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+        this.name = "HueConnectionError";
+    }
+}
+function identifier(value) {
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== "string" ||
+        !value ||
+        value.length > 4096 ||
+        value.includes("\u0000") ||
+        !value.isWellFormed())
+        throw new TypeError("Session/user identifiers must contain 1–4096 valid characters");
+    return value;
+}
+/** Local async context preserves nesting without registering or replacing global OTel providers. */
+class ContextualTracer {
+    source;
+    storage;
+    constructor(source, storage) {
+        this.source = source;
+        this.storage = storage;
+    }
+    startSpan(name, options = {}, parent) {
+        const active = this.storage.getStore();
+        return this.source.startSpan(name, {
+            ...options,
+            attributes: {
+                ...options.attributes,
+                ...(active?.sessionId ? { "gen_ai.conversation.id": active.sessionId } : {}),
+                ...(active?.userId ? { "user.id": active.userId } : {}),
+            },
+        }, parent ?? active?.context ?? context.active());
+    }
+    startActiveSpan(name, optionsOrFn, contextOrFn, fn) {
+        const options = typeof optionsOrFn === "function" ? {} : optionsOrFn;
+        const callback = typeof optionsOrFn === "function"
+            ? optionsOrFn
+            : typeof contextOrFn === "function"
+                ? contextOrFn
+                : fn;
+        if (!callback)
+            throw new TypeError("A span callback is required");
+        const parent = typeof contextOrFn === "object"
+            ? contextOrFn
+            : (this.storage.getStore()?.context ?? context.active());
+        const span = this.startSpan(name, options, parent);
+        return this.storage.run({ ...this.storage.getStore(), context: trace.setSpan(parent, span) }, () => callback(span));
+    }
+}
+export class HueClient {
+    transport;
+    tracer;
+    captureContent;
+    logger;
+    storage = new AsyncLocalStorage();
+    tracerProvider;
+    loggerProvider;
+    ownedProviders;
+    closed = false;
+    shutdownPromise;
+    flushPromise;
+    constructor(options) {
+        if ("transport" in options) {
+            this.transport = options.transport;
+            this.tracerProvider = options.tracerProvider;
+            this.loggerProvider = options.loggerProvider;
+        }
+        else {
+            this.transport = createHueTransport(options);
+            const resource = resourceFromAttributes({
+                "service.name": options.serviceName,
+                ...(options.serviceVersion ? { "service.version": options.serviceVersion } : {}),
+            });
+            const tracer = new TracerProvider({
+                resource,
+                spanProcessors: [this.transport.spanProcessor],
+            });
+            const logger = new LoggerProvider({
+                resource,
+                processors: [this.transport.logRecordProcessor],
+            });
+            this.ownedProviders = { tracer, logger };
+            this.tracerProvider = tracer;
+            this.loggerProvider = logger;
+        }
+        this.captureContent = this.transport.options.captureContent;
+        this.tracer = new ContextualTracer(this.tracerProvider.getTracer("@hue-run/sdk", "0.1.2"), this.storage);
+        this.logger = this.loggerProvider.getLogger("@hue-run/sdk", "0.1.2");
+    }
+    getContext() {
+        return this.storage.getStore()?.context ?? context.active();
+    }
+    async withSpan(name, callback, options = {}) {
+        if (this.closed)
+            throw new Error("Hue client is shut down");
+        const inherited = this.storage.getStore();
+        const sessionId = identifier(options.sessionId ?? inherited?.sessionId);
+        const userId = identifier(options.userId ?? inherited?.userId);
+        const parent = options.parentContext ?? inherited?.context ?? context.active();
+        const active = { context: parent, sessionId, userId };
+        return this.storage.run(active, async () => {
+            const span = this.tracer.startSpan(name, { kind: options.kind ?? SpanKind.INTERNAL, attributes: options.attributes }, parent);
+            const spanContext = trace.setSpan(parent, span);
+            const handle = {
+                span,
+                context: spanContext,
+                traceId: span.spanContext().traceId,
+                spanId: span.spanContext().spanId,
+                setInput: (value) => this.setContent(span, "input.value", value),
+                setOutput: (value) => this.setContent(span, "output.value", value),
+            };
+            return this.storage.run({ ...active, context: spanContext }, async () => {
+                try {
+                    if (options.input !== undefined)
+                        handle.setInput(options.input);
+                    return await callback(handle);
+                }
+                catch (error) {
+                    this.recordError(span, error);
+                    throw error;
+                }
+                finally {
+                    span.end();
+                }
+            });
+        });
+    }
+    async tool(name, input, execute) {
+        return this.withSpan(name, async ({ span }) => {
+            this.setContent(span, "gen_ai.tool.call.arguments", input);
+            const result = await execute();
+            if (result !== undefined)
+                this.setContent(span, "gen_ai.tool.call.result", result);
+            return result;
+        }, { attributes: { "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": name } });
+    }
+    recordError(span, error) {
+        const type = error instanceof Error ? error.name : "Error";
+        span.setStatus({
+            code: SpanStatusCode.ERROR,
+            ...(this.captureContent
+                ? { message: error instanceof Error ? error.message : "Operation failed" }
+                : {}),
+        });
+        span.addEvent("exception", {
+            "exception.type": type,
+            ...(this.captureContent && error instanceof Error
+                ? {
+                    "exception.message": error.message,
+                    ...(error.stack ? { "exception.stacktrace": error.stack } : {}),
+                }
+                : {}),
+        });
+    }
+    recordMessages(messages, explicitContext) {
+        if (this.closed)
+            throw new Error("Hue client is shut down");
+        if (!this.captureContent)
+            return;
+        const active = explicitContext ?? this.getContext();
+        if (!trace.getSpanContext(active))
+            throw new Error("Message records require an active span or explicit span context");
+        const body = {};
+        if (messages.input !== undefined)
+            body["gen_ai.input.messages"] = messages.input;
+        if (messages.output !== undefined)
+            body["gen_ai.output.messages"] = messages.output;
+        this.logger.emit({
+            context: active,
+            severityNumber: SeverityNumber.INFO,
+            eventName: "gen_ai.client.inference.operation.details",
+            body,
+        });
+    }
+    setContent(span, key, value) {
+        if (!this.captureContent)
+            return;
+        const encoded = JSON.stringify(value, (_key, item) => {
+            if (typeof item === "number" && !Number.isFinite(item))
+                throw new TypeError("Captured JSON numbers must be finite");
+            return item;
+        });
+        if (encoded === undefined || Buffer.byteLength(encoded) > MAX_CONTENT_BYTES)
+            throw new RangeError("Captured content must be JSON and no more than 256 KiB");
+        span.setAttribute(key, encoded);
+    }
+    async checkConnection() {
+        const options = this.transport.options;
+        let response;
+        try {
+            response = await fetch(`${options.baseUrl}/api/v1/projects/current`, {
+                headers: { Authorization: `Bearer ${options.apiKey}` },
+                redirect: "error",
+                signal: AbortSignal.timeout(options.timeoutMillis),
+            });
+        }
+        catch {
+            throw new HueConnectionError("Unable to connect to Hue; check the endpoint and network");
+        }
+        if (!response.ok) {
+            await response.body?.cancel();
+            throw new HueConnectionError("Hue rejected the project connection", response.status);
+        }
+        try {
+            if (Number(response.headers.get("content-length") ?? 0) > 65536)
+                throw new Error("Oversized project response");
+            const reader = response.body?.getReader();
+            if (!reader)
+                throw new Error("Missing project response");
+            let size = 0;
+            const chunks = [];
+            try {
+                for (;;) {
+                    const { done, value } = await reader.read();
+                    if (done)
+                        break;
+                    size += value.byteLength;
+                    if (size > 65536)
+                        throw new Error("Oversized project response");
+                    chunks.push(value);
+                }
+            }
+            finally {
+                await reader.cancel();
+            }
+            const project = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+            if (!project || typeof project !== "object")
+                throw new Error("Invalid project response");
+            const fields = project;
+            for (const key of ["id", "name", "organizationId", "slug"])
+                if (typeof fields[key] !== "string")
+                    throw new Error("Invalid project response");
+            return {
+                id: fields.id,
+                name: fields.name,
+                organizationId: fields.organizationId,
+                slug: fields.slug,
+            };
+        }
+        catch {
+            throw new HueConnectionError("Hue returned an invalid project response");
+        }
+    }
+    flush() {
+        // Each caller needs a drain after its own preceding span/log emissions.
+        // Joining an earlier drain can acknowledge records that were not in its batch.
+        const from = this.transport.getFailureSequence();
+        const drain = async () => {
+            const report = await this.flushOnce();
+            if (this.transport.getFailureSequence() !== from)
+                throw new HueExportError(this.transport
+                    .getIssues()
+                    .filter((issue) => issue.sequence > from && issue.kind !== "warning"), report);
+            return report;
+        };
+        const next = (this.flushPromise ?? Promise.resolve()).then(drain, drain);
+        this.flushPromise = next;
+        const clear = () => {
+            if (this.flushPromise === next)
+                this.flushPromise = undefined;
+        };
+        void next.then(clear, clear);
+        return next;
+    }
+    async flushOnce() {
+        const results = await Promise.allSettled([
+            this.tracerProvider.forceFlush(),
+            this.loggerProvider.forceFlush(),
+        ]);
+        for (const [index, result] of results.entries())
+            if (result.status === "rejected")
+                this.transport.issue(index === 0 ? "traces" : "logs", "failed", 0, "OpenTelemetry provider flush failed");
+        return this.transport.flush();
+    }
+    shutdown() {
+        this.shutdownPromise ??= (async () => {
+            this.closed = true;
+            try {
+                return await this.flush();
+            }
+            finally {
+                if (this.ownedProviders) {
+                    await Promise.allSettled([
+                        this.ownedProviders.tracer.shutdown(),
+                        this.ownedProviders.logger.shutdown(),
+                    ]);
+                    await this.transport.shutdown();
+                }
+            }
+        })();
+        return this.shutdownPromise;
+    }
+}
+export function createHue(options) {
+    return new HueClient(options);
+}

package/dist/config.d.ts

@@ -0,0 +1,4 @@
+import type { HueOptions } from "./types.js";
+export declare const MAX_BODY_BYTES: number;
+export declare const MAX_CONTENT_BYTES: number;
+export declare function validateOptions(options: HueOptions): Required<Pick<HueOptions, "apiKey" | "serviceName" | "baseUrl" | "captureContent" | "timeoutMillis">> & HueOptions;

package/dist/config.js

@@ -0,0 +1,32 @@
+export const MAX_BODY_BYTES = 1024 * 1024;
+export const MAX_CONTENT_BYTES = 256 * 1024;
+export function validateOptions(options) {
+    if (typeof options.captureContent !== "boolean")
+        throw new TypeError("Choose captureContent explicitly: true or false");
+    if (typeof options.apiKey !== "string" ||
+        !options.apiKey ||
+        options.apiKey.length > 4096 ||
+        /\s/.test(options.apiKey) ||
+        options.apiKey.includes("\u0000"))
+        throw new TypeError("A valid Hue project API key is required");
+    if (typeof options.serviceName !== "string" ||
+        !options.serviceName.trim() ||
+        options.serviceName.length > 256)
+        throw new TypeError("A serviceName of 1–256 characters is required");
+    let url;
+    try {
+        url = new URL(options.baseUrl ?? "https://app.hue.run");
+    }
+    catch {
+        throw new TypeError("Invalid Hue baseUrl");
+    }
+    const loopback = url.hostname === "localhost" || url.hostname === "127.0.0.1" || url.hostname === "[::1]";
+    if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback))
+        throw new TypeError("Hue requires HTTPS except for a loopback development server");
+    if (url.username || url.password || url.pathname !== "/" || url.search || url.hash)
+        throw new TypeError("Hue baseUrl must be an origin without credentials, a path, query parameters or fragments");
+    const timeoutMillis = options.timeoutMillis ?? 10000;
+    if (!Number.isInteger(timeoutMillis) || timeoutMillis < 100 || timeoutMillis > 60000)
+        throw new TypeError("timeoutMillis must be 100–60000");
+    return { ...options, baseUrl: url.origin, timeoutMillis };
+}

package/dist/evals/checkpoint.d.ts

@@ -0,0 +1,9 @@
+/** One owner per directory. A crash leaves .lock for explicit operator recovery. */
+export declare class CheckpointStore {
+    readonly directory: string;
+    private constructor();
+    static acquire(directory: string, identity: unknown): Promise<CheckpointStore>;
+    read<T>(key: string): Promise<T | undefined>;
+    write(key: string, value: unknown): Promise<void>;
+    release(): Promise<void>;
+}

package/dist/evals/checkpoint.js

@@ -0,0 +1,89 @@
+import { constants } from "node:fs";
+import { lstat, mkdir, open, rename, rm } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { randomUUID } from "node:crypto";
+import { digest } from "./json.js";
+/** One owner per directory. A crash leaves .lock for explicit operator recovery. */
+export class CheckpointStore {
+    directory;
+    constructor(directory) {
+        this.directory = directory;
+    }
+    static async acquire(directory, identity) {
+        const root = resolve(directory);
+        await mkdir(root, { recursive: true, mode: 0o700 });
+        const info = await lstat(root);
+        if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o077) !== 0)
+            throw new Error("Use a private checkpoint directory (mode 0700, no symlink)");
+        const store = new CheckpointStore(root);
+        try {
+            await mkdir(join(root, ".lock"), { mode: 0o700 });
+        }
+        catch {
+            throw new Error("Checkpoint directory is locked; confirm its owner stopped before explicitly removing .lock");
+        }
+        try {
+            await store.write(".lock/owner", { pid: process.pid });
+            const expected = { format: 1, identity, digest: digest(identity) };
+            const prior = await store.read("manifest");
+            if (prior && (prior.format !== 1 || prior.digest !== expected.digest))
+                throw new Error("Checkpoint identity differs from this project, run, pins or content policy");
+            if (!prior)
+                await store.write("manifest", expected);
+            return store;
+        }
+        catch (error) {
+            await store.release();
+            throw error;
+        }
+    }
+    async read(key) {
+        let file;
+        try {
+            file = await open(join(this.directory, `${key}.json`), constants.O_RDONLY | constants.O_NOFOLLOW);
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return undefined;
+            throw error;
+        }
+        try {
+            const info = await file.stat();
+            if (!info.isFile() || info.size > 8 * 1024 * 1024 || (info.mode & 0o077) !== 0)
+                throw new Error("Unsafe or oversized checkpoint");
+            const envelope = JSON.parse(await file.readFile("utf8"));
+            if (digest(envelope.value) !== envelope.digest)
+                throw new Error("Checkpoint integrity check failed");
+            return envelope.value;
+        }
+        finally {
+            await file.close();
+        }
+    }
+    async write(key, value) {
+        const encoded = JSON.stringify({ value, digest: digest(value) });
+        if (Buffer.byteLength(encoded) > 8 * 1024 * 1024)
+            throw new RangeError("Checkpoint exceeds 8 MiB");
+        const destination = join(this.directory, `${key}.json`);
+        const temporary = `${destination}.${randomUUID()}.tmp`;
+        const file = await open(temporary, "wx", 0o600);
+        try {
+            await file.writeFile(encoded);
+            await file.sync();
+        }
+        finally {
+            await file.close();
+        }
+        await rename(temporary, destination);
+        const directory = await open(this.directory, "r");
+        try {
+            await directory.sync();
+        }
+        finally {
+            await directory.close();
+        }
+    }
+    async release() {
+        await rm(join(this.directory, ".lock"), { recursive: true });
+    }
+}

package/dist/evals/client.d.ts

@@ -0,0 +1,96 @@
+import type { ProjectConnection } from "../types.js";
+import type { CaseWrite, CompleteExecution, Completion, Dataset, DatasetCase, DatasetVersion, EvaluationItem, EvaluationRun, Execution, Experiment, ExperimentCase, ExperimentItem, Identity, JsonValue, JudgeBudget, JudgeJob, Page, PageOptions, Result, ResultSummary, Scorer, ScorerDefinition, ScorerVersion, StartExecution, Subject, StoredResult } from "./types.js";
+export interface EvaluationClientOptions {
+    apiKey: string;
+    baseUrl?: string;
+    timeoutMillis?: number;
+}
+export declare class HueApiError extends Error {
+    readonly status?: number | undefined;
+    constructor(status?: number | undefined);
+}
+/** No implicit mutation retry: callers retain stable idempotency keys for experiments/results. */
+export declare class EvaluationClient {
+    readonly baseUrl: string;
+    private readonly apiKey;
+    private readonly timeoutMillis;
+    constructor(options: EvaluationClientOptions);
+    private request;
+    private page;
+    checkConnection(): Promise<ProjectConnection>;
+    createDataset(input: Identity): Promise<Dataset>;
+    getDataset(id: string): Promise<Dataset>;
+    listDatasets(page?: PageOptions): Promise<Page<Omit<Dataset, "versions">>>;
+    createDatasetVersion(id: string, input?: {
+        fromVersionId?: string;
+    }): Promise<DatasetVersion>;
+    getDatasetVersion(id: string): Promise<DatasetVersion>;
+    listCases(id: string, page?: PageOptions): Promise<Page<DatasetCase>>;
+    addCase(id: string, input: CaseWrite): Promise<{
+        item: DatasetCase;
+        version: DatasetVersion;
+    }>;
+    freezeDatasetVersion(id: string, expectedRevision: number): Promise<DatasetVersion>;
+    createScorer(input: Identity): Promise<Scorer>;
+    getScorer(id: string): Promise<Scorer>;
+    listScorers(page?: PageOptions): Promise<Page<Scorer>>;
+    publishScorerVersion(id: string, definition: ScorerDefinition): Promise<ScorerVersion>;
+    getScorerVersion(id: string): Promise<ScorerVersion>;
+    createExperiment(input: {
+        idempotencyKey: string;
+        name: string;
+        datasetVersionId: string;
+        scorerVersionIds: string[];
+        config: JsonValue;
+    }): Promise<{
+        id: string;
+        evaluationRunId: string;
+    }>;
+    getExperiment(id: string): Promise<Experiment>;
+    listExperimentItems(id: string, page?: PageOptions): Promise<Page<ExperimentItem>>;
+    getExperimentCase(id: string, caseId: string): Promise<ExperimentCase>;
+    startExecution(id: string, caseId: string, input: StartExecution): Promise<Execution>;
+    getExecution(id: string): Promise<Execution>;
+    completeExecution(id: string, input: CompleteExecution): Promise<Completion>;
+    finishExperiment(id: string, idempotencyKey: string): Promise<{
+        id: string;
+        finishedAt: string;
+    }>;
+    createEvaluationRun(input: {
+        idempotencyKey: string;
+        name: string;
+        subjectIds: string[];
+        scorerVersionIds: string[];
+    }): Promise<{
+        id: string;
+    }>;
+    getEvaluationRun(id: string): Promise<EvaluationRun>;
+    listEvaluationItems(id: string, page?: PageOptions): Promise<Page<EvaluationItem>>;
+    getSubject(id: string): Promise<Subject>;
+    submitResults(id: string, input: {
+        idempotencyKey: string;
+        results: Result[];
+    }): Promise<{
+        ids: string[];
+    }>;
+    listResults(id: string, page?: PageOptions): Promise<Page<ResultSummary>>;
+    getResult(id: string): Promise<StoredResult>;
+    createJudgeJobs(id: string, input: {
+        idempotencyKey: string;
+        jobs: {
+            evaluationItemId: string;
+            scorerVersionId: string;
+        }[];
+    }): Promise<{
+        ids: string[];
+    }>;
+    listJudgeJobs(id: string, page?: PageOptions): Promise<Page<JudgeJob>>;
+    getJudgeJob(id: string): Promise<JudgeJob>;
+    cancelJudgeJob(id: string, reason: string): Promise<{
+        id: string;
+        state: JudgeJob["state"];
+        cancellationRequested?: boolean;
+    }>;
+    getJudgeBudget(): Promise<JudgeBudget>;
+}
+export declare function createEvaluationClient(options: EvaluationClientOptions): EvaluationClient;