@hue-run/sdk 0.1.2

package/dist/evals/schema-worker.js

@@ -0,0 +1,14 @@
+import { parentPort, workerData } from "node:worker_threads";
+import { Ajv2020 } from "ajv/dist/2020.js";
+try {
+    // No asynchronous loading, custom formats, coercion, defaults or data mutation.
+    const ajv = new Ajv2020({ strict: false, allErrors: false, validateFormats: false });
+    const validate = ajv.compile(workerData.schema);
+    if ("$async" in validate && validate.$async)
+        throw new Error("Async schemas are unsupported");
+    parentPort.postMessage({ match: validate(workerData.output) === true });
+}
+catch {
+    parentPort.postMessage({ error: true });
+}
+parentPort.close();

package/dist/evals/scorers.d.ts

@@ -0,0 +1,21 @@
+import type { JsonValue, LocalScorer, MetricDefinition, Score, ScoreContext, ScorerDefinition, ScorerVersion } from "./types.js";
+export declare const builtins: {
+    exactMatch: () => ScorerDefinition;
+    includes: (caseSensitive?: boolean) => ScorerDefinition;
+    jsonSchema: (schema: JsonValue) => ScorerDefinition;
+};
+/** The source hash is a caller declaration; closures and installed dependencies are not attested. */
+export declare function defineLocalScorer(options: {
+    source: string | Uint8Array;
+    entrypoint: string;
+    metrics: MetricDefinition[];
+    score: LocalScorer["score"];
+}): LocalScorer;
+/** Check local callbacks before target invocation; never execute downloaded source code. */
+export declare function validateScorerBindings(versions: ScorerVersion[], scorers?: LocalScorer[]): void;
+export declare function scoreLocally(version: ScorerVersion, context: ScoreContext, options?: {
+    scorers?: LocalScorer[];
+    schemaTimeoutMillis?: number;
+}): Promise<Score>;
+export declare function validateScore(value: Score, definition: ScorerDefinition): Score;
+export declare function persistedScore(score: Score, persistResultContent: boolean): Score;

package/dist/evals/scorers.js

@@ -0,0 +1,212 @@
+import { existsSync } from "node:fs";
+import { Worker } from "node:worker_threads";
+import { digest, json, sourceDigest } from "./json.js";
+export const builtins = {
+    exactMatch: () => ({
+        kind: "builtin",
+        entry: "hue.exact_match.v1",
+        config: {},
+    }),
+    includes: (caseSensitive = true) => ({
+        kind: "builtin",
+        entry: "hue.includes.v1",
+        config: { caseSensitive },
+    }),
+    jsonSchema: (schema) => ({
+        kind: "builtin",
+        entry: "hue.json_schema.v1",
+        config: { schema: json(schema) },
+    }),
+};
+/** The source hash is a caller declaration; closures and installed dependencies are not attested. */
+export function defineLocalScorer(options) {
+    return {
+        definition: {
+            kind: "local_code",
+            language: "typescript",
+            entrypoint: options.entrypoint,
+            sourceDigest: sourceDigest(options.source),
+            metrics: options.metrics,
+        },
+        score: options.score,
+    };
+}
+function schemaScore(schema, output, timeoutMillis) {
+    return new Promise((resolve) => {
+        const compiled = new URL("./schema-worker.js", import.meta.url);
+        const file = existsSync(compiled) ? compiled : new URL("./schema-worker.ts", import.meta.url);
+        const worker = new Worker(file, {
+            workerData: { schema, output },
+            resourceLimits: { maxOldGenerationSizeMb: 64 },
+        });
+        let settled = false;
+        const finish = async (result) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            await worker.terminate();
+            resolve(result);
+        };
+        const timer = setTimeout(() => void finish({ state: "error", error: { type: "SchemaTimeout" } }), timeoutMillis);
+        worker.once("message", (value) => {
+            void finish(typeof value.match === "boolean"
+                ? match(value.match, "JSON Schema conformance evaluated locally")
+                : { state: "error", error: { type: "InvalidSchema" } });
+        });
+        worker.once("error", () => void finish({ state: "error", error: { type: "SchemaWorkerError" } }));
+        worker.once("exit", () => {
+            if (!settled)
+                void finish({ state: "error", error: { type: "SchemaWorkerExit" } });
+        });
+    });
+}
+function match(value, explanation) {
+    return { state: "scored", metrics: [{ name: "match", value, passed: value }], explanation };
+}
+function skip(explanation) {
+    return { state: "skipped", explanation };
+}
+/** Check local callbacks before target invocation; never execute downloaded source code. */
+export function validateScorerBindings(versions, scorers = []) {
+    for (const { definition } of versions) {
+        if (definition.kind === "local_code" &&
+            !scorers.some((local) => digest(local.definition) === digest(definition)))
+            throw new Error("A pinned local scorer has no matching language/source/entrypoint/metric binding");
+    }
+}
+export async function scoreLocally(version, context, options = {}) {
+    const definition = version.definition;
+    if (definition.kind === "llm_judge")
+        throw new TypeError("Hosted judge pins require dispatch through createJudgeJobs; do not submit a local result");
+    const timeout = options.schemaTimeoutMillis ?? 2000;
+    if (!Number.isInteger(timeout) || timeout < 100 || timeout > 60_000)
+        throw new RangeError("schemaTimeoutMillis must be 100–60000");
+    if (!context.hasOutput)
+        return skip("Output evidence is unavailable");
+    if (context.output === undefined)
+        throw new TypeError("hasOutput requires a present JSON output");
+    try {
+        // Clone and validate inputs so a scorer cannot mutate another scorer's evidence.
+        json(context, 1024 * 1024);
+        const owned = structuredClone(context);
+        if (definition.kind === "manual")
+            return skip("Manual scoring requires a human session");
+        if (definition.kind === "local_code") {
+            const binding = options.scorers?.find((local) => digest(local.definition) === digest(definition));
+            if (!binding)
+                return { state: "error", error: { type: "ScorerBindingUnavailable" } };
+            return validateScore(await binding.score(owned), definition);
+        }
+        if (definition.entry === "hue.json_schema.v1")
+            return await schemaScore(definition.config.schema, owned.output, timeout);
+        if (!owned.hasExpected)
+            return skip("Reference evidence is unavailable");
+        if (definition.entry === "hue.exact_match.v1")
+            return match(digest(owned.output) === digest(owned.expected), "JSON exact match evaluated locally");
+        if (typeof owned.output !== "string" || typeof owned.expected !== "string")
+            return skip("Includes requires string output and reference");
+        const normalize = (value) => definition.config.caseSensitive ? value : value.toLowerCase();
+        return match(normalize(owned.output).includes(normalize(owned.expected)), "String inclusion evaluated locally");
+    }
+    catch {
+        // Arbitrary thrown messages can include secrets/output. Keep scorer failures typed and sanitized.
+        return { state: "error", error: { type: "LocalScorerError" } };
+    }
+}
+export function validateScore(value, definition) {
+    json(value);
+    const score = structuredClone(value);
+    const keys = (object, allowed) => {
+        if (Object.keys(object).some((key) => !allowed.includes(key)))
+            throw new TypeError("Unexpected scorer result field");
+    };
+    if (score.state === "error") {
+        keys(score, ["state", "error"]);
+        if (!score.error ||
+            typeof score.error.type !== "string" ||
+            !score.error.type ||
+            score.error.type.length > 200)
+            throw new TypeError("Scorer error requires a bounded type");
+        keys(score.error, ["type", "message"]);
+        if (score.error.message !== undefined &&
+            (typeof score.error.message !== "string" || score.error.message.length > 4000))
+            throw new TypeError("Invalid scorer error message");
+        return score;
+    }
+    if (score.state === "skipped") {
+        keys(score, ["state", "explanation"]);
+        if (typeof score.explanation !== "string" ||
+            !score.explanation.trim() ||
+            score.explanation.length > 4000)
+            throw new TypeError("Skipped score requires a bounded reason");
+        return score;
+    }
+    if (score.state !== "scored" || !Array.isArray(score.metrics))
+        throw new TypeError("Invalid scorer result");
+    keys(score, ["state", "metrics", "explanation", "evidence"]);
+    const metrics = definition.kind === "builtin" ? [{ name: "match", type: "boolean" }] : definition.metrics;
+    if (score.metrics.length !== metrics.length ||
+        new Set(score.metrics.map((m) => m.name)).size !== metrics.length)
+        throw new TypeError("Scorer result must contain every declared metric once");
+    for (const expected of metrics) {
+        const actual = score.metrics.find((item) => item.name === expected.name);
+        if (!actual || (actual.passed !== undefined && typeof actual.passed !== "boolean"))
+            throw new TypeError("Invalid metric");
+        keys(actual, ["name", "value", "passed"]);
+        const value = actual.value;
+        if (expected.type === "number") {
+            if (typeof value !== "number" ||
+                !Number.isFinite(value) ||
+                (expected.min !== undefined && value < expected.min) ||
+                (expected.max !== undefined && value > expected.max))
+                throw new TypeError("Invalid numeric metric");
+        }
+        else if (expected.type === "category") {
+            if (typeof value !== "string" || !expected.categories.includes(value))
+                throw new TypeError("Invalid category metric");
+        }
+        else if (typeof value !== (expected.type === "text" ? "string" : "boolean"))
+            throw new TypeError("Invalid metric type");
+        if (typeof value === "string" && value.length > 4000)
+            throw new TypeError("Metric text is too long");
+    }
+    if (score.explanation !== undefined &&
+        (typeof score.explanation !== "string" ||
+            !score.explanation.trim() ||
+            score.explanation.length > 4000))
+        throw new TypeError("Invalid explanation");
+    const evidence = score.evidence;
+    const hasEvidence = evidence !== undefined &&
+        evidence !== null &&
+        (typeof evidence !== "string" || evidence.trim().length > 0) &&
+        (typeof evidence !== "object" || Object.keys(evidence).length > 0);
+    if (!score.explanation && !hasEvidence)
+        throw new TypeError("Scored results require an explanation or evidence");
+    return score;
+}
+export function persistedScore(score, persistResultContent) {
+    if (persistResultContent)
+        return score;
+    if (score.state === "scored")
+        return {
+            state: "scored",
+            metrics: score.metrics,
+            explanation: "Local scoring completed; result content storage disabled",
+        };
+    if (score.state === "error")
+        return { state: "error", error: { type: "LocalScorerError" } };
+    // Preserve fixed unavailable reasons, never arbitrary caller explanations.
+    const safeReasons = [
+        "Output evidence is unavailable",
+        "Reference evidence is unavailable",
+        "Includes requires string output and reference",
+        "Manual scoring requires a human session",
+    ];
+    return {
+        state: "skipped",
+        explanation: safeReasons.includes(score.explanation)
+            ? score.explanation
+            : "Local scoring skipped; result content storage disabled",
+    };
+}

package/dist/evals/types.d.ts

@@ -0,0 +1,301 @@
+import type { JsonValue } from "../types.js";
+export type { JsonValue } from "../types.js";
+export interface Page<T> {
+    items: T[];
+    nextCursor: string | null;
+}
+export interface PageOptions {
+    after?: string;
+    limit?: number;
+}
+export interface Identity {
+    name: string;
+    slug: string;
+    description?: string;
+}
+export interface Dataset extends Identity {
+    id: string;
+    versions: DatasetVersion[];
+}
+export interface DatasetVersion {
+    id: string;
+    datasetId: string;
+    version: number;
+    revision: number;
+    frozenAt: string | null;
+    contentDigest: string | null;
+}
+export interface DatasetCase {
+    id: string;
+    externalKey: string;
+    inputs: JsonValue;
+    expected?: JsonValue;
+    metadata: Record<string, JsonValue>;
+    datasetVersionId: string;
+}
+export interface ExperimentCase extends DatasetCase {
+    hasExpected: boolean;
+}
+export interface CaseWrite {
+    expectedRevision: number;
+    externalKey: string;
+    inputs: JsonValue;
+    expected?: JsonValue;
+    metadata?: Record<string, JsonValue>;
+}
+export type MetricDefinition = {
+    name: string;
+    type: "boolean" | "text";
+} | {
+    name: string;
+    type: "number";
+    min?: number;
+    max?: number;
+} | {
+    name: string;
+    type: "category";
+    categories: string[];
+};
+export type ScorerDefinition = {
+    kind: "builtin";
+    entry: "hue.exact_match.v1";
+    config: Record<string, never>;
+} | {
+    kind: "builtin";
+    entry: "hue.includes.v1";
+    config: {
+        caseSensitive: boolean;
+    };
+} | {
+    kind: "builtin";
+    entry: "hue.json_schema.v1";
+    config: {
+        schema: JsonValue;
+    };
+} | {
+    kind: "local_code";
+    language: "typescript" | "python";
+    entrypoint: string;
+    sourceDigest: string;
+    metrics: MetricDefinition[];
+} | {
+    kind: "manual";
+    metrics: MetricDefinition[];
+} | {
+    kind: "llm_judge";
+    config: JudgeConfig;
+    metrics: MetricDefinition[];
+};
+export interface JudgeConfig {
+    model: string;
+    provider: string;
+    rubric: string;
+    bindings: {
+        name: string;
+        path: string;
+        required: boolean;
+    }[];
+    maxOutputTokens: number;
+    timeoutMs: number;
+    temperature?: number;
+}
+export interface ResultSummary {
+    id: string;
+    itemId: string;
+    scorerVersionId: string;
+    state: "scored" | "error" | "skipped";
+}
+export interface StoredResult extends ResultSummary {
+    runId: string;
+    metrics: Metric[];
+    explanation: string | null;
+    evidence: JsonValue;
+    error: TypedError | null;
+    sourceDigest: string | null;
+}
+export interface JudgeJob {
+    id: string;
+    runId: string;
+    itemId: string;
+    scorerVersionId: string;
+    state: "queued" | "running" | "completed" | "cancelled" | "uncertain";
+    chargeState: "unreserved" | "reserved" | "settled" | "uncertain";
+    /** Preserved charge state before any separately verified reconciliation. */
+    originalChargeState: JudgeJob["chargeState"];
+    reservationMicroUsd: number;
+    actualMicroUsd: number | null;
+    reconciliation: {
+        jobId: string;
+        actualMicroUsd: number;
+        evidenceReference: string;
+        reason: string;
+        createdAt: string;
+    } | null;
+    priceQuote: JsonValue;
+    receipt: JsonValue;
+    workflowId: string | null;
+    cancelRequestedAt: string | null;
+    cancellationReason: string | null;
+    createdAt: string;
+    startedAt: string | null;
+    finishedAt: string | null;
+}
+export interface JudgeBudget {
+    projectId: string;
+    configured: boolean;
+    /** Credential resolution does not establish provider acceptance or available funds. */
+    authentication?: {
+        status: "available" | "unavailable";
+        method: "api-key" | "oidc" | null;
+        verification: "credential_resolution";
+    };
+    enabled: boolean;
+    allowanceMicroUsd: number;
+    reservedMicroUsd: number;
+    spentMicroUsd: number;
+    maxInFlight: number;
+    blocked: boolean;
+}
+export interface Scorer extends Identity {
+    id: string;
+    versions?: ScorerVersion[];
+}
+export interface ScorerVersion {
+    id: string;
+    contentDigest: string;
+    definition: ScorerDefinition;
+}
+export type TerminalState = "succeeded" | "error" | "cancelled";
+export interface Execution {
+    id: string;
+    attempt: number;
+    state: TerminalState | "started" | "uncertain";
+    traceExternalId: string | null;
+    subjectId?: string | null;
+}
+export interface ExperimentItem {
+    id: string;
+    externalKey: string;
+    hasExpected: boolean;
+    execution: Execution | null;
+}
+export interface EvaluationRun {
+    id: string;
+    name: string;
+    scorerVersions: ScorerVersion[];
+    itemCount: number;
+    scores: {
+        scored: number;
+        error: number;
+        skipped: number;
+        pending: number;
+    };
+}
+export interface Experiment {
+    id: string;
+    name: string;
+    datasetVersionId: string;
+    config: JsonValue;
+    configDigest: string;
+    evaluation: EvaluationRun;
+    caseCount: number;
+    finishedAt: string | null;
+    execution: {
+        unstarted: number;
+        started: number;
+        uncertain: number;
+        succeeded: number;
+        error: number;
+        cancelled: number;
+    };
+}
+export interface TypedError {
+    type: string;
+    message?: string;
+}
+export interface StartExecution {
+    idempotencyKey: string;
+    traceExternalId?: string;
+    previousExecutionId?: string;
+    allowUncertainRetry?: boolean;
+}
+export interface CompleteExecution {
+    idempotencyKey: string;
+    state: TerminalState;
+    output?: JsonValue;
+    error?: TypedError;
+    expectedTraceRevision?: number;
+    traceEvidence?: "required" | "omit";
+    omissionReason?: string;
+}
+export interface Completion {
+    executionId: string;
+    subjectId: string;
+    traceSnapshotId: string | null;
+    evaluationItemId: string;
+}
+export interface EvaluationItem {
+    id: string;
+    subjectId: string;
+    hasOutput: boolean;
+    traceSnapshotId: string | null;
+}
+export interface Subject {
+    id: string;
+    inputs: JsonValue;
+    hasOutput: boolean;
+    output?: JsonValue;
+    hasExpected: boolean;
+    expected?: JsonValue;
+    metadata: Record<string, JsonValue>;
+    contentDigest: string;
+    outputEvidence: "available" | "unavailable";
+    executionState: TerminalState;
+    traceSnapshotId: string | null;
+    caseId: string;
+    datasetVersionId: string;
+    caseExternalKey: string;
+    experimentId: string;
+    attempt: number;
+    traceEvidence: "captured" | "omitted" | "not_requested";
+    traceExternalId: string | null;
+    omissionReason: string | null;
+}
+export interface Metric {
+    name: string;
+    value: boolean | number | string;
+    passed?: boolean;
+}
+export type Score = {
+    state: "scored";
+    metrics: Metric[];
+    explanation?: string;
+    evidence?: JsonValue;
+} | {
+    state: "error";
+    error: TypedError;
+} | {
+    state: "skipped";
+    explanation: string;
+};
+export type Result = Score & {
+    evaluationItemId: string;
+    scorerVersionId: string;
+    sourceDigest?: string;
+};
+export interface ScoreContext {
+    inputs: JsonValue;
+    output?: JsonValue;
+    expected?: JsonValue;
+    hasOutput: boolean;
+    hasExpected: boolean;
+    metadata: Record<string, JsonValue>;
+    executionState: TerminalState;
+}
+export interface LocalScorer {
+    definition: Extract<ScorerDefinition, {
+        kind: "local_code";
+    }>;
+    /** Trusted local code. There is no callback timeout or side-effect cancellation. */
+    score(context: ScoreContext): Score | Promise<Score>;
+}

package/dist/evals/types.js

@@ -0,0 +1 @@
+export {};

package/dist/evals.d.ts

@@ -0,0 +1,7 @@
+export { createEvaluationClient, EvaluationClient, HueApiError } from "./evals/client.js";
+export type { EvaluationClientOptions } from "./evals/client.js";
+export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, } from "./evals/runner.js";
+export type { RunExperimentOptions, RescoreOptions, RunnerReport } from "./evals/runner.js";
+export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";
+export { sourceDigest } from "./evals/json.js";
+export type * from "./evals/types.js";

package/dist/evals.js

@@ -0,0 +1,4 @@
+export { createEvaluationClient, EvaluationClient, HueApiError } from "./evals/client.js";
+export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, } from "./evals/runner.js";
+export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";
+export { sourceDigest } from "./evals/json.js";

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { createHue, HueClient, HueConnectionError, type ExistingHueProviders } from "./client.js";
+export { createHueTransport, HueTransport, HueExportError } from "./transport.js";
+export type * from "./types.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export { createHue, HueClient, HueConnectionError } from "./client.js";
+export { createHueTransport, HueTransport, HueExportError } from "./transport.js";

package/dist/privacy.d.ts

@@ -0,0 +1,7 @@
+import type { ReadableSpan } from "@opentelemetry/sdk-trace";
+import type { ReadableLogRecord } from "@opentelemetry/sdk-logs";
+import { type Resource } from "@opentelemetry/resources";
+import type { HueOptions } from "./types.js";
+export type ResourceCache = WeakMap<Resource, Resource>;
+export declare function redactSpan(span: ReadableSpan, options: HueOptions, cache: ResourceCache): ReadableSpan;
+export declare function redactLog(log: ReadableLogRecord, options: HueOptions, cache: ResourceCache): ReadableLogRecord;