@zhanla/sdk-ts 0.1.0

package/dist/manifest.js

@@ -0,0 +1,275 @@
+/**
+ * Manifest emission — serialize SDK component instances into ComponentManifest objects.
+ * The manifest shape mirrors the Python CLI's ComponentManifest dataclass.
+ */
+import { readFileSync } from "node:fs";
+// ---------------------------------------------------------------------------
+// Tree serialization helpers
+// ---------------------------------------------------------------------------
+function isComponentLike(value) {
+    return (value != null &&
+        typeof value === "object" &&
+        typeof value.componentType === "string" &&
+        typeof value.name === "string" &&
+        typeof value.description === "string" &&
+        typeof value.versionHash === "function");
+}
+function isConditionalLike(value) {
+    return (value != null &&
+        typeof value === "object" &&
+        typeof value.condition === "function" &&
+        typeof value.ifTrue === "string" &&
+        typeof value.ifFalse === "string");
+}
+function isLeafLike(value) {
+    return (value != null &&
+        typeof value === "object" &&
+        "eval" in value &&
+        !("threshold" in value));
+}
+function serializeEdge(edge) {
+    return {
+        weight: edge.weight,
+        node: isLeafLike(edge.node)
+            ? { type: "leaf", eval: edge.node.eval.name }
+            : serializeBranch(edge.node),
+    };
+}
+function serializeBranch(branch) {
+    return {
+        type: "branch",
+        eval: branch.eval.name,
+        threshold: branch.threshold,
+        if_pass: branch.ifPass.map(serializeEdge),
+        if_fail: branch.ifFail.map(serializeEdge),
+    };
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function findMatchingParen(source, startIndex) {
+    let depth = 0;
+    let quote = null;
+    let inLineComment = false;
+    let inBlockComment = false;
+    for (let i = startIndex; i < source.length; i += 1) {
+        const char = source[i];
+        const next = source[i + 1];
+        const prev = source[i - 1];
+        if (inLineComment) {
+            if (char === "\n")
+                inLineComment = false;
+            continue;
+        }
+        if (inBlockComment) {
+            if (prev === "*" && char === "/")
+                inBlockComment = false;
+            continue;
+        }
+        if (quote) {
+            if (char === quote && prev !== "\\")
+                quote = null;
+            continue;
+        }
+        if (char === "/" && next === "/") {
+            inLineComment = true;
+            i += 1;
+            continue;
+        }
+        if (char === "/" && next === "*") {
+            inBlockComment = true;
+            i += 1;
+            continue;
+        }
+        if (char === "'" || char === '"' || char === "`") {
+            quote = char;
+            continue;
+        }
+        if (char === "(") {
+            depth += 1;
+            continue;
+        }
+        if (char === ")") {
+            depth -= 1;
+            if (depth === 0)
+                return i;
+        }
+    }
+    return -1;
+}
+function extractExportedComponentSource(filePath, symbolName) {
+    if (!filePath || !symbolName)
+        return undefined;
+    let source = "";
+    try {
+        source = readFileSync(filePath, "utf8");
+    }
+    catch {
+        return undefined;
+    }
+    const pattern = new RegExp(String.raw `(?:^|\n)\s*export\s+(?:const|let|var)\s+${escapeRegExp(symbolName)}\s*=\s*new\s+[A-Za-z_$][\w$.]*\s*\(`, "m");
+    const match = pattern.exec(source);
+    if (!match || match.index == null)
+        return undefined;
+    const start = source.lastIndexOf("export", match.index + match[0].length);
+    const openParenIndex = source.indexOf("(", match.index + match[0].length - 1);
+    if (start < 0 || openParenIndex < 0)
+        return undefined;
+    const closeParenIndex = findMatchingParen(source, openParenIndex);
+    if (closeParenIndex < 0)
+        return undefined;
+    let end = closeParenIndex + 1;
+    while (end < source.length && /\s/.test(source[end]))
+        end += 1;
+    if (source[end] === ";")
+        end += 1;
+    return source.slice(start, end).trim();
+}
+// ---------------------------------------------------------------------------
+// to_manifest
+// ---------------------------------------------------------------------------
+export function toManifest(component, opts = {}) {
+    const componentSource = extractExportedComponentSource(opts.filePath, opts.symbolName);
+    const base = {
+        name: component.name,
+        description: component.description,
+        component_type: component.componentType,
+        language: "typescript",
+        is_runnable: component.isRunnable,
+        is_eval: component.isEval,
+        version_hash: component.versionHash(),
+        file_path: opts.filePath,
+        symbol_name: opts.symbolName,
+    };
+    switch (component.componentType) {
+        case "tool": {
+            const tool = component;
+            return {
+                ...base,
+                fn_present: true,
+                is_async: tool.isAsync,
+                output_schema: tool.outputSchema,
+                source_code: componentSource || tool.fn.toString(),
+                source_language: "typescript",
+                source_format: componentSource ? "component" : "function",
+            };
+        }
+        case "code_eval": {
+            const codeEval = component;
+            return {
+                ...base,
+                fn_present: true,
+                is_async: codeEval.isAsync,
+                source_code: componentSource || codeEval.fn.toString(),
+                source_language: "typescript",
+                source_format: componentSource ? "component" : "function",
+                model_response_format: codeEval.modelResponseFormat,
+            };
+        }
+        case "skill": {
+            const skill = component;
+            return {
+                ...base,
+                instructions: skill.instructions,
+                fn_present: skill.fn != null,
+                is_async: skill.isAsync,
+                tools: skill.tools.map((t) => t.name),
+                output_schema: skill.outputSchema,
+                source_code: componentSource || skill.fn?.toString(),
+                source_language: "typescript",
+                source_format: componentSource ? "component" : skill.fn ? "function" : undefined,
+            };
+        }
+        case "agent": {
+            const agent = component;
+            return {
+                ...base,
+                instructions: agent.instructions,
+                model: agent.model,
+                tools: agent.tools.map((t) => t.name),
+                skills: agent.skills.map((s) => s.name),
+                agents: agent.agents.map((a) => a.name),
+                output_schema: agent.outputSchema,
+            };
+        }
+        case "llm_processor": {
+            const llmProcessor = component;
+            return {
+                ...base,
+                instructions: llmProcessor.instructions,
+                model: llmProcessor.model,
+                output_schema: llmProcessor.outputSchema,
+            };
+        }
+        case "llm_eval": {
+            const llmEval = component;
+            return {
+                ...base,
+                instructions: llmEval.instructions,
+                model: llmEval.model,
+                output_schema: llmEval.outputSchema,
+            };
+        }
+        case "orchestration": {
+            const orchestration = component;
+            const steps = orchestration.steps.map((s) => {
+                if (isConditionalLike(s.component)) {
+                    return {
+                        name: s.name,
+                        component: "conditional",
+                        next: s.next,
+                        is_conditional: true,
+                        if_true: s.component.ifTrue,
+                        if_false: s.component.ifFalse,
+                    };
+                }
+                return {
+                    name: s.name,
+                    component: s.component.name,
+                    next: s.next,
+                };
+            });
+            return { ...base, steps };
+        }
+        case "checklist": {
+            const checklist = component;
+            return {
+                ...base,
+                evals: checklist.evals.map((e) => e.name),
+                weights: checklist.weights,
+            };
+        }
+        case "eval_tree": {
+            const evalTree = component;
+            return {
+                ...base,
+                root: serializeBranch(evalTree.root),
+            };
+        }
+        default:
+            return base;
+    }
+}
+// ---------------------------------------------------------------------------
+// Collect exported components from a module
+// ---------------------------------------------------------------------------
+export function collectExportedComponents(moduleExports) {
+    const components = [];
+    const seen = new Set();
+    const namespaces = [moduleExports];
+    const defaultNamespace = moduleExports.default;
+    if (defaultNamespace != null &&
+        typeof defaultNamespace === "object" &&
+        !Array.isArray(defaultNamespace)) {
+        namespaces.push(defaultNamespace);
+    }
+    for (const namespace of namespaces) {
+        for (const value of Object.values(namespace)) {
+            if (isComponentLike(value) && !seen.has(value)) {
+                seen.add(value);
+                components.push(value);
+            }
+        }
+    }
+    return components;
+}

package/dist/trace_store.d.ts

@@ -0,0 +1,38 @@
+/**
+ * In-memory LLM call trace store backed by AsyncLocalStorage.
+ *
+ * The CLI sets a TraceContext before calling a component's runner.
+ * The bench.wrap() interceptors read the active context and record each LLM call.
+ * After execution the CLI flushes the calls and writes them to Supabase.
+ */
+import { AsyncLocalStorage } from "async_hooks";
+export interface LLMCall {
+    traceId: string;
+    parentId: string | null;
+    sequenceOrder: number;
+    autoraterRunId: string | null;
+    datasetItemId: string | null;
+    provider: "anthropic" | "openai" | "gemini";
+    model: string;
+    inputMessages: unknown[];
+    output: unknown | null;
+    toolCalls: unknown[] | null;
+    rawResponse: unknown | null;
+    inputTokens: number | null;
+    outputTokens: number | null;
+    latencyMs: number;
+    stopReason: string | null;
+    metadata: Record<string, unknown> | null;
+}
+export declare class TraceContext {
+    readonly traceId: string;
+    readonly autoraterRunId: string | null;
+    readonly datasetItemId: string | null;
+    private calls;
+    private sequence;
+    constructor(traceId: string, autoraterRunId?: string | null, datasetItemId?: string | null);
+    nextSequence(): number;
+    record(call: LLMCall): void;
+    flush(): LLMCall[];
+}
+export declare const traceStorage: AsyncLocalStorage<TraceContext>;

package/dist/trace_store.js

@@ -0,0 +1,30 @@
+/**
+ * In-memory LLM call trace store backed by AsyncLocalStorage.
+ *
+ * The CLI sets a TraceContext before calling a component's runner.
+ * The bench.wrap() interceptors read the active context and record each LLM call.
+ * After execution the CLI flushes the calls and writes them to Supabase.
+ */
+import { AsyncLocalStorage } from "async_hooks";
+export class TraceContext {
+    traceId;
+    autoraterRunId;
+    datasetItemId;
+    calls = [];
+    sequence = 0;
+    constructor(traceId, autoraterRunId = null, datasetItemId = null) {
+        this.traceId = traceId;
+        this.autoraterRunId = autoraterRunId;
+        this.datasetItemId = datasetItemId;
+    }
+    nextSequence() {
+        return this.sequence++;
+    }
+    record(call) {
+        this.calls.push(call);
+    }
+    flush() {
+        return this.calls.splice(0);
+    }
+}
+export const traceStorage = new AsyncLocalStorage();

package/dist/types.d.ts

@@ -0,0 +1,283 @@
+/**
+ * Core types for the bench TypeScript SDK.
+ * Mirrors the Python SDK's class-based component model.
+ */
+export type JsonSchemaScalar = {
+    type: "string" | "number" | "boolean" | "null";
+};
+export type JsonSchemaObject = {
+    type: "object";
+    properties: Record<string, JsonSchema>;
+    required?: string[];
+    additionalProperties?: boolean;
+};
+export type JsonSchemaArray = {
+    type: "array";
+    items: JsonSchema;
+};
+export type JsonSchema = JsonSchemaScalar | JsonSchemaObject | JsonSchemaArray | Record<string, unknown>;
+export type ComponentType = "tool" | "code_eval" | "skill" | "agent" | "llm_processor" | "llm_eval" | "orchestration" | "checklist" | "eval_tree";
+export declare const RUNNABLE_TYPES: Set<ComponentType>;
+export declare const EVAL_TYPES: Set<ComponentType>;
+export interface ToolCall {
+    name: string;
+    input: Record<string, unknown>;
+    id: string;
+    _parseError?: string;
+}
+export interface LLMResponse {
+    text: string;
+    toolCalls: ToolCall[];
+    stopReason: string | null;
+}
+export interface RunnerMessage {
+    role: "system" | "user" | "assistant";
+    content: string;
+}
+export interface RunnerCallOptions {
+    messages: RunnerMessage[];
+    model: string;
+    tools?: Tool[];
+    outputSchema?: JsonSchema;
+    jsonRepair?: boolean;
+}
+export interface RunnerOptions {
+    client: unknown;
+}
+export declare class Runner {
+    readonly client: unknown;
+    constructor(opts: RunnerOptions);
+    buildMessages(component: Pick<BaseComponent, "description"> & {
+        instructions?: string;
+    }, row: Record<string, unknown>): RunnerMessage[];
+    private providerName;
+    private buildJsonRepairMessages;
+    private requestJsonRepair;
+    maybeRepairJson(args: {
+        model: string;
+        outputSchema?: JsonSchema;
+        jsonRepair?: boolean;
+        text: string;
+    }): Promise<string>;
+    callLlm(opts: RunnerCallOptions): Promise<LLMResponse>;
+}
+export declare abstract class BaseComponent {
+    abstract readonly componentType: ComponentType;
+    abstract readonly name: string;
+    abstract readonly description: string;
+    get isRunnable(): boolean;
+    get isEval(): boolean;
+    abstract versionHash(): string;
+}
+export interface ToolOptions {
+    name: string;
+    description: string;
+    fn: (...args: unknown[]) => unknown;
+    inputSchema: JsonSchema;
+    outputSchema?: JsonSchema;
+}
+export declare class Tool extends BaseComponent {
+    readonly componentType: "tool";
+    readonly name: string;
+    readonly description: string;
+    readonly fn: (...args: unknown[]) => unknown;
+    readonly inputSchema: JsonSchema;
+    readonly outputSchema?: JsonSchema;
+    readonly isAsync: boolean;
+    constructor(opts: ToolOptions);
+    versionHash(): string;
+}
+export interface CodeEvalOptions {
+    name: string;
+    description: string;
+    fn: (...args: unknown[]) => unknown;
+    modelResponseFormat?: "JSON" | "TEXT" | "YAML";
+}
+export declare class CodeEval extends BaseComponent {
+    readonly componentType: "code_eval";
+    readonly name: string;
+    readonly description: string;
+    readonly fn: (...args: unknown[]) => unknown;
+    readonly isAsync: boolean;
+    readonly modelResponseFormat: "JSON" | "TEXT" | "YAML";
+    constructor(opts: CodeEvalOptions);
+    versionHash(): string;
+}
+export interface SkillOptions {
+    name: string;
+    description: string;
+    instructions: string;
+    tools?: Tool[];
+    fn?: (...args: unknown[]) => unknown;
+    outputSchema?: JsonSchema;
+}
+export declare class Skill extends BaseComponent {
+    readonly componentType: "skill";
+    readonly name: string;
+    readonly description: string;
+    readonly instructions: string;
+    readonly tools: Tool[];
+    readonly fn?: (...args: unknown[]) => unknown;
+    readonly outputSchema?: JsonSchema;
+    readonly isAsync: boolean;
+    constructor(opts: SkillOptions);
+    versionHash(): string;
+}
+export interface AgentOptions {
+    name: string;
+    description: string;
+    instructions: string;
+    model: string;
+    runner?: Runner;
+    tools?: Tool[];
+    skills?: Skill[];
+    agents?: Agent[];
+    outputSchema?: JsonSchema;
+    jsonRepair?: boolean;
+}
+export declare class Agent extends BaseComponent {
+    readonly componentType: "agent";
+    readonly name: string;
+    readonly description: string;
+    readonly instructions: string;
+    readonly model: string;
+    readonly runner?: Runner;
+    readonly tools: Tool[];
+    readonly skills: Skill[];
+    readonly agents: Agent[];
+    readonly outputSchema?: JsonSchema;
+    readonly jsonRepair: boolean;
+    constructor(opts: AgentOptions);
+    versionHash(): string;
+}
+export interface LLMProcessorOptions {
+    name: string;
+    description: string;
+    instructions: string;
+    model: string;
+    runner?: Runner;
+    outputSchema?: JsonSchema;
+    jsonRepair?: boolean;
+}
+export declare class LLMProcessor extends BaseComponent {
+    readonly componentType: "llm_processor";
+    readonly name: string;
+    readonly description: string;
+    readonly instructions: string;
+    readonly model: string;
+    readonly runner?: Runner;
+    readonly outputSchema?: JsonSchema;
+    readonly jsonRepair: boolean;
+    constructor(opts: LLMProcessorOptions);
+    versionHash(): string;
+}
+export interface LLMEvalOptions {
+    name: string;
+    description: string;
+    instructions: string;
+    model: string;
+    runner?: Runner;
+    outputSchema?: JsonSchema;
+    jsonRepair?: boolean;
+}
+export declare class LLMEval extends BaseComponent {
+    readonly componentType: "llm_eval";
+    readonly name: string;
+    readonly description: string;
+    readonly instructions: string;
+    readonly model: string;
+    readonly runner?: Runner;
+    readonly outputSchema?: JsonSchema;
+    readonly jsonRepair: boolean;
+    constructor(opts: LLMEvalOptions);
+    versionHash(): string;
+}
+export interface ConditionalOptions {
+    condition: (ctx: Record<string, unknown>) => boolean;
+    ifTrue: string;
+    ifFalse: string;
+}
+export declare class Conditional {
+    readonly condition: (ctx: Record<string, unknown>) => boolean;
+    readonly ifTrue: string;
+    readonly ifFalse: string;
+    constructor(opts: ConditionalOptions);
+}
+export interface StepOptions {
+    name: string;
+    component: BaseComponent | Conditional;
+    next?: string[];
+}
+export declare class Step {
+    readonly name: string;
+    readonly component: BaseComponent | Conditional;
+    readonly next: string[];
+    constructor(opts: StepOptions);
+}
+export interface OrchestrationOptions {
+    name: string;
+    description: string;
+    steps: Step[];
+}
+export declare class Orchestration extends BaseComponent {
+    readonly componentType: "orchestration";
+    readonly name: string;
+    readonly description: string;
+    readonly steps: Step[];
+    constructor(opts: OrchestrationOptions);
+    versionHash(): string;
+}
+export declare class Leaf {
+    readonly eval: BaseComponent;
+    constructor(opts: {
+        eval: BaseComponent;
+    });
+}
+export declare class Edge {
+    readonly weight: number;
+    readonly node: Leaf | Branch;
+    constructor(opts: {
+        weight: number;
+        node: Leaf | Branch;
+    });
+}
+export declare class Branch {
+    readonly eval: BaseComponent;
+    readonly threshold: number;
+    readonly ifPass: Edge[];
+    readonly ifFail: Edge[];
+    constructor(opts: {
+        eval: BaseComponent;
+        threshold: number;
+        ifPass: Edge[];
+        ifFail: Edge[];
+    });
+}
+export interface ChecklistOptions {
+    name: string;
+    description: string;
+    evals: BaseComponent[];
+    weights?: number[];
+}
+export declare class Checklist extends BaseComponent {
+    readonly componentType: "checklist";
+    readonly name: string;
+    readonly description: string;
+    readonly evals: BaseComponent[];
+    readonly weights?: number[];
+    constructor(opts: ChecklistOptions);
+    versionHash(): string;
+}
+export interface EvalTreeOptions {
+    name: string;
+    description: string;
+    root: Branch;
+}
+export declare class EvalTree extends BaseComponent {
+    readonly componentType: "eval_tree";
+    readonly name: string;
+    readonly description: string;
+    readonly root: Branch;
+    constructor(opts: EvalTreeOptions);
+    versionHash(): string;
+}