@zhanla/sdk-ts 0.1.0

package/dist/types.js

@@ -0,0 +1,697 @@
+/**
+ * Core types for the bench TypeScript SDK.
+ * Mirrors the Python SDK's class-based component model.
+ */
+import { createHash } from "crypto";
+import { parseJsonResponse } from "./json.js";
+import { isAnthropicClient, isGeminiClient, isOpenAIClient, wrap } from "./wrap.js";
+export const RUNNABLE_TYPES = new Set([
+    "tool",
+    "agent",
+    "llm_processor",
+    "orchestration",
+]);
+export const EVAL_TYPES = new Set([
+    "code_eval",
+    "llm_eval",
+    "checklist",
+    "eval_tree",
+]);
+// ---------------------------------------------------------------------------
+// Hashing utilities
+// ---------------------------------------------------------------------------
+function hashFields(...parts) {
+    const h = createHash("sha256");
+    for (const p of parts) {
+        h.update(p, "utf8");
+    }
+    return h.digest("hex");
+}
+function sortedStringify(value) {
+    if (value === null || typeof value !== "object") {
+        return JSON.stringify(value);
+    }
+    if (Array.isArray(value)) {
+        return "[" + value.map(sortedStringify).join(",") + "]";
+    }
+    const keys = Object.keys(value).sort();
+    const parts = keys.map((k) => `${JSON.stringify(k)}:${sortedStringify(value[k])}`);
+    return "{" + parts.join(",") + "}";
+}
+function normalizeSchema(schema) {
+    if (schema == null)
+        return "";
+    return sortedStringify(schema);
+}
+function isPlainRecord(value) {
+    return value != null && typeof value === "object" && !Array.isArray(value);
+}
+function cloneSchemaObject(schema) {
+    return { ...schema };
+}
+function validateToolInputSchema(schema) {
+    if (!isPlainRecord(schema)) {
+        throw new Error("Tool inputSchema must be a JSON Schema object.");
+    }
+    if (schema.type !== "object") {
+        throw new Error("Tool inputSchema must be a JSON Schema object with type 'object'.");
+    }
+    if (!isPlainRecord(schema.properties)) {
+        throw new Error("Tool inputSchema must define a properties object.");
+    }
+    return schema;
+}
+function schemaSummary(schema) {
+    const schemaRecord = isPlainRecord(schema)
+        ? schema
+        : null;
+    const rawProperties = schemaRecord != null && isPlainRecord(schemaRecord["properties"])
+        ? schemaRecord["properties"]
+        : null;
+    const properties = rawProperties != null
+        ? Object.entries(rawProperties)
+            .map(([name, value]) => {
+            const type = isPlainRecord(value) && typeof value.type === "string"
+                ? value.type
+                : "unknown";
+            return `${name}: ${type}`;
+        })
+            .join(", ")
+        : "";
+    return properties ? `Expected JSON object fields: ${properties}` : "Expected a JSON object.";
+}
+function previewText(text, limit = 160) {
+    const normalized = text.replace(/\s+/g, " ").trim();
+    return normalized.length <= limit ? normalized : `${normalized.slice(0, limit)}...`;
+}
+function fnSource(fn) {
+    if (fn == null)
+        return "";
+    return fn.toString();
+}
+function contentPartToText(part) {
+    if (typeof part === "string") {
+        return part;
+    }
+    if (part != null && typeof part === "object" && typeof part.text === "string") {
+        return part.text;
+    }
+    return "";
+}
+function normalizeOpenAIContent(content) {
+    if (typeof content === "string") {
+        return content;
+    }
+    if (!Array.isArray(content)) {
+        return "";
+    }
+    return content.map((part) => {
+        if (part != null && typeof part === "object" && typeof part.text === "string") {
+            return part.text;
+        }
+        return "";
+    }).join("");
+}
+function normalizeGeminiText(response) {
+    if (typeof response["text"] === "string") {
+        return response["text"];
+    }
+    const candidates = Array.isArray(response["candidates"]) ? response["candidates"] : [];
+    const first = candidates[0];
+    if (first == null || typeof first !== "object") {
+        return "";
+    }
+    const content = first.content;
+    if (content == null || typeof content !== "object") {
+        return "";
+    }
+    const parts = Array.isArray(content.parts)
+        ? content.parts
+        : [];
+    return parts.map(contentPartToText).join("");
+}
+function withClosedAdditionalProperties(schema) {
+    const copy = cloneSchemaObject(schema);
+    copy["additionalProperties"] = false;
+    return copy;
+}
+function toGeminiSchema(schema) {
+    const source = cloneSchemaObject(schema);
+    const result = {};
+    for (const [key, value] of Object.entries(source)) {
+        if (key === "required" || key === "additionalProperties" || key === "$schema") {
+            continue;
+        }
+        if (key === "type" && typeof value === "string") {
+            result[key] = value.toUpperCase();
+            continue;
+        }
+        if (key === "properties" && value != null && typeof value === "object" && !Array.isArray(value)) {
+            const properties = {};
+            for (const [propertyName, propertySchema] of Object.entries(value)) {
+                properties[propertyName] = toGeminiSchema(propertySchema);
+            }
+            result[key] = properties;
+            continue;
+        }
+        if (key === "items" && value != null && typeof value === "object" && !Array.isArray(value)) {
+            result[key] = toGeminiSchema(value);
+            continue;
+        }
+        result[key] = value;
+    }
+    return result;
+}
+function parseToolArguments(argumentsValue) {
+    if (argumentsValue != null && typeof argumentsValue === "object" && !Array.isArray(argumentsValue)) {
+        return { input: argumentsValue };
+    }
+    if (typeof argumentsValue === "string") {
+        try {
+            const parsed = parseJsonResponse(argumentsValue);
+            if (parsed != null && typeof parsed === "object" && !Array.isArray(parsed)) {
+                return { input: parsed };
+            }
+        }
+        catch {
+            return { input: {}, parseError: argumentsValue };
+        }
+    }
+    return { input: {} };
+}
+function getGeminiFunctionCall(part) {
+    if (part == null || typeof part !== "object") {
+        return null;
+    }
+    const record = part;
+    const functionCall = record["functionCall"] ?? record["function_call"];
+    return functionCall != null && typeof functionCall === "object"
+        ? functionCall
+        : null;
+}
+export class Runner {
+    client;
+    constructor(opts) {
+        this.client = wrap(opts.client);
+    }
+    buildMessages(component, row) {
+        const messages = [];
+        if (component.instructions != null && component.instructions.trim() !== "") {
+            messages.push({
+                role: "system",
+                content: component.instructions,
+            });
+        }
+        else if (component.description.trim() !== "") {
+            messages.push({
+                role: "system",
+                content: component.description,
+            });
+        }
+        messages.push({
+            role: "user",
+            content: JSON.stringify(row),
+        });
+        return messages;
+    }
+    providerName() {
+        if (isAnthropicClient(this.client))
+            return "anthropic";
+        if (isOpenAIClient(this.client))
+            return "openai";
+        if (isGeminiClient(this.client))
+            return "gemini";
+        return "unknown";
+    }
+    buildJsonRepairMessages(text, outputSchema) {
+        return [
+            {
+                role: "system",
+                content: [
+                    "Repair malformed JSON.",
+                    "Return only valid JSON that conforms to the requested schema.",
+                    schemaSummary(outputSchema),
+                ].join("\n"),
+            },
+            {
+                role: "user",
+                content: `Malformed JSON:\n${text}`,
+            },
+        ];
+    }
+    async requestJsonRepair(model, outputSchema, text) {
+        const response = await this.callLlm({
+            messages: this.buildJsonRepairMessages(text, outputSchema),
+            model,
+            tools: [],
+            outputSchema: undefined,
+            jsonRepair: false,
+        });
+        return response.text;
+    }
+    async maybeRepairJson(args) {
+        const { model, outputSchema, jsonRepair, text } = args;
+        if (!jsonRepair || outputSchema == null || text.trim() === "") {
+            return text;
+        }
+        try {
+            parseJsonResponse(text);
+            return text;
+        }
+        catch {
+            const repairedText = await this.requestJsonRepair(model, outputSchema, text);
+            try {
+                parseJsonResponse(repairedText);
+                return repairedText;
+            }
+            catch {
+                const provider = this.providerName();
+                throw new Error(`JSON repair failed for provider=${provider} model=${model} original_response_preview="${previewText(text)}" repair_attempt_preview="${previewText(repairedText)}"`);
+            }
+        }
+    }
+    async callLlm(opts) {
+        if (isAnthropicClient(this.client)) {
+            const systemMessage = opts.messages.find((message) => message.role === "system")?.content ?? "";
+            const chatMessages = opts.messages
+                .filter((message) => message.role !== "system")
+                .map((message) => ({
+                role: message.role,
+                content: message.content,
+            }));
+            const response = await this.client.messages.create({
+                model: opts.model,
+                max_tokens: 1024,
+                system: systemMessage,
+                messages: chatMessages,
+                tools: (opts.tools ?? []).map((tool) => ({
+                    name: tool.name,
+                    description: tool.description,
+                    input_schema: withClosedAdditionalProperties(tool.inputSchema),
+                })),
+            });
+            const content = Array.isArray(response.content)
+                ? response.content
+                : [];
+            const text = content
+                .filter((part) => part != null && typeof part === "object" && part.type === "text")
+                .map(contentPartToText)
+                .join("");
+            const toolCalls = content
+                .filter((part) => part != null && typeof part === "object" && part.type === "tool_use")
+                .map((part, index) => {
+                const toolUse = part;
+                return {
+                    name: String(toolUse.name ?? ""),
+                    input: toolUse.input ?? {},
+                    id: String(toolUse.id ?? `tool_${index}`),
+                };
+            });
+            return {
+                text: await this.maybeRepairJson({
+                    model: opts.model,
+                    outputSchema: opts.outputSchema,
+                    jsonRepair: opts.jsonRepair,
+                    text,
+                }),
+                toolCalls,
+                stopReason: typeof response.stop_reason === "string"
+                    ? response.stop_reason
+                    : null,
+            };
+        }
+        if (isOpenAIClient(this.client)) {
+            const response = await this.client.chat.completions.create({
+                model: opts.model,
+                messages: opts.messages.map((message) => ({
+                    role: message.role,
+                    content: message.content,
+                })),
+                tools: (opts.tools ?? []).map((tool) => ({
+                    type: "function",
+                    function: {
+                        name: tool.name,
+                        description: tool.description,
+                        parameters: withClosedAdditionalProperties(tool.inputSchema),
+                    },
+                })),
+            });
+            const choice = Array.isArray(response.choices)
+                ? response.choices[0]
+                : null;
+            const message = choice != null && typeof choice === "object"
+                ? choice.message ?? {}
+                : {};
+            const rawToolCalls = Array.isArray(message["tool_calls"]) ? message["tool_calls"] : [];
+            return {
+                text: await this.maybeRepairJson({
+                    model: opts.model,
+                    outputSchema: opts.outputSchema,
+                    jsonRepair: opts.jsonRepair,
+                    text: normalizeOpenAIContent(message["content"]),
+                }),
+                toolCalls: rawToolCalls.map((toolCall, index) => {
+                    const toolCallObj = toolCall;
+                    const parsed = parseToolArguments(toolCallObj.function?.arguments);
+                    return {
+                        name: String(toolCallObj.function?.name ?? ""),
+                        input: parsed.input,
+                        id: String(toolCallObj.id ?? `tool_${index}`),
+                        ...(parsed.parseError ? { _parseError: parsed.parseError } : {}),
+                    };
+                }),
+                stopReason: choice != null && typeof choice === "object" && typeof choice.finish_reason === "string"
+                    ? choice.finish_reason
+                    : null,
+            };
+        }
+        if (isGeminiClient(this.client)) {
+            const systemMessage = opts.messages.find((message) => message.role === "system")?.content ?? "";
+            const contents = opts.messages
+                .filter((message) => message.role !== "system")
+                .map((message) => ({
+                role: message.role === "assistant" ? "model" : "user",
+                parts: [{ text: message.content }],
+            }));
+            const response = await this.client.models.generateContent({
+                model: opts.model,
+                contents,
+                config: {
+                    systemInstruction: systemMessage,
+                    tools: (opts.tools ?? []).map((tool) => ({
+                        functionDeclarations: [{
+                                name: tool.name,
+                                description: tool.description,
+                                parameters: toGeminiSchema(tool.inputSchema),
+                            }],
+                    })),
+                },
+            });
+            const candidates = Array.isArray(response.candidates)
+                ? response.candidates
+                : [];
+            const first = candidates[0];
+            const parts = first != null && typeof first === "object"
+                && first.content != null
+                && Array.isArray(first.content?.parts)
+                ? (first.content.parts)
+                : [];
+            const toolCalls = parts
+                .map((part) => ({ fnCall: getGeminiFunctionCall(part) }))
+                .filter((part) => part.fnCall != null)
+                .map((part, index) => {
+                const fnCall = part.fnCall ?? {};
+                const parsed = parseToolArguments(fnCall["args"] ?? fnCall["arguments"]);
+                return {
+                    name: String(fnCall["name"] ?? ""),
+                    input: parsed.input,
+                    id: String(fnCall["id"] ?? `tool_${index}`),
+                    ...(parsed.parseError ? { _parseError: parsed.parseError } : {}),
+                };
+            });
+            return {
+                text: await this.maybeRepairJson({
+                    model: opts.model,
+                    outputSchema: opts.outputSchema,
+                    jsonRepair: opts.jsonRepair,
+                    text: normalizeGeminiText(response),
+                }),
+                toolCalls,
+                stopReason: first != null && typeof first === "object" && first.finishReason != null
+                    ? String(first.finishReason)
+                    : null,
+            };
+        }
+        throw new TypeError(`Runner does not support ${Object.getPrototypeOf(this.client)?.constructor?.name ?? typeof this.client}.`);
+    }
+}
+// ---------------------------------------------------------------------------
+// Base component class
+// ---------------------------------------------------------------------------
+export class BaseComponent {
+    get isRunnable() {
+        return RUNNABLE_TYPES.has(this.componentType);
+    }
+    get isEval() {
+        return EVAL_TYPES.has(this.componentType);
+    }
+}
+export class Tool extends BaseComponent {
+    componentType = "tool";
+    name;
+    description;
+    fn;
+    inputSchema;
+    outputSchema;
+    isAsync;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.fn = opts.fn;
+        this.inputSchema = validateToolInputSchema(opts.inputSchema);
+        this.outputSchema = opts.outputSchema;
+        this.isAsync = opts.fn.constructor.name === "AsyncFunction";
+    }
+    versionHash() {
+        return hashFields(this.componentType, fnSource(this.fn), normalizeSchema(this.inputSchema), normalizeSchema(this.outputSchema));
+    }
+}
+export class CodeEval extends BaseComponent {
+    componentType = "code_eval";
+    name;
+    description;
+    fn;
+    isAsync;
+    modelResponseFormat;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.fn = opts.fn;
+        this.isAsync = opts.fn.constructor.name === "AsyncFunction";
+        this.modelResponseFormat = opts.modelResponseFormat ?? "JSON";
+    }
+    versionHash() {
+        return hashFields(this.componentType, fnSource(this.fn), this.modelResponseFormat);
+    }
+}
+export class Skill extends BaseComponent {
+    componentType = "skill";
+    name;
+    description;
+    instructions;
+    tools;
+    fn;
+    outputSchema;
+    isAsync;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.instructions = opts.instructions;
+        this.tools = opts.tools ?? [];
+        this.fn = opts.fn;
+        this.outputSchema = opts.outputSchema;
+        this.isAsync = opts.fn?.constructor.name === "AsyncFunction";
+    }
+    versionHash() {
+        const toolRefs = [...this.tools.map((t) => t.name)].sort().join(",");
+        return hashFields(this.componentType, this.instructions, toolRefs, fnSource(this.fn));
+    }
+}
+export class Agent extends BaseComponent {
+    componentType = "agent";
+    name;
+    description;
+    instructions;
+    model;
+    runner;
+    tools;
+    skills;
+    agents;
+    outputSchema;
+    jsonRepair;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.instructions = opts.instructions;
+        this.model = opts.model;
+        this.runner = opts.runner;
+        this.tools = opts.tools ?? [];
+        this.skills = opts.skills ?? [];
+        this.agents = opts.agents ?? [];
+        this.outputSchema = opts.outputSchema;
+        this.jsonRepair = opts.jsonRepair ?? false;
+    }
+    versionHash() {
+        const toolRefs = [...this.tools.map((t) => t.name)].sort().join(",");
+        const skillRefs = [...this.skills.map((s) => s.name)].sort().join(",");
+        const agentRefs = [...this.agents.map((a) => a.name)].sort().join(",");
+        return hashFields(this.componentType, this.instructions, this.model, toolRefs, skillRefs, agentRefs, normalizeSchema(this.outputSchema));
+    }
+}
+export class LLMProcessor extends BaseComponent {
+    componentType = "llm_processor";
+    name;
+    description;
+    instructions;
+    model;
+    runner;
+    outputSchema;
+    jsonRepair;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.instructions = opts.instructions;
+        this.model = opts.model;
+        this.runner = opts.runner;
+        this.outputSchema = opts.outputSchema;
+        this.jsonRepair = opts.jsonRepair ?? false;
+    }
+    versionHash() {
+        return hashFields(this.componentType, this.instructions, this.model, normalizeSchema(this.outputSchema));
+    }
+}
+export class LLMEval extends BaseComponent {
+    componentType = "llm_eval";
+    name;
+    description;
+    instructions;
+    model;
+    runner;
+    outputSchema;
+    jsonRepair;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.instructions = opts.instructions;
+        this.model = opts.model;
+        this.runner = opts.runner;
+        this.outputSchema = opts.outputSchema;
+        this.jsonRepair = opts.jsonRepair ?? false;
+    }
+    versionHash() {
+        return hashFields(this.componentType, this.instructions, this.model, normalizeSchema(this.outputSchema));
+    }
+}
+export class Conditional {
+    condition;
+    ifTrue;
+    ifFalse;
+    constructor(opts) {
+        this.condition = opts.condition;
+        this.ifTrue = opts.ifTrue;
+        this.ifFalse = opts.ifFalse;
+    }
+}
+export class Step {
+    name;
+    component;
+    next;
+    constructor(opts) {
+        this.name = opts.name;
+        this.component = opts.component;
+        this.next = opts.next ?? [];
+    }
+}
+export class Orchestration extends BaseComponent {
+    componentType = "orchestration";
+    name;
+    description;
+    steps;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.steps = opts.steps;
+    }
+    versionHash() {
+        const stepParts = this.steps.map((s) => {
+            const compRef = s.component instanceof Conditional ? "conditional" : s.component.name;
+            return `${s.name}:${compRef}:${s.next.join(",")}`;
+        });
+        return hashFields(this.componentType, stepParts.join("|"));
+    }
+}
+// ---------------------------------------------------------------------------
+// Eval composition: Leaf, Edge, Branch
+// ---------------------------------------------------------------------------
+export class Leaf {
+    eval;
+    constructor(opts) {
+        this.eval = opts.eval;
+    }
+}
+export class Edge {
+    weight;
+    node;
+    constructor(opts) {
+        this.weight = opts.weight;
+        this.node = opts.node;
+    }
+}
+export class Branch {
+    eval;
+    threshold;
+    ifPass;
+    ifFail;
+    constructor(opts) {
+        this.eval = opts.eval;
+        this.threshold = opts.threshold;
+        this.ifPass = opts.ifPass;
+        this.ifFail = opts.ifFail;
+    }
+}
+export class Checklist extends BaseComponent {
+    componentType = "checklist";
+    name;
+    description;
+    evals;
+    weights;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.evals = opts.evals;
+        this.weights = opts.weights;
+    }
+    versionHash() {
+        const evalRefs = this.evals.map((e) => e.name).join(",");
+        const w = this.weights ? this.weights.join(",") : "";
+        return hashFields(this.componentType, evalRefs, w);
+    }
+}
+// ---------------------------------------------------------------------------
+// EvalTree
+// ---------------------------------------------------------------------------
+function treeHashParts(node) {
+    if (node instanceof Leaf) {
+        return `leaf:${node.eval.name}`;
+    }
+    if (node instanceof Edge) {
+        return `edge:${node.weight}:${treeHashParts(node.node)}`;
+    }
+    // Branch
+    const passParts = node.ifPass.map(treeHashParts).join("|");
+    const failParts = node.ifFail.map(treeHashParts).join("|");
+    return `branch:${node.eval.name}:${node.threshold}:[${passParts}]:[${failParts}]`;
+}
+export class EvalTree extends BaseComponent {
+    componentType = "eval_tree";
+    name;
+    description;
+    root;
+    constructor(opts) {
+        super();
+        this.name = opts.name;
+        this.description = opts.description;
+        this.root = opts.root;
+    }
+    versionHash() {
+        return hashFields(this.componentType, treeHashParts(this.root));
+    }
+}