@zhanla/sdk-ts 0.1.0

package/dist/wrap.d.ts

@@ -0,0 +1,37 @@
+/**
+ * bench.wrap() — observe LLM clients without executing them.
+ *
+ * Wraps an existing LLM client so every call is recorded in the active
+ * TraceContext. The underlying client is unchanged; bench only observes.
+ *
+ * Supported clients:
+ *   - Anthropic (from @anthropic-ai/sdk)
+ *   - OpenAI (from openai)
+ *   - GoogleGenAI (from @google/genai)
+ */
+export declare function isAnthropicClient(client: unknown): client is {
+    messages: {
+        create: (...args: unknown[]) => unknown;
+    };
+};
+export declare function isOpenAIClient(client: unknown): client is {
+    chat: {
+        completions: {
+            create: (...args: unknown[]) => unknown;
+        };
+    };
+};
+export declare function isGeminiClient(client: unknown): client is {
+    models: {
+        generateContent: (...args: unknown[]) => unknown;
+    };
+};
+/**
+ * Wrap an LLM client to record every call in the active TraceContext.
+ *
+ * @example
+ * import Anthropic from "@anthropic-ai/sdk";
+ * import bench from "@benchlabs/sdk";
+ * const client = bench.wrap(new Anthropic());
+ */
+export declare function wrap<T>(client: T): T;

package/dist/wrap.js

@@ -0,0 +1,255 @@
+/**
+ * bench.wrap() — observe LLM clients without executing them.
+ *
+ * Wraps an existing LLM client so every call is recorded in the active
+ * TraceContext. The underlying client is unchanged; bench only observes.
+ *
+ * Supported clients:
+ *   - Anthropic (from @anthropic-ai/sdk)
+ *   - OpenAI (from openai)
+ *   - GoogleGenAI (from @google/genai)
+ */
+import { traceStorage } from "./trace_store.js";
+const WRAPPED_CLIENT = Symbol.for("bench.sdk_ts.wrapped_client");
+// ---------------------------------------------------------------------------
+// Serialisation helper
+// ---------------------------------------------------------------------------
+function serialize(value) {
+    if (value === null || value === undefined)
+        return null;
+    if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+        return value;
+    }
+    if (Array.isArray(value)) {
+        return value.map(serialize);
+    }
+    if (typeof value === "object") {
+        const obj = value;
+        const result = {};
+        for (const key of Object.keys(obj)) {
+            if (!key.startsWith("_")) {
+                result[key] = serialize(obj[key]);
+            }
+        }
+        return result;
+    }
+    return value;
+}
+function getGeminiFunctionCallPart(part) {
+    if (part == null || typeof part !== "object") {
+        return null;
+    }
+    const record = part;
+    const functionCall = record["functionCall"] ?? record["function_call"];
+    return functionCall != null && typeof functionCall === "object"
+        ? functionCall
+        : null;
+}
+function extractGeminiToolCallParts(response) {
+    const candidates = Array.isArray(response["candidates"]) ? response["candidates"] : [];
+    const first = candidates[0];
+    if (first == null || typeof first !== "object") {
+        return [];
+    }
+    const content = first.content;
+    if (content == null || typeof content !== "object") {
+        return [];
+    }
+    const parts = Array.isArray(content.parts)
+        ? content.parts
+        : [];
+    return parts.filter((part) => getGeminiFunctionCallPart(part) != null);
+}
+// ---------------------------------------------------------------------------
+// Anthropic
+// ---------------------------------------------------------------------------
+function recordAnthropicCall(ctx, kwargs, response, latencyMs) {
+    if (!ctx)
+        return;
+    const content = response["content"] ?? [];
+    const toolCalls = content.filter((b) => b !== null && typeof b === "object" && b["type"] === "tool_use");
+    const usage = response["usage"];
+    ctx.record({
+        traceId: ctx.traceId,
+        parentId: null,
+        sequenceOrder: ctx.nextSequence(),
+        autoraterRunId: ctx.autoraterRunId,
+        datasetItemId: ctx.datasetItemId,
+        provider: "anthropic",
+        model: String(response["model"] ?? (kwargs["model"] ?? "")),
+        inputMessages: serialize(kwargs["messages"] ?? []),
+        output: serialize(content),
+        toolCalls: toolCalls.length > 0 ? serialize(toolCalls) : null,
+        rawResponse: serialize(response),
+        inputTokens: usage ? usage["input_tokens"] : null,
+        outputTokens: usage ? usage["output_tokens"] : null,
+        latencyMs,
+        stopReason: response["stop_reason"] ?? null,
+        metadata: null,
+    });
+}
+function wrapAnthropic(client) {
+    const originalCreate = client.messages.create.bind(client.messages);
+    client.messages.create = async function (...args) {
+        const ctx = traceStorage.getStore();
+        const kwargs = (args[0] ?? {});
+        const start = performance.now();
+        const response = await originalCreate(...args);
+        const latencyMs = Math.round(performance.now() - start);
+        recordAnthropicCall(ctx, kwargs, response, latencyMs);
+        return response;
+    };
+    return client;
+}
+// ---------------------------------------------------------------------------
+// OpenAI
+// ---------------------------------------------------------------------------
+function recordOpenAICall(ctx, kwargs, response, latencyMs) {
+    if (!ctx)
+        return;
+    const choices = response["choices"] ?? [];
+    const firstChoice = (choices[0] ?? null);
+    const message = firstChoice ? firstChoice["message"] : null;
+    const toolCalls = message ? message["tool_calls"] : null;
+    const usage = response["usage"];
+    const finishReason = firstChoice ? firstChoice["finish_reason"] : null;
+    ctx.record({
+        traceId: ctx.traceId,
+        parentId: null,
+        sequenceOrder: ctx.nextSequence(),
+        autoraterRunId: ctx.autoraterRunId,
+        datasetItemId: ctx.datasetItemId,
+        provider: "openai",
+        model: String(response["model"] ?? (kwargs["model"] ?? "")),
+        inputMessages: serialize(kwargs["messages"] ?? []),
+        output: serialize(message),
+        toolCalls: toolCalls && toolCalls.length > 0 ? serialize(toolCalls) : null,
+        rawResponse: serialize(response),
+        inputTokens: usage ? usage["prompt_tokens"] : null,
+        outputTokens: usage ? usage["completion_tokens"] : null,
+        latencyMs,
+        stopReason: finishReason,
+        metadata: null,
+    });
+}
+function wrapOpenAI(client) {
+    const originalCreate = client.chat.completions.create.bind(client.chat.completions);
+    client.chat.completions.create = async function (...args) {
+        const ctx = traceStorage.getStore();
+        const kwargs = (args[0] ?? {});
+        const start = performance.now();
+        const response = await originalCreate(...args);
+        const latencyMs = Math.round(performance.now() - start);
+        recordOpenAICall(ctx, kwargs, response, latencyMs);
+        return response;
+    };
+    return client;
+}
+// ---------------------------------------------------------------------------
+// Gemini (@google/genai)
+// ---------------------------------------------------------------------------
+function recordGeminiCall(ctx, kwargs, response, latencyMs) {
+    if (!ctx)
+        return;
+    const candidates = response["candidates"] ?? [];
+    const first = (candidates[0] ?? null);
+    const content = first ? serialize(first["content"]) : null;
+    const toolCalls = extractGeminiToolCallParts(response);
+    const finishReason = first ? String(first["finish_reason"] ?? "") : null;
+    const usageMeta = response["usageMetadata"];
+    ctx.record({
+        traceId: ctx.traceId,
+        parentId: null,
+        sequenceOrder: ctx.nextSequence(),
+        autoraterRunId: ctx.autoraterRunId,
+        datasetItemId: ctx.datasetItemId,
+        provider: "gemini",
+        model: String(kwargs["model"] ?? ""),
+        inputMessages: serialize(kwargs["contents"] ?? []),
+        output: content,
+        toolCalls: toolCalls.length > 0 ? serialize(toolCalls) : null,
+        rawResponse: serialize(response),
+        inputTokens: usageMeta ? usageMeta["promptTokenCount"] : null,
+        outputTokens: usageMeta ? usageMeta["candidatesTokenCount"] : null,
+        latencyMs,
+        stopReason: finishReason,
+        metadata: null,
+    });
+}
+function wrapGemini(client) {
+    const originalGenerate = client.models.generateContent.bind(client.models);
+    client.models.generateContent = async function (...args) {
+        const ctx = traceStorage.getStore();
+        const kwargs = (args[0] ?? {});
+        const start = performance.now();
+        const response = await originalGenerate(...args);
+        const latencyMs = Math.round(performance.now() - start);
+        recordGeminiCall(ctx, kwargs, response, latencyMs);
+        return response;
+    };
+    return client;
+}
+// ---------------------------------------------------------------------------
+// Type guards
+// ---------------------------------------------------------------------------
+export function isAnthropicClient(client) {
+    return (client !== null &&
+        typeof client === "object" &&
+        "messages" in client &&
+        typeof client.messages === "object" &&
+        client.messages !== null &&
+        typeof client.messages.create === "function");
+}
+export function isOpenAIClient(client) {
+    return (client !== null &&
+        typeof client === "object" &&
+        "chat" in client &&
+        typeof client.chat === "object" &&
+        client.chat !== null &&
+        typeof client.chat.completions === "object");
+}
+export function isGeminiClient(client) {
+    return (client !== null &&
+        typeof client === "object" &&
+        "models" in client &&
+        typeof client.models === "object" &&
+        client.models !== null &&
+        typeof client.models.generateContent ===
+            "function");
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Wrap an LLM client to record every call in the active TraceContext.
+ *
+ * @example
+ * import Anthropic from "@anthropic-ai/sdk";
+ * import bench from "@benchlabs/sdk";
+ * const client = bench.wrap(new Anthropic());
+ */
+export function wrap(client) {
+    if (client != null && typeof client === "object" && client[WRAPPED_CLIENT] === true) {
+        return client;
+    }
+    let wrapped;
+    if (isAnthropicClient(client))
+        wrapped = wrapAnthropic(client);
+    else if (isOpenAIClient(client))
+        wrapped = wrapOpenAI(client);
+    else if (isGeminiClient(client))
+        wrapped = wrapGemini(client);
+    else {
+        throw new TypeError(`bench.wrap() does not support ${Object.getPrototypeOf(client)?.constructor?.name ?? typeof client}. ` +
+            "Supported clients: Anthropic, OpenAI, GoogleGenAI.");
+    }
+    if (wrapped != null && typeof wrapped === "object") {
+        Object.defineProperty(wrapped, WRAPPED_CLIENT, {
+            value: true,
+            configurable: false,
+            enumerable: false,
+            writable: false,
+        });
+    }
+    return wrapped;
+}

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@zhanla/sdk-ts",
+  "version": "0.1.0",
+  "description": "TypeScript SDK for the zhanla CLI — define and run AI components locally",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "bin": {
+    "zhanla-sdk-ts": "./bin/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit",
+    "postinstall": "node ./bin/postinstall.js"
+  },
+  "dependencies": {
+    "tsx": "^4.7.0"
+  },
+  "files": [
+    "dist",
+    "bin"
+  ],
+
+  "devDependencies": {
+    "typescript": "^5.4.0",
+    "vitest": "^1.4.0",
+    "@types/node": "^20.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}