@zhanla/sdk-ts 0.1.0

package/README.md

@@ -0,0 +1,347 @@
+# @zhanla/sdk-ts
+
+`@zhanla/sdk-ts` is the TypeScript SDK for defining bench components in code.
+
+You export component instances from a TypeScript module, then run them with `zhanla` or the bundled `@zhanla/sdk-ts` helper CLI.
+
+## Installation
+
+```bash
+npm install @zhanla/sdk-ts
+```
+
+Requires Node `>=18`.
+
+Provider packages such as `@anthropic-ai/sdk`, `openai`, and `@google/genai` are optional. Install them only if you use `Runner` with those clients.
+
+## Quick Start
+
+```ts
+import Anthropic from "@anthropic-ai/sdk";
+import { Agent, CodeEval, Runner } from "@zhanla/sdk-ts";
+
+const runner = new Runner({
+  client: new Anthropic(),
+});
+
+export const supportAgent = new Agent({
+  name: "support_agent",
+  description: "Respond to support requests.",
+  instructions: 'Answer clearly. Return JSON: {"answer": "..."}',
+  model: "claude-sonnet-4-6",
+  runner,
+  outputSchema: {
+    type: "object",
+    properties: {
+      answer: { type: "string" },
+    },
+    required: ["answer"],
+  },
+});
+
+export const supportEval = new CodeEval({
+  name: "support_eval",
+  description: "Check whether an answer was returned.",
+  fn: (kwargs: unknown) => {
+    const { model_response } = kwargs as { model_response?: string };
+    const parsed = model_response ? JSON.parse(model_response) : {};
+    return { score: typeof parsed.answer === "string" ? 1.0 : 0.0 };
+  },
+});
+```
+
+Run it with the CLI:
+
+```bash
+bench run components.ts:support_agent --dataset tickets.json --eval components.ts:support_eval
+```
+
+## Public API
+
+```ts
+import {
+  Tool,
+  CodeEval,
+  Skill,
+  Agent,
+  LLMProcessor,
+  LLMEval,
+  Runner,
+  Orchestration,
+  Step,
+  Conditional,
+  Checklist,
+  EvalTree,
+  Branch,
+  Edge,
+  Leaf,
+  wrap,
+  parseJsonResponse,
+} from "@zhanla/sdk-ts";
+```
+
+TypeScript discovery is export-based. Only exported component instances are discoverable.
+
+## Components
+
+### `Tool`
+
+Use a `Tool` for deterministic TypeScript logic.
+
+```ts
+export const lookupCustomer = new Tool({
+  name: "lookup_customer",
+  description: "Fetch a customer record by ID.",
+  inputSchema: { type: "object", properties: {} },
+  fn: (kwargs: unknown) => {
+    const { customerId } = kwargs as { customerId: string };
+    return { id: customerId, email: "customer@example.com" };
+  },
+  outputSchema: {
+    type: "object",
+    properties: {
+      id: { type: "string" },
+      email: { type: "string" },
+    },
+    required: ["id", "email"],
+  },
+});
+```
+
+Notes:
+
+- `fn` can be sync or async.
+- Local execution passes a single `kwargs` object.
+- Non-object return values are wrapped as `{ result: value }`.
+
+### `Skill`
+
+Use a `Skill` for reusable instructions and tool access.
+
+```ts
+export const summarizeSkill = new Skill({
+  name: "summarize_skill",
+  description: "Reusable summarization instructions.",
+  instructions: "Summarize the provided text in one short paragraph.",
+  tools: [lookupCustomer],
+});
+```
+
+`Skill` is a non-executable configuration construct. It is not a local runtime entry point.
+
+### `Agent`
+
+Use an `Agent` for LLM-backed execution with instructions, tools, skills, and nested agents.
+
+```ts
+import OpenAI from "openai";
+import { Agent, Runner } from "@zhanla/sdk-ts";
+
+const runner = new Runner({
+  client: new OpenAI(),
+});
+
+export const supportAgent = new Agent({
+  name: "support_agent",
+  description: "Respond to support requests.",
+  instructions: 'Answer clearly. Return JSON: {"answer": "..."}',
+  model: "gpt-4.1-mini",
+  runner,
+  tools: [lookupCustomer],
+  skills: [summarizeSkill],
+  outputSchema: {
+    type: "object",
+    properties: {
+      answer: { type: "string" },
+    },
+    required: ["answer"],
+  },
+});
+```
+
+### `LLMProcessor`
+
+Use an `LLMProcessor` for prompt-defined transformation steps.
+
+```ts
+import Anthropic from "@anthropic-ai/sdk";
+import { LLMProcessor, Runner } from "@zhanla/sdk-ts";
+
+const runner = new Runner({
+  client: new Anthropic(),
+});
+
+export const intentClassifier = new LLMProcessor({
+  name: "intent_classifier",
+  description: "Classify intent.",
+  instructions: 'Return JSON: {"intent": "billing|technical|other"}',
+  model: "claude-haiku-4-5",
+  runner,
+  outputSchema: {
+    type: "object",
+    properties: {
+      intent: { type: "string" },
+    },
+    required: ["intent"],
+  },
+});
+```
+
+### `LLMEval`
+
+Use an `LLMEval` for LLM-backed evaluation logic.
+
+```ts
+export const toneEval = new LLMEval({
+  name: "tone_eval",
+  description: "Evaluate tone.",
+  instructions: 'Return JSON: {"score": 0.0, "reason": "..."}',
+  model: "gpt-4.1-mini",
+  runner,
+  outputSchema: {
+    type: "object",
+    properties: {
+      score: { type: "number" },
+      reason: { type: "string" },
+    },
+    required: ["score", "reason"],
+  },
+});
+```
+
+### `Orchestration`
+
+Use an `Orchestration` to compose steps into a DAG.
+
+```ts
+export const supportPipeline = new Orchestration({
+  name: "support_pipeline",
+  description: "Classify intent, then draft a reply.",
+  steps: [
+    new Step({ name: "classify", component: intentClassifier, next: ["reply"] }),
+    new Step({ name: "reply", component: supportAgent }),
+  ],
+});
+```
+
+### `Conditional`
+
+Use `Conditional` inside an orchestration to branch.
+
+```ts
+new Step({
+  name: "route",
+  component: new Conditional({
+    condition: (state) => state.intent === "billing",
+    ifTrue: "billing_reply",
+    ifFalse: "general_reply",
+  }),
+});
+```
+
+### `CodeEval`, `Checklist`, `EvalTree`
+
+`CodeEval` runs deterministic TypeScript scoring logic. `Checklist` combines evals with optional weights. `EvalTree` supports score-based branching across eval nodes.
+
+## `Runner`
+
+`Runner` is the local execution bridge for `Agent`, `LLMProcessor`, and `LLMEval`.
+
+```ts
+import Anthropic from "@anthropic-ai/sdk";
+import { Runner } from "@zhanla/sdk-ts";
+
+const runner = new Runner({
+  client: new Anthropic(),
+});
+```
+
+`Runner` behavior:
+
+- `constructor({ client })` wraps the client internally with `wrap(...)`
+- `buildMessages(component, row)` defaults to `[system instructions, user JSON row]`
+- `callLlm({ messages, model, tools, outputSchema })` supports Anthropic, OpenAI-compatible chat clients, and Gemini
+
+Current local execution behavior:
+
+- `runner` is required for `Agent`, `LLMProcessor`, and `LLMEval`
+- `model` must be set explicitly
+- response text is parsed as JSON when possible, otherwise wrapped as `{ result: text }`
+- `outputSchema` is used for validation
+- returned tool calls raise a not-implemented error for now
+
+## Observability
+
+### `wrap(client)`
+
+Wrap an LLM client so calls are recorded against the active trace context.
+
+```ts
+import Anthropic from "@anthropic-ai/sdk";
+import { wrap } from "@zhanla/sdk-ts";
+
+const client = wrap(new Anthropic());
+```
+
+`wrap()` is observational only. It does not change your application logic. `Runner` calls `wrap()` internally, so you do not need to wrap the same client first.
+
+### Trace Context
+
+The CLI sets a `TraceContext` before runner execution. Wrapped clients read the active context automatically via `AsyncLocalStorage`.
+
+```ts
+import { traceStorage } from "@zhanla/sdk-ts";
+
+const ctx = traceStorage.getStore();
+if (ctx) {
+  console.log(ctx.traceId);
+}
+```
+
+## Discovery And CLI Usage
+
+TypeScript discovery loads your module and collects exported component instances.
+
+```bash
+bench run workflow.ts:support_pipeline --dataset tickets.json --eval evals.ts:answer_quality
+```
+
+The package also ships a helper CLI:
+
+```bash
+npx @zhanla/sdk-ts discover components.ts
+npx @zhanla/sdk-ts run components.ts:support_agent --eval evals.ts:support_eval
+```
+
+## Validation Rules
+
+When run through `zhanla`, TypeScript component manifests are validated before execution.
+
+- `Tool` must provide a callable implementation and an `outputSchema`
+- `CodeEval` must provide a callable implementation
+- `Skill`, `Agent`, `LLMProcessor`, and `LLMEval` must provide `instructions`
+- `Agent`, `LLMProcessor`, and `LLMEval` must provide `model`
+
+## Utilities
+
+The package also exports lower-level helpers:
+
+```ts
+import {
+  collectExportedComponents,
+  toManifest,
+  executeComponent,
+  executeRow,
+  wrap,
+  parseJsonResponse,
+  TraceContext,
+  traceStorage,
+} from "@zhanla/sdk-ts";
+import type { LLMCall, LLMResponse, ToolCall } from "@zhanla/sdk-ts";
+```
+
+- `collectExportedComponents(...)` collects component instances from a loaded module
+- `toManifest(...)` serializes a component into the CLI manifest shape
+- `executeComponent(...)` runs one component locally
+- `executeRow(...)` runs one component plus optional eval for a single row
+- `parseJsonResponse(text)` extracts JSON from raw or fenced model text

package/bin/cli.js

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+/**
+ * zhanla-sdk-ts CLI entry point.
+ * Re-execs itself under the tsx loader so it can import user-authored .ts
+ * files directly, then dispatches to discover or run subcommands.
+ *
+ * Usage:
+ *   zhanla-sdk-ts discover <file.ts>
+ *   zhanla-sdk-ts run <file.ts:name> [--eval <file.ts:name>]
+ */
+
+import { spawnSync } from "child_process";
+import { createRequire } from "module";
+
+if (process.env.BENCH_SDK_TS_LOADER_ACTIVE !== "1") {
+  const require = createRequire(import.meta.url);
+  const loaderPath = require.resolve("tsx");
+  const child = spawnSync(
+    process.execPath,
+    ["--import", loaderPath, new URL(import.meta.url).pathname, ...process.argv.slice(2)],
+    {
+      stdio: "inherit",
+      env: { ...process.env, BENCH_SDK_TS_LOADER_ACTIVE: "1" },
+    },
+  );
+
+  process.exit(child.status ?? 1);
+}
+
+const [, , subcommand, ...args] = process.argv;
+
+if (!subcommand) {
+  console.error("Usage: zhanla-sdk-ts <discover|run> <target>");
+  process.exit(1);
+}
+if (subcommand === "discover") {
+  const [filePath] = args;
+  if (!filePath) {
+    console.error("Usage: zhanla-sdk-ts discover <file.ts>");
+    process.exit(1);
+  }
+  const { runDiscover } = await import("./discover.js");
+  await runDiscover(filePath);
+} else if (subcommand === "run") {
+  const [target, ...rest] = args;
+  if (!target) {
+    console.error("Usage: zhanla-sdk-ts run <file.ts:name>");
+    process.exit(1);
+  }
+  let evalTarget = null;
+  for (let i = 0; i < rest.length; i += 1) {
+    if (rest[i] === "--eval") {
+      evalTarget = rest[i + 1] ?? null;
+      break;
+    }
+  }
+  const { runComponent } = await import("./run.js");
+  await runComponent(target, evalTarget);
+} else {
+  console.error(`Unknown subcommand: ${subcommand}. Use 'discover' or 'run'.`);
+  process.exit(1);
+}

package/bin/discover.js

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+/**
+ * zhanla-sdk-ts discover <file.ts>
+ *
+ * Loads a TypeScript module via tsx, collects exported BaseComponent instances,
+ * emits their manifests as a JSON array to stdout.
+ * Exits non-zero if no components are found or the file fails to load.
+ */
+
+import { pathToFileURL } from "url";
+import { resolve } from "path";
+
+export async function runDiscover(filePath) {
+  const absolutePath = resolve(filePath);
+  const fileUrl = pathToFileURL(absolutePath).href;
+
+  let moduleExports;
+  try {
+    moduleExports = await import(fileUrl);
+  } catch (err) {
+    console.error(`Failed to load ${filePath}: ${err.message}`);
+    process.exit(1);
+  }
+
+  // Dynamically import from the built SDK (or tsx will resolve from node_modules)
+  let collectExportedComponents, toManifest;
+  try {
+    const sdk = await import("@zhanla/sdk-ts");
+    collectExportedComponents = sdk.collectExportedComponents;
+    toManifest = sdk.toManifest;
+  } catch {
+    // Fallback: try loading from relative path (dev mode)
+    const sdk = await import(new URL("../dist/index.js", import.meta.url).href);
+    collectExportedComponents = sdk.collectExportedComponents;
+    toManifest = sdk.toManifest;
+  }
+
+  const components = collectExportedComponents(moduleExports);
+
+  if (components.length === 0) {
+    console.error(`No zhanla components found in ${filePath}`);
+    process.exit(1);
+  }
+
+  const symbolByComponent = new Map();
+  const namespaces = [moduleExports];
+  if (
+    moduleExports.default != null &&
+    typeof moduleExports.default === "object" &&
+    !Array.isArray(moduleExports.default)
+  ) {
+    namespaces.push(moduleExports.default);
+  }
+  for (const namespace of namespaces) {
+    for (const [exportName, value] of Object.entries(namespace)) {
+      if (!symbolByComponent.has(value)) {
+        symbolByComponent.set(value, exportName);
+      }
+    }
+  }
+
+  const manifests = components.map((comp) =>
+    toManifest(comp, {
+      filePath: absolutePath,
+      symbolName: symbolByComponent.get(comp) ?? comp.name,
+    })
+  );
+
+  process.stdout.write(JSON.stringify(manifests) + "\n");
+}

package/bin/postinstall.js

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+import fs from "fs";
+import path from "path";
+
+const ENTRIES = ["node_modules", "dist", "package-lock.json"];
+
+// Walk up from cwd to find the project root (where package.json lives, outside node_modules)
+function findProjectRoot() {
+  let dir = process.cwd();
+  // When running as a postinstall script the cwd is inside node_modules/<pkg>
+  // Walk up until we're outside of node_modules
+  while (dir.includes("node_modules")) {
+    dir = path.dirname(dir);
+  }
+  return dir;
+}
+
+const root = findProjectRoot();
+const gitignorePath = path.join(root, ".gitignore");
+
+let existing = "";
+try {
+  existing = fs.readFileSync(gitignorePath, "utf8");
+} catch {
+  // .gitignore doesn't exist yet — we'll create it
+}
+
+const existingLines = new Set(existing.split("\n").map((l) => l.trim()));
+const toAdd = ENTRIES.filter((e) => !existingLines.has(e));
+
+if (toAdd.length === 0) {
+  process.exit(0);
+}
+
+const addition = "\n# zhanla-sdk-ts\n" + toAdd.join("\n") + "\n";
+fs.appendFileSync(gitignorePath, addition);
+console.log(`zhanla-sdk-ts: added to .gitignore: ${toAdd.join(", ")}`);

package/bin/run.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+/**
+ * zhanla-sdk-ts run <file.ts:name> [--eval <file.ts:name>]
+ *
+ * Loads a TypeScript module, resolves the named component, reads rows as
+ * NDJSON from stdin, executes each row, emits results as NDJSON to stdout.
+ *
+ * Each result line: {"status": "ok", "output": {...}} or {"status": "error", "error": "..."}
+ */
+
+import { pathToFileURL } from "url";
+import { resolve } from "path";
+import readline from "readline";
+
+function redirectConsoleToStderr() {
+  const stringifyArg = (arg) => {
+    if (typeof arg === "string") return arg;
+    try {
+      return JSON.stringify(arg);
+    } catch {
+      return String(arg);
+    }
+  };
+
+  const write = (...args) => {
+    process.stderr.write(`${args.map(stringifyArg).join(" ")}\n`);
+  };
+
+  console.log = (...args) => write(...args);
+  console.info = (...args) => write(...args);
+  console.warn = (...args) => write(...args);
+  console.error = (...args) => write(...args);
+}
+
+function parseTarget(target) {
+  const colonIdx = target.lastIndexOf(":");
+  if (colonIdx === -1) {
+    console.error(`Invalid target format. Expected <file.ts:name>, got: ${target}`);
+    process.exit(1);
+  }
+
+  const filePath = target.slice(0, colonIdx);
+  const symbolName = target.slice(colonIdx + 1);
+
+  if (!symbolName) {
+    console.error(`Empty component name in target: ${target}`);
+    process.exit(1);
+  }
+
+  return { filePath, symbolName };
+}
+
+async function loadTargetComponent(target, collectExportedComponents) {
+  const { filePath, symbolName } = parseTarget(target);
+
+  const absolutePath = resolve(filePath);
+  const fileUrl = pathToFileURL(absolutePath).href;
+
+  let moduleExports;
+  try {
+    moduleExports = await import(fileUrl);
+  } catch (err) {
+    console.error(`Failed to load ${filePath}: ${err.message}`);
+    process.exit(1);
+  }
+
+  const components = collectExportedComponents(moduleExports);
+  const component = components.find((c) => c.name === symbolName);
+
+  if (!component) {
+    const available = components.map((c) => c.name).join(", ");
+    console.error(`Component '${symbolName}' not found. Available: ${available}`);
+    process.exit(1);
+  }
+
+  return component;
+}
+
+export async function runComponent(target, evalTarget = null) {
+  redirectConsoleToStderr();
+  // Dynamically import from SDK
+  let collectExportedComponents, executeRow;
+  try {
+    const sdk = await import("@zhanla/sdk-ts");
+    collectExportedComponents = sdk.collectExportedComponents;
+    executeRow = sdk.executeRow;
+  } catch {
+    const sdk = await import(new URL("../dist/index.js", import.meta.url).href);
+    collectExportedComponents = sdk.collectExportedComponents;
+    executeRow = sdk.executeRow;
+  }
+
+  // Read rows from stdin as NDJSON, emit results to stdout
+  const component = await loadTargetComponent(target, collectExportedComponents);
+  const evalComponent = evalTarget
+    ? await loadTargetComponent(evalTarget, collectExportedComponents)
+    : null;
+
+  const rl = readline.createInterface({
+    input: process.stdin,
+    crlfDelay: Infinity,
+  });
+
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    let row;
+    try {
+      row = JSON.parse(trimmed);
+    } catch (err) {
+      const result = { status: "error", error: `Invalid JSON row: ${err.message}` };
+      process.stdout.write(JSON.stringify(result) + "\n");
+      continue;
+    }
+
+    try {
+      const result = await executeRow(component, evalComponent, row);
+      const output =
+        result.output && typeof result.output === "object" ? { ...result.output } : result.output;
+      let evalOutput;
+      if (output && typeof output === "object" && "_eval" in output) {
+        evalOutput = output._eval;
+        delete output._eval;
+      }
+      process.stdout.write(
+        JSON.stringify({
+          status: result.status,
+          output,
+          pathTaken: result.pathTaken,
+          eval_output: evalOutput,
+          error: result.status === "error" ? result.error : undefined,
+          eval_error: result.status === "ok" ? result.error : undefined,
+        }) + "\n"
+      );
+    } catch (err) {
+      const result = {
+        status: "error",
+        error: err instanceof Error ? err.message : String(err),
+      };
+      process.stdout.write(JSON.stringify(result) + "\n");
+    }
+  }
+}

package/dist/executor.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Local execution engine for TypeScript SDK components.
+ * Mirrors the Python CLI executor's component-type dispatch semantics.
+ */
+import { BaseComponent } from "./types.js";
+export interface RowResult {
+    status: "ok" | "error";
+    output?: Record<string, unknown>;
+    error?: string;
+    pathTaken?: Record<string, unknown>;
+}
+export declare function executeComponent(comp: BaseComponent, kwargs: Record<string, unknown>): Promise<Record<string, unknown>>;
+export declare function executeRow(component: BaseComponent, evalComponent: BaseComponent | null, row: Record<string, unknown>): Promise<RowResult>;