@zhanla/sdk-ts 0.3.3 → 0.3.5

package/bin/cli.js

@@ -30,7 +30,7 @@ if (process.env.BENCH_SDK_TS_LOADER_ACTIVE !== "1") {
 const [, , subcommand, ...args] = process.argv;
 
 if (!subcommand) {
-  console.error("Usage: zhanla-sdk-ts <discover|run> <target>");
+  console.error("Usage: zhanla-sdk-ts <discover|run|eval-only> <target>");
   process.exit(1);
 }
 if (subcommand === "discover") {
@@ -41,6 +41,11 @@ if (subcommand === "discover") {
   }
   const { runDiscover } = await import("./discover.js");
   await runDiscover(filePath);
+} else if (subcommand === "version") {
+  const { createRequire } = await import("module");
+  const require = createRequire(import.meta.url);
+  const { version } = require("../package.json");
+  process.stdout.write(version + "\n");
 } else if (subcommand === "run") {
   const [target, ...rest] = args;
   if (!target) {
@@ -56,7 +61,15 @@ if (subcommand === "discover") {
   }
   const { runComponent } = await import("./run.js");
   await runComponent(target, evalTarget);
+} else if (subcommand === "eval-only") {
+  const [evalTarget] = args;
+  if (!evalTarget) {
+    console.error("Usage: zhanla-sdk-ts eval-only <file.ts:name>");
+    process.exit(1);
+  }
+  const { runEvalOnly } = await import("./run.js");
+  await runEvalOnly(evalTarget);
 } else {
-  console.error(`Unknown subcommand: ${subcommand}. Use 'discover' or 'run'.`);
+  console.error(`Unknown subcommand: ${subcommand}. Use 'discover', 'run', or 'eval-only'.`);
   process.exit(1);
 }

package/bin/discover.js

@@ -11,7 +11,17 @@
 
 import { pathToFileURL } from "url";
 import { resolve } from "path";
-import { loadDotenvLocal, splitFileSpec } from "./env.js";
+
+export function splitFileSpec(fileSpec) {
+  const colonIndex = fileSpec.lastIndexOf(":");
+  if (colonIndex > 1) {
+    return {
+      filePath: fileSpec.slice(0, colonIndex),
+      symbolName: fileSpec.slice(colonIndex + 1) || null,
+    };
+  }
+  return { filePath: fileSpec, symbolName: null };
+}
 
 function isComponentLike(value) {
   return (
@@ -216,7 +226,6 @@ export async function runDiscover(fileSpec) {
   const { filePath, symbolName } = splitFileSpec(fileSpec);
 
   const absolutePath = resolve(filePath);
-  loadDotenvLocal(absolutePath);
   const fileUrl = pathToFileURL(absolutePath).href;
 
   let moduleExports;

package/bin/run.js

@@ -9,9 +9,11 @@
  */
 
 import { pathToFileURL } from "url";
+import { readFileSync } from "fs";
 import { resolve } from "path";
+import vm from "vm";
 import readline from "readline";
-import { loadDotenvLocal } from "./env.js";
+import ts from "typescript";
 
 function redirectConsoleToStderr() {
   const stringifyArg = (arg) => {
@@ -55,7 +57,6 @@ async function loadTargetComponent(target, collectExportedComponents) {
   const { filePath, symbolName } = parseTarget(target);
 
   const absolutePath = resolve(filePath);
-  loadDotenvLocal(absolutePath);
   const fileUrl = pathToFileURL(absolutePath).href;
 
   let moduleExports;
@@ -78,6 +79,344 @@ async function loadTargetComponent(target, collectExportedComponents) {
   return component;
 }
 
+function isExportedStatement(statement) {
+  return statement.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword) ?? false;
+}
+
+function getStringLiteralValue(node) {
+  if (!node) return null;
+  if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) {
+    return node.text;
+  }
+  return null;
+}
+
+function getManagedCodeEvalName(initializer) {
+  if (!initializer || !ts.isNewExpression(initializer)) return null;
+  if (!ts.isIdentifier(initializer.expression) || initializer.expression.text !== "CodeEval") {
+    return null;
+  }
+  const [config] = initializer.arguments ?? [];
+  if (!config || !ts.isObjectLiteralExpression(config)) return null;
+  for (const property of config.properties) {
+    if (
+      ts.isPropertyAssignment(property) &&
+      ts.isIdentifier(property.name) &&
+      property.name.text === "name"
+    ) {
+      return getStringLiteralValue(property.initializer);
+    }
+  }
+  return null;
+}
+
+function buildManagedTsIsolationContext(code) {
+  const sourceFile = ts.createSourceFile(
+    "managed-eval.ts",
+    code,
+    ts.ScriptTarget.ES2020,
+    true,
+    ts.ScriptKind.TS,
+  );
+  const topLevelDeclarations = new Map();
+  const importedBindings = new Map();
+  const sdkImportStatements = [];
+
+  for (const statement of sourceFile.statements) {
+    if (ts.isImportDeclaration(statement)) {
+      const moduleSpecifier = getStringLiteralValue(statement.moduleSpecifier);
+      if (moduleSpecifier === "@zhanla/sdk-ts") {
+        sdkImportStatements.push(statement);
+      }
+      const importClause = statement.importClause;
+      if (moduleSpecifier && importClause) {
+        if (importClause.name) {
+          importedBindings.set(importClause.name.text, moduleSpecifier);
+        }
+        const namedBindings = importClause.namedBindings;
+        if (namedBindings && ts.isNamedImports(namedBindings)) {
+          for (const element of namedBindings.elements) {
+            importedBindings.set(element.name.text, moduleSpecifier);
+          }
+        } else if (namedBindings && ts.isNamespaceImport(namedBindings)) {
+          importedBindings.set(namedBindings.name.text, moduleSpecifier);
+        }
+      }
+      continue;
+    }
+
+    if (
+      ts.isFunctionDeclaration(statement) ||
+      ts.isClassDeclaration(statement) ||
+      ts.isEnumDeclaration(statement)
+    ) {
+      if (statement.name) {
+        topLevelDeclarations.set(statement.name.text, statement);
+      }
+      continue;
+    }
+
+    if (ts.isVariableStatement(statement)) {
+      for (const declaration of statement.declarationList.declarations) {
+        if (ts.isIdentifier(declaration.name)) {
+          topLevelDeclarations.set(declaration.name.text, statement);
+        }
+      }
+    }
+  }
+
+  return { sourceFile, topLevelDeclarations, importedBindings, sdkImportStatements };
+}
+
+function scanManagedTsDependencies(statement, context) {
+  const referencedDeclarations = new Set();
+  const disallowedImports = new Map();
+
+  const visit = (node) => {
+    if (ts.isIdentifier(node)) {
+      if (
+        (ts.isPropertyAccessExpression(node.parent) && node.parent.name === node) ||
+        (ts.isPropertyAssignment(node.parent) && node.parent.name === node)
+      ) {
+        return;
+      }
+
+      const localStatement = context.topLevelDeclarations.get(node.text);
+      if (localStatement && localStatement !== statement) {
+        referencedDeclarations.add(localStatement);
+      }
+
+      const importedFrom = context.importedBindings.get(node.text);
+      if (importedFrom && importedFrom !== "@zhanla/sdk-ts") {
+        const bindings = disallowedImports.get(importedFrom) ?? new Set();
+        bindings.add(node.text);
+        disallowedImports.set(importedFrom, bindings);
+      }
+    }
+
+    ts.forEachChild(node, visit);
+  };
+
+  visit(statement);
+  return { referencedDeclarations, disallowedImports };
+}
+
+function isolateManagedTypeScriptEvalSource(code, evalName) {
+  const context = buildManagedTsIsolationContext(code);
+
+  let targetStatement = null;
+  for (const statement of context.sourceFile.statements) {
+    if (!ts.isVariableStatement(statement) || !isExportedStatement(statement)) {
+      continue;
+    }
+    for (const declaration of statement.declarationList.declarations) {
+      if (!ts.isIdentifier(declaration.name)) continue;
+      const declaredEvalName = getManagedCodeEvalName(declaration.initializer);
+      if (declaredEvalName === evalName || declaration.name.text === evalName) {
+        targetStatement = statement;
+        break;
+      }
+    }
+    if (targetStatement) break;
+  }
+
+  if (!targetStatement) {
+    throw new Error(
+      `Managed TypeScript eval '${evalName}' was not found in the synced source file.`,
+    );
+  }
+
+  const includedStatements = new Set([targetStatement]);
+  const pendingStatements = [targetStatement];
+  const disallowedImports = new Map();
+
+  while (pendingStatements.length > 0) {
+    const statement = pendingStatements.pop();
+    if (!statement) continue;
+
+    const scan = scanManagedTsDependencies(statement, context);
+    for (const dependency of scan.referencedDeclarations) {
+      if (!includedStatements.has(dependency)) {
+        includedStatements.add(dependency);
+        pendingStatements.push(dependency);
+      }
+    }
+    for (const [moduleSpecifier, bindings] of scan.disallowedImports) {
+      const existing = disallowedImports.get(moduleSpecifier) ?? new Set();
+      for (const binding of bindings) {
+        existing.add(binding);
+      }
+      disallowedImports.set(moduleSpecifier, existing);
+    }
+  }
+
+  if (disallowedImports.size > 0) {
+    const formattedImports = Array.from(disallowedImports.entries())
+      .map(
+        ([moduleSpecifier, bindings]) =>
+          `${moduleSpecifier} (${Array.from(bindings).sort().join(", ")})`,
+      )
+      .join("; ");
+    throw new Error(
+      `Managed TypeScript eval '${evalName}' uses unsupported imports: ${formattedImports}. ` +
+      "Keep managed evals self-contained and move helpers into the same file.",
+    );
+  }
+
+  return context.sourceFile.statements
+    .filter(
+      (statement) =>
+        context.sdkImportStatements.includes(statement) || includedStatements.has(statement),
+    )
+    .map((statement) => statement.getFullText(context.sourceFile))
+    .join("");
+}
+
+function buildEvalOnlySandbox(code) {
+  const transpiled = ts.transpileModule(code, {
+    compilerOptions: {
+      module: ts.ModuleKind.CommonJS,
+      target: ts.ScriptTarget.ES2020,
+      esModuleInterop: true,
+    },
+  }).outputText;
+
+  const moduleRecord = { exports: {} };
+  const sdkStub = class {
+    constructor(opts) {
+      Object.assign(this, opts);
+    }
+  };
+
+  const sandbox = vm.createContext({
+    JSON,
+    Math,
+    Array,
+    Object,
+    String,
+    Number,
+    Boolean,
+    Date,
+    RegExp,
+    Promise,
+    Error,
+    TypeError,
+    RangeError,
+    ReferenceError,
+    SyntaxError,
+    Map,
+    Set,
+    WeakMap,
+    WeakSet,
+    Symbol,
+    parseInt,
+    parseFloat,
+    isNaN,
+    isFinite,
+    encodeURIComponent,
+    decodeURIComponent,
+    console,
+    exports: moduleRecord.exports,
+    module: moduleRecord,
+    require: (specifier) => {
+      if (specifier === "@zhanla/sdk-ts") {
+        return {
+          CodeEval: sdkStub,
+          LLMEval: sdkStub,
+          Checklist: sdkStub,
+          EvalTree: sdkStub,
+          Branch: sdkStub,
+          Leaf: sdkStub,
+          Edge: sdkStub,
+          Tool: sdkStub,
+          Skill: sdkStub,
+          Agent: sdkStub,
+          Orchestration: sdkStub,
+          Step: sdkStub,
+          ComponentValue: null,
+        };
+      }
+      throw new Error(
+        `Cannot import '${specifier}' in the managed TypeScript eval sandbox. ` +
+        "Only @zhanla/sdk-ts imports and same-file helpers are supported.",
+      );
+    },
+  });
+  sandbox.globalThis = sandbox;
+  new vm.Script(transpiled).runInContext(sandbox, { timeout: 10000 });
+  return moduleRecord;
+}
+
+function findEvalComponent(moduleRecord, evalName) {
+  for (const value of Object.values(moduleRecord.exports)) {
+    if (value && typeof value === "object" && value.name === evalName && typeof value.fn === "function") {
+      return value;
+    }
+  }
+  return null;
+}
+
+function loadIsolatedEvalComponent(target) {
+  const { filePath, symbolName } = parseTarget(target);
+  const absolutePath = resolve(filePath);
+  const sourceText = readFileSync(absolutePath, "utf8");
+  const isolatedSource = isolateManagedTypeScriptEvalSource(sourceText, symbolName);
+  const moduleRecord = buildEvalOnlySandbox(isolatedSource);
+  const component = findEvalComponent(moduleRecord, symbolName);
+  if (!component) {
+    throw new Error(`Component '${symbolName}' not found in isolated eval source.`);
+  }
+  return component;
+}
+
+export async function runEvalOnly(evalTarget) {
+  redirectConsoleToStderr();
+  let evalComponent;
+  try {
+    evalComponent = loadIsolatedEvalComponent(evalTarget);
+  } catch (err) {
+    console.error(err instanceof Error ? err.message : String(err));
+    process.exit(1);
+  }
+
+  const rl = readline.createInterface({
+    input: process.stdin,
+    crlfDelay: Infinity,
+  });
+
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    let context;
+    try {
+      context = JSON.parse(trimmed);
+    } catch (err) {
+      process.stdout.write(JSON.stringify({ status: "error", error: `Invalid JSON: ${err.message}` }) + "\n");
+      continue;
+    }
+
+    try {
+      const evalOutput = evalComponent.fn.length <= 1
+        ? await Promise.resolve(evalComponent.fn(context))
+        : await Promise.resolve(
+            evalComponent.fn(
+              context.model_response,
+              context.expected_output,
+              context.model_input,
+            ),
+          );
+      process.stdout.write(
+        JSON.stringify({ status: "ok", name: evalComponent.name, eval_output: evalOutput }) + "\n"
+      );
+    } catch (err) {
+      process.stdout.write(
+        JSON.stringify({ status: "error", name: evalComponent.name, error: err instanceof Error ? err.message : String(err) }) + "\n"
+      );
+    }
+  }
+}
+
 export async function runComponent(target, evalTarget = null) {
   redirectConsoleToStderr();
   // Dynamically import from SDK

package/dist/executor.d.ts

@@ -3,11 +3,13 @@
  * Mirrors the Python CLI executor's component-type dispatch semantics.
  */
 import { ZhanlaComponent } from "./types.js";
+import { LLMCall } from "./trace_store.js";
 export interface RowResult {
     status: "ok" | "error";
     output?: Record<string, unknown>;
     error?: string;
     pathTaken?: Record<string, unknown>;
+    componentLlmCalls?: LLMCall[];
 }
 export declare function executeComponent(comp: ZhanlaComponent, kwargs: Record<string, unknown>): Promise<Record<string, unknown>>;
 export declare function executeRow(component: ZhanlaComponent, evalComponent: ZhanlaComponent | null, row: Record<string, unknown>): Promise<RowResult>;

package/dist/executor.js

@@ -2,7 +2,9 @@
  * Local execution engine for TypeScript SDK components.
  * Mirrors the Python CLI executor's component-type dispatch semantics.
  */
+import { randomUUID } from "crypto";
 import { parseJsonResponse } from "./json.js";
+import { TraceContext, traceStorage } from "./trace_store.js";
 const COMPONENT_ERROR_KEY = "_component_error";
 const OUTPUT_SCHEMA_MISMATCH_REASON = "Model response doesn't match the requested output schema.";
 const JSON_OUTPUT_ERROR_MARKERS = [
@@ -69,6 +71,12 @@ function stringifyValue(value) {
         return String(value);
     }
 }
+function deriveModelInput(row) {
+    if ("input" in row) {
+        return stringifyValue(row["input"]);
+    }
+    return stringifyValue(row);
+}
 function deriveExpectedOutput(row) {
     if ("expected_output" in row) {
         return stringifyValue(row["expected_output"]);
@@ -76,9 +84,12 @@ function deriveExpectedOutput(row) {
     if ("output" in row) {
         return stringifyValue(row["output"]);
     }
-    return "";
+    return stringifyValue(row);
 }
 function deriveModelResponse(output) {
+    if (output == null) {
+        return stringifyValue(output);
+    }
     const keys = Object.keys(output);
     if (keys.length === 1 && keys[0] === "result") {
         return stringifyValue(output["result"]);
@@ -88,6 +99,13 @@ function deriveModelResponse(output) {
     }
     return stringifyValue(output);
 }
+function buildCodeEvalKwargs(output, row) {
+    return {
+        model_input: deriveModelInput(row),
+        model_response: deriveModelResponse(output),
+        expected_output: deriveExpectedOutput(row),
+    };
+}
 // ---------------------------------------------------------------------------
 // Component execution dispatch
 // ---------------------------------------------------------------------------
@@ -156,13 +174,11 @@ async function runTool(comp, kwargs) {
     return { result };
 }
 async function runCodeEval(comp, kwargs) {
-    const modelResponse = kwargs["model_response"];
-    const expectedResponse = kwargs["expected_response"] ?? null;
     const { value: result, logs } = await captureConsoleLogs(async () => {
         if (comp.isAsync) {
-            return await comp.fn(modelResponse, expectedResponse);
+            return await comp.fn(kwargs);
         }
-        return comp.fn(modelResponse, expectedResponse);
+        return comp.fn(kwargs);
     });
     if (result !== null && typeof result === "object" && !Array.isArray(result)) {
         return logs.length > 0
@@ -528,14 +544,19 @@ export async function executeRow(component, evalComponent, row) {
     let error;
     let demotedError;
     let orchestrationSteps;
+    let componentLlmCalls;
+    let componentFailedWithoutEval = false;
+    const traceContext = new TraceContext(randomUUID());
     try {
-        if (component.componentType === "orchestration") {
-            orchestrationSteps = [];
-            output = await runOrchestration(component, row, orchestrationSteps);
-        }
-        else {
-            output = await executeComponent(component, row);
-        }
+        await traceStorage.run(traceContext, async () => {
+            if (component.componentType === "orchestration") {
+                orchestrationSteps = [];
+                output = await runOrchestration(component, row, orchestrationSteps);
+            }
+            else {
+                output = await executeComponent(component, row);
+            }
+        });
     }
     catch (err) {
         error = err instanceof Error ? err.message : String(err);
@@ -545,9 +566,20 @@ export async function executeRow(component, evalComponent, row) {
             error = undefined;
         }
         else {
-            return { status: "error", error, pathTaken: buildPathTaken(component, row, undefined, error, orchestrationSteps) };
+            componentFailedWithoutEval = true;
         }
     }
+    finally {
+        componentLlmCalls = traceContext.flush();
+    }
+    if (componentFailedWithoutEval) {
+        return {
+            status: "error",
+            error,
+            pathTaken: buildPathTaken(component, row, undefined, error, orchestrationSteps),
+            componentLlmCalls,
+        };
+    }
     if (evalComponent != null) {
         if (output != null
             && typeof output[COMPONENT_ERROR_KEY] === "string"
@@ -556,18 +588,17 @@ export async function executeRow(component, evalComponent, row) {
                 status: "ok",
                 output: { ...output, _eval: buildSchemaMismatchEvalOutput(String(output[COMPONENT_ERROR_KEY])) },
                 pathTaken: buildPathTaken(component, row, output, demotedError, orchestrationSteps),
+                componentLlmCalls,
             };
         }
-        const evalKwargs = {
-            model_response: deriveModelResponse(output),
-            expected_response: deriveExpectedOutput(row) || null,
-        };
+        const evalKwargs = buildCodeEvalKwargs(output, row);
         try {
             const evalOutput = await executeComponent(evalComponent, evalKwargs);
             return {
                 status: "ok",
                 output: { ...output, _eval: evalOutput },
                 pathTaken: buildPathTaken(component, row, output, demotedError, orchestrationSteps),
+                componentLlmCalls,
             };
         }
         catch (err) {
@@ -577,6 +608,7 @@ export async function executeRow(component, evalComponent, row) {
                 output,
                 error: evalError,
                 pathTaken: buildPathTaken(component, row, output, demotedError, orchestrationSteps),
+                componentLlmCalls,
             };
         }
     }
@@ -584,6 +616,7 @@ export async function executeRow(component, evalComponent, row) {
         status: "ok",
         output,
         pathTaken: buildPathTaken(component, row, output, demotedError, orchestrationSteps),
+        componentLlmCalls,
     };
 }
 function buildPathTaken(component, inputData, output, error, orchestrationSteps) {

package/dist/index.d.ts

@@ -3,7 +3,7 @@
  * Import component classes to define zhanla components in TypeScript.
  */
 export { Tool, CodeEval, Skill, Agent, LLMProcessor, LLMEval, Runner, Conditional, Step, Orchestration, Leaf, Edge, Branch, Checklist, EvalTree, ZhanlaComponent, RUNNABLE_TYPES, EVAL_TYPES, } from "./types.js";
-export type { ComponentValue, ToolOptions, CodeEvalOptions, SkillOptions, AgentOptions, LLMProcessorOptions, LLMEvalOptions, OrchestrationOptions, ChecklistOptions, EvalTreeOptions, StepOptions, ComponentType, JsonSchema, JsonSchemaObject, JsonSchemaArray, JsonSchemaScalar, ToolCall, LLMResponse, RunnerOptions, RunnerCallOptions, RunnerMessage, } from "./types.js";
+export type { ComponentValue, ToolOptions, CodeEvalKwargs, CodeEvalOptions, SkillOptions, AgentOptions, LLMProcessorOptions, LLMEvalOptions, OrchestrationOptions, ChecklistOptions, EvalTreeOptions, StepOptions, ComponentType, JsonSchema, JsonSchemaObject, JsonSchemaArray, JsonSchemaScalar, ToolCall, LLMResponse, RunnerOptions, RunnerCallOptions, RunnerMessage, } from "./types.js";
 export { toManifest, collectExportedComponents } from "./manifest.js";
 export type { ComponentManifest, StepManifest, BranchManifest, EdgeManifest, LeafManifest } from "./manifest.js";
 export { executeComponent, executeRow } from "./executor.js";

package/dist/manifest.d.ts

@@ -57,10 +57,8 @@ export interface ComponentManifest {
     output_schema?: JsonSchema;
     file_path?: string;
     symbol_name?: string;
-    source_code?: string;
-    source_language?: string;
-    source_format?: string;
     model_response_format?: string;
+    questions?: string[];
 }
 export declare function toManifest(component: ZhanlaComponent, opts?: {
     filePath?: string;

package/dist/manifest.js

@@ -2,7 +2,6 @@
  * Manifest emission — serialize SDK component instances into ComponentManifest objects.
  * The manifest shape mirrors the Python CLI's ComponentManifest dataclass.
  */
-import { readFileSync } from "node:fs";
 // ---------------------------------------------------------------------------
 // Tree serialization helpers
 // ---------------------------------------------------------------------------
@@ -54,87 +53,6 @@ function serializeBranch(branch) {
         if_fail: branch.ifFail.map(serializeEdge),
     };
 }
-function escapeRegExp(value) {
-    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-function findMatchingParen(source, startIndex) {
-    let depth = 0;
-    let quote = null;
-    let inLineComment = false;
-    let inBlockComment = false;
-    for (let i = startIndex; i < source.length; i += 1) {
-        const char = source[i];
-        const next = source[i + 1];
-        const prev = source[i - 1];
-        if (inLineComment) {
-            if (char === "\n")
-                inLineComment = false;
-            continue;
-        }
-        if (inBlockComment) {
-            if (prev === "*" && char === "/")
-                inBlockComment = false;
-            continue;
-        }
-        if (quote) {
-            if (char === quote && prev !== "\\")
-                quote = null;
-            continue;
-        }
-        if (char === "/" && next === "/") {
-            inLineComment = true;
-            i += 1;
-            continue;
-        }
-        if (char === "/" && next === "*") {
-            inBlockComment = true;
-            i += 1;
-            continue;
-        }
-        if (char === "'" || char === '"' || char === "`") {
-            quote = char;
-            continue;
-        }
-        if (char === "(") {
-            depth += 1;
-            continue;
-        }
-        if (char === ")") {
-            depth -= 1;
-            if (depth === 0)
-                return i;
-        }
-    }
-    return -1;
-}
-function extractExportedComponentSource(filePath, symbolName) {
-    if (!filePath || !symbolName)
-        return undefined;
-    let source = "";
-    try {
-        source = readFileSync(filePath, "utf8");
-    }
-    catch {
-        return undefined;
-    }
-    const pattern = new RegExp(String.raw `(?:^|\n)\s*export\s+(?:const|let|var)\s+${escapeRegExp(symbolName)}\s*=\s*new\s+[A-Za-z_$][\w$.]*\s*\(`, "m");
-    const match = pattern.exec(source);
-    if (!match || match.index == null)
-        return undefined;
-    const start = source.lastIndexOf("export", match.index + match[0].length);
-    const openParenIndex = source.indexOf("(", match.index + match[0].length - 1);
-    if (start < 0 || openParenIndex < 0)
-        return undefined;
-    const closeParenIndex = findMatchingParen(source, openParenIndex);
-    if (closeParenIndex < 0)
-        return undefined;
-    let end = closeParenIndex + 1;
-    while (end < source.length && /\s/.test(source[end]))
-        end += 1;
-    if (source[end] === ";")
-        end += 1;
-    return source.slice(start, end).trim();
-}
 // ---------------------------------------------------------------------------
 // to_manifest
 // ---------------------------------------------------------------------------
@@ -150,7 +68,6 @@ const EXECUTION_MODES = {
     eval_tree: "composite",
 };
 export function toManifest(component, opts = {}) {
-    const componentSource = extractExportedComponentSource(opts.filePath, opts.symbolName);
     const key = component.key;
     const keySource = "explicit";
     const executionMode = EXECUTION_MODES[component.componentType] ?? "unknown";
@@ -175,9 +92,6 @@ export function toManifest(component, opts = {}) {
                 fn_present: true,
                 is_async: tool.isAsync,
                 output_schema: tool.outputSchema,
-                source_code: componentSource || tool.fn.toString(),
-                source_language: "typescript",
-                source_format: componentSource ? "component" : "function",
             };
         }
         case "code_eval": {
@@ -186,9 +100,6 @@ export function toManifest(component, opts = {}) {
                 ...base,
                 fn_present: true,
                 is_async: codeEval.isAsync,
-                source_code: componentSource || codeEval.fn.toString(),
-                source_language: "typescript",
-                source_format: componentSource ? "component" : "function",
                 model_response_format: codeEval.modelResponseFormat,
             };
         }
@@ -201,9 +112,6 @@ export function toManifest(component, opts = {}) {
                 is_async: skill.isAsync,
                 tool_refs: skill.tools.length > 0 ? skill.tools.map(resolveComponentRef) : undefined,
                 output_schema: skill.outputSchema,
-                source_code: componentSource || skill.fn?.toString(),
-                source_language: "typescript",
-                source_format: componentSource ? "component" : skill.fn ? "function" : undefined,
             };
         }
         case "agent": {
@@ -233,7 +141,8 @@ export function toManifest(component, opts = {}) {
             const llmEval = component;
             return {
                 ...base,
-                instructions: llmEval.instructions,
+                ...(llmEval.instructions !== undefined ? { instructions: llmEval.instructions } : {}),
+                ...(llmEval.questions !== undefined ? { questions: llmEval.questions } : {}),
                 model: llmEval.model,
                 temperature: llmEval.temperature,
                 output_schema: llmEval.outputSchema,

package/dist/trace_store.d.ts

@@ -12,7 +12,7 @@ export interface LLMCall {
     sequenceOrder: number;
     autoraterRunId: string | null;
     datasetItemId: string | null;
-    provider: "anthropic" | "openai" | "gemini";
+    provider: "anthropic" | "openai" | "gemini" | "openrouter";
     model: string;
     inputMessages: unknown[];
     output: unknown | null;

package/dist/types.d.ts

@@ -103,10 +103,15 @@ export declare class Tool extends ZhanlaComponent {
     readonly key: string;
     constructor(opts: ToolOptions);
 }
+export interface CodeEvalKwargs extends Record<string, unknown> {
+    model_input?: string;
+    model_response: string;
+    expected_output?: string;
+}
 export interface CodeEvalOptions {
     name: string;
     description: string;
-    fn: (modelResponse: ComponentValue, expectedResponse: ComponentValue | null) => unknown;
+    fn: (kwargs: CodeEvalKwargs) => unknown;
     modelResponseFormat?: "JSON" | "TEXT" | "YAML";
     key: string;
 }
@@ -114,7 +119,7 @@ export declare class CodeEval extends ZhanlaComponent {
     readonly componentType: "code_eval";
     readonly name: string;
     readonly description: string;
-    readonly fn: (modelResponse: ComponentValue, expectedResponse: ComponentValue | null) => unknown;
+    readonly fn: (kwargs: CodeEvalKwargs) => unknown;
     readonly isAsync: boolean;
     readonly modelResponseFormat: "JSON" | "TEXT" | "YAML";
     readonly key: string;
@@ -204,7 +209,8 @@ export declare class LLMProcessor extends ZhanlaComponent {
 export interface LLMEvalOptions {
     name: string;
     description: string;
-    instructions: string;
+    instructions?: string;
+    questions?: string[];
     model: string;
     client?: unknown;
     runner?: Runner;
@@ -218,7 +224,8 @@ export declare class LLMEval extends ZhanlaComponent {
     readonly componentType: "llm_eval";
     readonly name: string;
     readonly description: string;
-    readonly instructions: string;
+    readonly instructions?: string;
+    readonly questions?: string[];
     readonly model: string;
     readonly runner?: Runner;
     readonly outputSchema?: JsonSchema;

package/dist/types.js

@@ -3,7 +3,7 @@
  * Mirrors the Python SDK's class-based component model.
  */
 import { parseJsonResponse } from "./json.js";
-import { isAnthropicClient, isGeminiClient, isOpenAIClient, wrap } from "./wrap.js";
+import { isAnthropicClient, isGeminiClient, isOpenAIClient, isOpenRouterClient, wrap } from "./wrap.js";
 export const RUNNABLE_TYPES = new Set([
     "tool",
     "agent",
@@ -210,6 +210,8 @@ export class Runner {
     providerName() {
         if (isAnthropicClient(this.client))
             return "anthropic";
+        if (isOpenRouterClient(this.client))
+            return "openrouter";
         if (isOpenAIClient(this.client))
             return "openai";
         if (isGeminiClient(this.client))
@@ -583,6 +585,7 @@ export class LLMEval extends ZhanlaComponent {
     name;
     description;
     instructions;
+    questions;
     model;
     runner;
     outputSchema;
@@ -592,12 +595,21 @@ export class LLMEval extends ZhanlaComponent {
     key;
     constructor(opts) {
         super();
+        const hasInstructions = opts.instructions !== undefined && opts.instructions !== "";
+        const hasQuestions = opts.questions !== undefined && opts.questions.length > 0;
+        if (hasInstructions && hasQuestions) {
+            throw new Error(`LLMEval '${opts.name}': provide exactly one of 'instructions' or 'questions', not both.`);
+        }
+        if (!hasInstructions && !hasQuestions) {
+            throw new Error(`LLMEval '${opts.name}': must provide exactly one of 'instructions' or 'questions'.`);
+        }
         if (opts.client !== undefined && opts.runner !== undefined) {
             throw new Error("Specify client or runner, not both.");
         }
         this.name = opts.name;
         this.description = opts.description;
         this.instructions = opts.instructions;
+        this.questions = opts.questions;
         this.model = opts.model;
         this.runner = opts.client !== undefined ? new Runner({ client: opts.client }) : opts.runner;
         this.outputSchema = opts.outputSchema;

package/dist/wrap.d.ts

@@ -7,6 +7,7 @@
  * Supported clients:
  *   - Anthropic (from @anthropic-ai/sdk)
  *   - OpenAI (from openai)
+ *   - OpenRouter (OpenAI client with baseURL containing "openrouter")
  *   - GoogleGenAI (from @google/genai)
  */
 export declare function isAnthropicClient(client: unknown): client is {
@@ -21,6 +22,14 @@ export declare function isOpenAIClient(client: unknown): client is {
         };
     };
 };
+export declare function isOpenRouterClient(client: unknown): client is {
+    chat: {
+        completions: {
+            create: (...args: unknown[]) => unknown;
+        };
+    };
+    baseURL: string;
+};
 export declare function isGeminiClient(client: unknown): client is {
     models: {
         generateContent: (...args: unknown[]) => unknown;

package/dist/wrap.js

@@ -7,6 +7,7 @@
  * Supported clients:
  *   - Anthropic (from @anthropic-ai/sdk)
  *   - OpenAI (from openai)
+ *   - OpenRouter (OpenAI client with baseURL containing "openrouter")
  *   - GoogleGenAI (from @google/genai)
  */
 import { traceStorage } from "./trace_store.js";
@@ -104,7 +105,7 @@ function wrapAnthropic(client) {
 // ---------------------------------------------------------------------------
 // OpenAI
 // ---------------------------------------------------------------------------
-function recordOpenAICall(ctx, kwargs, response, latencyMs) {
+function recordOpenAICall(ctx, kwargs, response, latencyMs, provider = "openai") {
     if (!ctx)
         return;
     const choices = response["choices"] ?? [];
@@ -119,7 +120,7 @@ function recordOpenAICall(ctx, kwargs, response, latencyMs) {
         sequenceOrder: ctx.nextSequence(),
         autoraterRunId: ctx.autoraterRunId,
         datasetItemId: ctx.datasetItemId,
-        provider: "openai",
+        provider,
         model: String(response["model"] ?? (kwargs["model"] ?? "")),
         inputMessages: serialize(kwargs["messages"] ?? []),
         output: serialize(message),
@@ -132,7 +133,7 @@ function recordOpenAICall(ctx, kwargs, response, latencyMs) {
         metadata: null,
     });
 }
-function wrapOpenAI(client) {
+function wrapOpenAI(client, provider = "openai") {
     const originalCreate = client.chat.completions.create.bind(client.chat.completions);
     client.chat.completions.create = async function (...args) {
         const ctx = traceStorage.getStore();
@@ -140,7 +141,7 @@ function wrapOpenAI(client) {
         const start = performance.now();
         const response = await originalCreate(...args);
         const latencyMs = Math.round(performance.now() - start);
-        recordOpenAICall(ctx, kwargs, response, latencyMs);
+        recordOpenAICall(ctx, kwargs, response, latencyMs, provider);
         return response;
     };
     return client;
@@ -208,6 +209,12 @@ export function isOpenAIClient(client) {
         client.chat !== null &&
         typeof client.chat.completions === "object");
 }
+export function isOpenRouterClient(client) {
+    if (!isOpenAIClient(client))
+        return false;
+    const baseURL = client["baseURL"];
+    return typeof baseURL === "string" && baseURL.includes("openrouter");
+}
 export function isGeminiClient(client) {
     return (client !== null &&
         typeof client === "object" &&
@@ -235,13 +242,15 @@ export function wrap(client) {
     let wrapped;
     if (isAnthropicClient(client))
         wrapped = wrapAnthropic(client);
+    else if (isOpenRouterClient(client))
+        wrapped = wrapOpenAI(client, "openrouter");
     else if (isOpenAIClient(client))
         wrapped = wrapOpenAI(client);
     else if (isGeminiClient(client))
         wrapped = wrapGemini(client);
     else {
         throw new TypeError(`bench.wrap() does not support ${Object.getPrototypeOf(client)?.constructor?.name ?? typeof client}. ` +
-            "Supported clients: Anthropic, OpenAI, GoogleGenAI.");
+            "Supported clients: Anthropic, OpenAI, OpenRouter (OpenAI client with OpenRouter baseURL), GoogleGenAI.");
     }
     if (wrapped != null && typeof wrapped === "object") {
         Object.defineProperty(wrapped, WRAPPED_CLIENT, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zhanla/sdk-ts",
-  "version": "0.3.3",
+  "version": "0.3.5",
   "description": "TypeScript SDK for the zhanla CLI — define and run AI components locally",
   "homepage": "https://benchmark-black.vercel.app/",
   "repository": {
@@ -20,7 +20,8 @@
     "postinstall": "node ./bin/postinstall.js"
   },
   "dependencies": {
-    "tsx": "^4.7.0"
+    "tsx": "^4.7.0",
+    "typescript": "^5.4.0"
   },
   "files": [
     "dist",

package/bin/env.js

@@ -1,72 +0,0 @@
-#!/usr/bin/env node
-
-import { existsSync, readFileSync, statSync } from "fs";
-import { dirname, join, resolve } from "path";
-
-function isDirectory(path) {
-  try {
-    return statSync(path).isDirectory();
-  } catch {
-    return false;
-  }
-}
-
-export function splitFileSpec(fileSpec) {
-  const colonIndex = fileSpec.lastIndexOf(":");
-  // Preserve Windows drive letters (e.g. C:\...) by only splitting on later colons.
-  if (colonIndex > 1) {
-    return {
-      filePath: fileSpec.slice(0, colonIndex),
-      symbolName: fileSpec.slice(colonIndex + 1) || null,
-    };
-  }
-  return { filePath: fileSpec, symbolName: null };
-}
-
-export function findDotenvLocal(startPath) {
-  let current = resolve(startPath);
-  if (!isDirectory(current)) {
-    current = dirname(current);
-  }
-
-  while (true) {
-    const candidate = join(current, ".env.local");
-    if (existsSync(candidate)) {
-      return candidate;
-    }
-    const parent = dirname(current);
-    if (parent === current) {
-      return null;
-    }
-    current = parent;
-  }
-}
-
-export function loadDotenvLocal(startPath) {
-  const envPath = findDotenvLocal(startPath);
-  if (!envPath) {
-    return null;
-  }
-
-  const source = readFileSync(envPath, "utf8");
-  for (const line of source.split(/\r?\n/)) {
-    const trimmed = line.trim();
-    if (!trimmed || trimmed.startsWith("#")) continue;
-    const equalsIndex = trimmed.indexOf("=");
-    if (equalsIndex === -1) continue;
-
-    const key = trimmed.slice(0, equalsIndex).trim();
-    let value = trimmed.slice(equalsIndex + 1).trim();
-    if (
-      (value.startsWith('"') && value.endsWith('"'))
-      || (value.startsWith("'") && value.endsWith("'"))
-    ) {
-      value = value.slice(1, -1);
-    }
-    if (key && process.env[key] === undefined) {
-      process.env[key] = value;
-    }
-  }
-
-  return envPath;
-}