@axlsdk/axl 0.2.0 → 0.4.0

package/dist/index.d.cts

@@ -1075,6 +1075,7 @@ declare class AxlRuntime extends EventEmitter {
     private executions;
     private pendingDecisionResolvers;
     private abortControllers;
+    private registeredEvals;
     private mcpManager?;
     private memoryManager?;
     private spanManager;
@@ -1112,6 +1113,33 @@ declare class AxlRuntime extends EventEmitter {
     getAgents(): Agent[];
     /** Get a registered standalone agent by name. */
     getAgent(name: string): Agent | undefined;
+    /**
+     * Register an eval config for Studio introspection and execution.
+     * The config should be the result of `defineEval()` from `@axlsdk/eval`.
+     * An optional `executeWorkflow` function can override the default behavior
+     * of calling `runtime.execute()`.
+     */
+    registerEval(name: string, config: unknown, executeWorkflow?: (input: unknown) => Promise<{
+        output: unknown;
+        cost?: number;
+    }>): void;
+    /** Get metadata about all registered evals. */
+    getRegisteredEvals(): Array<{
+        name: string;
+        workflow: string;
+        dataset: string;
+        scorers: string[];
+    }>;
+    /** Get a registered eval config by name. */
+    getRegisteredEval(name: string): {
+        config: unknown;
+        executeWorkflow?: (input: unknown) => Promise<{
+            output: unknown;
+            cost?: number;
+        }>;
+    } | undefined;
+    /** Run a registered eval by name. */
+    runRegisteredEval(name: string): Promise<unknown>;
     /** Get all execution info (running + completed). */
     getExecutions(): ExecutionInfo[];
     /** Register a custom provider instance. */

package/dist/index.d.ts

@@ -1075,6 +1075,7 @@ declare class AxlRuntime extends EventEmitter {
     private executions;
     private pendingDecisionResolvers;
     private abortControllers;
+    private registeredEvals;
     private mcpManager?;
     private memoryManager?;
     private spanManager;
@@ -1112,6 +1113,33 @@ declare class AxlRuntime extends EventEmitter {
     getAgents(): Agent[];
     /** Get a registered standalone agent by name. */
     getAgent(name: string): Agent | undefined;
+    /**
+     * Register an eval config for Studio introspection and execution.
+     * The config should be the result of `defineEval()` from `@axlsdk/eval`.
+     * An optional `executeWorkflow` function can override the default behavior
+     * of calling `runtime.execute()`.
+     */
+    registerEval(name: string, config: unknown, executeWorkflow?: (input: unknown) => Promise<{
+        output: unknown;
+        cost?: number;
+    }>): void;
+    /** Get metadata about all registered evals. */
+    getRegisteredEvals(): Array<{
+        name: string;
+        workflow: string;
+        dataset: string;
+        scorers: string[];
+    }>;
+    /** Get a registered eval config by name. */
+    getRegisteredEval(name: string): {
+        config: unknown;
+        executeWorkflow?: (input: unknown) => Promise<{
+            output: unknown;
+            cost?: number;
+        }>;
+    } | undefined;
+    /** Run a registered eval by name. */
+    runRegisteredEval(name: string): Promise<unknown>;
     /** Get all execution info (running + completed). */
     getExecutions(): ExecutionInfo[];
     /** Register a custom provider instance. */

package/dist/index.js

@@ -1746,6 +1746,15 @@ function zodToJsonSchema(schema) {
 function estimateTokens(text) {
   return Math.ceil(text.length / 4);
 }
+function stripMarkdownFences(text) {
+  const trimmed = text.trim();
+  if (trimmed.startsWith("```")) {
+    const withoutOpening = trimmed.replace(/^```\w*\s*\n?/, "");
+    const withoutClosing = withoutOpening.replace(/\n?```\s*$/, "");
+    return withoutClosing.trim();
+  }
+  return trimmed;
+}
 function estimateMessagesTokens(messages) {
   let total = 0;
   for (const msg of messages) {
@@ -2439,7 +2448,7 @@ Please fix and try again.`;
       }
       if (options?.schema) {
         try {
-          const parsed = JSON.parse(content);
+          const parsed = JSON.parse(stripMarkdownFences(content));
           const validated = options.schema.parse(parsed);
           return validated;
         } catch (err) {
@@ -4577,6 +4586,7 @@ var AxlRuntime = class extends EventEmitter2 {
   executions = /* @__PURE__ */ new Map();
   pendingDecisionResolvers = /* @__PURE__ */ new Map();
   abortControllers = /* @__PURE__ */ new Map();
+  registeredEvals = /* @__PURE__ */ new Map();
   mcpManager;
   memoryManager;
   spanManager = new NoopSpanManager();
@@ -4672,6 +4682,52 @@ var AxlRuntime = class extends EventEmitter2 {
   getAgent(name) {
     return this.agents.get(name);
   }
+  /**
+   * Register an eval config for Studio introspection and execution.
+   * The config should be the result of `defineEval()` from `@axlsdk/eval`.
+   * An optional `executeWorkflow` function can override the default behavior
+   * of calling `runtime.execute()`.
+   */
+  registerEval(name, config, executeWorkflow) {
+    this.registeredEvals.set(name, { config, executeWorkflow });
+  }
+  /** Get metadata about all registered evals. */
+  getRegisteredEvals() {
+    const result = [];
+    for (const [name, { config }] of this.registeredEvals) {
+      const cfg = config;
+      result.push({
+        name,
+        workflow: cfg.workflow ?? "unknown",
+        dataset: cfg.dataset?.name ?? "unknown",
+        scorers: (cfg.scorers ?? []).map((s) => s.name ?? "unknown")
+      });
+    }
+    return result;
+  }
+  /** Get a registered eval config by name. */
+  getRegisteredEval(name) {
+    return this.registeredEvals.get(name);
+  }
+  /** Run a registered eval by name. */
+  async runRegisteredEval(name) {
+    const entry = this.registeredEvals.get(name);
+    if (!entry) throw new Error(`Eval "${name}" is not registered`);
+    if (entry.executeWorkflow) {
+      let runEvalFn;
+      try {
+        ({ runEval: runEvalFn } = await import("@axlsdk/eval"));
+      } catch {
+        throw new Error(
+          "axl-eval is required for AxlRuntime.runRegisteredEval(). Install it with: npm install @axlsdk/eval"
+        );
+      }
+      return runEvalFn(entry.config, entry.executeWorkflow);
+    }
+    return this.eval(
+      entry.config
+    );
+  }
   /** Get all execution info (running + completed). */
   getExecutions() {
     return [...this.executions.values()];
@@ -5055,10 +5111,10 @@ var AxlRuntime = class extends EventEmitter2 {
   async eval(config) {
     let runEval;
     try {
-      ({ runEval } = await import("axl-eval"));
+      ({ runEval } = await import("@axlsdk/eval"));
     } catch {
       throw new Error(
-        "axl-eval is required for AxlRuntime.eval(). Install it with: npm install axl-eval"
+        "axl-eval is required for AxlRuntime.eval(). Install it with: npm install @axlsdk/eval"
       );
     }
     const executeWorkflow = async (input) => {
@@ -5085,10 +5141,10 @@ var AxlRuntime = class extends EventEmitter2 {
   async evalCompare(baseline, candidate) {
     let evalCompareFn;
     try {
-      ({ evalCompare: evalCompareFn } = await import("axl-eval"));
+      ({ evalCompare: evalCompareFn } = await import("@axlsdk/eval"));
     } catch {
       throw new Error(
-        "axl-eval is required for AxlRuntime.evalCompare(). Install it with: npm install axl-eval"
+        "axl-eval is required for AxlRuntime.evalCompare(). Install it with: npm install @axlsdk/eval"
       );
     }
     return evalCompareFn(baseline, candidate);