@axlsdk/axl 0.11.6 → 0.13.0

package/README.md

@@ -505,7 +505,7 @@ const result = await ctx.ask(extractAgent, 'Extract user from this text', {
 
 ### State Stores
 
-Three built-in implementations. All persist the same data: workflow execution checkpoints, `awaitHuman` decisions, session history, memory entries, and the execution state needed for suspend/resume.
+Three built-in implementations. All persist the same data: workflow execution checkpoints, `awaitHuman` decisions, session history, memory entries, execution history, eval history, and the execution state needed for suspend/resume.
 
 **Memory** (default) — in-process, no persistence. Use for development and stateless workflows.
 

package/dist/index.cjs

@@ -2666,6 +2666,11 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
         model: modelUri,
         promptVersion: agent2._config.version,
         cost: response.cost,
+        tokens: response.usage ? {
+          input: response.usage.prompt_tokens,
+          output: response.usage.completion_tokens,
+          reasoning: response.usage.reasoning_tokens
+        } : void 0,
         duration: Date.now() - startTime,
         data: { prompt, response: response.content }
       });
@@ -4053,6 +4058,8 @@ var MemoryStore = class {
   decisions = /* @__PURE__ */ new Map();
   executionStates = /* @__PURE__ */ new Map();
   memories = /* @__PURE__ */ new Map();
+  executionHistory = /* @__PURE__ */ new Map();
+  evalHistory = /* @__PURE__ */ new Map();
   constructor() {
     this.loadPersistedState();
   }
@@ -4149,11 +4156,36 @@ var MemoryStore = class {
   async getAllMemory(scope) {
     const scopeMap = this.memories.get(scope);
     if (!scopeMap) return [];
-    return Array.from(scopeMap.entries()).map(([key, value]) => ({ key, value }));
+    return Array.from(scopeMap.entries()).map(([key, value]) => ({
+      key,
+      value: structuredClone(value)
+    }));
   }
   async deleteMemory(scope, key) {
     this.memories.get(scope)?.delete(key);
   }
+  // ── Execution History ──────────────────────────────────────────────
+  async saveExecution(execution) {
+    this.executionHistory.set(execution.executionId, structuredClone(execution));
+  }
+  async getExecution(executionId) {
+    const exec = this.executionHistory.get(executionId);
+    return exec ? structuredClone(exec) : null;
+  }
+  async listExecutions(limit) {
+    const sorted = [...this.executionHistory.values()].sort((a, b) => b.startedAt - a.startedAt);
+    const result = limit ? sorted.slice(0, limit) : sorted;
+    return result.map((e) => structuredClone(e));
+  }
+  // ── Eval History ──────────────────────────────────────────────────
+  async saveEvalResult(entry) {
+    this.evalHistory.set(entry.id, structuredClone(entry));
+  }
+  async listEvalResults(limit) {
+    const sorted = [...this.evalHistory.values()].sort((a, b) => b.timestamp - a.timestamp);
+    const result = limit ? sorted.slice(0, limit) : sorted;
+    return result.map((e) => structuredClone(e));
+  }
   // ── Sessions (Studio introspection) ─────────────────────────────────
   async listSessions() {
     return [...this.sessions.keys()];
@@ -4213,6 +4245,19 @@ function safeJsonParse(data) {
     return null;
   }
 }
+function rowToExecutionInfo(row) {
+  return {
+    executionId: row.execution_id,
+    workflow: row.workflow,
+    status: row.status,
+    totalCost: row.total_cost,
+    startedAt: row.started_at,
+    completedAt: row.completed_at ?? void 0,
+    duration: row.duration,
+    error: row.error ?? void 0,
+    steps: safeJsonParse(row.steps) ?? []
+  };
+}
 var SQLiteStore = class {
   db;
   constructor(filePath) {
@@ -4273,6 +4318,28 @@ var SQLiteStore = class {
         updated_at TEXT NOT NULL DEFAULT (datetime('now')),
         PRIMARY KEY (scope, key)
       );
+
+      CREATE TABLE IF NOT EXISTS execution_history (
+        execution_id TEXT PRIMARY KEY,
+        workflow TEXT NOT NULL,
+        status TEXT NOT NULL,
+        total_cost REAL NOT NULL DEFAULT 0,
+        started_at INTEGER NOT NULL,
+        completed_at INTEGER,
+        duration INTEGER NOT NULL DEFAULT 0,
+        error TEXT,
+        steps TEXT NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS eval_history (
+        id TEXT PRIMARY KEY,
+        eval_name TEXT NOT NULL,
+        timestamp INTEGER NOT NULL,
+        data TEXT NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_exec_history_started ON execution_history (started_at DESC);
+      CREATE INDEX IF NOT EXISTS idx_eval_history_timestamp ON eval_history (timestamp DESC);
     `);
   }
   // ── Checkpoints ────────────────────────────────────────────────────────
@@ -4386,6 +4453,47 @@ var SQLiteStore = class {
     const rows = stmt.all();
     return rows.map((r) => r.execution_id);
   }
+  // ── Execution History ────────────────────────────────────────────────────
+  async saveExecution(execution) {
+    this.db.prepare(
+      "INSERT OR REPLACE INTO execution_history (execution_id, workflow, status, total_cost, started_at, completed_at, duration, error, steps) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
+    ).run(
+      execution.executionId,
+      execution.workflow,
+      execution.status,
+      execution.totalCost,
+      execution.startedAt,
+      execution.completedAt ?? null,
+      execution.duration,
+      execution.error ?? null,
+      JSON.stringify(execution.steps)
+    );
+  }
+  async getExecution(executionId) {
+    const row = this.db.prepare("SELECT * FROM execution_history WHERE execution_id = ?").get(executionId);
+    return row ? rowToExecutionInfo(row) : null;
+  }
+  async listExecutions(limit) {
+    const sql = limit ? "SELECT * FROM execution_history ORDER BY started_at DESC LIMIT ?" : "SELECT * FROM execution_history ORDER BY started_at DESC";
+    const rows = limit ? this.db.prepare(sql).all(limit) : this.db.prepare(sql).all();
+    return rows.map(rowToExecutionInfo);
+  }
+  // ── Eval History ────────────────────────────────────────────────────────
+  async saveEvalResult(entry) {
+    this.db.prepare(
+      "INSERT OR REPLACE INTO eval_history (id, eval_name, timestamp, data) VALUES (?, ?, ?, ?)"
+    ).run(entry.id, entry.eval, entry.timestamp, JSON.stringify(entry.data));
+  }
+  async listEvalResults(limit) {
+    const sql = limit ? "SELECT * FROM eval_history ORDER BY timestamp DESC LIMIT ?" : "SELECT * FROM eval_history ORDER BY timestamp DESC";
+    const rows = limit ? this.db.prepare(sql).all(limit) : this.db.prepare(sql).all();
+    return rows.map((r) => ({
+      id: r.id,
+      eval: r.eval_name,
+      timestamp: r.timestamp,
+      data: safeJsonParse(r.data)
+    }));
+  }
   // ── Sessions (Studio introspection) ────────────────────────────────────
   async listSessions() {
     const rows = this.db.prepare("SELECT session_id FROM sessions").all();
@@ -5271,6 +5379,10 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
   mcpManager;
   memoryManager;
   spanManager = new NoopSpanManager();
+  historicalExecutions = /* @__PURE__ */ new Map();
+  historicalExecutionsLoadPromise = null;
+  evalHistory = [];
+  evalHistoryLoadPromise = null;
   constructor(config) {
     super();
     this.config = resolveConfig(config ?? {});
@@ -5315,6 +5427,20 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
         return new MemoryStore();
     }
   }
+  /**
+   * Persist a completed/failed execution to the state store (fire-and-forget)
+   * and move it from the active executions map to the historical cache.
+   */
+  persistExecution(execInfo) {
+    const snapshot = structuredClone(execInfo);
+    if (this.stateStore.saveExecution) {
+      this.stateStore.saveExecution(snapshot).catch(() => {
+      });
+    }
+    const id = execInfo.executionId;
+    this.historicalExecutions.set(id, snapshot);
+    this.executions.delete(id);
+  }
   /** Register a workflow with the runtime. */
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   register(workflow2) {
@@ -5393,6 +5519,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
   async runRegisteredEval(name) {
     const entry = this.registeredEvals.get(name);
     if (!entry) throw new Error(`Eval "${name}" is not registered`);
+    let result;
     if (entry.executeWorkflow) {
       let runEvalFn;
       try {
@@ -5404,20 +5531,50 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
       }
       const originalExecuteFn = entry.executeWorkflow;
       const wrappedExecuteFn = async (input, runtime) => {
-        const { result, cost: trackedCost } = await this.trackCost(async () => {
+        const { result: result2, cost: trackedCost } = await this.trackCost(async () => {
           return originalExecuteFn(input, runtime);
         });
-        return { output: result.output, cost: result.cost ?? trackedCost };
+        return { output: result2.output, cost: result2.cost ?? trackedCost };
       };
-      return runEvalFn(entry.config, wrappedExecuteFn, void 0, this);
+      result = await runEvalFn(entry.config, wrappedExecuteFn, this);
+    } else {
+      result = await this.eval(
+        entry.config
+      );
     }
-    return this.eval(
-      entry.config
-    );
+    const resultObj = result;
+    try {
+      await this.saveEvalResult({
+        id: resultObj.id ?? (0, import_node_crypto2.randomUUID)(),
+        eval: name,
+        timestamp: Date.now(),
+        data: structuredClone(result)
+      });
+    } catch {
+    }
+    return result;
   }
-  /** Get all execution info (running + completed). */
-  getExecutions() {
-    return [...this.executions.values()];
+  /** Get all execution info (running + completed + historical). */
+  async getExecutions() {
+    if (!this.historicalExecutionsLoadPromise && this.stateStore.listExecutions) {
+      this.historicalExecutionsLoadPromise = this.stateStore.listExecutions().then((stored) => {
+        for (const exec of stored) {
+          if (!this.executions.has(exec.executionId) && !this.historicalExecutions.has(exec.executionId)) {
+            this.historicalExecutions.set(exec.executionId, exec);
+          }
+        }
+      }).catch(() => {
+        this.historicalExecutionsLoadPromise = null;
+      });
+    }
+    if (this.historicalExecutionsLoadPromise) {
+      await this.historicalExecutionsLoadPromise;
+    }
+    const merged = new Map(this.historicalExecutions);
+    for (const [id, exec] of this.executions) {
+      merged.set(id, exec);
+    }
+    return [...merged.values()].sort((a, b) => b.startedAt - a.startedAt);
   }
   /**
    * Create a WorkflowContext for ad-hoc use (evals, tool testing, prototyping).
@@ -5458,6 +5615,10 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
   registerProvider(name, provider) {
     this.providerRegistry.registerInstance(name, provider);
   }
+  /** Resolve a provider:model URI to a Provider instance and model name. */
+  resolveProvider(uri) {
+    return this.providerRegistry.resolve(uri, this.config);
+  }
   /** Execute a workflow and return the result. */
   async execute(name, input, options) {
     const workflow2 = this.workflows.get(name);
@@ -5549,6 +5710,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
           }
           span.setAttribute("axl.workflow.cost", execInfo.totalCost);
           span.setAttribute("axl.workflow.duration", execInfo.duration);
+          this.persistExecution(execInfo);
           return output;
         } catch (err) {
           execInfo.status = "failed";
@@ -5556,6 +5718,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
           execInfo.duration = execInfo.completedAt - execInfo.startedAt;
           execInfo.error = err instanceof Error ? err.message : String(err);
           ctx.log("workflow_end", { workflow: name, status: "failed", error: execInfo.error });
+          this.persistExecution(execInfo);
           throw err;
         } finally {
           this.abortControllers.delete(executionId);
@@ -5569,6 +5732,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
     const controller = new AbortController();
     axlStream.on("close", () => controller.abort());
     let execInfo;
+    let ctx;
     const run = async () => {
       const workflow2 = this.workflows.get(name);
       if (!workflow2) throw new Error(`Workflow "${name}" not registered`);
@@ -5587,7 +5751,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
         duration: 0
       };
       this.executions.set(executionId, execInfo);
-      const ctx = new WorkflowContext({
+      const wfCtx = new WorkflowContext({
         input: validated,
         executionId,
         metadata: options?.metadata,
@@ -5675,6 +5839,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
           policy: "finish_and_stop"
         }
       });
+      ctx = wfCtx;
       return this.spanManager.withSpanAsync(
         "axl.workflow.execute",
         {
@@ -5684,16 +5849,23 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
         },
         async (span) => {
           try {
-            const rawResult = await workflow2.handler(ctx);
+            const rawResult = await workflow2.handler(wfCtx);
             const result = workflow2.outputSchema ? workflow2.outputSchema.parse(rawResult) : rawResult;
             execInfo.status = "completed";
             execInfo.completedAt = Date.now();
             execInfo.duration = execInfo.completedAt - execInfo.startedAt;
+            wfCtx.log("workflow_end", {
+              workflow: name,
+              status: "completed",
+              duration: execInfo.duration,
+              cost: execInfo.totalCost
+            });
             if (this.stateStore.deleteCheckpoints) {
               await this.stateStore.deleteCheckpoints(executionId);
             }
             span.setAttribute("axl.workflow.cost", execInfo.totalCost);
             span.setAttribute("axl.workflow.duration", execInfo.duration);
+            this.persistExecution(execInfo);
             return result;
           } finally {
             this.abortControllers.delete(executionId);
@@ -5707,6 +5879,12 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
         execInfo.completedAt = Date.now();
         execInfo.duration = execInfo.completedAt - execInfo.startedAt;
         execInfo.error = err instanceof Error ? err.message : String(err);
+        ctx?.log("workflow_end", {
+          workflow: name,
+          status: "failed",
+          error: execInfo.error
+        });
+        this.persistExecution(execInfo);
       }
       axlStream._error(err instanceof Error ? err : new Error(String(err)));
     });
@@ -5750,7 +5928,45 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
   }
   /** Get execution details by ID. */
   async getExecution(executionId) {
-    return this.executions.get(executionId);
+    const inMemory = this.executions.get(executionId);
+    if (inMemory) return inMemory;
+    const cached = this.historicalExecutions.get(executionId);
+    if (cached) return cached;
+    if (this.stateStore.getExecution) {
+      const stored = await this.stateStore.getExecution(executionId);
+      if (stored) {
+        this.historicalExecutions.set(executionId, stored);
+        return stored;
+      }
+    }
+    return void 0;
+  }
+  /** Save an eval result to history. */
+  async saveEvalResult(entry) {
+    this.evalHistory.unshift(entry);
+    if (this.stateStore.saveEvalResult) {
+      await this.stateStore.saveEvalResult(entry);
+    }
+  }
+  /** Get eval result history (most recent first). */
+  async getEvalHistory() {
+    if (!this.evalHistoryLoadPromise && this.stateStore.listEvalResults) {
+      this.evalHistoryLoadPromise = this.stateStore.listEvalResults().then((stored) => {
+        const ids = new Set(this.evalHistory.map((e) => e.id));
+        for (const entry of stored) {
+          if (!ids.has(entry.id)) {
+            this.evalHistory.push(entry);
+          }
+        }
+        this.evalHistory.sort((a, b) => b.timestamp - a.timestamp);
+      }).catch(() => {
+        this.evalHistoryLoadPromise = null;
+      });
+    }
+    if (this.evalHistoryLoadPromise) {
+      await this.evalHistoryLoadPromise;
+    }
+    return [...this.evalHistory];
   }
   /** List pending human decisions. */
   async getPendingDecisions() {
@@ -5859,7 +6075,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
       });
       return { output: result, cost };
     };
-    return runEvalFn(config, executeWorkflow, void 0, this);
+    return runEvalFn(config, executeWorkflow, this);
   }
   /**
    * Compare two evaluation results to detect regressions and improvements.
@@ -6058,6 +6274,18 @@ var RedisStore = class _RedisStore {
   pendingExecSetKey() {
     return "axl:pending-executions";
   }
+  execHistoryKey(executionId) {
+    return `axl:exec-history:${executionId}`;
+  }
+  execHistorySetKey() {
+    return "axl:exec-history-ids";
+  }
+  evalHistoryKey(id) {
+    return `axl:eval-history:${id}`;
+  }
+  evalHistorySetKey() {
+    return "axl:eval-history-ids";
+  }
   // ── Checkpoints ──────────────────────────────────────────────────────
   async saveCheckpoint(executionId, step, data) {
     await this.client.hSet(this.checkpointKey(executionId), String(step), JSON.stringify(data));
@@ -6129,6 +6357,42 @@ var RedisStore = class _RedisStore {
   async listPendingExecutions() {
     return this.client.sMembers(this.pendingExecSetKey());
   }
+  // ── Execution History ────────────────────────────────────────────────
+  async saveExecution(execution) {
+    await this.client.sAdd(this.execHistorySetKey(), execution.executionId);
+    await this.client.set(this.execHistoryKey(execution.executionId), JSON.stringify(execution));
+  }
+  async getExecution(executionId) {
+    const raw = await this.client.get(this.execHistoryKey(executionId));
+    return raw ? JSON.parse(raw) : null;
+  }
+  async listExecutions(limit) {
+    const ids = await this.client.sMembers(this.execHistorySetKey());
+    if (ids.length === 0) return [];
+    const entries = [];
+    for (const id of ids) {
+      const raw = await this.client.get(this.execHistoryKey(id));
+      if (raw) entries.push(JSON.parse(raw));
+    }
+    entries.sort((a, b) => b.startedAt - a.startedAt);
+    return limit ? entries.slice(0, limit) : entries;
+  }
+  // ── Eval History ────────────────────────────────────────────────────
+  async saveEvalResult(entry) {
+    await this.client.sAdd(this.evalHistorySetKey(), entry.id);
+    await this.client.set(this.evalHistoryKey(entry.id), JSON.stringify(entry));
+  }
+  async listEvalResults(limit) {
+    const ids = await this.client.sMembers(this.evalHistorySetKey());
+    if (ids.length === 0) return [];
+    const entries = [];
+    for (const id of ids) {
+      const raw = await this.client.get(this.evalHistoryKey(id));
+      if (raw) entries.push(JSON.parse(raw));
+    }
+    entries.sort((a, b) => b.timestamp - a.timestamp);
+    return limit ? entries.slice(0, limit) : entries;
+  }
   // ── Sessions (Studio introspection) ────────────────────────────────────
   async listSessions() {
     return this.client.sMembers("axl:session-ids");