@duckflux/core 0.6.8 → 0.7.0

package/dist/engine/index.js

@@ -26590,6 +26590,248 @@ var init_validate = __esm(() => {
   BUILTIN_ONERROR = new Set(["fail", "skip", "retry"]);
 });
 
+// src/tracer/writers/json.ts
+var exports_json = {};
+__export(exports_json, {
+  JsonTraceWriter: () => JsonTraceWriter
+});
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+class JsonTraceWriter {
+  dir;
+  filePath = "";
+  trace = {
+    execution: {
+      id: "",
+      startedAt: "",
+      finishedAt: "",
+      duration: 0,
+      status: "running",
+      inputs: null,
+      output: null
+    },
+    steps: []
+  };
+  constructor(dir) {
+    this.dir = dir;
+  }
+  async open(meta) {
+    this.filePath = join(this.dir, `${meta.id}.json`);
+    this.trace = {
+      execution: {
+        id: meta.id,
+        workflowId: meta.workflowId,
+        workflowName: meta.workflowName,
+        workflowVersion: meta.workflowVersion,
+        startedAt: meta.startedAt,
+        finishedAt: "",
+        duration: 0,
+        status: "running",
+        inputs: meta.inputs,
+        output: null
+      },
+      steps: []
+    };
+    await this.flush();
+  }
+  writeStep(step) {
+    this.trace.steps.push(step);
+    this.flushSync();
+  }
+  async finalize(meta) {
+    this.trace.execution.status = meta.status;
+    this.trace.execution.output = meta.output;
+    this.trace.execution.finishedAt = meta.finishedAt;
+    this.trace.execution.duration = meta.duration;
+    await this.flush();
+  }
+  flushSync() {
+    this.flush().catch(() => {});
+  }
+  async flush() {
+    if (!this.filePath)
+      return;
+    await writeFile(this.filePath, JSON.stringify(this.trace, null, 2), "utf-8");
+  }
+}
+var init_json = () => {};
+
+// src/tracer/writers/txt.ts
+var exports_txt = {};
+__export(exports_txt, {
+  TxtTraceWriter: () => TxtTraceWriter
+});
+import { appendFile, readFile as readFile2, writeFile as writeFile2 } from "node:fs/promises";
+import { join as join2 } from "node:path";
+function serializeValue(value) {
+  if (value === undefined || value === null)
+    return "none";
+  if (typeof value === "string")
+    return value;
+  return JSON.stringify(value, null, 2);
+}
+function formatStep(step) {
+  const lines = [
+    `## [${step.seq}] ${step.name} (${step.type})`,
+    `startedAt: ${step.startedAt}`
+  ];
+  if (step.finishedAt)
+    lines.push(`finishedAt: ${step.finishedAt}`);
+  if (step.duration !== undefined)
+    lines.push(`duration: ${step.duration}ms`);
+  lines.push(`status: ${step.status}`);
+  if (step.loopIndex !== undefined)
+    lines.push(`loopIndex: ${step.loopIndex}`);
+  if (step.retries !== undefined && step.retries > 0)
+    lines.push(`retries: ${step.retries}`);
+  lines.push(`input: ${serializeValue(step.input)}`);
+  lines.push(`output: ${serializeValue(step.output)}`);
+  if (step.error)
+    lines.push(`error: ${step.error}`);
+  lines.push("");
+  return lines.join(`
+`);
+}
+
+class TxtTraceWriter {
+  dir;
+  filePath = "";
+  constructor(dir) {
+    this.dir = dir;
+  }
+  async open(meta) {
+    this.filePath = join2(this.dir, `${meta.id}.txt`);
+    const versionStr = meta.workflowVersion !== undefined ? ` (v${meta.workflowVersion})` : "";
+    const workflowLabel = meta.workflowName ?? meta.workflowId ?? "unnamed";
+    const header = [
+      "# execution",
+      `id: ${meta.id}`,
+      `workflow: ${workflowLabel}${versionStr}`,
+      `startedAt: ${meta.startedAt}`,
+      "status: running",
+      "",
+      "# inputs",
+      serializeValue(meta.inputs),
+      "",
+      "# steps",
+      ""
+    ].join(`
+`);
+    await writeFile2(this.filePath, header, "utf-8");
+  }
+  async writeStep(step) {
+    if (!this.filePath)
+      return;
+    await appendFile(this.filePath, formatStep(step), "utf-8");
+  }
+  async finalize(meta) {
+    if (!this.filePath)
+      return;
+    const outputSection = [
+      "# output",
+      serializeValue(meta.output),
+      ""
+    ].join(`
+`);
+    await appendFile(this.filePath, outputSection, "utf-8");
+    const content = await readFile2(this.filePath, "utf-8");
+    const updated = content.replace(/^status: running$/m, `status: ${meta.status}
+finishedAt: ${meta.finishedAt}
+duration: ${meta.duration}ms`);
+    await writeFile2(this.filePath, updated, "utf-8");
+  }
+}
+var init_txt = () => {};
+
+// src/tracer/writers/sqlite.ts
+var exports_sqlite = {};
+__export(exports_sqlite, {
+  SqliteTraceWriter: () => SqliteTraceWriter
+});
+import { join as join3 } from "node:path";
+function toJson(value) {
+  if (value === undefined || value === null)
+    return null;
+  if (typeof value === "string")
+    return value;
+  return JSON.stringify(value);
+}
+
+class SqliteTraceWriter {
+  dir;
+  db = null;
+  executionId = "";
+  constructor(dir) {
+    this.dir = dir;
+  }
+  async open(meta) {
+    this.executionId = meta.id;
+    const filePath = join3(this.dir, `${meta.id}.sqlite`);
+    const { Database } = await import("bun:sqlite");
+    this.db = new Database(filePath);
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS executions (
+        id TEXT PRIMARY KEY,
+        workflow_id TEXT,
+        workflow_name TEXT,
+        workflow_version TEXT,
+        started_at TEXT NOT NULL,
+        finished_at TEXT,
+        duration_ms INTEGER,
+        status TEXT NOT NULL,
+        inputs TEXT,
+        output TEXT
+      );
+
+      CREATE TABLE IF NOT EXISTS steps (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        execution_id TEXT NOT NULL REFERENCES executions(id),
+        seq INTEGER NOT NULL,
+        name TEXT NOT NULL,
+        type TEXT NOT NULL,
+        started_at TEXT,
+        finished_at TEXT,
+        duration_ms INTEGER,
+        status TEXT NOT NULL,
+        input TEXT,
+        output TEXT,
+        error TEXT,
+        retries INTEGER,
+        loop_index INTEGER
+      );
+    `);
+    const insert = this.db.prepare(`
+      INSERT INTO executions (id, workflow_id, workflow_name, workflow_version, started_at, status, inputs)
+      VALUES (?, ?, ?, ?, ?, 'running', ?)
+    `);
+    insert.run(meta.id, meta.workflowId ?? null, meta.workflowName ?? null, meta.workflowVersion !== undefined ? String(meta.workflowVersion) : null, meta.startedAt, toJson(meta.inputs));
+  }
+  writeStep(step) {
+    if (!this.db)
+      return;
+    const insert = this.db.prepare(`
+      INSERT INTO steps
+        (execution_id, seq, name, type, started_at, finished_at, duration_ms, status, input, output, error, retries, loop_index)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    insert.run(this.executionId, step.seq, step.name, step.type, step.startedAt ?? null, step.finishedAt ?? null, step.duration ?? null, step.status, toJson(step.input), toJson(step.output), step.error ?? null, step.retries ?? null, step.loopIndex ?? null);
+  }
+  async finalize(meta) {
+    if (!this.db)
+      return;
+    const update = this.db.prepare(`
+      UPDATE executions
+      SET status = ?, output = ?, finished_at = ?, duration_ms = ?
+      WHERE id = ?
+    `);
+    update.run(meta.status, toJson(meta.output), meta.finishedAt, Math.round(meta.duration), this.executionId);
+    this.db.close();
+    this.db = null;
+  }
+}
+var init_sqlite = () => {};
+
 // src/engine/errors.ts
 function sleep(ms) {
   return new Promise((resolve2) => setTimeout(resolve2, ms));
@@ -27189,6 +27431,10 @@ async function executeStep(workflow, state, step, basePath = process.cwd(), engi
           output: "",
           duration: 0
         });
+        const loopIndex2 = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+        const skippedSeq = state.tracer?.startStep(stepName, participant.type, undefined, loopIndex2);
+        if (skippedSeq !== undefined)
+          state.tracer?.endStep(skippedSeq, "skipped");
       }
       return chain;
     }
@@ -27197,6 +27443,8 @@ async function executeStep(workflow, state, step, basePath = process.cwd(), engi
   const overrideInput = override?.input !== undefined ? resolveParticipantInput(override.input, state) : undefined;
   const mergedWithBase = mergeChainedInput(chain, baseInput);
   const mergedInput = mergeChainedInput(mergedWithBase, overrideInput);
+  const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+  const traceSeq = state.tracer?.startStep(stepName ?? "<anonymous>", participant.type, mergedInput, loopIndex);
   state.currentInput = mergedInput;
   const strategy = resolveErrorStrategy(override ?? null, participant, workflow.defaults ?? null);
   const timeoutMs = resolveTimeout(override ?? null, participant, workflow.defaults ?? null);
@@ -27274,6 +27522,8 @@ async function executeStep(workflow, state, step, basePath = process.cwd(), engi
     }
     const outputValue = result.parsedOutput ?? result.output;
     state.currentOutput = outputValue;
+    if (traceSeq !== undefined)
+      state.tracer?.endStep(traceSeq, result.status, outputValue, undefined, retries);
     return outputValue;
   } catch (error) {
     const message = String(error?.message ?? error);
@@ -27299,6 +27549,8 @@ async function executeStep(workflow, state, step, basePath = process.cwd(), engi
       if (stepName) {
         state.setResult(stepName, skipResult);
       }
+      if (traceSeq !== undefined)
+        state.tracer?.endStep(traceSeq, "skipped", undefined, message);
       return chain;
     }
     if (strategy !== "fail" && strategy !== "retry") {
@@ -27317,6 +27569,8 @@ async function executeStep(workflow, state, step, basePath = process.cwd(), engi
           ...httpMeta
         });
       }
+      if (traceSeq !== undefined)
+        state.tracer?.endStep(traceSeq, "failure", undefined, message);
       const fallbackResult = await executeStep(workflow, state, fallbackName, basePath, engineExecutor, [...fallbackStack, stepName ?? "<anonymous>"], chain, hub);
       return fallbackResult;
     }
@@ -27331,6 +27585,8 @@ async function executeStep(workflow, state, step, basePath = process.cwd(), engi
         ...httpMeta
       });
     }
+    if (traceSeq !== undefined)
+      state.tracer?.endStep(traceSeq, "failure", undefined, message);
     throw error;
   }
 }
@@ -27443,21 +27699,38 @@ async function executeControlStep(workflow, state, step, basePath = process.cwd(
   }
   const obj = step;
   if ("wait" in obj && Object.keys(obj).length === 1) {
-    const { executeWait: executeWait2 } = await Promise.resolve().then(() => (init_wait(), exports_wait));
-    return executeWait2(state, obj.wait, chain, hub, signal);
+    const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const traceSeq = state.tracer?.startStep("wait", "wait", undefined, loopIndex);
+    try {
+      const { executeWait: executeWait2 } = await Promise.resolve().then(() => (init_wait(), exports_wait));
+      const result = await executeWait2(state, obj.wait, chain, hub, signal);
+      if (traceSeq !== undefined)
+        state.tracer?.endStep(traceSeq, "success", result);
+      return result;
+    } catch (err) {
+      if (traceSeq !== undefined)
+        state.tracer?.endStep(traceSeq, "failure", undefined, String(err?.message ?? err));
+      throw err;
+    }
   }
   if ("set" in obj && Object.keys(obj).length === 1) {
     const setDef = obj.set;
     const ctx = state.toCelContext();
+    const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const traceSeq = state.tracer?.startStep("set", "set", setDef, loopIndex);
     if (!state.executionMeta.context) {
       state.executionMeta.context = {};
     }
     for (const [key, expr] of Object.entries(setDef)) {
       if (RESERVED_SET_KEYS.has(key)) {
+        if (traceSeq !== undefined)
+          state.tracer?.endStep(traceSeq, "failure", undefined, `set key '${key}' uses a reserved name`);
         throw new Error(`set key '${key}' uses a reserved name`);
       }
       state.executionMeta.context[key] = evaluateCel(expr, ctx);
     }
+    if (traceSeq !== undefined)
+      state.tracer?.endStep(traceSeq, "success", state.executionMeta.context);
     return chain;
   }
   if ("loop" in obj && Object.keys(obj).length === 1) {
@@ -27474,6 +27747,8 @@ async function executeControlStep(workflow, state, step, basePath = process.cwd(
       maxIterations = loopDef.max ?? Number.POSITIVE_INFINITY;
     }
     const hasMax = loopDef.max !== undefined;
+    const outerLoopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const loopTraceSeq = state.tracer?.startStep(loopAs ?? "loop", "loop", undefined, outerLoopIndex);
     state.pushLoop(loopAs);
     let loopChain = chain;
     try {
@@ -27497,6 +27772,12 @@ async function executeControlStep(workflow, state, step, basePath = process.cwd(
         iterations += 1;
         state.incrementLoop();
       }
+      if (loopTraceSeq !== undefined)
+        state.tracer?.endStep(loopTraceSeq, "success", loopChain);
+    } catch (err) {
+      if (loopTraceSeq !== undefined)
+        state.tracer?.endStep(loopTraceSeq, "failure", undefined, String(err?.message ?? err));
+      throw err;
     } finally {
       state.popLoop();
     }
@@ -27505,29 +27786,54 @@ async function executeControlStep(workflow, state, step, basePath = process.cwd(
   if ("parallel" in obj && Object.keys(obj).length === 1) {
     const parallelSteps = obj.parallel;
     const controller = new AbortController;
+    const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const parallelTraceSeq = state.tracer?.startStep("parallel", "parallel", undefined, loopIndex);
     const branchSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
-    const results = await Promise.all(parallelSteps.map(async (parallelStep) => {
-      try {
-        return await executeControlStep(workflow, state, parallelStep, basePath, engineExecutor, chain, hub, branchSignal);
-      } catch (error) {
-        controller.abort();
-        throw error;
-      }
-    }));
-    return results;
+    try {
+      const results = await Promise.all(parallelSteps.map(async (parallelStep) => {
+        try {
+          return await executeControlStep(workflow, state, parallelStep, basePath, engineExecutor, chain, hub, branchSignal);
+        } catch (error) {
+          controller.abort();
+          throw error;
+        }
+      }));
+      if (parallelTraceSeq !== undefined)
+        state.tracer?.endStep(parallelTraceSeq, "success", results);
+      return results;
+    } catch (err) {
+      if (parallelTraceSeq !== undefined)
+        state.tracer?.endStep(parallelTraceSeq, "failure", undefined, String(err?.message ?? err));
+      throw err;
+    }
   }
   if ("if" in obj && Object.keys(obj).length === 1) {
     const ifDef = obj.if;
     const condition = evaluateCel(ifDef.condition, state.toCelContext());
+    const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const ifTraceSeq = state.tracer?.startStep("if", "if", { condition: ifDef.condition }, loopIndex);
     if (typeof condition !== "boolean") {
+      if (ifTraceSeq !== undefined)
+        state.tracer?.endStep(ifTraceSeq, "failure", undefined, `if.condition must evaluate to boolean, got ${typeof condition}`);
       throw new Error(`if.condition must evaluate to boolean, got ${typeof condition}`);
     }
-    if (condition) {
-      return executeSequential(workflow, state, ifDef.then, basePath, engineExecutor, chain, hub, signal);
-    } else if (ifDef.else) {
-      return executeSequential(workflow, state, ifDef.else, basePath, engineExecutor, chain, hub, signal);
+    try {
+      let result;
+      if (condition) {
+        result = await executeSequential(workflow, state, ifDef.then, basePath, engineExecutor, chain, hub, signal);
+      } else if (ifDef.else) {
+        result = await executeSequential(workflow, state, ifDef.else, basePath, engineExecutor, chain, hub, signal);
+      } else {
+        result = chain;
+      }
+      if (ifTraceSeq !== undefined)
+        state.tracer?.endStep(ifTraceSeq, "success", result);
+      return result;
+    } catch (err) {
+      if (ifTraceSeq !== undefined)
+        state.tracer?.endStep(ifTraceSeq, "failure", undefined, String(err?.message ?? err));
+      throw err;
     }
-    return chain;
   }
   return executeStep(workflow, state, step, basePath, engineExecutor, [], chain, hub, signal);
 }
@@ -27707,6 +28013,80 @@ function validateInputs(inputDefs, provided) {
   };
 }
 
+// src/tracer/index.ts
+import { mkdir } from "node:fs/promises";
+
+class TraceCollector {
+  openSteps = new Map;
+  seq = 0;
+  truncateAt;
+  writer;
+  constructor(truncateAt = 1e6) {
+    this.truncateAt = truncateAt;
+  }
+  startStep(name, type3, input, loopIndex) {
+    this.seq += 1;
+    this.openSteps.set(this.seq, {
+      name,
+      type: type3,
+      startedAt: new Date().toISOString(),
+      startMs: performance.now(),
+      input: input !== undefined ? this.truncate(input) : undefined,
+      loopIndex
+    });
+    return this.seq;
+  }
+  endStep(seq, status, output, error, retries) {
+    const open = this.openSteps.get(seq);
+    if (!open)
+      return;
+    this.openSteps.delete(seq);
+    const finishedAt = new Date().toISOString();
+    const duration = Math.max(0, performance.now() - open.startMs);
+    const step = {
+      seq,
+      name: open.name,
+      type: open.type,
+      startedAt: open.startedAt,
+      finishedAt,
+      duration: Math.round(duration),
+      status,
+      ...open.input !== undefined ? { input: open.input } : {},
+      ...output !== undefined ? { output: this.truncate(output) } : {},
+      ...error !== undefined ? { error } : {},
+      ...retries !== undefined && retries > 0 ? { retries } : {},
+      ...open.loopIndex !== undefined ? { loopIndex: open.loopIndex } : {}
+    };
+    this.writer?.writeStep(step);
+  }
+  truncate(value) {
+    if (value == null)
+      return value;
+    const str = typeof value === "string" ? value : JSON.stringify(value);
+    const bytes = new TextEncoder().encode(str);
+    if (bytes.length <= this.truncateAt)
+      return value;
+    const cut = new TextDecoder().decode(bytes.slice(0, this.truncateAt));
+    return cut + "...[truncated]";
+  }
+}
+async function createTraceWriter(dir, format) {
+  await mkdir(dir, { recursive: true });
+  if (format === "json") {
+    const { JsonTraceWriter: JsonTraceWriter2 } = await Promise.resolve().then(() => (init_json(), exports_json));
+    return new JsonTraceWriter2(dir);
+  }
+  if (format === "txt") {
+    const { TxtTraceWriter: TxtTraceWriter2 } = await Promise.resolve().then(() => (init_txt(), exports_txt));
+    return new TxtTraceWriter2(dir);
+  }
+  if (format === "sqlite") {
+    const { SqliteTraceWriter: SqliteTraceWriter2 } = await Promise.resolve().then(() => (init_sqlite(), exports_sqlite));
+    return new SqliteTraceWriter2(dir);
+  }
+  throw new Error(`unknown trace format: ${format}`);
+}
+
 // src/engine/engine.ts
 init_control();
 
@@ -27725,6 +28105,7 @@ class WorkflowState {
   chainValue;
   eventPayload;
   ancestorPaths;
+  tracer;
   constructor(inputs = {}) {
     this.inputs = { ...inputs };
     this.workflowInputs = { ...inputs };
@@ -27775,6 +28156,9 @@ class WorkflowState {
     if (top)
       top.last = last2;
   }
+  isInsideLoop() {
+    return this.loopStack.length > 0;
+  }
   currentLoopContext() {
     const top = this.loopStack[this.loopStack.length - 1];
     if (!top)
@@ -27865,38 +28249,66 @@ async function executeWorkflow(workflow, inputs = {}, basePath = process.cwd(),
   if (options.cwd) {
     state.executionMeta.cwd = options.cwd;
   }
-  const startedAt = performance.now();
+  const startedAtMs = performance.now();
+  const startedAtIso = state.executionMeta.startedAt;
   if (options._ancestorPaths) {
     state.ancestorPaths = options._ancestorPaths;
   }
+  if (options.traceDir && !options._ancestorPaths) {
+    const collector = new TraceCollector;
+    const writer = await createTraceWriter(options.traceDir, options.traceFormat ?? "json");
+    collector.writer = writer;
+    state.tracer = collector;
+    await writer.open({
+      id: state.executionMeta.id,
+      workflowId: workflow.id,
+      workflowName: workflow.name,
+      workflowVersion: workflow.version,
+      startedAt: startedAtIso,
+      inputs: resolved
+    });
+  }
   const engineExecutor = async (subWorkflow, subInputs, subBasePath) => {
     return executeWorkflow(subWorkflow, subInputs, subBasePath, {
       ...options,
       _ancestorPaths: state.ancestorPaths
     });
   };
-  let chain;
-  for (const step of workflow.flow) {
-    chain = await executeControlStep(workflow, state, step, basePath, engineExecutor, chain, options.hub);
-  }
   let output;
-  if (workflow.output !== undefined) {
-    output = state.resolveOutput(workflow.output, evaluateCel);
-    if (typeof workflow.output === "object" && "schema" in workflow.output && "map" in workflow.output && typeof output === "object" && output !== null) {
-      validateOutputSchema(workflow.output.schema, output);
+  let success = false;
+  try {
+    let chain;
+    for (const step of workflow.flow) {
+      chain = await executeControlStep(workflow, state, step, basePath, engineExecutor, chain, options.hub);
+    }
+    if (workflow.output !== undefined) {
+      output = state.resolveOutput(workflow.output, evaluateCel);
+      if (typeof workflow.output === "object" && "schema" in workflow.output && "map" in workflow.output && typeof output === "object" && output !== null) {
+        validateOutputSchema(workflow.output.schema, output);
+      }
+    } else {
+      output = chain;
+    }
+    const steps = state.getAllResults();
+    success = !Object.values(steps).some((step) => step.status === "failure");
+    state.executionMeta.status = success ? "success" : "failure";
+    return {
+      success,
+      output,
+      steps,
+      duration: Math.max(0, performance.now() - startedAtMs)
+    };
+  } finally {
+    if (state.tracer?.writer) {
+      const duration = Math.max(0, performance.now() - startedAtMs);
+      await state.tracer.writer.finalize({
+        status: success ? "success" : "failure",
+        output: output ?? null,
+        finishedAt: new Date().toISOString(),
+        duration
+      }).catch(() => {});
     }
-  } else {
-    output = chain;
   }
-  const steps = state.getAllResults();
-  const success = !Object.values(steps).some((step) => step.status === "failure");
-  state.executionMeta.status = success ? "success" : "failure";
-  return {
-    success,
-    output,
-    steps,
-    duration: Math.max(0, performance.now() - startedAt)
-  };
 }
 async function runWorkflowFromFile(filePath, inputs = {}, options = {}) {
   const workflow = await parseWorkflowFile(filePath);