@duckflux/core 0.6.8 → 0.7.0

package/src/engine/control.ts

@@ -33,14 +33,25 @@ export async function executeControlStep(
 
   // Wait step
   if ("wait" in obj && Object.keys(obj).length === 1) {
-    const { executeWait } = await import("./wait");
-    return executeWait(state, (obj as { wait: Record<string, unknown> }).wait, chain, hub, signal);
+    const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const traceSeq = state.tracer?.startStep("wait", "wait", undefined, loopIndex);
+    try {
+      const { executeWait } = await import("./wait");
+      const result = await executeWait(state, (obj as { wait: Record<string, unknown> }).wait, chain, hub, signal);
+      if (traceSeq !== undefined) state.tracer?.endStep(traceSeq, "success", result);
+      return result;
+    } catch (err) {
+      if (traceSeq !== undefined) state.tracer?.endStep(traceSeq, "failure", undefined, String((err as Error)?.message ?? err));
+      throw err;
+    }
   }
 
   // Set step — write values to execution.context
   if ("set" in obj && Object.keys(obj).length === 1) {
     const setDef = (obj as { set: Record<string, string> }).set;
     const ctx = state.toCelContext();
+    const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const traceSeq = state.tracer?.startStep("set", "set", setDef, loopIndex);
 
     if (!state.executionMeta.context) {
       state.executionMeta.context = {};
@@ -48,11 +59,13 @@ export async function executeControlStep(
 
     for (const [key, expr] of Object.entries(setDef)) {
       if (RESERVED_SET_KEYS.has(key)) {
+        if (traceSeq !== undefined) state.tracer?.endStep(traceSeq, "failure", undefined, `set key '${key}' uses a reserved name`);
         throw new Error(`set key '${key}' uses a reserved name`);
       }
       state.executionMeta.context[key] = evaluateCel(expr, ctx);
     }
 
+    if (traceSeq !== undefined) state.tracer?.endStep(traceSeq, "success", state.executionMeta.context);
     // set does not produce output; chain passes through unchanged
     return chain;
   }
@@ -75,6 +88,8 @@ export async function executeControlStep(
     }
 
     const hasMax = loopDef.max !== undefined;
+    const outerLoopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const loopTraceSeq = state.tracer?.startStep(loopAs ?? "loop", "loop", undefined, outerLoopIndex);
 
     state.pushLoop(loopAs);
     let loopChain = chain;
@@ -114,6 +129,10 @@ export async function executeControlStep(
         iterations += 1;
         state.incrementLoop();
       }
+      if (loopTraceSeq !== undefined) state.tracer?.endStep(loopTraceSeq, "success", loopChain);
+    } catch (err) {
+      if (loopTraceSeq !== undefined) state.tracer?.endStep(loopTraceSeq, "failure", undefined, String((err as Error)?.message ?? err));
+      throw err;
     } finally {
       state.popLoop();
     }
@@ -124,44 +143,62 @@ export async function executeControlStep(
   if ("parallel" in obj && Object.keys(obj).length === 1) {
     const parallelSteps = (obj as { parallel: unknown[] }).parallel;
     const controller = new AbortController();
+    const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const parallelTraceSeq = state.tracer?.startStep("parallel", "parallel", undefined, loopIndex);
     // Combine parent signal with local controller
     const branchSignal = signal
       ? AbortSignal.any([signal, controller.signal])
       : controller.signal;
 
-    const results = await Promise.all(
-      parallelSteps.map(async (parallelStep) => {
-        try {
-          // Each branch starts with the same incoming chain
-          return await executeControlStep(workflow, state, parallelStep, basePath, engineExecutor, chain, hub, branchSignal);
-        } catch (error) {
-          controller.abort();
-          throw error;
-        }
-      }),
-    );
-
-    // Chain after parallel is ordered array of branch outputs
-    return results;
+    try {
+      const results = await Promise.all(
+        parallelSteps.map(async (parallelStep) => {
+          try {
+            // Each branch starts with the same incoming chain
+            return await executeControlStep(workflow, state, parallelStep, basePath, engineExecutor, chain, hub, branchSignal);
+          } catch (error) {
+            controller.abort();
+            throw error;
+          }
+        }),
+      );
+
+      if (parallelTraceSeq !== undefined) state.tracer?.endStep(parallelTraceSeq, "success", results);
+      // Chain after parallel is ordered array of branch outputs
+      return results;
+    } catch (err) {
+      if (parallelTraceSeq !== undefined) state.tracer?.endStep(parallelTraceSeq, "failure", undefined, String((err as Error)?.message ?? err));
+      throw err;
+    }
   }
 
   // If step
   if ("if" in obj && Object.keys(obj).length === 1) {
     const ifDef = (obj as { if: { condition: string; then: unknown[]; else?: unknown[] } }).if;
     const condition = evaluateCel(ifDef.condition, state.toCelContext());
+    const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+    const ifTraceSeq = state.tracer?.startStep("if", "if", { condition: ifDef.condition }, loopIndex);
 
     if (typeof condition !== "boolean") {
+      if (ifTraceSeq !== undefined) state.tracer?.endStep(ifTraceSeq, "failure", undefined, `if.condition must evaluate to boolean, got ${typeof condition}`);
       throw new Error(`if.condition must evaluate to boolean, got ${typeof condition}`);
     }
 
-    if (condition) {
-      return executeSequential(workflow, state, ifDef.then, basePath, engineExecutor, chain, hub, signal);
-    } else if (ifDef.else) {
-      return executeSequential(workflow, state, ifDef.else, basePath, engineExecutor, chain, hub, signal);
+    try {
+      let result: unknown;
+      if (condition) {
+        result = await executeSequential(workflow, state, ifDef.then, basePath, engineExecutor, chain, hub, signal);
+      } else if (ifDef.else) {
+        result = await executeSequential(workflow, state, ifDef.else, basePath, engineExecutor, chain, hub, signal);
+      } else {
+        result = chain;
+      }
+      if (ifTraceSeq !== undefined) state.tracer?.endStep(ifTraceSeq, "success", result);
+      return result;
+    } catch (err) {
+      if (ifTraceSeq !== undefined) state.tracer?.endStep(ifTraceSeq, "failure", undefined, String((err as Error)?.message ?? err));
+      throw err;
     }
-
-    // False without else: chain passes through
-    return chain;
   }
 
   // Inline participant (has `type` field) or participant override

package/src/engine/engine.ts

@@ -6,6 +6,7 @@ import { validateSchema } from "../parser/schema";
 import { validateSemantic } from "../parser/validate";
 import { validateInputs } from "../parser/validate_inputs";
 import type { WorkflowEngineExecutor } from "../participant/workflow";
+import { TraceCollector, createTraceWriter } from "../tracer/index";
 import { executeControlStep } from "./control";
 import { validateOutputSchema } from "./output";
 import { WorkflowState } from "./state";
@@ -23,6 +24,10 @@ export interface ExecuteOptions {
   executionNumber?: number;
   verbose?: boolean;
   quiet?: boolean;
+  /** Directory to write trace files; one file per execution named <executionId>.<ext> */
+  traceDir?: string;
+  /** Trace output format (default: "json") */
+  traceFormat?: "json" | "txt" | "sqlite";
   /** @internal Tracks ancestor workflow paths for circular detection */
   _ancestorPaths?: Set<string>;
 }
@@ -51,13 +56,30 @@ export async function executeWorkflow(
     state.executionMeta.cwd = options.cwd;
   }
 
-  const startedAt = performance.now();
+  const startedAtMs = performance.now();
+  const startedAtIso = state.executionMeta.startedAt;
 
   // Propagate ancestor paths for circular sub-workflow detection
   if (options._ancestorPaths) {
     state.ancestorPaths = options._ancestorPaths;
   }
 
+  // Set up incremental tracing if requested (only on the root workflow, not sub-workflows)
+  if (options.traceDir && !options._ancestorPaths) {
+    const collector = new TraceCollector();
+    const writer = await createTraceWriter(options.traceDir, options.traceFormat ?? "json");
+    collector.writer = writer;
+    state.tracer = collector;
+    await writer.open({
+      id: state.executionMeta.id,
+      workflowId: workflow.id,
+      workflowName: workflow.name,
+      workflowVersion: workflow.version,
+      startedAt: startedAtIso,
+      inputs: resolved,
+    });
+  }
+
   const engineExecutor: WorkflowEngineExecutor = async (subWorkflow, subInputs, subBasePath) => {
     // Sub-workflows share parent hub, propagate ancestor paths for circular detection
     return executeWorkflow(subWorkflow, subInputs, subBasePath, {
@@ -66,43 +88,56 @@ export async function executeWorkflow(
     });
   };
 
-  // Execute flow with chain threading
-  let chain: unknown;
-  for (const step of workflow.flow) {
-    chain = await executeControlStep(workflow, state, step, basePath, engineExecutor, chain, options.hub);
-  }
-
-  // Output resolution
   let output: unknown;
-  if (workflow.output !== undefined) {
-    output = state.resolveOutput(workflow.output, evaluateCel);
-
-    // Validate output against schema if defined
-    if (
-      typeof workflow.output === "object" &&
-      "schema" in workflow.output &&
-      "map" in workflow.output &&
-      typeof output === "object" &&
-      output !== null
-    ) {
-      validateOutputSchema(workflow.output.schema, output as Record<string, unknown>);
-    }
-  } else {
-    // Default: return final chain value
-    output = chain;
-  }
+  let success = false;
 
-  const steps = state.getAllResults();
-  const success = !Object.values(steps).some((step) => step.status === "failure");
+  try {
+    // Execute flow with chain threading
+    let chain: unknown;
+    for (const step of workflow.flow) {
+      chain = await executeControlStep(workflow, state, step, basePath, engineExecutor, chain, options.hub);
+    }
 
-  state.executionMeta.status = success ? "success" : "failure";
+    // Output resolution
+    if (workflow.output !== undefined) {
+      output = state.resolveOutput(workflow.output, evaluateCel);
+
+      // Validate output against schema if defined
+      if (
+        typeof workflow.output === "object" &&
+        "schema" in workflow.output &&
+        "map" in workflow.output &&
+        typeof output === "object" &&
+        output !== null
+      ) {
+        validateOutputSchema(workflow.output.schema, output as Record<string, unknown>);
+      }
+    } else {
+      // Default: return final chain value
+      output = chain;
+    }
 
-  return {
-    success,
-    output,
-    steps,
-    duration: Math.max(0, performance.now() - startedAt),
-  };
+    const steps = state.getAllResults();
+    success = !Object.values(steps).some((step) => step.status === "failure");
+    state.executionMeta.status = success ? "success" : "failure";
+
+    return {
+      success,
+      output,
+      steps,
+      duration: Math.max(0, performance.now() - startedAtMs),
+    };
+  } finally {
+    if (state.tracer?.writer) {
+      const duration = Math.max(0, performance.now() - startedAtMs);
+      await state.tracer.writer.finalize({
+        status: success ? "success" : "failure",
+        output: output ?? null,
+        finishedAt: new Date().toISOString(),
+        duration,
+      }).catch(() => {});
+    }
+  }
 }
 
 export async function runWorkflowFromFile(

package/src/engine/sequential.ts

@@ -151,6 +151,9 @@ export async function executeStep(
           output: "",
           duration: 0,
         });
+        const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+        const skippedSeq = state.tracer?.startStep(stepName, participant.type, undefined, loopIndex);
+        if (skippedSeq !== undefined) state.tracer?.endStep(skippedSeq, "skipped");
       }
       return chain; // Skipped steps preserve chain
     }
@@ -165,6 +168,9 @@ export async function executeStep(
   const mergedWithBase = mergeChainedInput(chain, baseInput);
   const mergedInput = mergeChainedInput(mergedWithBase, overrideInput);
 
+  const loopIndex = state.isInsideLoop() ? state.currentLoopIndex() : undefined;
+  const traceSeq = state.tracer?.startStep(stepName ?? "<anonymous>", participant.type, mergedInput, loopIndex);
+
   // Set participant-scoped input in state
   state.currentInput = mergedInput;
 
@@ -280,6 +286,7 @@ export async function executeStep(
     // Update participant-scoped output and chain
     const outputValue = result.parsedOutput ?? result.output;
     state.currentOutput = outputValue;
+    if (traceSeq !== undefined) state.tracer?.endStep(traceSeq, result.status, outputValue, undefined, retries);
     return outputValue;
   } catch (error) {
     const message = String((error as Error)?.message ?? error);
@@ -305,6 +312,7 @@ export async function executeStep(
       if (stepName) {
         state.setResult(stepName, skipResult);
       }
+      if (traceSeq !== undefined) state.tracer?.endStep(traceSeq, "skipped", undefined, message);
       return chain; // Skipped steps preserve chain
     }
 
@@ -315,7 +323,7 @@ export async function executeStep(
         throw new Error(`fallback cycle detected on participant '${fallbackName}'`);
       }
 
-      // Keep original step as failure (Go runner behavior)
+      // Keep original step as failure
       if (stepName) {
         state.setResult(stepName, {
           status: "failure",
@@ -327,6 +335,7 @@ export async function executeStep(
           ...httpMeta,
         });
       }
+      if (traceSeq !== undefined) state.tracer?.endStep(traceSeq, "failure", undefined, message);
 
       // Execute fallback
       const fallbackResult = await executeStep(
@@ -354,6 +363,7 @@ export async function executeStep(
         ...httpMeta,
       });
     }
+    if (traceSeq !== undefined) state.tracer?.endStep(traceSeq, "failure", undefined, message);
 
     throw error;
   }

package/src/engine/state.ts

@@ -1,5 +1,6 @@
 import { env } from "node:process";
 import type { StepResult } from "../model/index";
+import type { TraceCollector } from "../tracer/index";
 
 export type { StepResult };
 
@@ -25,6 +26,8 @@ export class WorkflowState {
   eventPayload: unknown;
   /** @internal Tracks ancestor workflow paths for circular sub-workflow detection */
   ancestorPaths: Set<string>;
+  /** @internal Optional trace collector; set by engine when --trace-dir is active */
+  tracer?: TraceCollector;
 
   constructor(inputs: Record<string, unknown> = {}) {
     this.inputs = { ...inputs };
@@ -82,6 +85,10 @@ export class WorkflowState {
     if (top) top.last = last;
   }
 
+  isInsideLoop(): boolean {
+    return this.loopStack.length > 0;
+  }
+
   currentLoopContext(): { index: number; iteration: number; first: boolean; last: boolean; as?: string } {
     const top = this.loopStack[this.loopStack.length - 1];
     if (!top) return { index: 0, iteration: 1, first: true, last: false };
@@ -148,7 +155,7 @@ export class WorkflowState {
     // Event payload
     ctx["event"] = this.eventPayload ?? {};
 
-    // Now — epoch seconds to match timestamp() and Go runner behavior (Spec §12.9)
+    // Now — epoch seconds to match timestamp() (Spec §12.9)
     ctx["now"] = Math.floor(Date.now() / 1000);
 
     return ctx;

package/src/model/index.ts

@@ -181,3 +181,34 @@ export interface ValidationError {
   path: string;
   message: string;
 }
+
+export interface StepTrace {
+  seq: number;
+  name: string;
+  type: string;
+  startedAt: string;
+  finishedAt?: string;
+  duration?: number;
+  status: "success" | "failure" | "skipped";
+  input?: unknown;
+  output?: unknown;
+  error?: string;
+  retries?: number;
+  loopIndex?: number;
+}
+
+export interface ExecutionTrace {
+  execution: {
+    id: string;
+    workflowId?: string;
+    workflowName?: string;
+    workflowVersion?: string | number;
+    startedAt: string;
+    finishedAt: string;
+    duration: number;
+    status: "success" | "failure" | "running";
+    inputs: unknown;
+    output: unknown;
+  };
+  steps: StepTrace[];
+}

package/src/parser/schema/duckflux.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://duckflux.dev/schema/v0.4/duckflux.schema.json",
+  "$id": "https://raw.githubusercontent.com/duckflux/spec/main/duckflux.schema.json",
   "title": "duckflux Workflow",
   "description": "Schema for duckflux — a minimal, deterministic, runtime-agnostic workflow DSL.",
   "type": "object",

package/src/tracer/index.ts

@@ -0,0 +1,108 @@
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import type { StepTrace } from "../model/index";
+
+export type { StepTrace };
+
+export interface TraceOpenMeta {
+  id: string;
+  workflowId?: string;
+  workflowName?: string;
+  workflowVersion?: string | number;
+  startedAt: string;
+  inputs: unknown;
+}
+
+export interface TraceFinalizeMeta {
+  status: "success" | "failure";
+  output: unknown;
+  finishedAt: string;
+  duration: number;
+}
+
+export interface TraceWriter {
+  open(meta: TraceOpenMeta): Promise<void>;
+  writeStep(step: StepTrace): void | Promise<void>;
+  finalize(meta: TraceFinalizeMeta): Promise<void>;
+}
+
+export class TraceCollector {
+  private openSteps: Map<number, { name: string; type: string; startedAt: string; startMs: number; input?: unknown; loopIndex?: number }> = new Map();
+  private seq = 0;
+  readonly truncateAt: number;
+  writer?: TraceWriter;
+
+  constructor(truncateAt = 1_000_000) {
+    this.truncateAt = truncateAt;
+  }
+
+  startStep(name: string, type: string, input?: unknown, loopIndex?: number): number {
+    this.seq += 1;
+    this.openSteps.set(this.seq, {
+      name,
+      type,
+      startedAt: new Date().toISOString(),
+      startMs: performance.now(),
+      input: input !== undefined ? this.truncate(input) : undefined,
+      loopIndex,
+    });
+    return this.seq;
+  }
+
+  endStep(seq: number, status: StepTrace["status"], output?: unknown, error?: string, retries?: number): void {
+    const open = this.openSteps.get(seq);
+    if (!open) return;
+    this.openSteps.delete(seq);
+
+    const finishedAt = new Date().toISOString();
+    const duration = Math.max(0, performance.now() - open.startMs);
+
+    const step: StepTrace = {
+      seq,
+      name: open.name,
+      type: open.type,
+      startedAt: open.startedAt,
+      finishedAt,
+      duration: Math.round(duration),
+      status,
+      ...(open.input !== undefined ? { input: open.input } : {}),
+      ...(output !== undefined ? { output: this.truncate(output) } : {}),
+      ...(error !== undefined ? { error } : {}),
+      ...(retries !== undefined && retries > 0 ? { retries } : {}),
+      ...(open.loopIndex !== undefined ? { loopIndex: open.loopIndex } : {}),
+    };
+
+    this.writer?.writeStep(step);
+  }
+
+  truncate(value: unknown): unknown {
+    if (value == null) return value;
+    const str = typeof value === "string" ? value : JSON.stringify(value);
+    const bytes = new TextEncoder().encode(str);
+    if (bytes.length <= this.truncateAt) return value;
+    const cut = new TextDecoder().decode(bytes.slice(0, this.truncateAt));
+    return cut + "...[truncated]";
+  }
+}
+
+export async function createTraceWriter(
+  dir: string,
+  format: "json" | "txt" | "sqlite",
+): Promise<TraceWriter> {
+  await mkdir(dir, { recursive: true });
+
+  if (format === "json") {
+    const { JsonTraceWriter } = await import("./writers/json");
+    return new JsonTraceWriter(dir);
+  }
+  if (format === "txt") {
+    const { TxtTraceWriter } = await import("./writers/txt");
+    return new TxtTraceWriter(dir);
+  }
+  if (format === "sqlite") {
+    const { SqliteTraceWriter } = await import("./writers/sqlite");
+    return new SqliteTraceWriter(dir);
+  }
+
+  throw new Error(`unknown trace format: ${format}`);
+}

package/src/tracer/writers/json.ts

@@ -0,0 +1,68 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { ExecutionTrace, StepTrace } from "../../model/index";
+import type { TraceFinalizeMeta, TraceOpenMeta, TraceWriter } from "../index";
+
+export class JsonTraceWriter implements TraceWriter {
+  private dir: string;
+  private filePath = "";
+  private trace: ExecutionTrace = {
+    execution: {
+      id: "",
+      startedAt: "",
+      finishedAt: "",
+      duration: 0,
+      status: "running",
+      inputs: null,
+      output: null,
+    },
+    steps: [],
+  };
+
+  constructor(dir: string) {
+    this.dir = dir;
+  }
+
+  async open(meta: TraceOpenMeta): Promise<void> {
+    this.filePath = join(this.dir, `${meta.id}.json`);
+    this.trace = {
+      execution: {
+        id: meta.id,
+        workflowId: meta.workflowId,
+        workflowName: meta.workflowName,
+        workflowVersion: meta.workflowVersion,
+        startedAt: meta.startedAt,
+        finishedAt: "",
+        duration: 0,
+        status: "running",
+        inputs: meta.inputs,
+        output: null,
+      },
+      steps: [],
+    };
+    await this.flush();
+  }
+
+  writeStep(step: StepTrace): void {
+    this.trace.steps.push(step);
+    this.flushSync();
+  }
+
+  async finalize(meta: TraceFinalizeMeta): Promise<void> {
+    this.trace.execution.status = meta.status;
+    this.trace.execution.output = meta.output;
+    this.trace.execution.finishedAt = meta.finishedAt;
+    this.trace.execution.duration = meta.duration;
+    await this.flush();
+  }
+
+  private flushSync(): void {
+    // Fire-and-forget async write; errors are silently ignored to not disrupt execution
+    this.flush().catch(() => {});
+  }
+
+  private async flush(): Promise<void> {
+    if (!this.filePath) return;
+    await writeFile(this.filePath, JSON.stringify(this.trace, null, 2), "utf-8");
+  }
+}