@duckflux/core 0.6.8

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@duckflux/core",
+  "version": "0.6.8",
+  "type": "module",
+  "main": "src/index.ts",
+  "exports": {
+    ".": {
+      "bun": "./src/index.ts",
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./parser": {
+      "bun": "./src/parser/index.ts",
+      "import": "./dist/parser/index.js",
+      "types": "./dist/parser/index.d.ts"
+    },
+    "./cel": {
+      "bun": "./src/cel/index.ts",
+      "import": "./dist/cel/index.js",
+      "types": "./dist/cel/index.d.ts"
+    },
+    "./engine": {
+      "bun": "./src/engine/index.ts",
+      "import": "./dist/engine/index.js",
+      "types": "./dist/engine/index.d.ts"
+    },
+    "./eventhub": {
+      "bun": "./src/eventhub/index.ts",
+      "import": "./dist/eventhub/index.js",
+      "types": "./dist/eventhub/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "test": "bun test",
+    "build": "bun build src/index.ts --outdir dist --target node --format esm && tsc --project tsconfig.build.json",
+    "prepublishOnly": "bun run build"
+  },
+  "dependencies": {
+    "ajv": "^8.12.0",
+    "ajv-formats": "^2.1.1",
+    "cel-js": "^0.8.2",
+    "yaml": "^2.2.1"
+  }
+}

package/src/cel/index.ts

@@ -0,0 +1,156 @@
+import { evaluate, parse } from "cel-js";
+
+// Expression cache: parsed AST reuse
+const expressionCache = new Map<string, ReturnType<typeof parse>>();
+
+function getParsed(expr: string): ReturnType<typeof parse> {
+  let cached = expressionCache.get(expr);
+  if (!cached) {
+    cached = parse(expr);
+    expressionCache.set(expr, cached);
+  }
+  return cached;
+}
+
+export function validateCelExpression(expr: string): { valid: boolean; error?: string } {
+  const parsed = getParsed(expr);
+  if (parsed.isSuccess) {
+    return { valid: true };
+  }
+
+  return {
+    valid: false,
+    error: parsed.errors.join("; "),
+  };
+}
+
+// Custom functions registered as cel-js macros (third parameter to evaluate).
+// These are available as function-style calls in CEL expressions: e.g. timestamp("..."), matches("...", "...").
+const customFunctions: Record<string, CallableFunction> = {
+  // Timestamp: converts ISO string to epoch seconds
+  timestamp: (s: string) => {
+    const d = new Date(s);
+    if (Number.isNaN(d.getTime())) throw new Error(`invalid timestamp: ${s}`);
+    return Math.floor(d.getTime() / 1000);
+  },
+  // Duration: converts duration string to seconds
+  duration: (s: string) => {
+    const match = s.match(/^(\d+)(ms|s|m|h|d)$/);
+    if (!match) throw new Error(`invalid duration: ${s}`);
+    const val = Number(match[1]);
+    switch (match[2]) {
+      case "ms": return val / 1000;
+      case "s": return val;
+      case "m": return val * 60;
+      case "h": return val * 3600;
+      case "d": return val * 86400;
+      default: throw new Error(`invalid duration unit: ${match[2]}`);
+    }
+  },
+  // CEL standard library — string functions (§11.1)
+  matches: (s: string, pattern: string) => new RegExp(pattern).test(s),
+  lowerAscii: (s: string) => String(s).toLowerCase(),
+  upperAscii: (s: string) => String(s).toUpperCase(),
+  replace: (s: string, old: string, replacement: string) =>
+    String(s).split(old).join(replacement),
+  split: (s: string, sep: string) => String(s).split(sep),
+  join: (list: unknown[], sep: string) => list.join(sep ?? ","),
+  contains: (s: string, substr: string) => String(s).includes(substr),
+  startsWith: (s: string, prefix: string) => String(s).startsWith(prefix),
+  endsWith: (s: string, suffix: string) => String(s).endsWith(suffix),
+};
+
+// Names of custom functions (for filtering in error messages)
+const CUSTOM_FUNCTION_NAMES = new Set(Object.keys(customFunctions));
+
+export function evaluateCel(expr: unknown, context: Record<string, unknown>): unknown {
+  if (typeof expr !== "string") {
+    return expr;
+  }
+
+  const parsed = getParsed(expr);
+  if (!parsed.isSuccess) {
+    const contextKeys = Object.keys(context).filter((k) => !CUSTOM_FUNCTION_NAMES.has(k));
+    throw new Error(
+      `CEL parse error in expression '${expr}': ${parsed.errors.join("; ")}. Available context keys: [${contextKeys.join(", ")}]`,
+    );
+  }
+
+  try {
+    return evaluate(parsed.cst, context, customFunctions);
+  } catch (err) {
+    const contextKeys = Object.keys(context).filter((k) => !CUSTOM_FUNCTION_NAMES.has(k));
+    // Strip context values from cel-js error messages to avoid leaking sensitive data
+    const rawMessage = (err as Error).message;
+    const sanitized = rawMessage.replace(/in context: \{[^}]*\}/g, "in context: {…}");
+    throw new Error(
+      `CEL evaluation error in '${expr}': ${sanitized}. Available context keys: [${contextKeys.join(", ")}]`,
+    );
+  }
+}
+
+export function evaluateCelStrict(expr: string, context: Record<string, unknown>): boolean {
+  const result = evaluateCel(expr, context);
+  if (typeof result !== "boolean") {
+    throw new Error(`CEL expression must evaluate to boolean, got ${typeof result}: ${expr}`);
+  }
+  return result;
+}
+
+/**
+ * Tries to evaluate a value as CEL. If the value is a string and CEL parsing
+ * fails, returns the literal string. For objects/arrays, recurses into values.
+ * Non-string scalars are returned as-is.
+ */
+export function evalMaybeCel(value: unknown, ctx: Record<string, unknown>): unknown {
+  if (typeof value === "string") {
+    try {
+      return evaluateCel(value, ctx);
+    } catch {
+      return value;
+    }
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((item) => evalMaybeCel(item, ctx));
+  }
+
+  if (value !== null && typeof value === "object") {
+    const result: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      result[k] = evalMaybeCel(v, ctx);
+    }
+    return result;
+  }
+
+  return value;
+}
+
+export function buildCelContext(state: {
+  toCelContext?: () => Record<string, unknown>;
+  getAllResults?: () => Record<string, { output?: unknown; parsedOutput?: unknown; status: string }>;
+  inputs?: Record<string, unknown>;
+  currentLoopIndex?: () => number;
+}): Record<string, unknown> {
+  if (typeof state.toCelContext === "function") {
+    return state.toCelContext();
+  }
+
+  const ctx: Record<string, unknown> = {};
+
+  const results = state.getAllResults ? state.getAllResults() : {};
+  for (const [name, result] of Object.entries(results)) {
+    ctx[name] = {
+      output: result.parsedOutput ?? result.output,
+      status: result.status,
+    };
+  }
+
+  ctx.input = state.inputs ?? {};
+  ctx.env = { ...process.env };
+  ctx.loop = {
+    index: state.currentLoopIndex ? state.currentLoopIndex() : 0,
+  };
+
+  return ctx;
+}

package/src/engine/control.ts

@@ -0,0 +1,169 @@
+import { evaluateCel } from "../cel/index";
+import type { EventHub } from "../eventhub/types";
+import type { Workflow } from "../model/index";
+import type { WorkflowEngineExecutor } from "../participant/workflow";
+import { executeSequential, executeStep } from "./sequential";
+import type { WorkflowState } from "./state";
+
+const RESERVED_SET_KEYS = new Set(["workflow", "execution", "input", "output", "env", "loop", "event"]);
+
+export async function executeControlStep(
+  workflow: Workflow,
+  state: WorkflowState,
+  step: unknown,
+  basePath = process.cwd(),
+  engineExecutor?: WorkflowEngineExecutor,
+  chain?: unknown,
+  hub?: EventHub,
+  signal?: AbortSignal,
+): Promise<unknown> {
+  if (signal?.aborted) {
+    throw new Error("execution aborted");
+  }
+
+  if (typeof step === "string") {
+    return executeStep(workflow, state, step, basePath, engineExecutor, [], chain, hub, signal);
+  }
+
+  if (!step || typeof step !== "object" || Array.isArray(step)) {
+    return executeStep(workflow, state, step, basePath, engineExecutor, [], chain, hub, signal);
+  }
+
+  const obj = step as Record<string, unknown>;
+
+  // Wait step
+  if ("wait" in obj && Object.keys(obj).length === 1) {
+    const { executeWait } = await import("./wait");
+    return executeWait(state, (obj as { wait: Record<string, unknown> }).wait, chain, hub, signal);
+  }
+
+  // Set step — write values to execution.context
+  if ("set" in obj && Object.keys(obj).length === 1) {
+    const setDef = (obj as { set: Record<string, string> }).set;
+    const ctx = state.toCelContext();
+
+    if (!state.executionMeta.context) {
+      state.executionMeta.context = {};
+    }
+
+    for (const [key, expr] of Object.entries(setDef)) {
+      if (RESERVED_SET_KEYS.has(key)) {
+        throw new Error(`set key '${key}' uses a reserved name`);
+      }
+      state.executionMeta.context[key] = evaluateCel(expr, ctx);
+    }
+
+    // set does not produce output; chain passes through unchanged
+    return chain;
+  }
+
+  // Loop step
+  if ("loop" in obj && Object.keys(obj).length === 1) {
+    const loopDef = (obj as { loop: { as?: string; until?: string; max?: number | string; steps: unknown[] } }).loop;
+    const loopAs = loopDef.as;
+
+    // Resolve max (can be CEL string)
+    let maxIterations: number;
+    if (typeof loopDef.max === "string") {
+      const resolved = evaluateCel(loopDef.max, state.toCelContext());
+      maxIterations = Number(resolved);
+      if (!Number.isFinite(maxIterations)) {
+        throw new Error(`loop.max CEL expression resolved to non-number: ${resolved}`);
+      }
+    } else {
+      maxIterations = loopDef.max ?? Number.POSITIVE_INFINITY;
+    }
+
+    const hasMax = loopDef.max !== undefined;
+
+    state.pushLoop(loopAs);
+    let loopChain = chain;
+    try {
+      let iterations = 0;
+
+      while (iterations < maxIterations) {
+        if (signal?.aborted) {
+          throw new Error("execution aborted");
+        }
+
+        // Set loop.last before executing steps
+        const isLast = hasMax && iterations + 1 === maxIterations;
+        state.setLoopLast(isLast);
+
+        loopChain = await executeSequential(
+          workflow,
+          state,
+          loopDef.steps,
+          basePath,
+          engineExecutor,
+          loopChain,
+          hub,
+          signal,
+        );
+
+        if (loopDef.until) {
+          const untilValue = evaluateCel(loopDef.until, state.toCelContext());
+          if (untilValue !== true && typeof untilValue !== "boolean") {
+            throw new Error(`loop.until must evaluate to boolean, got ${typeof untilValue}`);
+          }
+          if (untilValue === true) {
+            break;
+          }
+        }
+
+        iterations += 1;
+        state.incrementLoop();
+      }
+    } finally {
+      state.popLoop();
+    }
+    return loopChain;
+  }
+
+  // Parallel step
+  if ("parallel" in obj && Object.keys(obj).length === 1) {
+    const parallelSteps = (obj as { parallel: unknown[] }).parallel;
+    const controller = new AbortController();
+    // Combine parent signal with local controller
+    const branchSignal = signal
+      ? AbortSignal.any([signal, controller.signal])
+      : controller.signal;
+
+    const results = await Promise.all(
+      parallelSteps.map(async (parallelStep) => {
+        try {
+          // Each branch starts with the same incoming chain
+          return await executeControlStep(workflow, state, parallelStep, basePath, engineExecutor, chain, hub, branchSignal);
+        } catch (error) {
+          controller.abort();
+          throw error;
+        }
+      }),
+    );
+
+    // Chain after parallel is ordered array of branch outputs
+    return results;
+  }
+
+  // If step
+  if ("if" in obj && Object.keys(obj).length === 1) {
+    const ifDef = (obj as { if: { condition: string; then: unknown[]; else?: unknown[] } }).if;
+    const condition = evaluateCel(ifDef.condition, state.toCelContext());
+
+    if (typeof condition !== "boolean") {
+      throw new Error(`if.condition must evaluate to boolean, got ${typeof condition}`);
+    }
+
+    if (condition) {
+      return executeSequential(workflow, state, ifDef.then, basePath, engineExecutor, chain, hub, signal);
+    } else if (ifDef.else) {
+      return executeSequential(workflow, state, ifDef.else, basePath, engineExecutor, chain, hub, signal);
+    }
+
+    // False without else: chain passes through
+    return chain;
+  }
+
+  // Inline participant (has `type` field) or participant override
+  return executeStep(workflow, state, step, basePath, engineExecutor, [], chain, hub, signal);
+}

package/src/engine/engine.ts

@@ -0,0 +1,127 @@
+import { dirname, resolve } from "node:path";
+import { evaluateCel } from "../cel/index";
+import type { Workflow, WorkflowResult } from "../model/index";
+import { parseWorkflowFile } from "../parser/parser";
+import { validateSchema } from "../parser/schema";
+import { validateSemantic } from "../parser/validate";
+import { validateInputs } from "../parser/validate_inputs";
+import type { WorkflowEngineExecutor } from "../participant/workflow";
+import { executeControlStep } from "./control";
+import { validateOutputSchema } from "./output";
+import { WorkflowState } from "./state";
+
+export interface EventHub {
+  publish(event: string, payload: unknown): Promise<void>;
+  publishAndWaitAck(event: string, payload: unknown, timeoutMs: number): Promise<void>;
+  subscribe(event: string, signal?: AbortSignal): AsyncIterable<{ name: string; payload: unknown }>;
+  close(): Promise<void>;
+}
+
+export interface ExecuteOptions {
+  hub?: EventHub;
+  cwd?: string;
+  executionNumber?: number;
+  verbose?: boolean;
+  quiet?: boolean;
+  /** @internal Tracks ancestor workflow paths for circular detection */
+  _ancestorPaths?: Set<string>;
+}
+
+export async function executeWorkflow(
+  workflow: Workflow,
+  inputs: Record<string, unknown> = {},
+  basePath = process.cwd(),
+  options: ExecuteOptions = {},
+): Promise<WorkflowResult> {
+  const { result: inputResult, resolved } = validateInputs(workflow.inputs, inputs);
+  if (!inputResult.valid) {
+    throw new Error(`input validation failed: ${JSON.stringify(inputResult.errors)}`);
+  }
+
+  const state = new WorkflowState(resolved);
+
+  // Set workflow metadata
+  state.workflowMeta = {
+    id: workflow.id,
+    name: workflow.name,
+    version: workflow.version,
+  };
+  state.executionMeta.number = options.executionNumber ?? 1;
+  if (options.cwd) {
+    state.executionMeta.cwd = options.cwd;
+  }
+
+  const startedAt = performance.now();
+
+  // Propagate ancestor paths for circular sub-workflow detection
+  if (options._ancestorPaths) {
+    state.ancestorPaths = options._ancestorPaths;
+  }
+
+  const engineExecutor: WorkflowEngineExecutor = async (subWorkflow, subInputs, subBasePath) => {
+    // Sub-workflows share parent hub, propagate ancestor paths for circular detection
+    return executeWorkflow(subWorkflow, subInputs, subBasePath, {
+      ...options,
+      _ancestorPaths: state.ancestorPaths,
+    });
+  };
+
+  // Execute flow with chain threading
+  let chain: unknown;
+  for (const step of workflow.flow) {
+    chain = await executeControlStep(workflow, state, step, basePath, engineExecutor, chain, options.hub);
+  }
+
+  // Output resolution
+  let output: unknown;
+  if (workflow.output !== undefined) {
+    output = state.resolveOutput(workflow.output, evaluateCel);
+
+    // Validate output against schema if defined
+    if (
+      typeof workflow.output === "object" &&
+      "schema" in workflow.output &&
+      "map" in workflow.output &&
+      typeof output === "object" &&
+      output !== null
+    ) {
+      validateOutputSchema(workflow.output.schema, output as Record<string, unknown>);
+    }
+  } else {
+    // Default: return final chain value
+    output = chain;
+  }
+
+  const steps = state.getAllResults();
+  const success = !Object.values(steps).some((step) => step.status === "failure");
+
+  state.executionMeta.status = success ? "success" : "failure";
+
+  return {
+    success,
+    output,
+    steps,
+    duration: Math.max(0, performance.now() - startedAt),
+  };
+}
+
+export async function runWorkflowFromFile(
+  filePath: string,
+  inputs: Record<string, unknown> = {},
+  options: ExecuteOptions = {},
+): Promise<WorkflowResult> {
+  const workflow = await parseWorkflowFile(filePath);
+
+  const schemaValidation = validateSchema(workflow);
+  if (!schemaValidation.valid) {
+    throw new Error(`schema validation failed: ${JSON.stringify(schemaValidation.errors)}`);
+  }
+
+  const workflowBasePath = dirname(resolve(filePath));
+  const semanticValidation = await validateSemantic(workflow, workflowBasePath);
+  if (!semanticValidation.valid) {
+    throw new Error(`semantic validation failed: ${JSON.stringify(semanticValidation.errors)}`);
+  }
+
+  return executeWorkflow(workflow, inputs, workflowBasePath, options);
+}

package/src/engine/errors.ts

@@ -0,0 +1,90 @@
+import type { ErrorStrategy, RetryConfig } from "../model/index";
+
+export class WorkflowError extends Error {
+  stepName: string;
+  strategy: ErrorStrategy;
+  retriesAttempted: number;
+
+  constructor(message: string, stepName: string, strategy: ErrorStrategy, retriesAttempted = 0) {
+    super(message);
+    this.name = "WorkflowError";
+    this.stepName = stepName;
+    this.strategy = strategy;
+    this.retriesAttempted = retriesAttempted;
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function parseDuration(duration: string): number {
+  const match = duration.match(/^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)\s*$/i);
+  if (!match) {
+    throw new Error(`unsupported duration format: ${duration}`);
+  }
+
+  const value = Number(match[1]);
+  const unit = match[2].toLowerCase();
+
+  switch (unit) {
+    case "ms":
+      return Math.round(value);
+    case "s":
+      return Math.round(value * 1000);
+    case "m":
+      return Math.round(value * 60 * 1000);
+    case "h":
+      return Math.round(value * 60 * 60 * 1000);
+    case "d":
+      return Math.round(value * 24 * 60 * 60 * 1000);
+    default:
+      throw new Error(`unsupported duration unit: ${unit}`);
+  }
+}
+
+export function resolveErrorStrategy(
+  stepOverride?: { onError?: ErrorStrategy } | null,
+  participant?: { onError?: ErrorStrategy } | null,
+  defaults?: { onError?: ErrorStrategy } | null,
+): ErrorStrategy {
+  if (stepOverride?.onError) {
+    return stepOverride.onError;
+  }
+  if (participant?.onError) {
+    return participant.onError;
+  }
+  if (defaults?.onError) {
+    return defaults.onError;
+  }
+  return "fail";
+}
+
+export async function executeWithRetry<T>(
+  fn: () => Promise<T>,
+  retryConfig: RetryConfig | undefined,
+  stepName = "<unknown>",
+): Promise<{ result: T; attempts: number }> {
+  const maxRetries = retryConfig?.max ?? 0;
+  const baseBackoffMs = retryConfig?.backoff ? parseDuration(retryConfig.backoff) : 0;
+  const factor = retryConfig?.factor ?? 1;
+
+  let attempt = 0;
+  while (true) {
+    try {
+      const result = await fn();
+      return { result, attempts: attempt };
+    } catch (error) {
+      if (attempt >= maxRetries) {
+        throw new WorkflowError(String((error as Error)?.message ?? error), stepName, "retry", attempt);
+      }
+
+      const delay = Math.round(baseBackoffMs * Math.pow(factor, attempt));
+      if (delay > 0) {
+        await sleep(delay);
+      }
+
+      attempt += 1;
+    }
+  }
+}

package/src/engine/index.ts

@@ -0,0 +1,8 @@
+export { executeWorkflow, runWorkflowFromFile } from "./engine";
+export type { EventHub, ExecuteOptions } from "./engine";
+export { executeControlStep } from "./control";
+export { executeSequential, executeStep, mergeChainedInput } from "./sequential";
+export { WorkflowState } from "./state";
+export { WorkflowError, parseDuration, resolveErrorStrategy, executeWithRetry } from "./errors";
+export { TimeoutError, withTimeout, resolveTimeout } from "./timeout";
+export { executeWait } from "./wait";