@xdelivered/emberflow 0.2.0

package/server/redact.ts

@@ -0,0 +1,35 @@
+/**
+ * Pure value-based secret redaction. Deep-clones a JSON-safe payload,
+ * replacing every occurrence of each secret VALUE (raw and its
+ * encodeURIComponent form) inside any string leaf with «secret:KEY».
+ */
+export function redactSecrets<T>(payload: T, secrets: Record<string, string>): T {
+  const replacements: Array<[string, string]> = [];
+  for (const [key, value] of Object.entries(secrets)) {
+    if (value.length < 6) continue;
+    const token = `«secret:${key}»`;
+    const encoded = encodeURIComponent(value);
+    const base64 = Buffer.from(value).toString('base64');
+    replacements.push([value, token]);
+    if (encoded !== value) replacements.push([encoded, token]);
+    if (base64 !== value && base64 !== encoded) replacements.push([base64, token]);
+  }
+  // Longest value first, so an encoded superstring is replaced before the raw substring it contains.
+  replacements.sort((a, b) => b[0].length - a[0].length);
+
+  const redactString = (input: string): string =>
+    replacements.reduce((acc, [value, token]) => acc.split(value).join(token), input);
+
+  const walk = (node: unknown): unknown => {
+    if (typeof node === 'string') return redactString(node);
+    if (Array.isArray(node)) return node.map(walk);
+    if (node !== null && typeof node === 'object') {
+      const out: Record<string, unknown> = {};
+      for (const [k, v] of Object.entries(node)) out[k] = walk(v);
+      return out;
+    }
+    return node;
+  };
+
+  return walk(payload) as T;
+}

package/server/redactEgress.test.ts

@@ -0,0 +1,281 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { spawn, type ChildProcess } from 'node:child_process';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+// Verifies secret values never reach the studio via SSE run events or
+// GET /samples, per Task 2 (egress-choke-point redaction). Boots the actual
+// runner subprocess (server/index.ts) against scratch project dirs, mirroring
+// the harness in runsAuth.test.ts.
+
+const PORT = 8130;
+const base = `http://127.0.0.1:${PORT}`;
+const SECRET = 'supersecretvalue123';
+
+async function waitHealthy(url: string, tries = 40): Promise<void> {
+  for (let i = 0; i < tries; i++) {
+    try {
+      if ((await fetch(url)).ok) return;
+    } catch {
+      /* not up yet */
+    }
+    await new Promise((r) => setTimeout(r, 150));
+  }
+  throw new Error('runner did not become healthy');
+}
+
+function bootRunner(port: number, env: Record<string, string | undefined>): ChildProcess {
+  return spawn('npx', ['tsx', 'server/index.ts'], {
+    env: { ...process.env, EMBERFLOW_RUNNER_PORT: String(port), ...env },
+    stdio: 'ignore',
+  });
+}
+
+// Single-node flow: an Input node that just echoes whatever input the run
+// was invoked with (including `headers`).
+const echoFlow = {
+  id: 'echo-input',
+  name: 'Echo Input',
+  version: 1,
+  createdAt: '2026-01-01T00:00:00Z',
+  updatedAt: '2026-01-01T00:00:00Z',
+  nodes: [
+    {
+      id: 'input',
+      type: 'Input',
+      label: 'Input',
+      position: { x: 0, y: 0 },
+      config: { fields: [], defaults: {} },
+    },
+  ],
+  edges: [],
+};
+
+/** Collects every SSE event emitted for a run until it finishes. */
+async function collectSseEvents(runId: string): Promise<Array<Record<string, unknown>>> {
+  const res = await fetch(`${base}/api/runs/${runId}/events`);
+  if (!res.body) throw new Error('no SSE body');
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  let buf = '';
+  const events: Array<Record<string, unknown>> = [];
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    buf += decoder.decode(value, { stream: true });
+    let idx: number;
+    while ((idx = buf.indexOf('\n\n')) !== -1) {
+      const chunk = buf.slice(0, idx);
+      buf = buf.slice(idx + 2);
+      const dataLine = chunk.split('\n').find((l) => l.startsWith('data: '));
+      if (dataLine) {
+        const parsed = JSON.parse(dataLine.slice('data: '.length));
+        events.push(parsed);
+        if (parsed.type === 'finished') {
+          reader.cancel();
+          return events;
+        }
+      }
+    }
+  }
+  return events;
+}
+
+describe('secret redaction at egress choke points', () => {
+  describe('with a configured auth secret', () => {
+    let projectDir: string;
+    let proc: ChildProcess;
+
+    beforeAll(async () => {
+      projectDir = mkdtempSync(join(tmpdir(), 'redactegress-'));
+      mkdirSync(join(projectDir, 'emberflow', 'apis', 'default'), { recursive: true });
+      writeFileSync(join(projectDir, 'emberflow.config.mjs'), 'export default {};\n');
+      writeFileSync(
+        join(projectDir, 'emberflow.environments.json'),
+        JSON.stringify({
+          defaultEnvironment: 'local',
+          environments: {
+            local: {
+              vars: {},
+              secrets: { sessionCookie: SECRET },
+              auth: { attach: { as: 'cookie', name: 'session', secretRef: 'sessionCookie' } },
+            },
+          },
+        }),
+      );
+      proc = bootRunner(PORT, { EMBERFLOW_PROJECT: projectDir });
+      await waitHealthy(`${base}/healthz`);
+    }, 20_000);
+
+    afterAll(() => {
+      proc?.kill();
+      if (projectDir) rmSync(projectDir, { recursive: true, force: true });
+    });
+
+    it('never leaks the raw secret value over SSE, and redacts the attached cookie', async () => {
+      const createRes = await fetch(`${base}/api/runs`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ flow: echoFlow, mode: 'run', input: {} }),
+      });
+      expect(createRes.status).toBe(201);
+      const { runId } = await createRes.json();
+
+      const events = await collectSseEvents(runId);
+      expect(events.length).toBeGreaterThan(0);
+
+      const raw = JSON.stringify(events);
+      expect(raw).not.toContain(SECRET);
+
+      const nodeStateEvents = events.filter(
+        (e) => e.type === 'nodeState' && (e as { nodeId?: string }).nodeId === 'input',
+      );
+      const succeeded = nodeStateEvents.find(
+        (e) => (e.state as { status?: string } | undefined)?.status === 'succeeded',
+      );
+      expect(succeeded).toBeDefined();
+      const output = (succeeded!.state as { output?: { headers?: { cookie?: string } } }).output;
+      expect(output?.headers?.cookie).toContain('session=«secret:sessionCookie»');
+    });
+
+    it('redacts the sample output returned via GET /samples', async () => {
+      const createRes = await fetch(`${base}/api/runs`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ flow: echoFlow, mode: 'step', input: {} }),
+      });
+      expect(createRes.status).toBe(201);
+      const { runId } = await createRes.json();
+
+      const stepRes = await fetch(`${base}/api/runs/${runId}/step`, { method: 'POST' });
+      expect(stepRes.status).toBe(200);
+
+      const samplesRes = await fetch(`${base}/api/samples?nodeId=input`);
+      expect(samplesRes.status).toBe(200);
+      const body = await samplesRes.json();
+      const raw = JSON.stringify(body);
+      expect(raw).not.toContain(SECRET);
+
+      const sample = body.samples.find((s: { runId: string }) => s.runId === runId);
+      expect(sample).toBeDefined();
+      expect(sample.output.headers.cookie).toContain('session=«secret:sessionCookie»');
+    });
+  });
+
+  describe('with two environments sharing a secret key (different values)', () => {
+    let projectDir: string;
+    let proc: ChildProcess;
+    const port = 8141;
+    const base3 = `http://127.0.0.1:${port}`;
+    const DEV_SECRET = 'devsecretvalue-abc123';
+    const PROD_SECRET = 'prodsecretvalue-xyz789';
+
+    beforeAll(async () => {
+      projectDir = mkdtempSync(join(tmpdir(), 'redactegress-multienv-'));
+      mkdirSync(join(projectDir, 'emberflow', 'apis', 'default'), { recursive: true });
+      writeFileSync(join(projectDir, 'emberflow.config.mjs'), 'export default {};\n');
+      writeFileSync(
+        join(projectDir, 'emberflow.environments.json'),
+        JSON.stringify({
+          defaultEnvironment: 'dev',
+          environments: {
+            // Both envs define the SAME key with DIFFERENT values. A merged
+            // redaction map keyed by name would keep only prod's value and
+            // let dev's pass through /samples raw.
+            dev: {
+              vars: {},
+              secrets: { sessionCookie: DEV_SECRET },
+              auth: { attach: { as: 'cookie', name: 'session', secretRef: 'sessionCookie' } },
+            },
+            prod: {
+              vars: {},
+              secrets: { sessionCookie: PROD_SECRET },
+              auth: { attach: { as: 'cookie', name: 'session', secretRef: 'sessionCookie' } },
+            },
+          },
+        }),
+      );
+      proc = bootRunner(port, { EMBERFLOW_PROJECT: projectDir });
+      await waitHealthy(`${base3}/healthz`);
+    }, 20_000);
+
+    afterAll(() => {
+      proc?.kill();
+      if (projectDir) rmSync(projectDir, { recursive: true, force: true });
+    });
+
+    it('GET /samples leaks NEITHER environment value raw, even on key collision', async () => {
+      // Run against dev (the default env): dev's value lands in the sample
+      // via the auto-attached cookie.
+      const createRes = await fetch(`${base3}/api/runs`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ flow: echoFlow, mode: 'step', input: {} }),
+      });
+      expect(createRes.status).toBe(201);
+      const { runId } = await createRes.json();
+
+      const stepRes = await fetch(`${base3}/api/runs/${runId}/step`, { method: 'POST' });
+      expect(stepRes.status).toBe(200);
+
+      const samplesRes = await fetch(`${base3}/api/samples?nodeId=input`);
+      expect(samplesRes.status).toBe(200);
+      const body = await samplesRes.json();
+      const raw = JSON.stringify(body);
+      expect(raw).not.toContain(DEV_SECRET);
+      expect(raw).not.toContain(PROD_SECRET);
+
+      const sample = body.samples.find((s: { runId: string }) => s.runId === runId);
+      expect(sample).toBeDefined();
+      expect(sample.output.headers.cookie).toContain('session=«secret:sessionCookie»');
+    });
+  });
+
+  describe('without any secrets configured', () => {
+    let projectDir: string;
+    let proc: ChildProcess;
+    const port = 8140;
+    const base2 = `http://127.0.0.1:${port}`;
+
+    beforeAll(async () => {
+      projectDir = mkdtempSync(join(tmpdir(), 'redactegress-nosecrets-'));
+      mkdirSync(join(projectDir, 'emberflow', 'apis', 'default'), { recursive: true });
+      writeFileSync(join(projectDir, 'emberflow.config.mjs'), 'export default {};\n');
+      writeFileSync(
+        join(projectDir, 'emberflow.environments.json'),
+        JSON.stringify({
+          defaultEnvironment: 'local',
+          environments: { local: { vars: {}, secrets: {} } },
+        }),
+      );
+      proc = bootRunner(port, { EMBERFLOW_PROJECT: projectDir });
+      await waitHealthy(`${base2}/healthz`);
+    }, 20_000);
+
+    afterAll(() => {
+      proc?.kill();
+      if (projectDir) rmSync(projectDir, { recursive: true, force: true });
+    });
+
+    it('run output is untouched: no redaction tokens appear anywhere', async () => {
+      const createRes = await fetch(`${base2}/api/runs`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ flow: echoFlow, mode: 'step', input: { headers: { 'x-foo': 'bar' } } }),
+      });
+      expect(createRes.status).toBe(201);
+      const { runId } = await createRes.json();
+
+      const stepRes = await fetch(`${base2}/api/runs/${runId}/step`, { method: 'POST' });
+      expect(stepRes.status).toBe(200);
+
+      const samplesRes = await fetch(`${base2}/api/samples?nodeId=input`);
+      const { samples } = await samplesRes.json();
+      const sample = samples.find((s: { runId: string }) => s.runId === runId);
+      expect(sample).toBeDefined();
+      expect(sample.output.headers['x-foo']).toBe('bar');
+      expect(JSON.stringify(sample)).not.toContain('«secret:');
+    });
+  });
+});

package/server/runRegistry.ts

@@ -0,0 +1,270 @@
+import type {
+  LogLine, NodeExecutionSample, NodeRunState,
+  WorkflowDefinition, WorkflowRun,
+} from '../src/engine';
+import { InMemoryTraceSink, startRun, type FlowRun, type NodeRegistry } from '../src/engine';
+import { createDefaultRegistry } from '../src/nodes';
+import { redactSecrets } from './redact';
+import { makeSubflowRunner } from './subflowRunner';
+
+/** SSE-shaped events, mirroring ExecutorEvents 1:1. */
+export type RunEvent =
+  | { type: 'nodeState'; nodeId: string; state: NodeRunState }
+  | { type: 'log'; line: LogLine }
+  | {
+      type: 'finished';
+      run: WorkflowRun;
+      /** Present only when this run is an error-handler invocation: the
+       *  workflowId of the run whose failure fired it. */
+      errorHandler?: { firedBy: string };
+    };
+
+type Listener = (event: RunEvent) => void;
+
+interface LiveRun {
+  handle: FlowRun;
+  buffer: RunEvent[];
+  listeners: Set<Listener>;
+  finished: boolean;
+  cleanupTimer?: ReturnType<typeof setTimeout>;
+}
+
+const RETAIN_FINISHED_MS = 10 * 60 * 1000; // 10 minutes
+const MAX_LIVE_RUNS = 100;
+
+/** Manages live runs: buffers engine events per run and fans them out to SSE subscribers. */
+export class RunRegistry {
+  private readonly registry: NodeRegistry;
+  private readonly runs = new Map<string, LiveRun>();
+  readonly sink = new InMemoryTraceSink();
+
+  /**
+   * @param loadFlow Resolves a workflow by id for Subflow nodes (server hosts
+   *   this via flowStore.load). Absent → Subflow nodes fail with a clear message.
+   * @param registry Execution registry to run against. Defaults to the
+   *   built-in node set; project mode passes one preloaded with consumer nodes.
+   * @param errorOperation Op id (resolved via `loadFlow`, same as Subflow
+   *   nodes) to run whenever a run finishes with status 'failed'. Best-effort:
+   *   never throws, never affects the failed run. Absent → disabled.
+   */
+  constructor(
+    private readonly loadFlow?: (id: string) => WorkflowDefinition | undefined,
+    registry: NodeRegistry = createDefaultRegistry(),
+    private readonly errorOperation?: string,
+  ) {
+    this.registry = registry;
+  }
+
+  /** Number of registered node types (for boot logging). */
+  get nodeCount(): number {
+    return this.registry.list().length;
+  }
+
+  /** The registry runs execute against — used to verify artifact node hashes. */
+  get executionRegistry(): NodeRegistry {
+    return this.registry;
+  }
+
+  /** Samples recorded for a node across all runs on this registry (newest first, capped). */
+  samplesFor(nodeId: string): NodeExecutionSample[] {
+    return this.sink.samplesFor(nodeId).slice(0, 100);
+  }
+
+  /** Create a run; wires engine events into a per-run buffer + attached subscribers. */
+  create(
+    flow: WorkflowDefinition,
+    opts: {
+      secrets: Record<string, string>;
+      vars: Record<string, string>;
+      environment: string;
+      safeMode: boolean;
+      pins?: Record<string, unknown>;
+      input?: Record<string, unknown>;
+      /** When true, runs in Mock mode: mocked nodes return their canned
+       *  output, unmocked infra nodes fail loud. See `StartRunOptions.mockRun`. */
+      mockRun?: boolean;
+      /** nodeId -> canned output, consulted only when `mockRun` is true. */
+      mocks?: Record<string, unknown>;
+      /** Internal: marks this run as an error-workflow invocation so it never
+       *  itself triggers `errorOperation` (recursion guard — a flag, not an id
+       *  comparison, since an error op can legitimately fail on its own). */
+      isErrorHandler?: boolean;
+      /** Internal: the workflowId of the run that failed and fired this
+       *  error-handler run — stamped onto this run's `finished` event as
+       *  `errorHandler: { firedBy }`. Only meaningful alongside isErrorHandler. */
+      errorHandlerFiredBy?: string;
+    },
+  ): { runId: string; handle: FlowRun } {
+    this.evictIfNeeded();
+
+    // Placeholder entry so the ExecutorEvents closures can reach the live run.
+    const entry: LiveRun = { handle: undefined as unknown as FlowRun, buffer: [], listeners: new Set(), finished: false };
+
+    // Redact secret values from every event before it reaches the buffer
+    // (replayed to SSE subscribers) or any listener — internal run state
+    // (`entry.handle`, `handle.run.nodeStates`) is never routed through
+    // `emit` and stays raw.
+    const scrub = <T>(e: T): T => redactSecrets(e, opts.secrets ?? {});
+
+    const emit = (event: RunEvent): void => {
+      const scrubbed = scrub(event);
+      entry.buffer.push(scrubbed);
+      for (const listener of entry.listeners) listener(scrubbed);
+    };
+
+    // Subflow runner (shared factory — see server/subflowRunner.ts): resolves
+    // the child via loadFlow, runs it to completion on the same registry
+    // sharing the parent's environment, and forwards its (prefixed) logs into
+    // this run's SSE stream via `onLog`.
+    const subflowRunner = makeSubflowRunner(
+      {
+        loadFlow: (id) => this.loadFlow?.(id),
+        registry: this.registry,
+        secrets: opts.secrets,
+        vars: opts.vars,
+        environment: opts.environment,
+        safeMode: opts.safeMode,
+        mockRun: opts.mockRun,
+        trace: this.sink,
+        onLog: (line) => emit({ type: 'log', line }),
+      },
+      [flow.id],
+    );
+
+    const handle = startRun({
+      flow,
+      registry: this.registry,
+      secrets: opts.secrets,
+      vars: opts.vars,
+      environment: opts.environment,
+      safeMode: opts.safeMode,
+      pins: opts.pins,
+      input: opts.input,
+      mockRun: opts.mockRun,
+      mocks: opts.mocks,
+      trace: this.sink,
+      subflowRunner,
+      events: {
+        onNodeStateChange: (nodeId, state) => emit({ type: 'nodeState', nodeId, state }),
+        onLog: (line) => emit({ type: 'log', line }),
+        onRunFinished: (run) => {
+          emit({
+            type: 'finished',
+            run,
+            ...(opts.isErrorHandler && opts.errorHandlerFiredBy
+              ? { errorHandler: { firedBy: opts.errorHandlerFiredBy } }
+              : {}),
+          });
+          entry.finished = true;
+          this.scheduleCleanup(run.id);
+          if (run.status === 'failed' && !opts.isErrorHandler) this.fireErrorWorkflow(run, flow, opts);
+        },
+      },
+    });
+
+    entry.handle = handle;
+    this.runs.set(handle.run.id, entry);
+    return { runId: handle.run.id, handle };
+  }
+
+  /**
+   * Best-effort: fires `errorOperation` (resolved via the same `loadFlow`
+   * callback Subflow nodes use) as a new run when `failedRun` finished with
+   * status 'failed'. Never throws and never affects the failed run — any
+   * problem (unset config, unknown op id, the error op itself failing to
+   * start) is logged and swallowed.
+   */
+  private fireErrorWorkflow(
+    failedRun: WorkflowRun,
+    failedFlow: WorkflowDefinition,
+    opts: {
+      secrets: Record<string, string>;
+      vars: Record<string, string>;
+      environment: string;
+      safeMode: boolean;
+      /** A failed MOCK run's error op must also run mocked — otherwise the
+       *  most common mock failure (the fail-loud infra boundary) would fire
+       *  an error op that touches real infrastructure. */
+      mockRun?: boolean;
+    },
+  ): void {
+    if (!this.errorOperation) return;
+    try {
+      const errorFlow = this.loadFlow?.(this.errorOperation);
+      if (!errorFlow) {
+        console.error(`[error-workflow] unknown errorOperation "${this.errorOperation}"`);
+        return;
+      }
+      // Report the failure that HALTED the run: an `optional` (fail-soft) node
+      // can fail earlier without aborting, so prefer the first failed state
+      // whose node is not optional. Fall back to the first failed state only
+      // when every failed node is optional.
+      const failedEntries = Object.entries(failedRun.nodeStates).filter(([, s]) => s.status === 'failed');
+      const failedEntry =
+        failedEntries.find(([id]) => !failedFlow.nodes.find((n) => n.id === id)?.optional)
+        ?? failedEntries[0];
+      const failedNodeId = failedEntry?.[0];
+      const error = failedEntry?.[1] && 'error' in failedEntry[1] ? failedEntry[1].error : undefined;
+      const { handle } = this.create(errorFlow, {
+        secrets: opts.secrets,
+        vars: opts.vars,
+        environment: opts.environment,
+        safeMode: opts.safeMode,
+        // Mock mode propagates with the ERROR op's own op-level mocks
+        // (nodeIds are flow-scoped — the failed run's map doesn't apply).
+        mockRun: opts.mockRun,
+        mocks: (errorFlow as { mocks?: Record<string, unknown> }).mocks ?? {},
+        input: {
+          failedRunId: failedRun.id,
+          failedWorkflowId: failedRun.workflowId,
+          failedNodeId,
+          error,
+          environment: opts.environment,
+        },
+        isErrorHandler: true,
+        errorHandlerFiredBy: failedRun.workflowId,
+      });
+      handle.runToEnd().catch((err) => {
+        console.error('[error-workflow] error op run failed', err instanceof Error ? err.message : String(err));
+      });
+    } catch (err) {
+      console.error('[error-workflow] failed to fire error operation', err instanceof Error ? err.message : String(err));
+    }
+  }
+
+  get(runId: string): FlowRun | undefined {
+    return this.runs.get(runId)?.handle;
+  }
+
+  /** Replay the buffer to the listener, then stream live events. Returns an unsubscribe fn. */
+  subscribe(runId: string, listener: Listener): (() => void) | undefined {
+    const entry = this.runs.get(runId);
+    if (!entry) return undefined;
+    for (const event of entry.buffer) listener(event);
+    // A run that already finished has its full buffer replayed above; no live events follow.
+    entry.listeners.add(listener);
+    return () => entry.listeners.delete(listener);
+  }
+
+  private scheduleCleanup(runId: string): void {
+    const entry = this.runs.get(runId);
+    if (!entry) return;
+    if (entry.cleanupTimer) clearTimeout(entry.cleanupTimer);
+    entry.cleanupTimer = setTimeout(() => this.runs.delete(runId), RETAIN_FINISHED_MS);
+  }
+
+  /** Cap live entries: drop the oldest finished run, or the oldest run outright. */
+  private evictIfNeeded(): void {
+    if (this.runs.size < MAX_LIVE_RUNS) return;
+    let victim: string | undefined;
+    for (const [id, entry] of this.runs) {
+      if (entry.finished) { victim = id; break; }
+    }
+    if (!victim) victim = this.runs.keys().next().value;
+    if (victim) {
+      const entry = this.runs.get(victim);
+      if (entry?.cleanupTimer) clearTimeout(entry.cleanupTimer);
+      this.runs.delete(victim);
+    }
+  }
+}