amalgm 0.1.51 → 0.1.52

package/runtime/scripts/amalgm-mcp/workflows/runner.js

@@ -0,0 +1,593 @@
+'use strict';
+
+const crypto = require('crypto');
+const fs = require('fs');
+const { spawn } = require('child_process');
+const vm = require('vm');
+
+const { DEFAULT_CWD } = require('../config');
+const { recordEventRun } = require('../events/store');
+const { compileWorkflowText } = require('./compiler');
+const { loadWorkflows } = require('./store');
+const { callToolboxMcpTool, toolboxMcpToolName } = require('../toolbox/runner');
+const { resolveAmalgmCoreTool, resolveToolboxAction, suggestWorkflowToolActions } = require('../automations/tool-actions');
+
+const DEFAULT_CELL_TIMEOUT_MS = 120_000;
+const DEFAULT_CODE_TIMEOUT_MS = 15_000;
+const MAX_OUTPUT_BYTES = 512 * 1024;
+const workflowRunQueues = new Map();
+
+class WorkflowStop extends Error {
+  constructor(message = 'Workflow stopped') {
+    super(message);
+    this.name = 'WorkflowStop';
+  }
+}
+
+function isObject(value) {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isFunctionRef(value) {
+  return isObject(value) && typeof value.$fn === 'string';
+}
+
+function durationSince(startedAt) {
+  return Date.now() - new Date(startedAt).getTime();
+}
+
+function positiveInteger(value, fallback = null) {
+  const number = Number(value);
+  if (!Number.isFinite(number)) return fallback;
+  return Math.max(1, Math.floor(number));
+}
+
+function queueLimitValue(value) {
+  const number = Number(value);
+  if (!Number.isFinite(number)) return 0;
+  return Math.max(0, Math.floor(number));
+}
+
+function withTimeout(promise, timeoutMs, label) {
+  let timer = null;
+  return Promise.race([
+    promise,
+    new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
+    }),
+  ]).finally(() => {
+    if (timer) clearTimeout(timer);
+  });
+}
+
+function publicTrigger(trigger) {
+  return {
+    id: trigger.id,
+    name: trigger.name,
+    source: trigger.source,
+    event: trigger.event,
+  };
+}
+
+function publicWorkflow(workflow) {
+  return {
+    id: workflow.id,
+    name: workflow.name,
+  };
+}
+
+function buildSecrets(allowlist = {}) {
+  const names = Array.isArray(allowlist.secrets) ? allowlist.secrets : [];
+  const secrets = {};
+  for (const name of names) {
+    const key = String(name || '').trim();
+    if (key && process.env[key] != null) secrets[key] = process.env[key];
+  }
+  return secrets;
+}
+
+function buildInput(ctx, stop) {
+  return {
+    event: ctx.event,
+    payload: ctx.event?.payload,
+    headers: ctx.event?.headers || {},
+    trigger: ctx.trigger,
+    workflow: ctx.workflow,
+    previous: ctx.previous,
+    outputs: ctx.outputs,
+    secrets: ctx.secrets,
+    stop,
+    ...ctx.outputs,
+  };
+}
+
+async function runFunctionSource(fnSource, input, options = {}) {
+  const timeoutMs = Number(options.timeoutMs) || DEFAULT_CODE_TIMEOUT_MS;
+  const sandbox = {
+    __input: input,
+    __result: undefined,
+    console,
+    fetch: globalThis.fetch,
+    URL,
+    URLSearchParams,
+    TextDecoder,
+    TextEncoder,
+    setTimeout,
+    clearTimeout,
+  };
+  const context = vm.createContext(sandbox, {
+    name: 'amalgm-workflow-cell',
+    codeGeneration: { strings: false, wasm: false },
+  });
+  const script = new vm.Script(`__result = (${fnSource})(__input)`, {
+    filename: 'workflow-cell.js',
+  });
+
+  try {
+    const result = script.runInContext(context, { timeout: Math.min(timeoutMs, 1000) });
+    if (result && typeof result.then === 'function') {
+      return await withTimeout(result, timeoutMs, 'Workflow function');
+    }
+    return result;
+  } catch (error) {
+    if (error instanceof WorkflowStop || error?.name === 'WorkflowStop') throw error;
+    throw new Error(error?.message || String(error));
+  }
+}
+
+async function resolveDynamic(value, input, options = {}) {
+  if (isFunctionRef(value)) return runFunctionSource(value.$fn, input, options);
+  if (Array.isArray(value)) {
+    const next = [];
+    for (const item of value) next.push(await resolveDynamic(item, input, options));
+    return next;
+  }
+  if (isObject(value)) {
+    const next = {};
+    for (const [key, child] of Object.entries(value)) {
+      next[key] = await resolveDynamic(child, input, options);
+    }
+    return next;
+  }
+  return value;
+}
+
+function appendBuffer(current, chunk) {
+  const next = Buffer.concat([current, Buffer.from(chunk)]);
+  return next.length > MAX_OUTPUT_BYTES ? next.subarray(next.length - MAX_OUTPUT_BYTES) : next;
+}
+
+function safeEnv(extraEnv, allowlist = {}) {
+  const env = {};
+  for (const key of ['PATH', 'HOME', 'USER', 'LOGNAME', 'SHELL', 'LANG', 'LC_ALL', 'TERM', 'TMPDIR']) {
+    if (process.env[key]) env[key] = process.env[key];
+  }
+  for (const key of Array.isArray(allowlist.secrets) ? allowlist.secrets : []) {
+    if (process.env[key] != null) env[key] = process.env[key];
+  }
+  if (isObject(extraEnv)) {
+    for (const [key, value] of Object.entries(extraEnv)) {
+      if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) && value != null) env[key] = String(value);
+    }
+  }
+  return env;
+}
+
+async function runShellCommand(config, allowlist) {
+  const command = String(config.command || config.run || '').trim();
+  if (!command) throw new Error('CLI cell needs a command.');
+  if (allowlist.localCompute !== true) {
+    throw new Error('CLI cells require allowlist.localCompute=true.');
+  }
+
+  const timeoutMs = Number(config.timeoutMs || config.timeout_ms) || DEFAULT_CELL_TIMEOUT_MS;
+  const cwd = config.cwd || DEFAULT_CWD;
+  if (!fs.existsSync(cwd)) fs.mkdirSync(cwd, { recursive: true });
+  const child = spawn(command, {
+    cwd,
+    env: safeEnv(config.env, allowlist),
+    shell: true,
+    stdio: ['pipe', 'pipe', 'pipe'],
+    windowsHide: true,
+  });
+
+  if (config.stdin != null) child.stdin.end(String(config.stdin));
+  else child.stdin.end();
+
+  let stdout = Buffer.alloc(0);
+  let stderr = Buffer.alloc(0);
+  let timedOut = false;
+  const result = await new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      timedOut = true;
+      try { child.kill('SIGTERM'); } catch {}
+      setTimeout(() => {
+        try { child.kill('SIGKILL'); } catch {}
+      }, 1500).unref?.();
+    }, timeoutMs);
+
+    child.stdout.on('data', (chunk) => { stdout = appendBuffer(stdout, chunk); });
+    child.stderr.on('data', (chunk) => { stderr = appendBuffer(stderr, chunk); });
+    child.on('error', (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+    child.on('close', (code, signal) => {
+      clearTimeout(timer);
+      resolve({
+        exitCode: code,
+        signal,
+        timedOut,
+        stdout: stdout.toString('utf8'),
+        stderr: stderr.toString('utf8'),
+      });
+    });
+  });
+
+  if (result.timedOut || result.exitCode !== 0) {
+    const status = result.timedOut ? 'timed out' : `failed with exit code ${result.exitCode ?? result.signal ?? 'unknown'}`;
+    throw new Error([
+      `Command ${status}: ${command}`,
+      result.stdout ? `stdout:\n${result.stdout.trim()}` : null,
+      result.stderr ? `stderr:\n${result.stderr.trim()}` : null,
+    ].filter(Boolean).join('\n\n'));
+  }
+
+  return {
+    command,
+    cwd,
+    exitCode: result.exitCode,
+    stdout: result.stdout,
+    stderr: result.stderr,
+  };
+}
+
+async function runHttpTool(args) {
+  const method = String(args.method || 'GET').toUpperCase();
+  if (!args.url) throw new Error('http.fetch needs a url.');
+  const response = await fetch(args.url, {
+    method,
+    headers: args.headers || {},
+    body: ['GET', 'HEAD'].includes(method) ? undefined : (
+      typeof args.body === 'string' ? args.body : JSON.stringify(args.body ?? {})
+    ),
+  });
+  const text = await response.text();
+  let body = text;
+  try {
+    body = JSON.parse(text);
+  } catch {
+    // Keep raw text.
+  }
+  if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}: ${text.slice(0, 12000)}`);
+  return {
+    status: response.status,
+    headers: Object.fromEntries(response.headers.entries()),
+    body,
+  };
+}
+
+function mcpResultToOutput(result, actionKey) {
+  if (!result) throw new Error(`Tool action returned no result: ${actionKey}.`);
+  if (result.isError) {
+    const text = result.content?.map((part) => part.text).filter(Boolean).join('\n') || `${actionKey} failed.`;
+    throw new Error(text);
+  }
+  return result.structuredContent ?? {
+    text: result.content?.map((part) => part.text).filter(Boolean).join('\n') || '',
+  };
+}
+
+async function runToolCell(cell, args, allowlist) {
+  const actionKey = `${cell.toolId}.${cell.actionName}`;
+  const allowedActions = Array.isArray(allowlist.actions) ? allowlist.actions : [];
+  if (allowedActions.length > 0 && !allowedActions.includes(actionKey)) {
+    throw new Error(`Workflow is not allowed to call ${actionKey}.`);
+  }
+
+  if (cell.toolId === 'http' && cell.actionName === 'fetch') {
+    if (allowlist.network !== true) throw new Error('http.fetch requires allowlist.network=true.');
+    return runHttpTool(args);
+  }
+
+  if (cell.toolId === 'amalgm') {
+    const coreTool = resolveAmalgmCoreTool(cell.actionName);
+    if (!coreTool?.handler) throw new Error(`Amalgm action not found: ${cell.actionName}.`);
+    const result = await coreTool.handler(args, {
+      workflow: true,
+      sessionMetadata: {
+        origin: 'event-workflow',
+      },
+    });
+    return mcpResultToOutput(result, actionKey);
+  }
+
+  const resolved = resolveToolboxAction(cell.toolId, cell.actionName);
+  if (!resolved) {
+    const suggestions = suggestWorkflowToolActions(cell.toolId, cell.actionName);
+    throw new Error([
+      `Tool action not found: ${actionKey}.`,
+      suggestions.length > 0 ? `Did you mean: ${suggestions.join(', ')}?` : 'Use automations_list_actions to inspect callable workflow actions.',
+    ].join(' '));
+  }
+
+  const result = await callToolboxMcpTool(toolboxMcpToolName(resolved.action), args, {
+    sessionMetadata: {
+      tools: {
+        mode: 'selected',
+        selectedToolIds: [resolved.tool.id, resolved.action.id],
+      },
+    },
+  });
+  return mcpResultToOutput(result, actionKey);
+}
+
+function workflowIrForTarget(target) {
+  if (target.workflowIr && target.workflowIr.version === 1) return target.workflowIr;
+  if (target.workflowText || target.workflow) return compileWorkflowText(target.workflowText || target.workflow);
+  throw new Error('Workflow has no script.');
+}
+
+function workflowTargetsForTrigger(trigger) {
+  const workflowIds = Array.isArray(trigger.workflowIds)
+    ? trigger.workflowIds.map((id) => String(id || '').trim()).filter(Boolean)
+    : [];
+  if (workflowIds.length === 0) return [trigger];
+
+  const workflowsById = new Map(loadWorkflows().workflows.map((workflow) => [workflow.id, workflow]));
+  return workflowIds.map((workflowId) => {
+    const workflow = workflowsById.get(workflowId);
+    if (!workflow) throw new Error(`Linked workflow not found: ${workflowId}`);
+    return workflow;
+  }).filter((workflow) => workflow.enabled !== false);
+}
+
+async function executeWorkflowCell(cell, ctx, allowlist, limits) {
+  const timeoutMs = Number(cell.timeoutMs || limits.cellTimeoutMs) || DEFAULT_CELL_TIMEOUT_MS;
+  const stop = () => {
+    throw new WorkflowStop();
+  };
+  const input = buildInput(ctx, stop);
+
+  if (cell.if !== undefined) {
+    const condition = await resolveDynamic(cell.if, input, { timeoutMs });
+    if (!condition) return { skipped: true, output: null };
+  }
+
+  if (cell.kind === 'code') {
+    const output = await runFunctionSource(cell.code, input, {
+      timeoutMs: Number(cell.timeoutMs || limits.codeTimeoutMs) || DEFAULT_CODE_TIMEOUT_MS,
+    });
+    return { output };
+  }
+
+  if (cell.kind === 'cli') {
+    const config = await resolveDynamic(cell.config || {}, input, { timeoutMs });
+    const output = await runShellCommand(config, allowlist);
+    return { output };
+  }
+
+  if (cell.kind === 'tool') {
+    const args = await resolveDynamic(cell.args || {}, input, { timeoutMs });
+    const output = await runToolCell(cell, args, allowlist);
+    return { output };
+  }
+
+  throw new Error(`Unsupported workflow cell kind "${cell.kind}".`);
+}
+
+async function runWorkflowTargetNow(target, eventObj, opts = {}) {
+  const runId = crypto.randomUUID();
+  const startedAt = new Date().toISOString();
+  const trigger = opts.trigger || target;
+  const triggerId = opts.triggerId || trigger.id;
+  const ir = workflowIrForTarget(target);
+  const allowlist = {
+    ...(ir.allowlist || {}),
+    ...(target.allowlist || {}),
+    ...(trigger.allowlist || {}),
+  };
+  const limits = {
+    ...(ir.limits || {}),
+    ...(target.limits || {}),
+    ...(trigger.limits || {}),
+  };
+  const ctx = {
+    event: eventObj,
+    trigger: publicTrigger(trigger),
+    workflow: publicWorkflow(target),
+    previous: null,
+    outputs: {},
+    secrets: buildSecrets(allowlist),
+  };
+  const projectPath = target.projectPath || trigger.projectPath || null;
+
+  recordEventRun(triggerId, {
+    runId,
+    triggerName: trigger.name,
+    workflowId: target.id,
+    workflowName: target.name,
+    startedAt,
+    status: 'running',
+    projectPath,
+    workflow: {
+      id: target.id,
+      name: target.name,
+      version: ir.version,
+      cells: ir.cells.map((cell) => ({ name: cell.name, kind: cell.kind })),
+    },
+  }, { source: 'event_runs:workflow:start' });
+
+  let status = 'completed';
+  try {
+    for (const cell of ir.cells) {
+      const cellStartedAt = new Date().toISOString();
+      recordEventRun(triggerId, {
+        runId,
+        status: 'running',
+        cell: { name: cell.name, kind: cell.kind, status: 'running', startedAt: cellStartedAt },
+      }, { source: 'event_runs:workflow:cell-start' });
+
+      try {
+        const result = await executeWorkflowCell(cell, ctx, allowlist, limits);
+        const cellStatus = result.skipped ? 'skipped' : 'completed';
+        if (!result.skipped) {
+          ctx.previous = result.output;
+          ctx.outputs[cell.name] = result.output;
+        }
+        recordEventRun(triggerId, {
+          runId,
+          status: 'running',
+          cell: {
+            name: cell.name,
+            kind: cell.kind,
+            status: cellStatus,
+            startedAt: cellStartedAt,
+            finishedAt: new Date().toISOString(),
+            durationMs: durationSince(cellStartedAt),
+            output: result.output,
+          },
+        }, { source: 'event_runs:workflow:cell-complete' });
+      } catch (error) {
+        if (error instanceof WorkflowStop || error?.name === 'WorkflowStop') {
+          status = 'stopped';
+          recordEventRun(triggerId, {
+            runId,
+            status,
+            cell: {
+              name: cell.name,
+              kind: cell.kind,
+              status: 'stopped',
+              startedAt: cellStartedAt,
+              finishedAt: new Date().toISOString(),
+              durationMs: durationSince(cellStartedAt),
+            },
+          }, { source: 'event_runs:workflow:cell-stopped' });
+          break;
+        }
+        recordEventRun(triggerId, {
+          runId,
+          status: 'failed',
+          cell: {
+            name: cell.name,
+            kind: cell.kind,
+            status: 'failed',
+            startedAt: cellStartedAt,
+            finishedAt: new Date().toISOString(),
+            durationMs: durationSince(cellStartedAt),
+            error: error.message || String(error),
+          },
+        }, { source: 'event_runs:workflow:cell-failed' });
+        throw error;
+      }
+    }
+
+    recordEventRun(triggerId, {
+      runId,
+      triggerName: trigger.name,
+      workflowId: target.id,
+      workflowName: target.name,
+      finishedAt: new Date().toISOString(),
+      status,
+      durationMs: durationSince(startedAt),
+      output: ctx.outputs,
+    }, { source: `event_runs:workflow:${status}` });
+  } catch (error) {
+    status = 'failed';
+    recordEventRun(triggerId, {
+      runId,
+      triggerName: trigger.name,
+      workflowId: target.id,
+      workflowName: target.name,
+      finishedAt: new Date().toISOString(),
+      status,
+      error: error.message || String(error),
+      durationMs: durationSince(startedAt),
+    }, { source: 'event_runs:workflow:failed' });
+    throw error;
+  }
+}
+
+function runQueuedWorkflow(key, state, item) {
+  state.active += 1;
+  runWorkflowTargetNow(item.target, item.eventObj, item.opts)
+    .then(item.resolve, item.reject)
+    .finally(() => {
+      state.active = Math.max(0, state.active - 1);
+      const next = state.queue.shift();
+      if (next) {
+        runQueuedWorkflow(key, state, next);
+      } else if (state.active === 0) {
+        workflowRunQueues.delete(key);
+      }
+    });
+}
+
+function executeWorkflowTarget(target, eventObj, opts = {}) {
+  const trigger = opts.trigger || target;
+  const ir = workflowIrForTarget(target);
+  const limits = {
+    ...(ir.limits || {}),
+    ...(target.limits || {}),
+    ...(trigger.limits || {}),
+  };
+  const maxConcurrentRuns = positiveInteger(limits.maxConcurrentRuns);
+  if (!maxConcurrentRuns) return runWorkflowTargetNow(target, eventObj, opts);
+
+  const triggerId = opts.triggerId || trigger.id;
+  const key = target.id || triggerId || 'event-workflow';
+  let state = workflowRunQueues.get(key);
+  if (!state) {
+    state = { active: 0, queue: [] };
+    workflowRunQueues.set(key, state);
+  }
+
+  return new Promise((resolve, reject) => {
+    const item = { target, eventObj, opts, resolve, reject };
+    if (state.active < maxConcurrentRuns) {
+      runQueuedWorkflow(key, state, item);
+      return;
+    }
+
+    const queueLimit = queueLimitValue(limits.queueLimit);
+    if (state.queue.length >= queueLimit) {
+      const timestamp = new Date().toISOString();
+      const error = new Error(`Workflow queue is full for "${target.name || target.id || triggerId}" (${queueLimit} queued).`);
+      recordEventRun(triggerId, {
+        runId: crypto.randomUUID(),
+        triggerName: trigger.name,
+        workflowId: target.id,
+        workflowName: target.name,
+        startedAt: timestamp,
+        finishedAt: timestamp,
+        status: 'rejected',
+        projectPath: target.projectPath || trigger.projectPath || null,
+        error: error.message,
+      }, { source: 'event_runs:workflow:queue-full' });
+      reject(error);
+      return;
+    }
+
+    state.queue.push(item);
+  });
+}
+
+function executeEventWorkflow(trigger, eventObj, opts = {}) {
+  try {
+    const targets = workflowTargetsForTrigger(trigger);
+    if (targets.length === 0) return Promise.resolve([]);
+    return Promise.all(targets.map((target) => executeWorkflowTarget(target, eventObj, {
+      ...opts,
+      trigger,
+      triggerId: opts.triggerId || trigger.id,
+    })));
+  } catch (error) {
+    return Promise.reject(error);
+  }
+}
+
+module.exports = {
+  executeEventWorkflow,
+  executeWorkflowTarget,
+  runFunctionSource,
+};