@pipefy/pipefy-process-coder 0.1.0

package/src/commands/flows.ts

@@ -0,0 +1,414 @@
+import { dim, info, warn } from '../util/log.ts';
+import { Progress } from '../util/tui.ts';
+import { assertFlowRoundTrip, unlinkFlow } from '../codec/flow.ts';
+import {
+  readFlows,
+  writeFlows,
+  writeFlowBaseline,
+  readFlowBaseline,
+  unknownFlows,
+  flowIdOf,
+  type FlowsSection,
+} from '../workspace/flows.ts';
+import { diffFlows, type FlowChangeSet } from '../diff/flow.ts';
+import { buildFlowOperations, compileFlows, type FlowPlan, type FlowStep } from '../apply/flowops.ts';
+import { IpaasClient, IpaasUnavailable, type ConnectionRow } from '../pipefy/ipaas.ts';
+
+/**
+ * The flow half of the pipe loop: read on pull, diff, compile, apply.
+ *
+ * Kept in one module because the flow path is small and shares nothing with the
+ * pipe mutation compiler — the operations are per node against a different
+ * endpoint, and pretending otherwise would force both through a registry that
+ * describes GraphQL mutations.
+ */
+
+/** Read a pipe's whole iPaaS workspace, or record why it could not be read. */
+export const pullFlowsFor = async (
+  client: IpaasClient,
+  pipeId: number,
+  opts: { skipRoundTrip?: boolean } = {},
+): Promise<FlowsSection> => {
+  const probe = await client.available(pipeId);
+  if (!probe.ok) {
+    /**
+     * Not an error. iPaaS is gated per organization and per pipe, so "no token"
+     * is the normal answer for most pipes — and it must never be read as "this
+     * pipe has no flows", which is what would license deleting them later.
+     */
+    return unknownFlows(probe.reason);
+  }
+
+  const { projectId } = probe.session;
+  const rows = await client.listFlows(pipeId);
+  const flows = [];
+  const notes: string[] = [];
+
+  for (const row of rows) {
+    const full = await client.getFlow(pipeId, String(row['id']));
+    if (!opts.skipRoundTrip) {
+      const rt = assertFlowRoundTrip(full);
+      if (!rt.ok) {
+        // The fold is not lossless for this flow, so the split files would not
+        // rebuild it. Reported, and the raw row is what the diff will use.
+        notes.push(
+          `flow ${row['id']}: codec round trip is not clean (${rt.differences
+            .map((d) => `${d.where}: ${d.detail}`)
+            .join('; ')})`,
+        );
+      }
+    }
+    flows.push(unlinkFlow(full));
+  }
+
+  const [connections, pieces, runs] = await Promise.all([
+    client.listConnections(pipeId),
+    client.listPieces(pipeId),
+    client.listRuns(pipeId),
+  ]);
+
+  return {
+    meta: {
+      coverage: 'complete',
+      projectId,
+      readAt: new Date().toISOString(),
+      flows: flows.length,
+      connections: connections.length,
+      pieces: Object.keys(pieces).length,
+      notes,
+    },
+    flows,
+    connections,
+    pieces,
+    runs,
+  };
+};
+
+/** Pull and write, with a line per flow. Returns what was read. */
+export const pullAndWriteFlows = async (
+  client: IpaasClient,
+  pipeId: number,
+  repoDir: string,
+  opts: { skipRoundTrip?: boolean; workspaceRoot?: string } = {},
+): Promise<FlowsSection> => {
+  const section = await pullFlowsFor(client, pipeId, opts);
+  await writeFlows(repoDir, section);
+  /**
+   * And record it as the baseline.
+   *
+   * Without this the flow diff had no past to compare against and re-read the
+   * pipe live on every invocation — so there was no offline diff, and no record
+   * of what a pull actually brought back. The pipe half has stored its baseline
+   * since the first version; this is the flow half catching up.
+   */
+  if (opts.workspaceRoot) await writeFlowBaseline(opts.workspaceRoot, pipeId, section);
+  return section;
+};
+
+/**
+ * The flows diff: what is on disk against what was recorded at the last pull.
+ *
+ * Mirrors the pipe half — `baseline` needs no network, `live` re-reads the pipe.
+ * The default is the baseline, so `pipe diff` stays offline and answers the same
+ * question for both halves of a process: what did *I* change.
+ *
+ * `live` is what an apply uses, because what matters there is the current state
+ * of the pipe, not the state someone pulled an hour ago.
+ */
+export const diffFlowsFor = async (
+  client: IpaasClient | null,
+  pipeId: number,
+  repoDir: string,
+  opts: { against?: 'baseline' | 'live'; workspaceRoot: string },
+): Promise<{ changeSet: FlowChangeSet; before: FlowsSection | null; authored: FlowsSection | null }> => {
+  const authored = await readFlows(repoDir);
+  if ((opts.against ?? 'baseline') === 'baseline') {
+    const before = await readFlowBaseline(opts.workspaceRoot, pipeId);
+    if (!before) {
+      const reason =
+        'no flow baseline for this repo — re-pull to record one, or use --against live';
+      return { changeSet: { changes: [], skipped: [{ reason }] }, before: null, authored };
+    }
+    return { changeSet: diffFlows(before, authored), before, authored };
+  }
+  if (!client) {
+    const reason = 'a live flow diff needs the network, and no client was supplied';
+    return { changeSet: { changes: [], skipped: [{ reason }] }, before: null, authored };
+  }
+  const live = await pullFlowsFor(client, pipeId, { skipRoundTrip: true });
+  return { changeSet: diffFlows(live, authored), before: live, authored };
+};
+
+/** The flows diff against the live pipe. What an apply compiles from. */
+export const diffFlowsAgainstLive = async (
+  client: IpaasClient,
+  pipeId: number,
+  repoDir: string,
+): Promise<{ changeSet: FlowChangeSet; live: FlowsSection; authored: FlowsSection | null }> => {
+  const authored = await readFlows(repoDir);
+  const live = await pullFlowsFor(client, pipeId, { skipRoundTrip: true });
+  return { changeSet: diffFlows(live, authored), live, authored };
+};
+
+/**
+ * The revert plan, given both sides already read — pure, no network, so it
+ * is the part `test/flows-revert.test.ts` can actually exercise without a
+ * live `IpaasClient`.
+ *
+ * Direction is the mirror of an ordinary apply: `live` is the *current*
+ * state (baseline), `before` is the old pre-apply one (intended target). A
+ * node the failed apply added and `before` never had comes out as a delete;
+ * one it removed comes out as a create — the same diff engine, just pointed
+ * backwards.
+ */
+export const buildFlowRevertPlan = (live: FlowsSection, before: FlowsSection): FlowPlan => {
+  const changeSet = diffFlows(live, before);
+  /**
+   * Allowed unconditionally, unlike an ordinary apply's --allow-flow-enable
+   * gate: putting a flow back to enabled is restoring what was already true
+   * before this apply touched it, not arming something new. The
+   * confirmation prompt renders the plan either way, so an enable is never
+   * silent.
+   */
+  return planFlows(changeSet, { allowArm: true, connections: live.connections });
+};
+
+/**
+ * What reverting to a pre-apply flow state would take — computed, not sent.
+ *
+ * `before` is the live read `diffFlowsAgainstLive` already took right
+ * before this apply's own diff was computed — the only rollback point flows
+ * have at all, since Pipefy's snapshot restore never covered them (it is
+ * pipe structure only; see AGENTS.md). `live` here is read fresh, because
+ * some steps may have landed since `before` was taken.
+ */
+export const planFlowRevert = async (
+  client: IpaasClient,
+  pipeId: number,
+  before: FlowsSection,
+): Promise<{ plan: FlowPlan; live: FlowsSection }> => {
+  const live = await pullFlowsFor(client, pipeId, { skipRoundTrip: true });
+  return { plan: buildFlowRevertPlan(live, before), live };
+};
+
+export const planFlows = (
+  cs: FlowChangeSet,
+  opts: { allowArm?: boolean; connections?: ConnectionRow[] } = {},
+): FlowPlan =>
+  compileFlows(cs, {
+    allowArm: opts.allowArm,
+    connections: new Map(
+      (opts.connections ?? []).map((c) => [
+        c.externalId,
+        // A placeholder is reported as its own status so the compiler can refuse to
+        // arm a flow that depends on one.
+        { pieceName: c.pieceName, status: c._placeholder ? 'PLACEHOLDER' : c.status },
+      ]),
+    ),
+  });
+
+export const renderFlowPlan = (plan: FlowPlan): string => {
+  const lines: string[] = [];
+  if (plan.blocked.length) {
+    lines.push('');
+    lines.push(`  ${plan.blocked.length} flow change(s) cannot be applied:`);
+    for (const b of plan.blocked) {
+      lines.push(`    x ${b.label}`);
+      lines.push(`        ${b.reason}`);
+    }
+  }
+  if (!plan.steps.length) {
+    if (!plan.blocked.length) lines.push('  flows: nothing to apply');
+    return lines.join('\n');
+  }
+  lines.push('');
+  lines.push(`  ${plan.steps.length} flow step(s)`);
+  for (const s of plan.steps) {
+    const tags = [s.destructive ? 'destructive' : '', s.arms ? 'ARMS THE FLOW' : ''].filter(Boolean);
+    lines.push(`    ${s.id} ${s.description}${tags.length ? `  ${tags.join(' ')}` : ''}`);
+  }
+  return lines.join('\n');
+};
+
+export type FlowApplyResult = { done: number; failed: number; skipped: number; steps: FlowStep[] };
+
+/**
+ * Execute the flow steps.
+ *
+ * A created flow is laid down in one step from the outside — create, then trigger,
+ * then each node in execution order — because the flow id does not exist until the
+ * create returns, and every subsequent operation needs it. The alternative is
+ * placeholders in a second id space, which buys nothing here: the whole sequence
+ * is one flow's contents and either all of it lands or the flow is visibly
+ * half-built and reported as such.
+ */
+export const applyFlows = async (
+  client: IpaasClient,
+  pipeId: number,
+  plan: FlowPlan,
+  opts: { dryRun?: boolean; continueOnError?: boolean } = {},
+): Promise<FlowApplyResult> => {
+  const ui = new Progress({
+    title: opts.dryRun ? `dry run · ${plan.steps.length} flow steps` : `applying ${plan.steps.length} flow steps`,
+  });
+  for (const s of plan.steps) {
+    ui.add({
+      id: s.id,
+      label: s.description,
+      state: 'pending',
+      tags: [...(s.destructive ? ['destructive'] : []), ...(s.arms ? ['arms'] : [])],
+    });
+  }
+  ui.start();
+
+  for (const step of plan.steps) {
+    if (opts.dryRun) {
+      step.status = 'skipped';
+      ui.update(step.id, { state: 'done', detail: 'not sent' });
+      continue;
+    }
+
+    step.status = 'running';
+    ui.update(step.id, { state: 'running' });
+    try {
+      if (step.createFlow) {
+        const flowId = await client.createFlow(pipeId, step.createFlow.displayName);
+        step.flowId = flowId;
+        const ops = buildFlowOperations(step.createFlow.tree);
+        for (const [i, op] of ops.entries()) {
+          ui.update(step.id, { detail: `${i + 1}/${ops.length} ${op.describe}` });
+          await client.operate(pipeId, flowId, op.type, op.request);
+        }
+        step.result = { flowId, operations: ops.length };
+        ui.update(step.id, { state: 'done', detail: `flow ${flowId}, ${ops.length} operations` });
+      } else if (step.createConnection) {
+        const made = await client.createPlaceholderConnection(pipeId, step.createConnection);
+        step.result = made;
+        ui.update(step.id, {
+          state: 'done',
+          detail: `${made.externalId} — placeholder, reconnect before enabling`,
+        });
+      } else if (step.deleteFlow) {
+        await client.deleteFlow(pipeId, step.flowId as string);
+        step.status = 'done';
+        ui.update(step.id, { state: 'done' });
+      } else if (step.operation) {
+        step.result = await client.operate(
+          pipeId,
+          step.flowId as string,
+          step.operation.type,
+          step.operation.request,
+        );
+        ui.update(step.id, { state: 'done' });
+      }
+      step.status = 'done';
+    } catch (e) {
+      step.status = 'failed';
+      step.error = (e as Error).message;
+      ui.update(step.id, { state: 'failed', detail: step.error });
+      if (!opts.continueOnError) break;
+    }
+  }
+
+  const count = (s: FlowStep['status']) => plan.steps.filter((x) => x.status === s).length;
+  ui.stop();
+
+  /**
+   * Placeholders are announced after the run, not buried in the step list.
+   *
+   * The host marks any connection ACTIVE without validating its credential, so
+   * nothing downstream will tell you these do not work — the flow simply fails on
+   * every trigger once enabled.
+   */
+  const placeholders = plan.steps.filter((s) => s.createConnection && s.status === 'done');
+  if (placeholders.length && !opts.dryRun) {
+    info('');
+    warn(
+      `${placeholders.length} placeholder connection(s) were created — the flow is built but not ` +
+        'authenticated',
+    );
+    for (const s of placeholders) {
+      info(dim(`   ${s.createConnection!.externalId}  ${s.createConnection!.pieceName}`));
+    }
+    info(dim('   reconnect each under Advanced Automations > Connections before enabling a flow'));
+  }
+
+  return { done: count('done'), failed: count('failed'), skipped: count('skipped'), steps: plan.steps };
+};
+
+/**
+ * After applying, read the flows back and confirm every datapill survived exactly.
+ *
+ * This is the check nothing else can make. A migrating write path — `IMPORT_FLOW`
+ * is one — rewrites `{{step_1['output'].x}}` into `{{step_1['output']['output'].x}}`,
+ * which is structurally valid, passes every schema check, passes the platform's own
+ * validator, and resolves to nothing at runtime. Only comparing the stored string
+ * against the intended one finds it.
+ */
+export const verifyFlowDatapills = async (
+  client: IpaasClient,
+  pipeId: number,
+  authored: FlowsSection,
+): Promise<Array<{ flow: string; node: string; at: string; sent: string; stored: string }>> => {
+  const drift: Array<{ flow: string; node: string; at: string; sent: string; stored: string }> = [];
+
+  const pillsOf = (v: unknown, at: string, out: Array<{ at: string; pill: string }> = []) => {
+    if (typeof v === 'string') {
+      for (const m of v.matchAll(/\{\{[^}]+\}\}/g)) out.push({ at, pill: m[0] });
+      return out;
+    }
+    if (Array.isArray(v)) {
+      v.forEach((x, i) => pillsOf(x, `${at}[${i}]`, out));
+      return out;
+    }
+    if (v && typeof v === 'object') for (const [k, val] of Object.entries(v)) pillsOf(val, at ? `${at}.${k}` : k, out);
+    return out;
+  };
+
+  for (const tree of authored.flows) {
+    const id = flowIdOf(tree);
+    if (!id) continue;
+    let live;
+    try {
+      live = unlinkFlow(await client.getFlow(pipeId, id));
+    } catch {
+      continue;
+    }
+    const liveByName = new Map(live.nodes.map((n) => [n.name, n]));
+    for (const node of tree.nodes) {
+      const other = liveByName.get(node.name);
+      if (!other) continue;
+      const sent = pillsOf(node['settings'], 'settings');
+      const stored = pillsOf(other['settings'], 'settings');
+      for (const one of sent) {
+        const sameKey = stored.filter((x) => x.at === one.at);
+        if (!sameKey.length) continue;
+        if (!sameKey.some((x) => x.pill === one.pill)) {
+          drift.push({
+            flow: String(tree.version['displayName'] ?? id),
+            node: node.name,
+            at: one.at,
+            sent: one.pill,
+            stored: sameKey[0]!.pill,
+          });
+        }
+      }
+    }
+  }
+  return drift;
+};
+
+export const reportDatapillDrift = (drift: Awaited<ReturnType<typeof verifyFlowDatapills>>) => {
+  if (!drift.length) return;
+  warn(`${drift.length} datapill(s) are not stored as they were sent`);
+  for (const d of drift.slice(0, 8)) {
+    info(dim(`   ${d.flow} / ${d.node} ${d.at}`));
+    info(dim(`     sent   ${d.sent}`));
+    info(dim(`     stored ${d.stored}`));
+  }
+  if (drift.some((d) => d.stored.includes("['output']['output']"))) {
+    info(dim("   ['output']['output'] is a double migration — the write path migrated an already-live flow"));
+  }
+};
+
+export { IpaasUnavailable };