@pipefy/pipefy-process-coder 0.1.0

package/src/pipefy/reconstruct.ts

@@ -0,0 +1,775 @@
+import { LIMITS } from '../config.ts';
+import { debug, progress, progressDone, step, warn } from '../util/log.ts';
+import { mapPool } from '../util/pool.ts';
+import {
+  ENTITY_NAMES,
+  emptyRelations,
+  fullCoverage,
+  type Coverage,
+  type EntityName,
+  type PayloadEnvelope,
+  type RepoPayload,
+  type Row,
+} from '../model/payload.ts';
+import type { PipefyClient } from './client.ts';
+import type { InternalApi } from './internal.ts';
+
+/**
+ * Reconstruct a snapshot-shaped payload from the public GraphQL API plus the
+ * internal API — the read path for organizations without snapshots.
+ *
+ * Every query here is taken from the Change Migrator recipes, which do exactly
+ * this in production. What the recipes read, we read; what they cannot see, we
+ * mark `unknown` rather than empty, so a later pack or diff never reads absence
+ * as deletion.
+ *
+ * Field names the plan lists as unverified are requested through introspection
+ * pruning (client.prune), so an unknown column costs us that column instead of
+ * the whole query.
+ */
+
+type Spec = {
+  /** Field name on the GraphQL type. */
+  gql: string;
+  /** Sub-selection, for object-valued fields. */
+  sub?: string;
+  /** Column name in the snapshot payload. */
+  to: string;
+  /** Pull the payload value out of the GraphQL value. */
+  pick?: (v: unknown) => unknown;
+};
+
+const firstId = (v: unknown): unknown => {
+  if (v && typeof v === 'object' && 'id' in (v as Row)) return (v as Row)['id'];
+  return v;
+};
+
+const numOrNull = (v: unknown): number | null => {
+  if (v === null || v === undefined || v === '') return null;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : null;
+};
+
+/** Build a pruned selection plus the mapper that turns the result into a payload row. */
+const build = async (client: PipefyClient, typeName: string, specs: Spec[]) => {
+  const have = await client.typeFields(typeName);
+  const kept = specs.filter((s) => !have || have.has(s.gql));
+  const skipped = specs.filter((s) => have && !have.has(s.gql)).map((s) => s.gql);
+  if (skipped.length) debug(`${typeName}: not in schema — ${skipped.join(', ')}`);
+  const selection = kept.map((s) => (s.sub ? `${s.gql} ${s.sub}` : s.gql)).join('\n        ');
+  const map = (src: Row): Row => {
+    const row: Row = {};
+    for (const s of kept) {
+      if (!(s.gql in src)) continue;
+      row[s.to] = s.pick ? s.pick(src[s.gql]) : src[s.gql];
+    }
+    return row;
+  };
+  return { selection, map, skipped, missing: skipped.length > 0 };
+};
+
+/** The API returns `settings` as a JSON string; the payload stores the object. */
+/** An object whose every value is null carries no more meaning than null. */
+const emptyToNull = (v: unknown): unknown => {
+  if (v === null || v === undefined) return null;
+  if (Array.isArray(v)) return v.length ? v : null;
+  if (typeof v === 'object') {
+    const values = Object.values(v as Row);
+    return values.length && values.some((x) => x !== null && x !== undefined) ? v : null;
+  }
+  return v;
+};
+
+/** The payload stores "no value" here as {}, the API as [] or null. */
+const emptyToObject = (v: unknown): unknown => {
+  if (v === null || v === undefined) return {};
+  if (Array.isArray(v)) return v.length ? v : {};
+  if (typeof v === 'object' && Object.keys(v as Row).length === 0) return {};
+  return v;
+};
+
+const parseJsonish = (v: unknown): unknown => {
+  if (typeof v !== 'string') return v;
+  try {
+    return JSON.parse(v);
+  } catch {
+    return v;
+  }
+};
+
+const PIPE_SPECS: Spec[] = [
+  { gql: 'id', to: 'id', pick: numOrNull },
+  { gql: 'name', to: 'name' },
+  { gql: 'uuid', to: 'uuid' },
+  { gql: 'noun', to: 'noun' },
+  { gql: 'icon', to: 'icon' },
+  // The payload's colour column is `color_uuid`, which the API does not expose;
+  // `color` is a different representation of the same thing, so it is metadata.
+  { gql: 'color', to: '_color_name' },
+  { gql: 'public', to: 'public' },
+  { gql: 'description', to: 'description' },
+  { gql: 'expiration_time', to: 'expiration_time' },
+  { gql: 'create_card_label', to: 'create_card_label' },
+  { gql: 'anyone_can_create_card', to: 'anyone_can_create_card' },
+  { gql: 'only_admin_can_remove_cards', to: 'only_admin_can_remove_cards' },
+  { gql: 'only_assignees_can_edit_cards', to: 'only_assignees_can_edit_cards' },
+  { gql: 'count_only_week_days', to: 'count_only_week_days' },
+  { gql: 'suid', to: 'suid' },
+  { gql: 'cards_count', to: 'cards_count' },
+  { gql: 'users_count', to: 'users_count' },
+  { gql: 'organizationId', to: 'organization_id', pick: numOrNull },
+  { gql: 'organization', sub: '{ id uuid }', to: 'organization_uuid', pick: (v) => (v as Row | null)?.['uuid'] ?? null },
+  { gql: 'title_field', sub: '{ id internal_id }', to: 'title_field_id', pick: (v) => numOrNull((v as Row | null)?.['internal_id'] ?? null) },
+  { gql: 'startFormPhaseId', to: '_start_form_phase_id' },
+];
+
+/**
+ * Verified against api.pipefy.com by introspection, not assumed.
+ *
+ * `next_phase_ids` is public, but it is a *forward-only* view of what the
+ * payload calls `phase_jumps` — see readPhaseJumps for the measurement. It is
+ * read here so it can serve as a marked-partial fallback, never as the primary
+ * source.
+ */
+const PHASE_SPECS: Spec[] = [
+  { gql: 'id', to: 'id', pick: numOrNull },
+  { gql: 'name', to: 'name' },
+  { gql: 'uuid', to: 'uuid' },
+  { gql: 'done', to: 'done' },
+  { gql: 'description', to: 'description' },
+  { gql: 'lateness_time', to: 'lateness_time' },
+  { gql: 'index', to: 'index' },
+  { gql: 'can_receive_card_directly_from_draft', to: 'can_receive_card_directly_from_draft' },
+  { gql: 'identifyTask', to: 'identify_task' },
+  { gql: 'custom_sorting_preferences', to: 'custom_sorting_preferences' },
+  { gql: 'cards_count', to: 'cards_count' },
+  { gql: 'next_phase_ids', to: '_next_phase_ids' },
+];
+
+/**
+ * The type is `PhaseField`, not `Field` — pruning against a type name that does
+ * not exist filters nothing, so the query goes out with every speculative
+ * column and fails wholesale. That is how this list came to be verified.
+ *
+ * `internal_id` is the numeric id the snapshot payload calls `id`, and the
+ * public `id` is the label-derived slug. Getting this backwards would make every
+ * field reference in every automation meaningless — recipe steps 46/47 key their
+ * whole dev→prod map on `internal_id`.
+ *
+ * Several payload columns are camelCase here and there is no `unique` at all.
+ */
+const FIELD_SPECS: Spec[] = [
+  { gql: 'internal_id', to: 'id', pick: numOrNull },
+  { gql: 'id', to: 'slug' },
+  { gql: 'uuid', to: 'uuid' },
+  { gql: 'label', to: 'label' },
+  { gql: 'type', to: 'type_id' },
+  { gql: 'options', to: 'options' },
+  { gql: 'description', to: 'description' },
+  { gql: 'help', to: 'help' },
+  { gql: 'required', to: 'required' },
+  { gql: 'editable', to: 'editable' },
+  { gql: 'minimal_view', to: 'minimal_view' },
+  { gql: 'custom_validation', to: 'custom_validation' },
+  { gql: 'index', to: 'index' },
+  { gql: 'is_multiple', to: '_is_multiple' },
+  { gql: 'settings', to: 'settings', pick: parseJsonish },
+  { gql: 'synced_with_card', to: 'card_synced' },
+  { gql: 'archived', to: '_archived' },
+  { gql: 'connectedRepo', sub: '{ __typename ... on PublicPipe { id } ... on PublicTable { id internal_id } }', to: 'connected_pipe_id', pick: (v) => connectedRepoId(v) },
+  { gql: 'canCreateNewConnected', to: 'can_create_connected_cards' },
+  { gql: 'canConnectExisting', to: 'can_search_connected_cards' },
+  { gql: 'canConnectMultiples', to: 'can_connect_multiple_cards' },
+  { gql: 'childMustExistToFinishParent', to: 'child_must_exist_to_finish_parent' },
+  { gql: 'allChildrenMustBeDoneToMoveParent', to: 'all_children_must_be_done_to_move_parent' },
+  { gql: 'allChildrenMustBeDoneToFinishParent', to: 'all_children_must_be_done_to_finish_parent' },
+];
+
+/**
+ * A connector's target, as the numeric id the payload uses. A connected pipe
+ * answers with its numeric `id`; a connected database answers with its slug and
+ * carries the numeric form in `internal_id` (PLAN.md §2.2, id duality).
+ */
+const connectedRepoId = (v: unknown): unknown => {
+  if (!v || typeof v !== 'object') return null;
+  const repo = v as Row;
+  return numOrNull(repo['internal_id'] ?? repo['id']);
+};
+
+export type ReconstructResult = PayloadEnvelope;
+
+export type ReconstructOpts = {
+  /** Organization id, required by the internal automations query. */
+  organizationId?: string | null;
+  /** Skip the internal API entirely (automations + phase jumps become unknown). */
+  noInternal?: boolean;
+};
+
+export class Reconstructor {
+  private client: PipefyClient;
+  private internal: InternalApi;
+  /** Decided once on the first automation, then reused. */
+  private richAutomationParams: boolean | null = null;
+  /** Public forward-only jump sets, kept as a fallback for phase_jumps. */
+  private publicNextPhaseIds = new Map<number, number[]>();
+
+  constructor(client: PipefyClient, internal: InternalApi) {
+    this.client = client;
+    this.internal = internal;
+  }
+
+  async reconstruct(repoId: string | number, opts: ReconstructOpts = {}): Promise<ReconstructResult> {
+    const coverage = fullCoverage('unknown');
+    const notes: string[] = [];
+    const relations = emptyRelations();
+
+    const mark = (n: EntityName, c: Coverage) => {
+      coverage[n] = c;
+    };
+
+    step(`reading pipe ${repoId} over GraphQL (no snapshot)`);
+
+    // ── root + phases + fields ───────────────────────────────────────────────
+    const [pipe, phase, field] = await Promise.all([
+      build(this.client, 'Pipe', PIPE_SPECS),
+      build(this.client, 'Phase', PHASE_SPECS),
+      build(this.client, 'PhaseField', FIELD_SPECS),
+    ]);
+
+    const data = await this.client.raw<{ pipe: Row | null }>(
+      `query($id: ID!) {
+        pipe(id: $id) {
+        ${pipe.selection}
+          phases {
+        ${phase.selection}
+            fields {
+        ${field.selection}
+            }
+          }
+          start_form_fields {
+        ${field.selection}
+          }
+        }
+      }`,
+      { id: String(repoId) },
+      'reconstruct pipe',
+    );
+
+    const src = data.pipe;
+    if (!src) throw new Error(`pipe ${repoId} not found, or the token cannot read it`);
+
+    const root = pipe.map(src);
+    const startFormPhaseId = numOrNull(root['_start_form_phase_id']);
+    delete root['_start_form_phase_id'];
+    root['id'] = Number(root['id'] ?? repoId);
+    if (pipe.missing) notes.push(`pipe columns not in this schema: ${pipe.skipped.join(', ')}`);
+
+    const gqlPhases = (src['phases'] as Row[] | undefined) ?? [];
+    const startFormFields = (src['start_form_fields'] as Row[] | undefined) ?? [];
+
+    /**
+     * The start form is a phase (PLAN.md §4, at index 0). The public
+     * API splits it out as `start_form_fields`, so we put it back — and only
+     * synthesise a phase row for it when `phases` does not already include it.
+     */
+    const phaseRows: Row[] = [];
+    const haveStartFormPhase = gqlPhases.some((p) => numOrNull(p['id']) === startFormPhaseId);
+    if (startFormPhaseId !== null && !haveStartFormPhase) {
+      phaseRows.push({ id: startFormPhaseId, name: 'Start form', index: 0, done: false, repo_id: root['id'] });
+    }
+    for (const p of gqlPhases) phaseRows.push({ ...phase.map(p), repo_id: root['id'] });
+
+    // Public phases come back in order; the payload's `index` is authoritative
+    // when present, otherwise array order is the only truth we have.
+    const anyIndex = phaseRows.some((p) => typeof p['index'] === 'number');
+    phaseRows.sort((a, b) => {
+      if (anyIndex) return Number(a['index'] ?? 0) - Number(b['index'] ?? 0);
+      return numOrNull(a['id'])! - numOrNull(b['id'])!;
+    });
+    phaseRows.forEach((p, i) => {
+      if (!anyIndex) p['index'] = i;
+    });
+    relations.phases = phaseRows;
+    mark('phases', phase.missing ? 'partial' : 'complete');
+
+    /**
+     * `Phase.next_phase_ids` is public, but it is **not** the payload's
+     * `phase_jumps`. Checked against a snapshot of the same pipe:
+     *
+     *   payload phase_jumps   1,2,3,1,2,2,2,2,1  targets per phase
+     *   next_phase_ids        0,1,1,0,1,1,1,1,0
+     *
+     * The payload merges forward *and backward* jumps into one array; the public
+     * field carries the forward set only. Mapping it straight onto `phase_jumps`
+     * produced a tree that, on apply, would have deleted every backward jump —
+     * so it is kept only as a fallback, and marked `partial` when used.
+     */
+    const publicNext = new Map<number, number[]>();
+    for (const p of phaseRows) {
+      const targets = p['_next_phase_ids'];
+      delete p['_next_phase_ids'];
+      const id = numOrNull(p['id']);
+      if (id !== null && Array.isArray(targets)) {
+        publicNext.set(id, targets.map(numOrNull).filter((t): t is number => t !== null));
+      }
+    }
+    this.publicNextPhaseIds = publicNext;
+
+    const fieldRows: Row[] = [];
+    const pushFields = (rows: Row[], phaseId: number | null) => {
+      rows.forEach((f, i) => {
+        const row = field.map(f);
+        row['phase_id'] = phaseId;
+        row['repo_id'] = root['id'];
+        if (row['index'] === undefined) row['index'] = i;
+
+        /**
+         * The payload records archiving as a timestamp (`archived_at`); the API
+         * exposes a boolean. The distinction matters because the diff must tell
+         * archive from delete or it destroys data when the FDE meant to hide a
+         * field (PLAN.md §9) — so the flag is carried across, and the timestamp
+         * is left null rather than invented.
+         */
+        if ('_archived' in row) {
+          const archived = row['_archived'] === true;
+          delete row['_archived'];
+          row['archived_at'] = archived ? (row['archived_at'] ?? null) : null;
+          if (archived) row['_archived'] = true;
+        }
+        // A connector field's target is the discovery edge PLAN.md §2.2 depends on.
+        if (row['connected_pipe_id'] !== undefined) row['connected_pipe_id'] = numOrNull(row['connected_pipe_id']);
+        fieldRows.push(row);
+      });
+    };
+    pushFields(startFormFields, startFormPhaseId);
+    for (const p of gqlPhases) pushFields((p['fields'] as Row[] | undefined) ?? [], numOrNull(p['id']));
+    relations.fields = fieldRows;
+    mark('fields', field.missing ? 'partial' : 'complete');
+    if (field.missing) notes.push(`field columns not in this schema: ${field.skipped.join(', ')}`);
+
+    // ── labels ──────────────────────────────────────────────────────────────
+    const labels = await this.client.tryRaw<{ pipe: { labels: Row[] | null } | null }>(
+      `query($id: ID!) { pipe(id: $id) { labels { id name color } } }`,
+      { id: String(repoId) },
+      'labels',
+    );
+    if (labels?.pipe?.labels) {
+      relations.labels = labels.pipe.labels.map((l) => ({
+        id: numOrNull(l['id']),
+        name: l['name'],
+        color: l['color'],
+        repo_id: root['id'],
+      }));
+      // No `uuid` over the public API, and uuid is how labels are identified in
+      // a snapshot — so this is partial, not complete.
+      mark('labels', 'partial');
+    } else {
+      notes.push('labels: not readable over this endpoint');
+    }
+
+    // ── webhooks ────────────────────────────────────────────────────────────
+    const wh = await this.client.tryRaw<{ pipe: { webhooks: Row[] | null } | null }>(
+      `query($id: ID!) { pipe(id: $id) { webhooks { id name url email actions headers } } }`,
+      { id: String(repoId) },
+      'webhooks',
+    );
+    if (wh?.pipe?.webhooks) {
+      relations.webhooks = wh.pipe.webhooks.map((w) => ({
+        id: numOrNull(w['id']),
+        name: w['name'],
+        url: w['url'],
+        email: w['email'] ?? null,
+        actions: w['actions'] ?? [],
+        // Returned as a JSON string, stored as an object — same as field settings.
+        headers: parseJsonish(w['headers']) ?? {},
+        repo_id: root['id'],
+      }));
+      mark('webhooks', 'complete');
+    } else {
+      notes.push('webhooks: not readable over this endpoint');
+    }
+
+    // ── pipe_relations ──────────────────────────────────────────────────────
+    const prSel = await this.client.prune('Pipe', [
+      ['pipe_relations', '{ id name parent { id } child { id } canCreateNewItems canConnectExistingItems canConnectMultipleItems childMustExistToMoveParent childMustExistToFinishParent allChildrenMustBeDoneToMoveParent allChildrenMustBeDoneToFinishParent }'],
+    ]);
+    if (prSel) {
+      const pr = await this.client.tryRaw<{ pipe: { pipe_relations: Row[] | null } | null }>(
+        `query($id: ID!) { pipe(id: $id) { ${prSel} } }`,
+        { id: String(repoId) },
+        'pipe_relations',
+      );
+      if (pr?.pipe?.pipe_relations) {
+        relations.pipe_relations = pr.pipe.pipe_relations.map((r) => ({
+          id: numOrNull(r['id']),
+          name: r['name'],
+          parent_id: numOrNull(firstId(r['parent'])),
+          child_id: numOrNull(firstId(r['child'])),
+          can_create_new_items: r['canCreateNewItems'],
+          can_connect_existing_items: r['canConnectExistingItems'],
+          can_connect_multiple_items: r['canConnectMultipleItems'],
+          child_must_exist_to_move_parent: r['childMustExistToMoveParent'],
+          child_must_exist_to_finish_parent: r['childMustExistToFinishParent'],
+          all_children_must_be_done_to_move_parent: r['allChildrenMustBeDoneToMoveParent'],
+          all_children_must_be_done_to_finish_parent: r['allChildrenMustBeDoneToFinishParent'],
+        }));
+        mark('pipe_relations', 'partial');
+      }
+    }
+
+    // ── field conditions (+ actions, condition, expressions) ────────────────
+    await this.readFieldConditions(repoId, root, relations, mark, notes);
+
+    // ── automations (+ conditions, expressions, field_maps) ─────────────────
+    if (opts.noInternal) {
+      notes.push('automations and phase jumps skipped (--no-internal)');
+    } else {
+      await this.readAutomations(repoId, root, relations, mark, notes, opts.organizationId ?? null);
+      await this.readPhaseJumps(relations, mark, notes);
+    }
+
+    for (const n of ['public_forms', 'repo_preferences', 'visibilities', 'email_templates', 'email_inboxes'] as EntityName[]) {
+      if (coverage[n] === 'unknown') {
+        notes.push(`${n}: no read path outside a snapshot — left unknown, never treated as empty`);
+      }
+    }
+
+    /**
+     * No reconstructed group is ever 'complete'.
+     *
+     * Row coverage and *column* coverage are different things: the public API
+     * returns every phase, but not `color_uuid`, `only_admin_can_move_to_previous`
+     * or `repo_id`. Marking such a group complete made a snapshot-vs-reconstruction
+     * diff report ten phantom changes per phase, each one a proposal to null a
+     * column that had simply not been read. 'partial' is the honest ceiling, and it
+     * is what tells the diff to treat absence here as no information.
+     */
+    for (const name of ENTITY_NAMES) {
+      if (coverage[name] === 'complete') coverage[name] = 'partial';
+    }
+
+    const payload: RepoPayload = { ...root, id: Number(root['id']), relations };
+    return {
+      source: 'reconstructed',
+      versionId: null,
+      readAt: new Date().toISOString(),
+      repoKind: 'pipe',
+      coverage,
+      repo_columns: pipe.missing ? 'partial' : 'complete',
+      notes,
+      payload,
+    };
+  }
+
+  /**
+   * Recipe `Pipe Data Treatment` steps 49/50, split into the four payload
+   * arrays a snapshot would have produced.
+   */
+  private async readFieldConditions(
+    repoId: string | number,
+    root: Row,
+    relations: ReturnType<typeof emptyRelations>,
+    mark: (n: EntityName, c: Coverage) => void,
+    notes: string[],
+  ) {
+    const q = `query($id: ID!) {
+      pipe(id: $id) {
+        fieldConditions {
+          id
+          name
+          phase { id }
+          actions {
+            actionId
+            whenEvaluator
+            phaseField { internal_id label }
+          }
+          condition {
+            expressions_structure
+            expressions { field_address operation value structure_id }
+          }
+        }
+      }
+    }`;
+    const data = await this.client.tryRaw<{ pipe: { fieldConditions: Row[] | null } | null }>(
+      q,
+      { id: String(repoId) },
+      'fieldConditions',
+    );
+    const fcs = data?.pipe?.fieldConditions;
+    if (!fcs) {
+      notes.push('field_conditions: pipe.fieldConditions not readable — left unknown');
+      return;
+    }
+
+    fcs.forEach((fc, i) => {
+      const fcId = numOrNull(fc['id']);
+      relations.field_conditions.push({
+        id: fcId,
+        name: fc['name'],
+        phase_id: numOrNull(firstId(fc['phase'])),
+        index: i + 1,
+        repo_id: root['id'],
+      });
+
+      for (const a of (fc['actions'] as Row[] | undefined) ?? []) {
+        relations.field_condition_actions.push({
+          field_condition_id: fcId,
+          action_id: a['actionId'],
+          when_evaluator: a['whenEvaluator'],
+          field_id: numOrNull((a['phaseField'] as Row | null)?.['internal_id'] ?? null),
+          phase_id: null,
+        });
+      }
+
+      const cond = fc['condition'] as Row | null;
+      if (cond) {
+        relations.conditions.push({
+          source_type: 'FieldCondition',
+          source_id: fcId,
+          expressions_structure: cond['expressions_structure'] ?? [],
+        });
+        for (const e of (cond['expressions'] as Row[] | undefined) ?? []) {
+          const addr = e['field_address'];
+          relations.condition_expressions.push({
+            /** condition_id is resolved at pack time from the owner. */
+            condition_id: null,
+            _condition_owner: { source_type: 'FieldCondition', source_id: fcId },
+            field_address: addr,
+            field_id: numOrNull(addr),
+            operation: e['operation'],
+            value: e['value'],
+            structure_id: numOrNull(e['structure_id']),
+          });
+        }
+      }
+    });
+    mark('field_conditions', 'complete');
+    mark('field_condition_actions', 'complete');
+    mark('conditions', 'partial');
+    mark('condition_expressions', 'partial');
+  }
+
+  /**
+   * Recipe: list over the internal `getAutomations`, then `loadAutomationToEdit`
+   * per automation. The list alone omits `action_params`, so there is no
+   * shortcut — this is N+1 by necessity, bounded by readConcurrency.
+   */
+  private async readAutomations(
+    repoId: string | number,
+    root: Row,
+    relations: ReturnType<typeof emptyRelations>,
+    mark: (n: EntityName, c: Coverage) => void,
+    notes: string[],
+    organizationId: string | null,
+  ) {
+    const orgId = organizationId ?? (root['organization_id'] ? String(root['organization_id']) : null);
+    if (!orgId) {
+      notes.push('automations: no organization id available for the internal automations query');
+      return;
+    }
+
+    const list = await this.internal.listAutomations(orgId, repoId);
+    if (!list) {
+      notes.push('automations: internal API unavailable — left unknown, never treated as empty (PLAN.md §2.1)');
+      return;
+    }
+    if (list.length === 0) {
+      mark('automations', 'complete');
+      return;
+    }
+
+    let failures = 0;
+    const details = await mapPool(list, LIMITS.readConcurrency, async (a, i) => {
+      progress(`automation ${i + 1}/${list.length}: ${String(a['name'] ?? a['id'])}`);
+      const full = await this.internal.loadAutomation(String(a['id']));
+      if (!full) failures++;
+      return { summary: a, full };
+    });
+    progressDone();
+
+    for (const { summary, full } of details) {
+      const id = numOrNull(summary['id']);
+      if (!full) {
+        // Recording the automation without its params is honest; silently
+        // dropping it would let a later diff propose deleting it.
+        relations.automations.push({
+          id,
+          name: summary['name'],
+          event_id: summary['event_id'],
+          action_id: summary['action_id'],
+          active: summary['active'],
+          event_repo_id: numOrNull(firstId(summary['event_repo'])),
+          action_repo_id: numOrNull(firstId(summary['action_repo'])),
+          _incomplete: true,
+        });
+        continue;
+      }
+
+      const ep = (full['event_params'] as Row | null) ?? {};
+      const ap = (full['action_params'] as Row | null) ?? {};
+
+      relations.automations.push({
+        id,
+        name: full['name'],
+        event_id: full['event_id'],
+        action_id: full['action_id'],
+        active: full['active'],
+        event_repo_id: numOrNull(firstId(full['event_repo'])),
+        action_repo_id: numOrNull(firstId(full['action_repo'])),
+        organization_id: numOrNull(orgId),
+        scheduler_frequency: full['scheduler_frequency'] ?? null,
+        // An all-null cron object means "no schedule", which the payload stores
+        // as null; an empty searchFor list means "no filter", stored as {}.
+        scheduler_cron: emptyToNull(full['schedulerCron']),
+        search_for: emptyToObject(full['searchFor']),
+        response_schema: full['responseSchema'] ?? null,
+        event_params: snakeParams(ep),
+        action_params: snakeParams(ap),
+      });
+
+      for (const fm of ((ap['fieldMap'] as Row[] | undefined) ?? [])) {
+        relations.field_maps.push({
+          owner_type: 'Automation',
+          owner_id: id,
+          field_id: numOrNull(fm['fieldId']),
+          value: fm['value'] ?? '',
+          input_mode: fm['inputMode'] ?? null,
+        });
+      }
+
+      const cond = full['condition'] as Row | null;
+      if (cond && ((cond['expressions'] as Row[] | undefined)?.length || cond['expressions_structure'])) {
+        relations.conditions.push({
+          source_type: 'Automation',
+          source_id: id,
+          expressions_structure: cond['expressions_structure'] ?? [],
+        });
+        for (const e of (cond['expressions'] as Row[] | undefined) ?? []) {
+          relations.condition_expressions.push({
+            condition_id: null,
+            _condition_owner: { source_type: 'Automation', source_id: id },
+            field_address: e['field_address'],
+            field_id: numOrNull(e['field_address']),
+            operation: e['operation'],
+            value: e['value'],
+            structure_id: numOrNull(e['structure_id']),
+          });
+        }
+      }
+    }
+
+    mark('automations', failures ? 'partial' : 'complete');
+    mark('field_maps', failures ? 'partial' : 'complete');
+    if (relations.conditions.length) mark('conditions', 'partial');
+    if (failures) notes.push(`automations: ${failures}/${list.length} could not be loaded in full`);
+  }
+
+  /** One `GET /internal_api/settings/phases/:id` per phase. Recipe step 23. */
+  private async readPhaseJumps(
+    relations: ReturnType<typeof emptyRelations>,
+    mark: (n: EntityName, c: Coverage) => void,
+    notes: string[],
+  ) {
+    const phases = relations.phases;
+    if (!phases.length) return;
+    let failures = 0;
+    const results = await mapPool(phases, LIMITS.readConcurrency, async (p, i) => {
+      progress(`phase jumps ${i + 1}/${phases.length}`);
+      const s = await this.internal.getPhaseSettings(String(p['id']));
+      if (!s) failures++;
+      return { phase: p, settings: s };
+    });
+    progressDone();
+
+    for (const { phase, settings } of results) {
+      if (!settings) continue;
+      // `jump_targets` is already the merged set the payload stores.
+      for (const t of settings.jump_targets) {
+        relations.phase_jumps.push({ source_phase_id: numOrNull(phase['id']), target_phase_id: t });
+      }
+    }
+
+    if (failures === phases.length) {
+      // Fall back to the public forward-only set, which is better than nothing
+      // and honestly marked as incomplete.
+      if (this.publicNextPhaseIds.size) {
+        for (const [source, targets] of this.publicNextPhaseIds) {
+          for (const t of targets) relations.phase_jumps.push({ source_phase_id: source, target_phase_id: t });
+        }
+        mark('phase_jumps', 'partial');
+        notes.push(
+          'phase_jumps: read from the public Phase.next_phase_ids because the internal settings ' +
+            'endpoint was unavailable — forward jumps only, backward jumps unknown',
+        );
+        return;
+      }
+      notes.push('phase_jumps: internal settings endpoint unavailable — left unknown');
+      return;
+    }
+
+    mark('phase_jumps', failures ? 'partial' : 'complete');
+    if (failures) notes.push(`phase_jumps: ${failures}/${phases.length} phases unreadable`);
+  }
+}
+
+/** camelCase params from the internal API back to the payload's snake_case. */
+const CAMEL_TO_SNAKE: Record<string, string> = {
+  toPhaseId: 'to_phase_id',
+  fromPhaseId: 'from_phase_id',
+  inPhaseId: 'in_phase_id',
+  triggerFieldIds: 'trigger_field_ids',
+  kindOfSla: 'kind_of_sla',
+  triggerAutomationId: 'trigger_automation_id',
+  fieldsMapOrder: 'fields_map_order',
+  cardId: 'card_id',
+  httpMethod: 'http_method',
+  authenticationKey: 'authentication_key',
+  hasAuthenticationValue: 'has_authentication_value',
+  authenticationAddTo: 'authentication_add_to',
+  aiParams: 'ai_params',
+  slaParams: 'sla_params',
+  taskParams: 'task_params',
+  fieldIds: 'field_ids',
+  skillsIds: 'skills_ids',
+  startHour: 'start_hour',
+  endHour: 'end_hour',
+  fieldId: 'field_id',
+  inputMode: 'input_mode',
+  dayOfWeek: 'day_of_week',
+  dayOfMonth: 'day_of_month',
+};
+
+/**
+ * Keys the internal API derives and the payload does not store. Carrying them
+ * across made every automation report a phantom `action_params` change.
+ */
+const DERIVED_PARAM_KEYS = new Set(['hasAuthenticationValue', 'has_authentication_value']);
+
+const snakeParams = (v: unknown): unknown => {
+  if (Array.isArray(v)) return v.map(snakeParams);
+  if (v && typeof v === 'object') {
+    const out: Row = {};
+    for (const [k, val] of Object.entries(v as Row)) {
+      if (k === 'fieldMap') continue; // hoisted into field_maps
+      if (DERIVED_PARAM_KEYS.has(k)) continue;
+      if (val === null || val === undefined) continue;
+      out[CAMEL_TO_SNAKE[k] ?? k] = snakeParams(val);
+    }
+    return out;
+  }
+  return v;
+};
+
+/** Reported by `pipe doctor` so an FDE knows what a reconstructed pull cannot see. */
+export const RECONSTRUCTION_GAPS = [
+  'public_forms — no public read path',
+  'repo_preferences — no public read path',
+  'visibilities — no public read path (read-only anyway)',
+  'email_templates / email_inboxes — no public read path (read-only anyway)',
+  'labels.uuid — not exposed publicly, so labels are matched by id and name',
+  'response_schemas — kept on the automation row, not as a separate array',
+] as const;
+
+export const warnAboutGaps = () => {
+  warn('reconstructed read: ' + RECONSTRUCTION_GAPS.length + ' entity groups are unknown rather than empty');
+};