@pipefy/pipefy-process-coder 0.1.0

package/src/pipefy/discovery.ts

@@ -0,0 +1,209 @@
+import { debug, step, warn } from '../util/log.ts';
+import { rel, type RepoPayload, type Row } from '../model/payload.ts';
+import type { PipefyClient } from './client.ts';
+
+/**
+ * Connection discovery. PLAN.md §2.2 — the finding that would have silently
+ * broken the core requirement:
+ *
+ *   - `relations.pipe_relations` is [] even for a pipe with two live connections
+ *   - `pipeMapGraph(depth: 3)` returns one node and zero edges for that pipe
+ *
+ * So neither is the source of truth. Discovery is the union of three sources,
+ * walked transitively and deduplicated by repo identity.
+ */
+
+export type RepoKind = 'pipe' | 'table';
+
+export type RepoRef = {
+  kind: RepoKind;
+  /** Numeric id, which both the pipe and table APIs accept. */
+  id: number;
+  /** Table API ids round-trip as a slug (`dzmxcuxz`); pipes return numbers. */
+  apiId: string;
+  name?: string | null;
+  uuid?: string | null;
+  /** How this repo was found, for the manifest and the UI. */
+  via: string[];
+  depth: number;
+};
+
+export type DiscoveryResult = {
+  /** Every repo to pull, root first. */
+  repos: RepoRef[];
+  /** Edges, for the graph view and for MAPPING.md. */
+  edges: Array<{ from: number; to: number; kind: string; label?: string }>;
+  warnings: string[];
+};
+
+const asNum = (v: unknown): number | null => {
+  if (v === null || v === undefined || v === '') return null;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : null;
+};
+
+/** The three sources, applied to one already-read payload. */
+export const edgesFromPayload = (p: RepoPayload): Array<{ to: number; kind: string; label?: string }> => {
+  const self = Number(p.id);
+  const out: Array<{ to: number; kind: string; label?: string }> = [];
+
+  // 1. pipe_relations
+  for (const r of rel(p, 'pipe_relations')) {
+    for (const key of ['parent_id', 'child_id']) {
+      const id = asNum(r[key]);
+      if (id !== null && id !== self) out.push({ to: id, kind: 'pipe_relation', label: String(r['name'] ?? '') });
+    }
+  }
+
+  // 2. connector fields — the only source that finds the test pipe's two
+  //    connections, and the reason this function exists.
+  for (const f of rel(p, 'fields')) {
+    if (f['type_id'] !== 'connector') continue;
+    const id = asNum(f['connected_pipe_id']);
+    if (id !== null && id !== self) out.push({ to: id, kind: 'connector_field', label: String(f['label'] ?? '') });
+  }
+
+  // 3. cross-repo automations
+  for (const a of rel(p, 'automations')) {
+    for (const key of ['event_repo_id', 'action_repo_id']) {
+      const id = asNum(a[key]);
+      if (id !== null && id !== self) out.push({ to: id, kind: 'automation', label: String(a['name'] ?? '') });
+    }
+  }
+
+  return out;
+};
+
+/**
+ * Resolve a numeric repo id to what it actually is.
+ *
+ * **Neither root field discriminates.** Verified live: `pipe(id:)` and
+ * `table(id:)` both answer for *every* repo id, and both return plausible data —
+ * `pipe.phases` on a database returns its status phases, and `table.table_fields`
+ * on a pipe returns its start-form fields. A repo is one object viewed two ways.
+ * Trying `pipe` first and taking the win, as an earlier version did, classified
+ * every database as a pipe.
+ *
+ * `Table.url` is the real discriminator: a pipe lives at `/pipes/<id>` and a
+ * database at `/apollo_databases/<id>`. One query settles both the kind and
+ * PLAN.md §2.2's id duality, since `Table` carries the slug as `id` and the
+ * numeric form as `internal_id`.
+ */
+/**
+ * Resolve one repo id, in **either** spelling.
+ *
+ * `table(id:)` accepts a database's numeric internal id and its api id (a slug
+ * like `dzmxcuxz`) and always answers with the slug — measured both ways. So
+ * `pipe pull 302908356` and `pipe pull 3LhkZu_h` must reach the same database,
+ * and the `RepoRef` this returns normalises to the numeric id either way, which
+ * is what keeps one database from being pulled twice under two identities.
+ */
+export const resolveRepo = async (client: PipefyClient, id: number | string): Promise<RepoRef | null> => {
+  const res = await client.tryRaw<{
+    table: { id: string; internal_id?: string | null; name: string; uuid?: string | null; url?: string | null } | null;
+  }>(
+    `query($id: ID!) { table(id: $id) { id internal_id name uuid url } }`,
+    { id: String(id) },
+    `resolve repo ${id}`,
+  );
+  const repo = res?.table;
+  if (!repo) return null;
+
+  const url = String(repo.url ?? '');
+  const kind: RepoKind = /apollo_databases/.test(url) ? 'table' : 'pipe';
+  debug(`repo ${id} is a ${kind} (${url || 'no url'})`);
+
+  return {
+    kind,
+    id: Number(repo.internal_id ?? id),
+    /** The slug for a database, the numeric string for a pipe. */
+    apiId: kind === 'table' ? String(repo.id) : String(repo.internal_id ?? id),
+    name: repo.name,
+    uuid: repo.uuid ?? null,
+    via: [],
+    depth: 0,
+  };
+};
+
+export type DiscoverOpts = {
+  depth: number;
+  /** Read a repo's payload; supplied by the pull command so both read paths work. */
+  read: (ref: RepoRef) => Promise<RepoPayload | null>;
+};
+
+/**
+ * Walk outward from a root repo. `read` is injected so discovery works
+ * identically whether payloads come from snapshots or from reconstruction.
+ */
+export const discover = async (
+  client: PipefyClient,
+  rootId: number,
+  opts: DiscoverOpts,
+): Promise<DiscoveryResult> => {
+  const found = new Map<number, RepoRef>();
+  const edges: DiscoveryResult['edges'] = [];
+  const warnings: string[] = [];
+
+  const root = await resolveRepo(client, rootId);
+  if (!root) throw new Error(`repo ${rootId} is neither a readable pipe nor a readable table`);
+  root.via = ['root'];
+  found.set(rootId, root);
+
+  let frontier: RepoRef[] = [root];
+  for (let depth = 0; depth <= opts.depth && frontier.length; depth++) {
+    const next: RepoRef[] = [];
+    for (const ref of frontier) {
+      if (depth === opts.depth) break;
+      const payload = await opts.read(ref);
+      if (!payload) {
+        warnings.push(`${ref.kind} ${ref.id} (${ref.name ?? '?'}) could not be read — its connections are unexplored`);
+        continue;
+      }
+      for (const e of edgesFromPayload(payload)) {
+        edges.push({ from: ref.id, to: e.to, kind: e.kind, label: e.label });
+        const seen = found.get(e.to);
+        if (seen) {
+          if (!seen.via.includes(e.kind)) seen.via.push(e.kind);
+          continue;
+        }
+        const resolved = await resolveRepo(client, e.to);
+        if (!resolved) {
+          warnings.push(
+            `${ref.kind} ${ref.id} references repo ${e.to} via ${e.kind}, but the token cannot read it — ` +
+              `the pull is partial (PLAN.md §9, cross-repo permission)`,
+          );
+          continue;
+        }
+        resolved.via = [e.kind];
+        resolved.depth = depth + 1;
+        found.set(e.to, resolved);
+        next.push(resolved);
+        debug(`discovered ${resolved.kind} ${resolved.id} "${resolved.name}" via ${e.kind}`);
+      }
+    }
+    frontier = next;
+  }
+
+  const repos = [...found.values()].sort((a, b) => a.depth - b.depth || a.id - b.id);
+  step(`discovered ${repos.length} repo${repos.length === 1 ? '' : 's'} within depth ${opts.depth}`);
+  for (const w of warnings) warn(w);
+  return { repos, edges, warnings };
+};
+
+/**
+ * `pipeMapGraph` is kept for the graph view and for `hasHiddenAutomations` —
+ * never as the discovery mechanism. PLAN.md §2.2 and ask 7.
+ */
+export const mapGraph = async (client: PipefyClient, repoId: number, depth = 2) => {
+  const data = await client.tryRaw<{ pipeMapGraph: unknown }>(
+    `query($id: ID!, $depth: Int) {
+      pipeMapGraph(pipeId: $id, depth: $depth) {
+        nodes { id name __typename hasHiddenAutomations hasHiddenIntegrations }
+        edges { from to }
+      }
+    }`,
+    { id: String(repoId), depth },
+    'pipeMapGraph',
+  );
+  return data?.pipeMapGraph ?? null;
+};

package/src/pipefy/internal.ts

@@ -0,0 +1,380 @@
+import { ENDPOINTS, type TokenProvider } from '../config.ts';
+import { debug, warn } from '../util/log.ts';
+import { retry } from '../util/pool.ts';
+import type { Row } from '../model/payload.ts';
+
+/**
+ * The internal-API adapter. PLAN.md §2.1 flags this as an unversioned endpoint
+ * with no compatibility contract, so it is isolated here, behind one module,
+ * and every entry point degrades to `null`/`unsupported` rather than throwing.
+ *
+ * Two things live only here — both taken from the Change Migrator recipes,
+ * which have been running them in production:
+ *
+ *  1. Automations. Public `getAutomations` returns a summary; the recipes use
+ *     `loadAutomationToEdit` to get `action_params.field_map`, `searchFor`,
+ *     `responseSchema`, `schedulerCron` and `aiParams` — everything a
+ *     round trip needs and the summary omits.
+ *  2. Phase jumps. `GET/PUT /internal_api/settings/phases/:id`, set-replacing.
+ *
+ * Both accept the same personal API token as the public endpoint, so there is
+ * no session-cookie or CSRF subsystem here.
+ */
+
+export type InternalStatus = 'unknown' | 'ok' | 'unavailable';
+
+export class InternalApi {
+  private readonly tokenProvider: TokenProvider;
+  status: InternalStatus = 'unknown';
+  requestCount = 0;
+  /** Set once, so a broken endpoint produces one warning and not one per call. */
+  private warned = false;
+
+  constructor(token: string | TokenProvider) {
+    this.tokenProvider = typeof token === 'string' ? async () => token : token;
+  }
+
+  private async bearer(): Promise<string> {
+    return this.tokenProvider();
+  }
+
+  private markUnavailable(reason: string) {
+    this.status = 'unavailable';
+    if (!this.warned) {
+      this.warned = true;
+      warn(
+        `internal API unavailable (${reason}). Automation details and phase jumps ` +
+          `will be reported as unknown rather than empty — see PLAN.md §2.1.`,
+      );
+    }
+  }
+
+  /** GraphQL over the internal endpoint. Returns null when the endpoint fails. */
+  async gql<T>(query: string, variables: Record<string, unknown>, label: string): Promise<T | null> {
+    try {
+      const data = await retry(
+        async () => {
+          this.requestCount++;
+          const res = await fetch(ENDPOINTS.internal, {
+            method: 'POST',
+            headers: {
+              authorization: `Bearer ${await this.bearer()}`,
+              'content-type': 'application/json',
+              accept: 'application/json',
+            },
+            body: JSON.stringify({ query, variables }),
+          });
+          const text = await res.text();
+          if (!res.ok) {
+            const err = new Error(`HTTP ${res.status}: ${text.slice(0, 160)}`);
+            (err as { retryable?: boolean }).retryable = res.status >= 500 || res.status === 429;
+            throw err;
+          }
+          const body = JSON.parse(text) as { data?: T; errors?: Array<{ message: string }> };
+          if (body.errors?.length) {
+            const err = new Error(body.errors.map((e) => e.message).join('; '));
+            (err as { retryable?: boolean }).retryable = false;
+            throw err;
+          }
+          return body.data ?? null;
+        },
+        { label },
+      );
+      if (data) this.status = 'ok';
+      return data;
+    } catch (e) {
+      debug(`internal gql ${label} failed: ${(e as Error).message}`);
+      this.markUnavailable(`${label}: ${(e as Error).message}`);
+      return null;
+    }
+  }
+
+  // ── Automations ────────────────────────────────────────────────────────────
+
+  /** Recipe `Pipe Data Treatment` step 156. */
+  async listAutomations(orgId: string, repoId: string | number): Promise<Row[] | null> {
+    const q = `
+      query getAutomations($orgId: ID!, $repoId: ID) {
+        automations(organizationId: $orgId, repoId: $repoId) {
+          id
+          name
+          action_id
+          event_id
+          created_at
+          active
+          action_repo: action_repo_v2 {
+            __typename
+            ... on Pipe { id }
+            ... on Table { id }
+          }
+          event_repo { __typename id }
+        }
+      }`;
+    const data = await this.gql<{ automations: Row[] }>(q, { orgId: String(orgId), repoId: String(repoId) }, 'listAutomations');
+    return data?.automations ?? null;
+  }
+
+  /**
+   * Recipe `Apply Changes` step 118 — the full editable definition. This is the
+   * only read path that returns `action_params.field_map`, `searchFor`,
+   * `responseSchema` and `aiParams`, which is why the reconstruction path
+   * cannot avoid the internal API for automations.
+   */
+  async loadAutomation(id: string | number): Promise<Row | null> {
+    const q = `
+      query loadAutomationToEdit($id: ID!) {
+        automation(id: $id) {
+          name
+          event_id
+          active
+          event_repo { id }
+          event_params {
+            toPhaseId: to_phase_id
+            fromPhaseId
+            inPhaseId
+            triggerFieldIds
+            kindOfSla
+            triggerAutomationId
+          }
+          action_id
+          action_repo: action_repo_v2 {
+            ... on Pipe { id uuid }
+            ... on Table { id uuid }
+          }
+          scheduler_frequency
+          schedulerCron { minute hour dayOfWeek dayOfMonth month }
+          searchFor { id field operation value }
+          responseSchema
+          action_params {
+            toPhaseId: to_phase_id
+            fieldsMapOrder: fields_map_order
+            email_template_id
+            fieldMap: field_map { fieldId value inputMode }
+            cardId: card_id
+            headers
+            body
+            url
+            httpMethod
+            authenticationKey
+            hasAuthenticationValue
+            authenticationAddTo
+            authenticationType
+            strategy
+            # taskParams is not in the recipe's query, and the omission is not
+            # cosmetic: every send_a_task automation came back with an empty
+            # action_params, so a reconstructed read lost its recipients and
+            # title, and a recreate would have rebuilt it blank. Introspected
+            # from AutomationActionParams on the internal endpoint.
+            taskParams { recipients title }
+            aiParams { fieldIds value skillsIds }
+            aiBehaviorParams
+            slaParams {
+              monday { startHour endHour enabled }
+              tuesday { startHour endHour enabled }
+              wednesday { startHour endHour enabled }
+              thursday { startHour endHour enabled }
+              friday { startHour endHour enabled }
+              saturday { startHour endHour enabled }
+              sunday { startHour endHour enabled }
+              timezone
+              holidays { date recurrence description }
+            }
+          }
+          condition {
+            expressions_structure
+            expressions { id field_address operation value structure_id }
+          }
+        }
+      }`;
+    const data = await this.gql<{ automation: Row | null }>(q, { id: String(id) }, `loadAutomation ${id}`);
+    return data?.automation ?? null;
+  }
+
+  /** Recipe `Apply Changes` step 121. `input` is the AutomationInput variable set. */
+  async createAutomation(vars: Record<string, unknown>): Promise<{ id: string } | null> {
+    const q = `
+      mutation createAutomation(
+        $name: String!, $action_id: ID!, $event_id: ID!,
+        $action_repo_id: ID, $event_repo_id: ID,
+        $event_params: AutomationEventParamsInput,
+        $action_params: AutomationActionParamsInput,
+        $condition: ConditionInput,
+        $scheduler_frequency: String,
+        $searchFor: [SearchConditionInput],
+        $schedulerCron: CronInput,
+        $responseSchema: JSON
+      ) {
+        createAutomation(input: {
+          name: $name, action_id: $action_id, event_id: $event_id,
+          action_repo_id: $action_repo_id, event_repo_id: $event_repo_id,
+          event_params: $event_params, action_params: $action_params,
+          condition: $condition, scheduler_frequency: $scheduler_frequency,
+          searchFor: $searchFor, schedulerCron: $schedulerCron,
+          responseSchema: $responseSchema
+        }) {
+          automation { id }
+          error_details { object_key messages }
+        }
+      }`;
+    const data = await this.gql<{
+      createAutomation: { automation: { id: string } | null; error_details?: Array<{ object_key: string; messages: string[] }> };
+    }>(q, vars, 'createAutomation');
+    const payload = data?.createAutomation;
+    if (!payload) return null;
+    if (payload.error_details?.length) {
+      const detail = payload.error_details.map((d) => `${d.object_key}: ${d.messages.join(', ')}`).join(' | ');
+      throw new Error(`createAutomation rejected — ${detail}`);
+    }
+    return payload.automation;
+  }
+
+  /** Recipe `Apply Changes` step 126. */
+  async deleteAutomation(id: string | number): Promise<boolean> {
+    const q = `
+      mutation deleteAutomation($id: ID!) {
+        deleteAutomation(input: { id: $id }) { success }
+      }`;
+    const data = await this.gql<{ deleteAutomation: { success: boolean } }>(q, { id: String(id) }, `deleteAutomation ${id}`);
+    return Boolean(data?.deleteAutomation?.success);
+  }
+
+  // ── Phase jumps ────────────────────────────────────────────────────────────
+
+  /**
+   * `GET /internal_api/settings/phases/:id` — recipe step 23.
+   *
+   * **The GET and the PUT answer in different shapes.** Both captured live:
+   *
+   *   GET  { data: { attributes: { name, phases: [[id, name, done], …],
+   *                                jump_targets: [344054275] } } }
+   *   PUT  { data: { name, back_phase_ids: [], next_phase_ids: [344054275] } }
+   *
+   * The GET's `jump_targets` is one merged set, which is exactly what the
+   * payload's `phase_jumps` array holds. Reading it as `data.next_phase_ids` —
+   * the shape PLAN.md §2.1 recorded, from the PUT — yields an empty set for
+   * every phase without any error, and a diff that then proposes deleting every
+   * jump in the pipe.
+   */
+  async getPhaseSettings(
+    phaseId: string | number,
+  ): Promise<{ jump_targets: number[]; phases: Array<{ id: number; name: string; done: boolean }> } | null> {
+    try {
+      const res = await retry(
+        async () => {
+          this.requestCount++;
+          const r = await fetch(`${ENDPOINTS.internalSettings}/phases/${phaseId}`, {
+            method: 'GET',
+            headers: { authorization: `Bearer ${await this.bearer()}`, accept: 'application/json' },
+          });
+          if (!r.ok) {
+            const err = new Error(`HTTP ${r.status}`);
+            (err as { retryable?: boolean }).retryable = r.status >= 500 || r.status === 429;
+            throw err;
+          }
+          return (await r.json()) as {
+            data?: {
+              attributes?: { jump_targets?: number[]; phases?: Array<[number, string, boolean]> };
+              /** The PUT-style shape, accepted in case a deployment answers with it here. */
+              jump_targets?: number[];
+              next_phase_ids?: number[];
+              back_phase_ids?: number[];
+            };
+          };
+        },
+        { label: `getPhaseSettings ${phaseId}` },
+      );
+
+      const d = res.data ?? {};
+      const attrs = d.attributes ?? {};
+      const targets = attrs.jump_targets ?? d.jump_targets ?? [...(d.next_phase_ids ?? []), ...(d.back_phase_ids ?? [])];
+
+      this.status = 'ok';
+      return {
+        jump_targets: [...new Set(targets.map(Number).filter((n) => Number.isFinite(n)))],
+        phases: (attrs.phases ?? []).map(([id, name, done]) => ({ id: Number(id), name: String(name), done: Boolean(done) })),
+      };
+    } catch (e) {
+      debug(`getPhaseSettings ${phaseId} failed: ${(e as Error).message}`);
+      this.markUnavailable(`getPhaseSettings: ${(e as Error).message}`);
+      return null;
+    }
+  }
+
+  /**
+   * `PUT /internal_api/settings/phases/:id` with the complete desired jump set.
+   * Recipe step 24 does the same thing in Python. Set-replacing, so we send the
+   * whole set and assert on the echoed `next_phase_ids` rather than on the HTTP
+   * status — PLAN.md §2.1.
+   */
+  async setPhaseJumpTargets(
+    phaseId: string | number,
+    targetPhaseIds: Array<string | number>,
+  ): Promise<{ ok: boolean; echoed: number[]; error?: string }> {
+    const params = new URLSearchParams();
+    if (targetPhaseIds.length === 0) {
+      // An empty set still has to be expressible, or "remove the last jump"
+      // becomes impossible. The form convention for an empty array is a single
+      // blank entry.
+      params.append('phase[jump_target_ids][]', '');
+    }
+    for (const t of targetPhaseIds) params.append('phase[jump_target_ids][]', String(t));
+
+    try {
+      const res = await retry(
+        async () => {
+          this.requestCount++;
+          const r = await fetch(`${ENDPOINTS.internalSettings}/phases/${phaseId}`, {
+            method: 'PUT',
+            headers: {
+              authorization: `Bearer ${await this.bearer()}`,
+              'content-type': 'application/x-www-form-urlencoded',
+              accept: 'application/json',
+            },
+            body: params.toString(),
+          });
+          const text = await r.text();
+          if (!r.ok) {
+            const err = new Error(`HTTP ${r.status}: ${text.slice(0, 160)}`);
+            (err as { retryable?: boolean }).retryable = r.status >= 500 || r.status === 429;
+            throw err;
+          }
+          return JSON.parse(text) as {
+            data?: {
+              next_phase_ids?: number[];
+              back_phase_ids?: number[];
+              jump_targets?: number[];
+              attributes?: { jump_targets?: number[] };
+            };
+          };
+        },
+        { label: `setPhaseJumpTargets ${phaseId}` },
+      );
+
+      /**
+       * The echo is the union of both directions.
+       *
+       * `next_phase_ids` alone is forward-only, so a jump to an *earlier* phase
+       * echoes back as an empty next list even though the write succeeded — which
+       * this assertion reported as a failed write. Asserting on the union is what
+       * the sent set actually means.
+       */
+      const d = res.data ?? {};
+      const echoed = [
+        ...(d.attributes?.jump_targets ?? []),
+        ...(d.jump_targets ?? []),
+        ...(d.next_phase_ids ?? []),
+        ...(d.back_phase_ids ?? []),
+      ].map(Number);
+      const want = [...new Set(targetPhaseIds.map(Number))].sort((a, b) => a - b);
+      const got = [...new Set(echoed)].sort((a, b) => a - b);
+      const ok = want.length === got.length && want.every((v, i) => v === got[i]);
+      this.status = 'ok';
+      return ok
+        ? { ok, echoed }
+        : { ok, echoed, error: `echo mismatch: wanted [${want.join(',')}], endpoint reports [${got.join(',')}]` };
+    } catch (e) {
+      this.markUnavailable(`setPhaseJumpTargets: ${(e as Error).message}`);
+      return { ok: false, echoed: [], error: (e as Error).message };
+    }
+  }
+}