@pipefy/pipefy-process-coder 0.1.0

package/src/validate/integrity.ts

@@ -0,0 +1,387 @@
+import type { Row } from '../model/payload.ts';
+import type { Tree } from '../model/tree.ts';
+import type { Finding } from './schema.ts';
+import type { AgentsSection } from '../workspace/agents.ts';
+
+/**
+ * Referential integrity — PLAN.md §5.1.
+ *
+ * The rule that earns its keep: **deleting a field obliges you to fix every
+ * reference to it.** References hide inside opaque JSON as `%{<fieldId>}`,
+ * `"field_address": "<fieldId>"` and `"trigger_field_ids": ["<fieldId>"]`, so
+ * they are found by walking the structure rather than by checking a column.
+ */
+
+const REFERENCE_RE = /%\{(\d+)\}/g;
+const PLACEHOLDER_RE = /%\{_new:([^}]+)\}/g;
+
+/** Every field id referenced anywhere inside a value, with its location. */
+const collectReferences = (
+  value: unknown,
+  where: string,
+  out: Array<{ id: string; where: string; kind: 'template' | 'bare' }>,
+  keyIsIdBearing = false,
+) => {
+  if (typeof value === 'string') {
+    for (const m of value.matchAll(REFERENCE_RE)) out.push({ id: m[1] as string, where, kind: 'template' });
+    if (keyIsIdBearing && /^\d+$/.test(value)) out.push({ id: value, where, kind: 'bare' });
+    return;
+  }
+  if (typeof value === 'number' && keyIsIdBearing) {
+    out.push({ id: String(value), where, kind: 'bare' });
+    return;
+  }
+  if (Array.isArray(value)) {
+    value.forEach((v, i) => collectReferences(v, `${where}[${i}]`, out, keyIsIdBearing));
+    return;
+  }
+  if (value && typeof value === 'object') {
+    for (const [k, v] of Object.entries(value as Row)) {
+      const bearing = k === 'field_address' || k === 'field_id' || k === 'fieldId' || k === 'field_ids' || k === 'fieldIds' || k === 'trigger_field_ids' || k === 'triggerFieldIds';
+      collectReferences(v, `${where}.${k}`, out, bearing);
+    }
+  }
+};
+
+/** Phase ids referenced by an automation's params. */
+const phaseReferences = (value: unknown, where: string, out: Array<{ id: string; where: string }>) => {
+  if (Array.isArray(value)) {
+    value.forEach((v, i) => phaseReferences(v, `${where}[${i}]`, out));
+    return;
+  }
+  if (value && typeof value === 'object') {
+    for (const [k, v] of Object.entries(value as Row)) {
+      if (/^(to_phase_id|from_phase_id|in_phase_id)$/.test(k) && (typeof v === 'string' || typeof v === 'number')) {
+        if (String(v)) out.push({ id: String(v), where: `${where}.${k}` });
+      }
+      phaseReferences(v, `${where}.${k}`, out);
+    }
+  }
+};
+
+const AI_ACTIONS = new Set(['ai_behavior', 'generate_with_ai']);
+
+/** `%{_new:<key>}` — a reference to an entity created in the same edit. */
+const PLACEHOLDER_REF = /^%{_new:([^}]+)}$/;
+
+const placeholderKey = (v: unknown): string | null => {
+  if (typeof v !== 'string') return null;
+  return PLACEHOLDER_REF.exec(v)?.[1] ?? null;
+};
+
+export const validateIntegrity = (
+  tree: Tree,
+  opts: { knownRepoIds?: number[]; agents?: AgentsSection | null } = {},
+): Finding[] => {
+  const findings: Finding[] = [];
+
+  const fields = tree.phases.flatMap((p) => p.fields.map((f) => ({ f, phase: p })));
+  const fieldIds = new Set(fields.map(({ f }) => String(f['id'])).filter((id) => id && id !== 'null'));
+  const phaseIds = new Set(tree.phases.map((p) => String(p['id'])).filter((id) => id && id !== 'null'));
+  const repoId = Number(tree.repo['id']);
+  const knownRepos = new Set((opts.knownRepoIds ?? [repoId]).map(String));
+
+  const newFieldAliases = new Set<string>();
+  for (const { f } of fields) {
+    if (f['id'] === null || f['_new'] === true) {
+      if (f['uuid']) newFieldAliases.add(String(f['uuid']));
+      if (f['label']) newFieldAliases.add(String(f['label']));
+    }
+  }
+
+  const newPhaseAliases = new Set<string>();
+  for (const p of tree.phases) {
+    if (p['id'] === null || p['_new'] === true) {
+      if (p['uuid']) newPhaseAliases.add(String(p['uuid']));
+      if (p['name']) newPhaseAliases.add(String(p['name']));
+    }
+  }
+
+  /**
+   * `otherRepo` names the repo whose fields a value refers to, when that is not
+   * this one.
+   *
+   * A cross-repo automation's action side is written in the *target* pipe's field
+   * ids: `create_card` in another pipe maps values into that pipe's fields, so
+   * every id in `field_maps` and `fields_map_order` is foreign by construction.
+   * Checking them against this repo reported nine dangling references for an
+   * automation that was correct, and blocked the plan — which made a handoff
+   * between two pipes impossible to express. The reference is reported as
+   * unverifiable instead, which is what it is: the other pipe may not even be in
+   * this workspace.
+   */
+  const checkRefs = (value: unknown, label: string, file: string, entity: string, otherRepo?: number | null) => {
+    const refs: Array<{ id: string; where: string; kind: 'template' | 'bare' }> = [];
+    collectReferences(value, label, refs);
+    for (const r of refs) {
+      if (fieldIds.has(r.id)) continue;
+      if (otherRepo) {
+        findings.push({
+          severity: 'warning',
+          file,
+          entity,
+          message:
+            `references field ${r.id} at ${r.where}, which belongs to repo ${otherRepo} — ` +
+            'not checked here',
+          fix: `pull repo ${otherRepo} into this workspace to have it verified`,
+        });
+        continue;
+      }
+      findings.push({
+        severity: 'error',
+        file,
+        entity,
+        message: `references field ${r.id} at ${r.where}, and no field with that id exists in this repo`,
+        fix: 'point it at a field that exists, or restore the deleted field',
+      });
+    }
+
+    // Placeholders for fields created in the same edit must name a real new field.
+    if (typeof value === 'object' && value !== null) {
+      const text = JSON.stringify(value);
+      for (const m of text.matchAll(PLACEHOLDER_RE)) {
+        const key = m[1] as string;
+        if (!newFieldAliases.has(key)) {
+          findings.push({
+            severity: 'error',
+            file,
+            entity,
+            message: `placeholder %{_new:${key}} does not match the uuid or label of any new field in this workspace`,
+            fix: 'use the new field\'s uuid, exactly as written in its phase file',
+          });
+        }
+      }
+    }
+  };
+
+  // ── automations ────────────────────────────────────────────────────────────
+  for (const a of tree.automations) {
+    const file = `automations/${String(a['name'] ?? '')}`;
+
+    /**
+     * The action side and the event side of an automation are read in different
+     * repos when `action_repo_id` points elsewhere. The event side is always this
+     * repo — that is where the trigger fires.
+     */
+    const actionRepo = Number(a['action_repo_id']);
+    const foreign = Number.isFinite(actionRepo) && actionRepo !== repoId ? actionRepo : null;
+
+    checkRefs(a['action_params'], 'action_params', file, 'automations', foreign);
+    checkRefs(a['event_params'], 'event_params', file, 'automations');
+    checkRefs(a['search_for'], 'search_for', file, 'automations');
+    checkRefs(a.field_maps, 'field_maps', file, 'automations', foreign);
+    if (a.condition) checkRefs(a.condition, 'condition', file, 'automations');
+
+    const phaseRefs: Array<{ id: string; where: string }> = [];
+    phaseReferences(a['action_params'], 'action_params', phaseRefs);
+    phaseReferences(a['event_params'], 'event_params', phaseRefs);
+    for (const r of phaseRefs) {
+      if (phaseIds.has(r.id)) continue;
+      // A cross-repo automation legitimately names a phase in the other repo.
+      if (Number(a['action_repo_id']) !== repoId) continue;
+      findings.push({
+        severity: 'error',
+        file,
+        entity: 'automations',
+        message: `references phase ${r.id} at ${r.where}, which is not a phase of this repo`,
+      });
+    }
+
+    for (const key of ['event_repo_id', 'action_repo_id']) {
+      const v = a[key];
+      if (v === null || v === undefined) continue;
+      if (knownRepos.has(String(v))) continue;
+      findings.push({
+        severity: 'warning',
+        file,
+        entity: 'automations',
+        message: `${key} is ${String(v)}, which is not a repo in this workspace`,
+        fix: 'pull with a greater --depth, or check that the token can administer that repo',
+      });
+    }
+
+    // AI config is absent from the snapshot payload (ROUTE-B-V2.md ask 5), so a
+    // round trip drops it silently. Refusing here is the only protection.
+    const usesAi =
+      AI_ACTIONS.has(String(a['action_id'])) ||
+      JSON.stringify(a['action_params'] ?? {}).includes('fill_with_ai') ||
+      a.field_maps.some((m) => String(m['input_mode']) === 'fill_with_ai');
+    if (usesAi && (opts.agents?.agents.length ?? 0) === 0) {
+      /**
+       * An error only when the agents were readable and there genuinely are
+       * none — then the automation references configuration that does not
+       * exist. When they could not be read at all (no `agents/` section, or
+       * one whose coverage is 'unknown'), their absence proves nothing, so
+       * this is a warning: the same distinction coverage makes for every
+       * other entity group.
+       */
+      const readable = opts.agents?.meta.coverage === 'complete';
+      findings.push({
+        severity: readable ? 'error' : 'warning',
+        file,
+        entity: 'automations',
+        message: readable
+          ? 'this automation uses AI, but this repo has no AI agents'
+          : 'this automation uses AI, and the AI agents could not be read on this pull',
+        fix: readable
+          ? 'point it at an agent that exists, or remove the AI action'
+          : 'the payload has no ai_agents array (ROUTE-B-V2.md ask 5), and agents/ is missing or unreadable, so a round trip can drop AI configuration silently — do not edit this automation',
+      });
+    }
+  }
+
+  // ── field conditions ───────────────────────────────────────────────────────
+  for (const fc of tree.field_conditions) {
+    const file = `field-conditions/${String(fc['name'] ?? '')}`;
+    const fcPhasePlaceholder = placeholderKey(fc['phase_id']);
+    if (fcPhasePlaceholder) {
+      if (!newPhaseAliases.has(fcPhasePlaceholder)) {
+        findings.push({
+          severity: 'error',
+          file,
+          entity: 'field_conditions',
+          message: `phase_id names %{_new:${fcPhasePlaceholder}}, which is not the uuid or name of any new phase here`,
+          fix: "use the new phase's uuid, exactly as written in its phase file",
+        });
+      }
+    } else if (!phaseIds.has(String(fc['phase_id']))) {
+      findings.push({ severity: 'error', file, entity: 'field_conditions', message: `phase_id ${String(fc['phase_id'])} is not a phase of this repo` });
+    }
+
+    for (const action of fc.actions) {
+      const raw = action['field_id'];
+      const placeholder = placeholderKey(raw);
+      if (placeholder) {
+        if (!newFieldAliases.has(placeholder)) {
+          findings.push({
+            severity: 'error',
+            file,
+            entity: 'field_condition_actions',
+            message: `action targets %{_new:${placeholder}}, which is not the uuid or label of any new field here`,
+          });
+        }
+        continue;
+      }
+      const fid = String(raw ?? '');
+      if (fid && !fieldIds.has(fid)) {
+        findings.push({ severity: 'error', file, entity: 'field_condition_actions', message: `action targets field ${fid}, which does not exist` });
+      }
+    }
+    if (fc.condition) checkRefs(fc.condition, 'condition', file, 'field_conditions');
+    if (fc.condition && (fc.condition.expressions ?? []).length === 0) {
+      findings.push({ severity: 'error', file, entity: 'conditions', message: 'condition has no expressions — it can never evaluate' });
+    }
+  }
+
+  /**
+   * The two hidden-attribute lists are closed enumerations, and the endpoint
+   * only says so in its rejection: "There is an invalid attribute, accepted
+   * attributes: Assignees, Attachments, Checklist, Comments, DueDate, Labels".
+   * The value that provoked it — a plausible-looking `"clone"` — cost a failed
+   * step in the middle of an otherwise clean apply, and the list is nowhere in
+   * the schema. Checking it offline is free.
+   */
+  const HIDEABLE = new Set(['Assignees', 'Attachments', 'Checklist', 'Comments', 'DueDate', 'Labels']);
+  if (tree.preferences) {
+    for (const key of ['hidden_top_buttons', 'hidden_start_form_attributes']) {
+      const list = tree.preferences[key];
+      if (!Array.isArray(list)) continue;
+      for (const v of list) {
+        if (HIDEABLE.has(String(v))) continue;
+        findings.push({
+          severity: 'error',
+          file: 'preferences.json',
+          entity: 'repo_preferences',
+          message: `${key} contains "${String(v)}", which the endpoint does not accept`,
+          fix: `use one of: ${[...HIDEABLE].join(', ')}`,
+        });
+      }
+    }
+  }
+
+  // ── phase jumps ────────────────────────────────────────────────────────────
+  /** `%{_new:<uuid or name>}` naming a phase this edit creates. */
+  const namesNewPhase = (t: unknown): boolean => {
+    const m = /^%\{_new:(.+)\}$/.exec(String(t));
+    return m ? newPhaseAliases.has(m[1] as string) : false;
+  };
+
+  for (const p of tree.phases) {
+    for (const t of p.jump_target_ids) {
+      if (namesNewPhase(t)) continue;
+      if (!phaseIds.has(String(t))) {
+        findings.push({
+          severity: 'error',
+          file: `phases/… ${String(p['name'] ?? '')}`,
+          entity: 'phase_jumps',
+          message: `jumps to phase ${t}, which is not a phase of this repo`,
+        });
+      }
+      if (String(t) === String(p['id'])) {
+        findings.push({
+          severity: 'error',
+          file: `phases/… ${String(p['name'] ?? '')}`,
+          entity: 'phase_jumps',
+          message: 'a phase cannot jump to itself',
+        });
+      }
+    }
+  }
+
+  // ── title field must be a start-form field ─────────────────────────────────
+  const titleFieldId = tree.repo['title_field_id'];
+  if (titleFieldId) {
+    const startForm = tree.phases.find((p) => Number(p['index']) === 0);
+    const inStartForm = (startForm?.fields ?? []).some((f) => String(f['id']) === String(titleFieldId));
+    if (!inStartForm) {
+      findings.push({
+        severity: 'error',
+        file: 'pipe.json',
+        entity: 'repo',
+        message: `title_field_id ${String(titleFieldId)} is not a field on the start form`,
+        fix: 'the card title must come from a start-form field',
+      });
+    }
+  }
+
+  // ── connector fields must point somewhere we pulled ────────────────────────
+  for (const { f, phase } of fields) {
+    if (f['type_id'] !== 'connector') continue;
+    const target = f['connected_pipe_id'];
+    if (target === null || target === undefined) continue;
+    if (knownRepos.has(String(target))) continue;
+    findings.push({
+      severity: 'warning',
+      file: `phases/… ${String(phase['name'] ?? '')} / ${String(f['label'] ?? '')}`,
+      entity: 'fields',
+      message: `connects to repo ${String(target)}, which is not in this workspace`,
+      fix: 'pull with a greater --depth so the connected repo is present and checkable',
+    });
+  }
+
+  // ── coverage: never edit what was never read ───────────────────────────────
+  for (const [entity, cov] of Object.entries(tree.meta.coverage ?? {})) {
+    if (cov !== 'unknown') continue;
+    const populated =
+      (entity === 'webhooks' && tree.webhooks.length > 0) ||
+      (entity === 'labels' && tree.labels.length > 0) ||
+      (entity === 'automations' && tree.automations.length > 0) ||
+      (entity === 'public_forms' && tree.public_form !== null) ||
+      (entity === 'repo_preferences' && tree.preferences !== null);
+    if (populated) {
+      findings.push({
+        severity: 'error',
+        entity,
+        message: `${entity} was recorded as unknown by the pull, but this workspace has rows for it`,
+        fix: 'the rows were added by hand into an entity group that has no read path — remove them, or re-pull from a snapshot',
+      });
+    }
+  }
+
+  return findings;
+};
+
+export const summarise = (findings: Finding[]) => ({
+  errors: findings.filter((f) => f.severity === 'error').length,
+  warnings: findings.filter((f) => f.severity === 'warning').length,
+});

package/src/validate/reference.ts

@@ -0,0 +1,136 @@
+import type { Row } from '../model/payload.ts';
+import type { Tree } from '../model/tree.ts';
+import { checkAutomationParams } from '../pipefy/automations.ts';
+import { toCatalogue, enumValues, type Reference } from '../pipefy/reference.ts';
+import type { Finding } from './schema.ts';
+
+/**
+ * Closed-set validation, offline, against what the API said on the last pull.
+ *
+ * The values checked here are all published by the endpoint and enforced by none
+ * of it: the inputs that consume them are typed `ID` and `String`, so a wrong
+ * value survives query validation and returns `is invalid` with nothing named.
+ * Checking it here means the refusal happens before a send, names the value, and
+ * lists what belongs there.
+ *
+ * Why not hardcode the lists? Because that was the previous approach and it
+ * silently rotted. `FIELD_TYPES` in `schema.ts` still carries `time_range`, which
+ * the API does not offer, and is missing `dynamic_content`, which it does — so it
+ * warned about a valid field type and accepted an invalid one, and nothing
+ * revealed either. A list read from the endpoint is stamped with when it was read.
+ *
+ * Everything is skipped when there is no reference. An unread closed set carries
+ * no information, exactly as an unread entity group does.
+ */
+
+/** Where a closed set is consumed, and which set governs it. */
+const ENUM_SITES: Array<{
+  enumName: string;
+  entity: string;
+  key: string;
+  /** A value the payload uses that the enum does not list, and which is fine. */
+  tolerated?: string[];
+}> = [
+  { enumName: 'FieldTypeId', entity: 'fields', key: 'type_id' },
+  { enumName: 'FieldConditionFilter', entity: 'field_conditions', key: 'filter' },
+];
+
+const checkEnum = (
+  ref: Reference,
+  row: Row,
+  site: (typeof ENUM_SITES)[number],
+  file: string,
+  findings: Finding[],
+) => {
+  const values = enumValues(ref, site.enumName);
+  if (!values) return;
+  const raw = row[site.key];
+  if (raw === null || raw === undefined || raw === '') return;
+  const value = String(raw);
+  if (values.includes(value) || site.tolerated?.includes(value)) return;
+  findings.push({
+    severity: 'error',
+    file,
+    entity: site.entity,
+    message: `${site.key} "${value}" is not a value the API accepts`,
+    fix: `accepted: ${values.join(', ')}`,
+  });
+};
+
+/**
+ * Colors are checked case-insensitively and only warned about.
+ *
+ * The payload stores a phase colour in more than one spelling across pipe
+ * vintages, and an apply that sends a colour the endpoint rejects fails on one
+ * cosmetic column — worth reporting, not worth blocking a structural change for.
+ */
+const checkColor = (ref: Reference, row: Row, entity: string, file: string, findings: Finding[]) => {
+  const values = enumValues(ref, 'Colors');
+  const raw = row['color'];
+  if (!values || raw === null || raw === undefined || raw === '') return;
+  const value = String(raw).toLowerCase();
+  if (values.some((v) => v.toLowerCase() === value)) return;
+  findings.push({
+    severity: 'warning',
+    file,
+    entity,
+    message: `color "${String(raw)}" is not one the API offers`,
+    fix: `accepted: ${values.join(', ')}`,
+  });
+};
+
+/**
+ * Validate every authored value that must come from a closed set.
+ *
+ * Automations reuse `checkAutomationParams` — the same function the compiler
+ * gates on — so `pipe validate` and `pipe plan` cannot disagree about whether an
+ * automation is sendable.
+ */
+export const validateAgainstReference = (tree: Tree, ref: Reference | null): Finding[] => {
+  if (!ref) return [];
+  const findings: Finding[] = [];
+  const catalogue = toCatalogue(ref);
+
+  for (const phase of tree.phases) {
+    const pfile = `phases/${String(phase['name'] ?? '')}`;
+    checkColor(ref, phase, 'phases', pfile, findings);
+    for (const f of phase.fields) {
+      const ffile = `${pfile} / ${String(f['label'] ?? '')}`;
+      for (const site of ENUM_SITES) {
+        if (site.entity === 'fields') checkEnum(ref, f, site, ffile, findings);
+      }
+    }
+  }
+
+  for (const fc of tree.field_conditions) {
+    const file = `field-conditions/${String(fc['name'] ?? fc['id'] ?? '')}`;
+    for (const site of ENUM_SITES) {
+      if (site.entity === 'field_conditions') checkEnum(ref, fc, site, file, findings);
+    }
+  }
+
+  for (const auto of tree.automations) {
+    /**
+     * A behavior (`action_id: "ai_behavior"`) is not in this pipe's automation
+     * catalogue at all — confirmed live — because it is never sent through
+     * `createAutomation`/`updateAutomation`. Checking it here would refuse
+     * every behavior, always, regardless of whether it is actually sendable
+     * through the real path (`createAiAgent`/`updateAiAgent`, diff/agents.ts).
+     */
+    if (auto['action_id'] === 'ai_behavior') continue;
+    const file = `automations/${String(auto['name'] ?? auto['id'] ?? '')}`;
+    for (const f of checkAutomationParams(auto, catalogue)) {
+      findings.push({
+        severity: f.severity,
+        file,
+        entity: 'automations',
+        message: f.message,
+        ...(f.fix ? { fix: f.fix } : {}),
+      });
+    }
+  }
+
+  checkColor(ref, tree.repo, 'repo', 'pipe.json', findings);
+
+  return findings;
+};