@pipefy/pipefy-process-coder 0.1.0

package/src/workspace/stamp.ts

@@ -0,0 +1,301 @@
+import { dim } from '../util/log.ts';
+import { IdMap } from '../apply/idmap.ts';
+import type { Row } from '../model/payload.ts';
+import type { Tree } from '../model/tree.ts';
+
+/**
+ * Stamp server-assigned ids onto entities the workspace still marks as new.
+ *
+ * After a *partial* apply the post-apply sync does not run — the files must keep
+ * the author's edits so `--resume` and a re-plan have something to work with.
+ * But the entities that did get created still read `"id": null, "_new": true`,
+ * so recompiling would create them a second time. That is the same duplication
+ * hazard the success path avoids by adopting live state, and a partial failure is
+ * exactly when someone re-runs.
+ *
+ * So: for every still-new row, find the entity that now exists on the pipe and
+ * adopt its id. Matching is uuid first — exact, and available for phases,
+ * fields, automations and labels — then the natural key, which is what the rest
+ * have. Nothing else about the row is touched, so an edit that has not been
+ * applied yet survives.
+ */
+
+export type StampResult = {
+  /** Whether anything at all changed and the files need writing. */
+  changed?: boolean;
+  stamped: Array<{ entity: string; label: string; id: number }>;
+  /** Still new after matching: these genuinely were not created. */
+  unmatched: Array<{ entity: string; label: string }>;
+  /**
+   * Server-owned groups adopted from live state because the author had expressed
+   * no intent about them. Reported so the adoption is never silent.
+   */
+  refreshed: string[];
+  /**
+   * References rewritten from `%{_new:<uuid>}` to the real id.
+   *
+   * Adopting an id without this leaves every reference to it dangling, and the
+   * validator — correctly — refuses to plan. The placeholder was a promise to
+   * substitute once the id existed; the id now exists.
+   */
+  rewrittenReferences: number;
+};
+
+const isNew = (row: Row) => row['id'] === null || row['id'] === undefined || row['_new'] === true;
+
+const adopt = (row: Row, live: Row) => {
+  /**
+   * The authored uuid is kept as tool metadata before being replaced.
+   *
+   * It is the only link between a reference written as `%{_new:<authored uuid>}`
+   * and the entity that now exists under a server-minted one. Discarding it
+   * without rewriting in the same pass strands every such reference permanently
+   * — which is exactly what happened once, and cost a hand edit to undo.
+   */
+  if (row['uuid'] && live['uuid'] && row['uuid'] !== live['uuid']) {
+    row['_authored_uuid'] = row['uuid'];
+  }
+  row['id'] = live['id'];
+  if (live['uuid']) row['uuid'] = live['uuid'];
+  if (live['slug']) row['slug'] = live['slug'];
+  delete row['_new'];
+
+  /**
+   * Fill in what the server filled in, and only that.
+   *
+   * An authored row states a label and a type; the pipe also holds `unique`,
+   * `card_synced`, `color_uuid` and the connector flags. Adopting the id without
+   * them leaves a row that the ordinary diff then reads as "set all of these to
+   * null" — a page of phantom changes on the next `pipe diff`. Keys the author
+   * did write are never touched, so an edit that has not been applied yet
+   * survives adoption.
+   */
+  for (const [k, v] of Object.entries(live)) {
+    if (k in row) continue;
+    if (k.startsWith('_')) continue;
+    row[k] = v;
+  }
+};
+
+/** uuid, else a natural key that does not include anything a create can get wrong. */
+const keysOf = (row: Row, natural: (r: Row) => string): string[] => {
+  const keys: string[] = [];
+  if (row['uuid']) keys.push(`uuid:${String(row['uuid'])}`);
+  keys.push(`natural:${natural(row)}`);
+  return keys;
+};
+
+const matchInto = (
+  entity: string,
+  authored: Row[],
+  live: Row[],
+  natural: (r: Row) => string,
+  result: StampResult,
+  /** Aliases → id, so references to what was adopted can be rewritten. */
+  resolved?: IdMap,
+) => {
+  const byKey = new Map<string, Row>();
+  for (const l of live) {
+    for (const k of keysOf(l, natural)) if (!byKey.has(k)) byKey.set(k, l);
+  }
+  const claimed = new Set<Row>();
+
+  for (const row of authored) {
+    if (!isNew(row)) continue;
+    const label = String(row['name'] ?? row['label'] ?? entity);
+
+    let hit: Row | undefined;
+    for (const k of keysOf(row, natural)) {
+      const candidate = byKey.get(k);
+      if (candidate && !claimed.has(candidate)) {
+        hit = candidate;
+        break;
+      }
+    }
+
+    if (!hit) {
+      result.unmatched.push({ entity, label });
+      continue;
+    }
+    claimed.add(hit);
+    const authoredUuid = row['uuid'] ? String(row['uuid']) : null;
+    const authoredLabel = String(row['label'] ?? row['name'] ?? '');
+    adopt(row, hit);
+    const id = Number(hit['id']);
+    result.stamped.push({ entity, label, id });
+    result.changed = true;
+
+    if (resolved && Number.isFinite(id)) {
+      // Both spellings an author may have used to name this entity.
+      const aliases = [authoredUuid ? `_new:${authoredUuid}` : '', authoredLabel ? `_new:${authoredLabel}` : ''].filter(Boolean);
+      for (const a of aliases) resolved.resolvePlaceholder(a, id);
+    }
+  }
+};
+
+/**
+ * Groups the server owns outright, refreshed from live state whenever the author
+ * did not touch them.
+ *
+ * Creating a pipe's start form mints a `public_forms` row and a `visibilities`
+ * row that nobody asked for. After a partial apply the files are deliberately
+ * left as written, so those rows exist in the new baseline and not in the
+ * workspace — and the next diff reads that as *delete the public form*, marked
+ * destructive. The author never expressed an opinion about either row, so the
+ * honest resolution is to adopt what the server did.
+ *
+ * "Did not touch" is decided against the previous baseline, so an author who
+ * genuinely edited the public form keeps their edit.
+ */
+const refreshUntouched = (authored: Tree, live: Tree, previous: Tree | null, result: StampResult) => {
+  const same = (a: unknown, b: unknown) => JSON.stringify(a ?? null) === JSON.stringify(b ?? null);
+
+  const singletons: Array<'public_form' | 'preferences'> = ['public_form', 'preferences'];
+  for (const key of singletons) {
+    if (same(authored[key], live[key])) continue;
+    // With no previous baseline there is no way to tell an edit from a drift, so
+    // nothing is touched: a false negative here costs a phantom diff, a false
+    // positive silently discards someone's edit.
+    if (!previous || !same(authored[key], previous[key])) continue;
+    authored[key] = live[key];
+    result.refreshed.push(key);
+    result.changed = true;
+  }
+
+  // Read-only groups are refreshed unconditionally: there is no edit to lose,
+  // because an edit to them is refused at plan time by path alone.
+  for (const key of ['email_templates', 'email_inboxes', 'visibilities'] as const) {
+    if (same(authored.readonly[key], live.readonly[key])) continue;
+    authored.readonly[key] = live.readonly[key];
+    result.refreshed.push(`_readonly/${key}`);
+    result.changed = true;
+  }
+};
+
+export const stampCreated = (authored: Tree, live: Tree, previous: Tree | null = null): StampResult => {
+  const result: StampResult = { stamped: [], unmatched: [], refreshed: [], rewrittenReferences: 0, changed: false };
+  const resolved = new IdMap();
+
+  matchInto('phases', authored.phases, live.phases, (r) => String(r['name'] ?? ''), result, resolved);
+
+  /**
+   * Fields are matched within their phase where the phase is known, and across
+   * the whole pipe otherwise: a field created inside a brand-new phase has a
+   * null phase_id in the file it was authored in.
+   */
+  const liveFields = live.phases.flatMap((p) => p.fields.map((f) => ({ ...f, _phase_name: p['name'] })));
+  for (const phase of authored.phases) {
+    matchInto(
+      'fields',
+      phase.fields,
+      liveFields,
+      (r) => String(r['label'] ?? ''),
+      result,
+      resolved,
+    );
+  }
+
+  matchInto('labels', authored.labels, live.labels, (r) => String(r['name'] ?? ''), result, resolved);
+  matchInto('webhooks', authored.webhooks, live.webhooks, (r) => `${String(r['name'] ?? '')}::${String(r['url'] ?? '')}`, result, resolved);
+  matchInto(
+    'pipe_relations',
+    authored.pipe_relations,
+    live.pipe_relations,
+    (r) => `${String(r['parent_id'] ?? '')}::${String(r['child_id'] ?? '')}::${String(r['name'] ?? '')}`,
+    result,
+    resolved,
+  );
+  matchInto('automations', authored.automations, live.automations, (r) => String(r['name'] ?? ''), result, resolved);
+  matchInto('field_conditions', authored.field_conditions, live.field_conditions, (r) => String(r['name'] ?? ''), result, resolved);
+
+  refreshUntouched(authored, live, previous, result);
+
+  /**
+   * The resolution map is built from everything that exists on the pipe, not
+   * only from what this pass stamped.
+   *
+   * A placeholder can outlive the adoption of the thing it names: adopt a field's
+   * id in one run and the automation referring to it by `%{_new:<uuid>}` is still
+   * dangling in the next, with nothing left to match it against. Resolving
+   * against live state fixes it whenever it is noticed.
+   */
+  for (const p of authored.phases) {
+    for (const f of p.fields) {
+      const fid = Number(f['id']);
+      if (Number.isFinite(fid) && f['_authored_uuid']) {
+        resolved.resolvePlaceholder(`_new:${String(f['_authored_uuid'])}`, fid);
+      }
+    }
+    const pid = Number(p['id']);
+    if (Number.isFinite(pid) && p['_authored_uuid']) {
+      resolved.resolvePlaceholder(`_new:${String(p['_authored_uuid'])}`, pid);
+    }
+  }
+
+  for (const p of live.phases) {
+    const id = Number(p['id']);
+    if (!Number.isFinite(id)) continue;
+    if (p['uuid']) resolved.resolvePlaceholder(`_new:${String(p['uuid'])}`, id);
+    if (p['name']) resolved.resolvePlaceholder(`_new:${String(p['name'])}`, id);
+    for (const f of p.fields) {
+      const fid = Number(f['id']);
+      if (!Number.isFinite(fid)) continue;
+      if (f['uuid']) resolved.resolvePlaceholder(`_new:${String(f['uuid'])}`, fid);
+      if (f['label']) resolved.resolvePlaceholder(`_new:${String(f['label'])}`, fid);
+    }
+  }
+
+  /**
+   * Rewrite every reference to something that now exists. The IdMap does the
+   * deep walk — the same one the executor uses at send time — so an automation's
+   * `field_maps`, a condition's `field_address` and an `action_params` blob are
+   * all covered by one pass.
+   */
+  const before = JSON.stringify({
+    phases: authored.phases,
+    automations: authored.automations,
+    field_conditions: authored.field_conditions,
+  });
+  const rewritten = resolved.rewrite({
+    phases: authored.phases,
+    automations: authored.automations,
+    field_conditions: authored.field_conditions,
+  });
+  if (JSON.stringify(rewritten) !== before) {
+    authored.phases = rewritten.phases;
+    authored.automations = rewritten.automations;
+    authored.field_conditions = rewritten.field_conditions;
+    result.rewrittenReferences = [...before.matchAll(/%{_new:/g)].length;
+    result.changed = true;
+  }
+
+  return result;
+};
+
+export const renderStamp = (r: StampResult): string[] => {
+  const lines: string[] = [];
+  if (r.stamped.length) {
+    lines.push(
+      `adopted ${r.stamped.length} server-assigned id${r.stamped.length === 1 ? '' : 's'} into the workspace, ` +
+        `so a re-plan will not create them again`,
+    );
+    for (const s of r.stamped.slice(0, 8)) lines.push(dim(`   ${s.entity} ${s.label} → ${s.id}`));
+    if (r.stamped.length > 8) lines.push(dim(`   …and ${r.stamped.length - 8} more`));
+  }
+  if (r.rewrittenReferences) {
+    lines.push(
+      `resolved ${r.rewrittenReferences} placeholder reference${r.rewrittenReferences === 1 ? '' : 's'} to those ids`,
+    );
+  }
+  if (r.refreshed.length) {
+    lines.push(
+      `adopted live state for ${r.refreshed.length} group${r.refreshed.length === 1 ? '' : 's'} ` +
+        `the workspace never edited: ${r.refreshed.join(', ')}`,
+    );
+  }
+  if (r.unmatched.length) {
+    lines.push(`${r.unmatched.length} entit${r.unmatched.length === 1 ? 'y was' : 'ies were'} not created and remain new:`);
+    for (const u of r.unmatched.slice(0, 8)) lines.push(dim(`   ${u.entity} ${u.label}`));
+  }
+  return lines;
+};

package/src/workspace/write.ts

@@ -0,0 +1,225 @@
+import { promises as fs } from 'node:fs';
+import { join } from 'node:path';
+import { mkdirp, exists, writeJson } from '../util/fsx.ts';
+import { indexed, slug, uniquify } from '../util/slug.ts';
+import type { Row } from '../model/payload.ts';
+import type { Tree } from '../model/tree.ts';
+import { repoPaths } from './layout.ts';
+import { AGENT_DIR_TAG } from './agents.ts';
+
+/**
+ * tree → files. PLAN.md §4:
+ *   - fields live inside their phase file, because that is what Claude edits
+ *     most and they only make sense in phase context
+ *   - conditions and field_maps fold into their owner's file, matching how the
+ *     mutations already take them nested
+ *   - `_readonly/` is first-class so the planner can reject edits by path alone
+ */
+
+const clearDir = async (dir: string) => {
+  await mkdirp(dir);
+  for (const name of await fs.readdir(dir)) {
+    if (name.endsWith('.json')) await fs.rm(join(dir, name));
+  }
+};
+
+export type WrittenFiles = { path: string; kind: string }[];
+
+export const writeTree = async (dir: string, tree: Tree): Promise<WrittenFiles> => {
+  const p = repoPaths(dir, tree.meta.repoKind);
+  const written: WrittenFiles = [];
+  const put = async (path: string, value: unknown, kind: string) => {
+    await writeJson(path, value);
+    written.push({ path, kind });
+  };
+
+  await mkdirp(dir);
+
+  await put(p.meta, tree.meta, 'meta');
+  await put(p.repo, tree.repo, 'repo');
+
+  await clearDir(p.phasesDir);
+  const phaseName = uniquify();
+  for (const phase of tree.phases) {
+    const name = phaseName(indexed(Number(phase['index'] ?? 0), String(phase['name'] ?? '')));
+    await put(join(p.phasesDir, `${name}.json`), phase, 'phase');
+  }
+
+  /**
+   * A database gets no `automations/` or `field-conditions/` directory.
+   *
+   * Both are always empty for one — a database has no workflow to automate — and
+   * an empty directory is an invitation. Dropping a file into it would author a
+   * change that `pipe plan` can only refuse, so the honest thing is not to offer
+   * the slot. The arrays stay in the tree and round-trip as `[]`; it is only the
+   * directory that is absent.
+   */
+  const hasWorkflow = tree.meta.repoKind !== 'table';
+  if (hasWorkflow || tree.automations.length) {
+    await clearDir(p.automationsDir);
+
+    /**
+     * A behavior (an automation carrying `_agent_dir`, PLAN.md's agents/
+     * doc) is not written here — it goes under its agent's `behaviors/`
+     * instead, same file shape, different folder. Every agent directory that
+     * already exists gets its `behaviors/` cleared up front, same as
+     * `automations/` above, so a behavior deleted upstream (or reassigned to
+     * a different agent) does not leave a stale file behind.
+     */
+    const agentNames = (await exists(p.agentsDir)) ? await fs.readdir(p.agentsDir, { withFileTypes: true }) : [];
+    const agentDirs = agentNames.filter((e) => e.isDirectory()).map((e) => e.name);
+    for (const name of agentDirs) await clearDir(join(p.agentsDir, name, 'behaviors'));
+
+    const autoName = uniquify();
+    const behaviorNameByDir = new Map<string, ReturnType<typeof uniquify>>();
+    for (const a of tree.automations) {
+      const agentDir = a[AGENT_DIR_TAG] as string | undefined;
+      const { [AGENT_DIR_TAG]: _drop, ...row } = a as Row & { [AGENT_DIR_TAG]?: string };
+
+      if (agentDir) {
+        let behaviorName = behaviorNameByDir.get(agentDir);
+        if (!behaviorName) {
+          behaviorName = uniquify();
+          behaviorNameByDir.set(agentDir, behaviorName);
+        }
+        const dir = join(p.agentsDir, agentDir, 'behaviors');
+        await mkdirp(dir);
+        const name = behaviorName(slug(String(row['name'] ?? `behavior-${row['id']}`)));
+        await put(join(dir, `${name}.json`), row, 'behavior');
+        continue;
+      }
+
+      const name = autoName(slug(String(row['name'] ?? `automation-${row['id']}`)));
+      await put(join(p.automationsDir, `${name}.json`), row, 'automation');
+    }
+  }
+
+  if (hasWorkflow || tree.field_conditions.length) {
+    await clearDir(p.fieldConditionsDir);
+    const fcName = uniquify();
+    for (const fc of tree.field_conditions) {
+      const name = fcName(slug(String(fc['name'] ?? `condition-${fc['id']}`)));
+      await put(join(p.fieldConditionsDir, `${name}.json`), fc, 'field-condition');
+    }
+  }
+
+  await put(p.labels, tree.labels, 'labels');
+  await put(p.webhooks, tree.webhooks, 'webhooks');
+  await put(p.publicForm, tree.public_form, 'public-form');
+  await put(p.preferences, tree.preferences, 'preferences');
+
+  /**
+   * relations.json carries the writable `pipe_relations` plus a derived,
+   * read-only summary of where this pipe actually connects — which for the test
+   * pipe comes only from connector fields (PLAN.md §2.2). `_`-prefixed keys are
+   * tool metadata and are ignored on read.
+   */
+  const connectors = tree.phases
+    .flatMap((ph) => ph.fields.map((f) => ({ phase: ph['name'], field: f })))
+    .filter(({ field }) => field['type_id'] === 'connector')
+    .map(({ phase, field }) => ({
+      field_id: field['id'],
+      field_label: field['label'],
+      in_phase: phase,
+      connected_repo_id: field['connected_pipe_id'] ?? null,
+      can_create_connected_cards: field['can_create_connected_cards'] ?? null,
+    }));
+  await put(
+    p.relations,
+    {
+      pipe_relations: tree.pipe_relations,
+      _connector_fields: connectors,
+      _note:
+        'Connector fields do not imply pipe_relations rows and pipeMapGraph does not ' +
+        'derive edges from them (PLAN.md §2.2). _connector_fields is derived on pull ' +
+        'and ignored on read — edit the connector field inside its phase file instead.',
+    },
+    'relations',
+  );
+
+  await clearDir(p.readonlyDir);
+  const ro: Array<[string, Row[]]> = [
+    ['email-templates.json', tree.readonly.email_templates],
+    ['email-inboxes.json', tree.readonly.email_inboxes],
+    ['visibilities.json', tree.readonly.visibilities],
+    ['field-maps.json', tree.readonly.field_maps],
+  ];
+  for (const [name, rows] of ro) await put(join(p.readonlyDir, name), rows, 'readonly');
+
+  await writeReadmes(p, tree.meta.repoKind);
+  return written;
+};
+
+const writeReadmes = async (p: ReturnType<typeof repoPaths>, kind: 'pipe' | 'table' = 'pipe') => {
+  /**
+   * A database gets its own README, because its files are honest but not
+   * self-explanatory.
+   *
+   * Pipefy stores a database as a repo: its columns are fields on a start form,
+   * and the phases after it are record statuses. That is the real shape and the
+   * codec round-trips it exactly — but nothing about `phases/01--ativo.json`
+   * tells a reader it is a status with no write path rather than a workflow step.
+   * The old code refused to write these files at all for that reason; writing
+   * them with an explanation is strictly better than reporting a gap.
+   */
+  if (kind === 'table') {
+    await fs.writeFile(
+      join(p.dir, 'README.md'),
+      [
+        '# This is a database, not a pipe',
+        '',
+        'Pipefy stores a database as a repo, so these files have the shape of a pipe.',
+        'That shape is real — it is what the API returns and what it accepts back — but',
+        'two parts of it mean something different here.',
+        '',
+        '| Path | What it actually is |',
+        '| --- | --- |',
+        '| `table.json` | the database row. `noun` is what one record is called |',
+        '| `phases/00--start-form.json` | **the columns of the database.** Its `fields[]` are what you edit |',
+        '| `phases/NN--*.json` (the rest) | **record statuses**, not workflow steps. No public write path |',
+        '',
+        '## What can be changed',
+        '',
+        'The database row and its columns. A column is created with',
+        '`createTableField`, changed with `updateTableField` and removed with',
+        '`deleteTableField` — all of which address a field by its **slug**, unlike the',
+        'pipe mutations. Order is a separate `setTableFieldOrder` call, because',
+        '`createTableField` has no `index` input.',
+        '',
+        '## What cannot',
+        '',
+        '- **Record statuses.** Readable, and absent from `UpdateTableInput`.',
+        '- **Records.** They are the data of this database, as cards are the data of a pipe.',
+        '  They are deliberately not pulled, so an apply can never rewrite them.',
+        '- **Automations, field conditions and phase jumps.** A database has no',
+        '  workflow. If you find yourself authoring one here, the change belongs on a',
+        '  pipe.',
+        '',
+        '`pipe plan` refuses each of these by name rather than sending it.',
+        '',
+      ].join('\n'),
+      'utf8',
+    );
+  }
+  await fs.writeFile(
+    join(p.readonlyDir, 'README.md'),
+    [
+      '# Read-only entities',
+      '',
+      'These have **no public write path** (PLAN.md §2). They are pulled so that',
+      'Claude can read them and reason correctly, and any edit here is rejected by',
+      '`pipe validate` on the basis of the path alone.',
+      '',
+      '| File | Why read-only |',
+      '| --- | --- |',
+      '| `email-templates.json` | `EmailTemplate` is read-only; no Input types exist |',
+      '| `email-inboxes.json` | no mutation found |',
+      '| `visibilities.json` | only settable via `updatePipe.public_form`; `slug` is not settable |',
+      '| `field-maps.json` | a field’s own field_maps (`owner_type` other than `Automation`) — a connector ' +
+        'field’s "copy this value from the connected card" configuration. No write path has been ' +
+        'measured; edit the connector field in the product instead |',
+      '',
+    ].join('\n'),
+    'utf8',
+  );
+};