@pipefy/pipefy-process-coder 0.1.0

package/src/workspace/layout.ts

@@ -0,0 +1,165 @@
+import { join } from 'node:path';
+import { slug } from '../util/slug.ts';
+import type { RepoRef } from '../pipefy/discovery.ts';
+
+/**
+ * Workspace layout — PLAN.md §4.
+ *
+ * Two deliberate deviations from the plan's sketch, both to make the edit loop
+ * shorter, and both noted in the generated AGENTS.md:
+ *
+ *  - `phase-jumps.json` is gone. A phase's jump set lives in the phase file as
+ *    `jump_target_ids`, because the internal endpoint is per-phase and
+ *    set-replacing (PLAN.md §2.1), so the set is the honest shape and there is
+ *    no second file to keep consistent.
+ *  - `webhooks.json` is one array file rather than a directory, matching
+ *    `labels.json`. Webhooks are few and have no sub-entities.
+ */
+
+export const PPC_DIR = '.ppc';
+
+export const paths = (root: string) => ({
+  root,
+  workspaceFile: join(root, 'workspace.json'),
+  /**
+   * `AGENTS.md`, not `CLAUDE.md`: the emerging cross-tool convention (this
+   * repo's own top-level docs already use it) — the file is read by whatever
+   * agent is editing the workspace, not only Claude Code.
+   */
+  agentsMd: join(root, 'AGENTS.md'),
+  mappingMd: join(root, 'MAPPING.md'),
+  capabilitiesMd: join(root, 'CAPABILITIES.md'),
+  referenceMd: join(root, 'REFERENCE.md'),
+  pipe: join(root, PPC_DIR),
+  lock: join(root, PPC_DIR, 'lock.json'),
+  capabilities: join(root, PPC_DIR, 'capabilities.json'),
+  /**
+   * The closed sets the API publishes, read on the last pull. Cached so
+   * validate, plan and apply can refuse a wrong value offline and name it.
+   */
+  reference: join(root, PPC_DIR, 'reference.json'),
+  baselineDir: join(root, PPC_DIR, 'baseline'),
+  plansDir: join(root, PPC_DIR, 'plans'),
+  pipesDir: join(root, 'pipes'),
+  databasesDir: join(root, 'databases'),
+  gitignore: join(root, '.gitignore'),
+});
+
+/**
+ * `pipes/<numericId>--<slug>` / `databases/<internalId>--<apiId>--<slug>`.
+ *
+ * A database carries two identities — a numeric internal id and an api id — and
+ * `table(id:)` answers to both. Putting both in the directory name is what makes
+ * the duality visible rather than a footnote: whichever spelling you pull with,
+ * you land in the same directory, and the one you did not use is right there when
+ * you need to cross-reference a connector field (which carries the numeric form)
+ * against an API response (which carries the slug).
+ */
+export const repoDirName = (ref: { kind: string; id: number; apiId?: string; name?: string | null }) =>
+  ref.kind === 'table' && ref.apiId && ref.apiId !== String(ref.id)
+    ? `${ref.id}--${ref.apiId}--${slug(ref.name)}`
+    : `${ref.id}--${slug(ref.name)}`;
+
+export const repoDir = (root: string, ref: RepoRef) =>
+  join(ref.kind === 'table' ? paths(root).databasesDir : paths(root).pipesDir, repoDirName(ref));
+
+/**
+ * The files of one repo.
+ *
+ * `kind` decides one name: a database's root row is `table.json`, not
+ * `pipe.json`. It is worth the parameter because a file called `pipe.json`
+ * inside `databases/` teaches a reader something false about the platform, and
+ * this workspace is read by agents that have no other source of truth.
+ *
+ * Required rather than defaulted: a caller who forgets would silently get pipe
+ * paths for a database and a mysteriously absent root file.
+ */
+export const repoPaths = (dir: string, kind: 'pipe' | 'table' = 'pipe') => ({
+  dir,
+  meta: join(dir, '_meta.json'),
+  repo: join(dir, kind === 'table' ? 'table.json' : 'pipe.json'),
+  /** Both spellings, so a directory can be recognised before its kind is known. */
+  repoCandidates: [join(dir, 'pipe.json'), join(dir, 'table.json')],
+  phasesDir: join(dir, 'phases'),
+  automationsDir: join(dir, 'automations'),
+  fieldConditionsDir: join(dir, 'field-conditions'),
+  labels: join(dir, 'labels.json'),
+  webhooks: join(dir, 'webhooks.json'),
+  publicForm: join(dir, 'public-form.json'),
+  preferences: join(dir, 'preferences.json'),
+  relations: join(dir, 'relations.json'),
+  readonlyDir: join(dir, '_readonly'),
+  agentsDir: join(dir, 'agents'),
+});
+
+/** Baseline is stored as the payload envelope: one file, and diff-ready. */
+export const baselineFile = (root: string, repoId: number) => join(paths(root).baselineDir, `${repoId}.json`);
+
+/**
+ * The agent baseline: what a repo's AI agents looked like at the last pull.
+ *
+ * Stored separately from the payload envelope for the same reason flows are:
+ * agents are not in it, they come from a different query, keyed by the pipe's
+ * uuid rather than its id (src/pipefy/agents.ts).
+ */
+export const agentBaselineFile = (root: string, repoId: number) =>
+  join(paths(root).baselineDir, `${repoId}.agents.json`);
+
+/**
+ * The flow baseline: what a repo's iPaaS flows looked like at the last pull.
+ *
+ * Stored separately from the payload envelope because flows are not in it —
+ * they come from a different endpoint. Without this the flow diff had no
+ * recorded past to compare against and had to re-read the pipe live every time,
+ * which meant no offline diff and no record of what was pulled.
+ */
+export const flowBaselineFile = (root: string, repoId: number) =>
+  join(paths(root).baselineDir, `${repoId}.flows.json`);
+
+export const planFile = (root: string, planId: string) => join(paths(root).plansDir, `${planId}.json`);
+
+/**
+ * How a finished plan is named.
+ *
+ * A plan file is the record `--resume` replays, and it is written before the
+ * first request and rewritten after every step, so a crash mid-apply is
+ * resumable. That made `.ppc/plans/` a list of filenames that all look alike,
+ * with no way to tell an applied plan from one still waiting — the outcome was
+ * only inside the JSON, or via `pipe status`.
+ *
+ * So the outcome goes in the name once the apply is over. **No suffix means it
+ * was never applied**, which is the state a bare `pipe plan` leaves behind.
+ * The plan id does not change, so every `--resume <id>` instruction still works.
+ */
+export const PLAN_SUFFIX: Record<string, string> = {
+  applied: '--applied',
+  'partially-applied': '--partial',
+  failed: '--failed',
+};
+
+export const planFileFor = (root: string, planId: string, status: string) =>
+  join(paths(root).plansDir, `${planId}${PLAN_SUFFIX[status] ?? ''}.json`);
+
+/**
+ * The file holding a plan, whatever it was renamed to. Returns null if there is
+ * none — the caller decides whether that is an error.
+ */
+export const findPlanFile = async (root: string, planId: string): Promise<string | null> => {
+  const { access } = await import('node:fs/promises');
+  const candidates = [planFile(root, planId), ...Object.values(PLAN_SUFFIX).map((sfx) => join(paths(root).plansDir, `${planId}${sfx}.json`))];
+  for (const c of candidates) {
+    try {
+      await access(c);
+      return c;
+    } catch {
+      // try the next spelling
+    }
+  }
+  return null;
+};
+
+/** Paths under a repo dir that must never be edited — checked by path alone. */
+export const isReadonlyPath = (relPath: string) => {
+  const p = relPath.replace(/\\/g, '/');
+  return p.includes('/_readonly/') || p.startsWith('_readonly/');
+};

package/src/workspace/lock.ts

@@ -0,0 +1,148 @@
+import { relative } from 'node:path';
+import { exists, readJson, writeJson } from '../util/fsx.ts';
+import type { Coverage, EntityName } from '../model/payload.ts';
+import type { RepoRef } from '../pipefy/discovery.ts';
+import type { Tree } from '../model/tree.ts';
+import { baselineFile, paths, repoDir } from './layout.ts';
+
+/**
+ * `.ppc/lock.json` — PLAN.md §4: id ↔ uuid ↔ path, the table-id duality, and
+ * the version ids of what was pulled.
+ *
+ * It is also where the applier looks up "what did this uuid become", which is
+ * the job the Change Migrator recipes do with a Pipefy table keyed
+ * `dev_pipe_field_internal_id → prod_pipe_field_internal_id` and one API call
+ * per id. Same idea, local and free.
+ */
+
+export type LockRepo = {
+  kind: 'pipe' | 'table';
+  /** Numeric id, which every API accepts. */
+  id: number;
+  /**
+   * What the API answers with. For tables this is a slug (`dzmxcuxz`) while the
+   * connector field says `301908697` — store both or a connected database gets
+   * pulled twice under two identities (PLAN.md §2.2).
+   */
+  apiId: string;
+  uuid: string | null;
+  name: string | null;
+  dir: string;
+  via: string[];
+  depth: number;
+  source: 'snapshot' | 'reconstructed';
+  versionId: string | null;
+  readAt: string;
+  coverage: Record<EntityName, Coverage>;
+  /** uuid → numeric id, for the four arrays that have a uuid. */
+  identity: Partial<Record<EntityName, Record<string, number>>>;
+  /** field numeric id → slug, so `%{433255467}` can be explained to a human. */
+  fieldSlugs: Record<string, string>;
+};
+
+export type Lock = {
+  version: 1;
+  updatedAt: string;
+  rootRepoId: number;
+  organizationId: string | null;
+  depth: number;
+  repos: LockRepo[];
+  edges: Array<{ from: number; to: number; kind: string; label?: string }>;
+};
+
+export const emptyLock = (rootRepoId: number, organizationId: string | null, depth: number): Lock => ({
+  version: 1,
+  updatedAt: new Date().toISOString(),
+  rootRepoId,
+  organizationId,
+  depth,
+  repos: [],
+  edges: [],
+});
+
+export const readLock = async (root: string): Promise<Lock | null> => {
+  const p = paths(root).lock;
+  if (!(await exists(p))) return null;
+  return readJson<Lock>(p);
+};
+
+export const writeLock = async (root: string, lock: Lock) => {
+  lock.updatedAt = new Date().toISOString();
+  await writeJson(paths(root).lock, lock);
+};
+
+const identityFor = (tree: Tree): LockRepo['identity'] => {
+  const grab = (rows: Array<Record<string, unknown>>): Record<string, number> => {
+    const out: Record<string, number> = {};
+    for (const r of rows) {
+      const uuid = r['uuid'];
+      const id = Number(r['id']);
+      if (typeof uuid === 'string' && uuid && Number.isFinite(id)) out[uuid] = id;
+    }
+    return out;
+  };
+  return {
+    phases: grab(tree.phases),
+    fields: grab(tree.phases.flatMap((p) => p.fields)),
+    automations: grab(tree.automations),
+    labels: grab(tree.labels),
+  };
+};
+
+export const upsertRepo = (lock: Lock, ref: RepoRef, root: string, tree: Tree): LockRepo => {
+  const entry: LockRepo = {
+    kind: ref.kind,
+    id: ref.id,
+    apiId: ref.apiId,
+    uuid: (tree.repo['uuid'] as string | undefined) ?? ref.uuid ?? null,
+    name: (tree.repo['name'] as string | undefined) ?? ref.name ?? null,
+    dir: relative(root, repoDir(root, ref)).replace(/\\/g, '/'),
+    via: ref.via,
+    depth: ref.depth,
+    source: tree.meta.source,
+    versionId: tree.meta.versionId ?? null,
+    readAt: tree.meta.readAt,
+    coverage: tree.meta.coverage,
+    identity: identityFor(tree),
+    fieldSlugs: Object.fromEntries(
+      tree.phases
+        .flatMap((p) => p.fields)
+        .filter((f) => f['slug'])
+        .map((f) => [String(f['id']), String(f['slug'])]),
+    ),
+  };
+  const i = lock.repos.findIndex((r) => r.id === ref.id);
+  if (i >= 0) lock.repos[i] = entry;
+  else lock.repos.push(entry);
+  return entry;
+};
+
+export const lookupRepo = (lock: Lock, idOrName: string): LockRepo | undefined => {
+  const byId = lock.repos.find((r) => String(r.id) === idOrName || r.apiId === idOrName);
+  if (byId) return byId;
+  const needle = idOrName.toLowerCase();
+  return lock.repos.find((r) => (r.name ?? '').toLowerCase() === needle || r.dir.toLowerCase().includes(needle));
+};
+
+export const rootRepo = (lock: Lock): LockRepo | undefined => lock.repos.find((r) => r.id === lock.rootRepoId);
+
+export const baselinePathFor = (root: string, repoId: number) => baselineFile(root, repoId);
+
+export type WorkspaceFile = {
+  name: string;
+  createdAt: string;
+  rootRepoId: number;
+  organizationId: string | null;
+  depth: number;
+  /** How the last pull read the structure — informational, per repo in lock.json. */
+  lastPullSource: 'snapshot' | 'reconstructed' | 'mixed';
+  tool: { name: 'pipefy-process-coder'; version: string };
+};
+
+export const readWorkspace = async (root: string): Promise<WorkspaceFile | null> => {
+  const p = paths(root).workspaceFile;
+  if (!(await exists(p))) return null;
+  return readJson<WorkspaceFile>(p);
+};
+
+export const writeWorkspace = (root: string, w: WorkspaceFile) => writeJson(paths(root).workspaceFile, w);

package/src/workspace/read.ts

@@ -0,0 +1,165 @@
+import { basename, dirname } from 'node:path';
+import { exists, listJson, readJson } from '../util/fsx.ts';
+import { UserError } from '../util/log.ts';
+import { fullCoverage, type Row } from '../model/payload.ts';
+import type { AutomationNode, FieldConditionNode, PhaseNode, Tree, TreeMeta } from '../model/tree.ts';
+import { repoPaths } from './layout.ts';
+import { AGENT_DIR_TAG } from './agents.ts';
+
+/**
+ * files → tree. The exact inverse of write.ts, so that
+ * `readTree(writeTree(t)) ≡ t` and the edited workspace can be diffed against
+ * the baseline with no special cases.
+ *
+ * Ordering never comes from filenames: the index-prefixed names are for humans,
+ * and `index` inside the file is authoritative. A phase Claude renumbered by
+ * editing `index` therefore works without renaming any file.
+ */
+
+const num = (v: unknown): number => {
+  const n = Number(v);
+  return Number.isFinite(n) ? n : 0;
+};
+
+/**
+ * An id, or the placeholder that stands for one.
+ *
+ * `num` collapses anything non-numeric to 0, which is right for an index and
+ * catastrophic for an id list: a phase jumping to a phase created in the same
+ * edit is written `"%{_new:<uuid>}"`, exactly as the generated AGENTS.md
+ * instructs, and `num` turned that into a jump to phase 0 — accepted by the
+ * validator, sent to the API, and impossible to explain. A typo met the same
+ * fate. Non-numeric values are kept verbatim so the validator can judge them.
+ */
+const idOrPlaceholder = (v: unknown): number | string => {
+  const n = Number(v);
+  if (Number.isFinite(n) && v !== '' && v !== null) return n;
+  return String(v);
+};
+
+const readArrayFile = async (path: string): Promise<Row[]> => {
+  if (!(await exists(path))) return [];
+  const v = await readJson<unknown>(path);
+  if (v === null) return [];
+  if (!Array.isArray(v)) throw new UserError(`${path}: expected a JSON array`);
+  return v as Row[];
+};
+
+const readObjectFile = async (path: string): Promise<Row | null> => {
+  if (!(await exists(path))) return null;
+  const v = await readJson<unknown>(path);
+  if (v === null) return null;
+  if (typeof v !== 'object' || Array.isArray(v)) throw new UserError(`${path}: expected a JSON object or null`);
+  return v as Row;
+};
+
+const readDirOf = async <T>(dir: string): Promise<Array<{ path: string; value: T }>> => {
+  const files = (await listJson(dir)).filter((f) => basename(f) !== '_meta.json');
+  const out: Array<{ path: string; value: T }> = [];
+  for (const f of files) out.push({ path: f, value: await readJson<T>(f) });
+  return out;
+};
+
+export const readTree = async (dir: string): Promise<Tree> => {
+  /**
+   * The kind comes from `_meta.json`, which is written first and whose path does
+   * not depend on the kind. Falling back to probing both spellings keeps a
+   * hand-made directory readable.
+   */
+  const meta = ((await readObjectFile(repoPaths(dir).meta)) ?? {}) as unknown as TreeMeta;
+  const p = repoPaths(dir, meta.repoKind === 'table' ? 'table' : 'pipe');
+  let repoFile = p.repo;
+  if (!(await exists(repoFile))) {
+    const found = [];
+    for (const c of p.repoCandidates) if (await exists(c)) found.push(c);
+    if (!found.length) {
+      throw new UserError(`${dir} is not a pulled repo directory (no pipe.json or table.json)`);
+    }
+    repoFile = found[0] as string;
+  }
+
+  const repo = (await readObjectFile(repoFile)) ?? {};
+
+  const phaseFiles = await readDirOf<PhaseNode>(p.phasesDir);
+  const phases = phaseFiles
+    .map(({ path, value }) => {
+      if (!value || typeof value !== 'object') throw new UserError(`${path}: expected a phase object`);
+      return {
+        ...value,
+        fields: Array.isArray(value.fields) ? value.fields : [],
+        jump_target_ids: Array.isArray(value.jump_target_ids) ? value.jump_target_ids.map(idOrPlaceholder) : [],
+        team_member_ids: Array.isArray(value.team_member_ids) ? value.team_member_ids.map(idOrPlaceholder) : [],
+      } as PhaseNode;
+    })
+    .sort((a, b) => num(a['index']) - num(b['index']) || num(a['id']) - num(b['id']));
+
+  const normaliseAutomation = (path: string, value: unknown): AutomationNode => {
+    if (!value || typeof value !== 'object') throw new UserError(`${path}: expected an automation object`);
+    const v = value as AutomationNode;
+    return {
+      ...v,
+      condition: v.condition ?? null,
+      field_maps: Array.isArray(v.field_maps) ? v.field_maps : [],
+      distribute_assignments: Array.isArray(v.distribute_assignments) ? v.distribute_assignments : [],
+      response_schemas: Array.isArray(v.response_schemas) ? v.response_schemas : [],
+    };
+  };
+
+  const automations = (await readDirOf<AutomationNode>(p.automationsDir)).map(({ path, value }) =>
+    normaliseAutomation(path, value),
+  );
+
+  /**
+   * A behavior — an automation folded under its agent's `behaviors/` instead
+   * of `automations/`, per workspace/agents.ts. `_agent_dir` is not read from
+   * the file (it is never written there); it is derived from where the file
+   * physically is, which is the only record of the link there is.
+   */
+  for (const path of await listJson(p.agentsDir)) {
+    if (basename(dirname(path)) !== 'behaviors') continue;
+    const agentDir = basename(dirname(dirname(path)));
+    const value = await readJson<unknown>(path);
+    automations.push({ ...normaliseAutomation(path, value), [AGENT_DIR_TAG]: agentDir } as AutomationNode);
+  }
+
+  const field_conditions = (await readDirOf<FieldConditionNode>(p.fieldConditionsDir))
+    .map(({ path, value }) => {
+      if (!value || typeof value !== 'object') throw new UserError(`${path}: expected a field condition object`);
+      return {
+        ...value,
+        actions: Array.isArray(value.actions) ? value.actions : [],
+        condition: value.condition ?? null,
+      } as FieldConditionNode;
+    })
+    .sort((a, b) => num(a['index']) - num(b['index']) || num(a['id']) - num(b['id']));
+
+  const relationsFile = (await readObjectFile(p.relations)) ?? {};
+  const pipe_relations = Array.isArray(relationsFile['pipe_relations']) ? (relationsFile['pipe_relations'] as Row[]) : [];
+
+  return {
+    meta: {
+      source: meta.source ?? 'reconstructed',
+      versionId: meta.versionId ?? null,
+      readAt: meta.readAt ?? new Date().toISOString(),
+      repoKind: meta.repoKind ?? 'pipe',
+      coverage: meta.coverage ?? fullCoverage('unknown'),
+      repo_columns: meta.repo_columns ?? 'complete',
+      notes: meta.notes ?? [],
+    },
+    repo,
+    phases,
+    automations,
+    field_conditions,
+    labels: await readArrayFile(p.labels),
+    webhooks: await readArrayFile(p.webhooks),
+    pipe_relations,
+    public_form: await readObjectFile(p.publicForm),
+    preferences: await readObjectFile(p.preferences),
+    readonly: {
+      email_templates: await readArrayFile(`${p.readonlyDir}/email-templates.json`),
+      email_inboxes: await readArrayFile(`${p.readonlyDir}/email-inboxes.json`),
+      visibilities: await readArrayFile(`${p.readonlyDir}/visibilities.json`),
+      field_maps: await readArrayFile(`${p.readonlyDir}/field-maps.json`),
+    },
+  };
+};

package/src/workspace/reference.ts

@@ -0,0 +1,24 @@
+import { readFile } from 'node:fs/promises';
+import { paths } from './layout.ts';
+import { debug } from '../util/log.ts';
+import type { Reference } from '../pipefy/reference.ts';
+
+/**
+ * Load the cached closed sets, or null.
+ *
+ * Null on anything: no file, unreadable, malformed. Every consumer treats null
+ * as "these checks are skipped" and never as "this value is fine", so a
+ * workspace pulled by an older build loses the check rather than gaining a
+ * wrong answer.
+ */
+export const loadReference = async (root: string): Promise<Reference | null> => {
+  try {
+    const raw = await readFile(paths(root).reference, 'utf8');
+    const ref = JSON.parse(raw) as Reference;
+    if (!ref || typeof ref !== 'object' || !Array.isArray(ref.events)) return null;
+    return ref;
+  } catch (e) {
+    debug(`no cached reference: ${(e as Error).message}`);
+    return null;
+  }
+};