@pipefy/pipefy-process-coder 0.1.0

package/src/util/pool.ts

@@ -0,0 +1,48 @@
+import { debug } from './log.ts';
+
+export const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+/** Bounded-concurrency map that preserves input order. */
+export const mapPool = async <T, R>(
+  items: readonly T[],
+  limit: number,
+  fn: (item: T, index: number) => Promise<R>,
+): Promise<R[]> => {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
+    for (;;) {
+      const i = next++;
+      if (i >= items.length) return;
+      results[i] = await fn(items[i] as T, i);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+};
+
+export type RetryOpts = { tries?: number; baseMs?: number; label?: string };
+
+/**
+ * Exponential backoff with jitter. PLAN.md §9: rate limits are unmeasured, so
+ * every network call goes through this and every retry is logged - which is how
+ * we measure the ceiling instead of discovering it during a client apply.
+ */
+export const retry = async <T>(fn: () => Promise<T>, opts: RetryOpts = {}): Promise<T> => {
+  const tries = opts.tries ?? 5;
+  const base = opts.baseMs ?? 600;
+  let lastErr: unknown;
+  for (let attempt = 1; attempt <= tries; attempt++) {
+    try {
+      return await fn();
+    } catch (e) {
+      lastErr = e;
+      const retryable = (e as { retryable?: boolean }).retryable !== false;
+      if (!retryable || attempt === tries) break;
+      const wait = Math.round(base * 2 ** (attempt - 1) * (0.7 + Math.random() * 0.6));
+      debug(`retry ${attempt}/${tries - 1} in ${wait}ms${opts.label ? ` (${opts.label})` : ''}: ${(e as Error).message}`);
+      await sleep(wait);
+    }
+  }
+  throw lastErr;
+};

package/src/util/slug.ts

@@ -0,0 +1,26 @@
+/** Filesystem-safe, stable, human-readable name segment. */
+export const slug = (s: string | null | undefined, fallback = 'untitled'): string => {
+  const base = (s ?? '').normalize('NFKD').replace(/[̀-ͯ]/g, '');
+  const out = base
+    .toLowerCase()
+    .replace(/['"`]/g, '')
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 60)
+    .replace(/-+$/g, '');
+  return out || fallback;
+};
+
+/** `01--new-request` - index-prefixed so directory order matches pipe order. */
+export const indexed = (i: number, name: string | null | undefined): string =>
+  `${String(Math.trunc(i)).padStart(2, '0')}--${slug(name)}`;
+
+/** Disambiguate repeated slugs deterministically: name, name-2, name-3. */
+export const uniquify = (): ((s: string) => string) => {
+  const seen = new Map<string, number>();
+  return (s: string) => {
+    const n = (seen.get(s) ?? 0) + 1;
+    seen.set(s, n);
+    return n === 1 ? s : `${s}-${n}`;
+  };
+};

package/src/util/tui.ts

@@ -0,0 +1,335 @@
+import { bold, claimLogSink, dim, green, red, yellow, blue, magenta } from './log.ts';
+
+/**
+ * A small live progress display for the long commands — pull, apply, dry-run.
+ *
+ * Two renderers behind one interface, chosen by whether stderr is a terminal:
+ *
+ *  - **live**: redraws a block of lines in place, with a spinner and elapsed
+ *    times, so a 63-second snapshot wait looks like work rather than a hang.
+ *  - **plain**: one line per state change, no cursor control, no repainting.
+ *    This is what CI, `2>file`, and a backgrounded run get — and it is not an
+ *    afterthought, since that is how this tool is most often watched.
+ *
+ * Everything goes to stderr, leaving stdout clean for `--json`.
+ */
+
+const ESC = '\x1b[';
+const HIDE_CURSOR = `${ESC}?25l`;
+const SHOW_CURSOR = `${ESC}?25h`;
+const CLEAR_LINE = `${ESC}2K`;
+const cursorUp = (n: number) => (n > 0 ? `${ESC}${n}A` : '');
+
+/** Braille spinner: 8 frames, reads as motion even at 80ms. */
+const FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧'];
+
+export type TaskState = 'pending' | 'running' | 'done' | 'failed' | 'skipped';
+
+export type Task = {
+  id: string;
+  label: string;
+  state: TaskState;
+  /** Replaces the label's tail while running: "polling — 42s (~63s typical)". */
+  detail?: string;
+  /** Nesting depth, for fields under a phase or steps under a repo. */
+  depth?: number;
+  startedAt?: number;
+  endedAt?: number;
+  /** Shown after the label, dimmed: "destructive", "internal-api". */
+  tags?: string[];
+};
+
+const MARK: Record<TaskState, (s: string) => string> = {
+  pending: dim,
+  running: blue,
+  done: green,
+  failed: red,
+  skipped: yellow,
+};
+
+const GLYPH: Record<TaskState, string> = {
+  pending: '·',
+  running: '',
+  done: '+',
+  failed: 'x',
+  skipped: '-',
+};
+
+const secs = (ms: number) => `${(ms / 1000).toFixed(ms < 10_000 ? 1 : 0)}s`;
+
+export type ProgressOpts = {
+  title: string;
+  /** Force plain output regardless of tty — used by tests and `--no-tui`. */
+  plain?: boolean;
+  /** Where to write. Defaults to stderr. */
+  stream?: NodeJS.WriteStream;
+};
+
+export class Progress {
+  private tasks: Task[] = [];
+  private byId = new Map<string, Task>();
+  private title: string;
+  private stream: NodeJS.WriteStream;
+  private live: boolean;
+  private frame = 0;
+  private timer: NodeJS.Timeout | null = null;
+  private linesDrawn = 0;
+  private startedAt = Date.now();
+  private finished = false;
+  /** Extra lines under the task list: warnings that must survive a repaint. */
+  private notes: string[] = [];
+  private cleanup: (() => void) | null = null;
+  /**
+   * Renders are coalesced onto the spinner tick rather than run on every state
+   * change. Building a 200-step plan calls add() 200 times, and repainting the
+   * whole block each time is both wasted work and visible flicker; a 90ms delay
+   * before a change appears is not perceptible.
+   */
+  private dirty = false;
+
+  constructor(opts: ProgressOpts) {
+    this.title = opts.title;
+    this.stream = opts.stream ?? process.stderr;
+    this.live = !opts.plain && Boolean(this.stream.isTTY) && !process.env.PPC_NO_TUI && !process.env.NO_TUI;
+  }
+
+  /** Begin painting. Safe to call once; a second call is ignored. */
+  start() {
+    if (this.timer || this.finished) return;
+
+    /**
+     * Claimed in both modes, so the semantics do not depend on whether stderr
+     * is a terminal. Live, it stops an interleaved write garbling a repaint;
+     * plain, it stops the task list and the old progress chatter saying the same
+     * thing twice.
+     */
+    claimLogSink({ note: (text: string) => this.note(text) });
+
+    if (!this.live) {
+      this.stream.write(bold(this.title) + '\n');
+      return;
+    }
+    this.stream.write(HIDE_CURSOR);
+    // Leaving the cursor hidden after a Ctrl-C would break the user's terminal.
+    const restore = () => this.stop();
+    process.once('SIGINT', restore);
+    process.once('SIGTERM', restore);
+    process.once('exit', restore);
+    this.cleanup = () => {
+      process.off('SIGINT', restore);
+      process.off('SIGTERM', restore);
+      process.off('exit', restore);
+    };
+    this.timer = setInterval(() => {
+      this.frame++;
+      // The spinner has to move even when nothing changed, so a running task
+      // always repaints; an idle list repaints only when it is stale.
+      if (this.dirty || this.tasks.some((t) => t.state === 'running')) this.render();
+    }, 90);
+    this.timer.unref?.();
+    this.render();
+  }
+
+  add(task: Omit<Task, 'state'> & { state?: TaskState }): Task {
+    const t: Task = { state: 'pending', ...task };
+    this.tasks.push(t);
+    this.byId.set(t.id, t);
+    this.touch();
+    return t;
+  }
+
+  update(id: string, patch: Partial<Omit<Task, 'id'>>) {
+    const t = this.byId.get(id);
+    if (!t) return;
+    const before = t.state;
+    Object.assign(t, patch);
+    if (patch.state === 'running' && !t.startedAt) {
+      t.startedAt = Date.now();
+      // Seeded so the first heartbeat is one interval away, not immediate:
+      // otherwise every task announces its first stage and fast work is noise.
+      this.lastHeartbeat.set(t.id, t.startedAt);
+    }
+    if (patch.state && patch.state !== 'running' && !t.endedAt) t.endedAt = Date.now();
+    if (!this.live) {
+      if (patch.state && patch.state !== before) this.onPlainTransition(t);
+      else if (patch.detail !== undefined) this.plainHeartbeat(t);
+    } else {
+      this.touch();
+    }
+  }
+
+  /**
+   * One line per task in plain mode, plus a heartbeat for slow work.
+   *
+   * Every task reports its outcome, so nothing starts and then vanishes from the
+   * log. Work that announces a stage while running — a snapshot poll, a download
+   * — also reports that, throttled, because a log that goes silent for 63
+   * seconds is indistinguishable from a hang. A 200-step plan of fast calls is
+   * 200 lines.
+   */
+  private static readonly HEARTBEAT_MS = 15_000;
+  private lastHeartbeat = new Map<string, number>();
+
+  private onPlainTransition(t: Task) {
+    // Terminal states always speak.
+    if (t.state === 'done' || t.state === 'failed' || t.state === 'skipped') this.emitPlain(t);
+  }
+
+  private plainHeartbeat(t: Task) {
+    if (this.live || t.state !== 'running') return;
+    const now = Date.now();
+    const last = this.lastHeartbeat.get(t.id) ?? 0;
+    if (now - last < Progress.HEARTBEAT_MS) return;
+    this.lastHeartbeat.set(t.id, now);
+    this.emitPlain(t);
+  }
+
+  /**
+   * Mark the frame stale. Before start() there is no timer, so paint at once —
+   * that is what makes a short-lived Progress still show something.
+   */
+  private touch() {
+    if (!this.live) return;
+    this.dirty = true;
+    if (!this.timer) this.render();
+  }
+
+  /** Whether a task with this id already exists. */
+  has(id: string) {
+    return this.byId.has(id);
+  }
+
+  /** The last few only: a frame that grows without bound scrolls the terminal. */
+  private static readonly MAX_NOTES = 5;
+
+  /** A line that stays visible under the task list. */
+  note(text: string) {
+    this.notes.push(text);
+    if (this.notes.length > Progress.MAX_NOTES) this.notes.splice(0, this.notes.length - Progress.MAX_NOTES);
+    if (!this.live) this.stream.write(text + '\n');
+    else this.render();
+  }
+
+  /** Stop painting and leave the final frame on screen. */
+  stop(summary?: string) {
+    if (this.finished) return;
+    this.finished = true;
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    this.cleanup?.();
+    claimLogSink(null);
+    if (this.live) {
+      this.render(true);
+      this.stream.write(SHOW_CURSOR);
+    }
+    if (summary) this.stream.write(summary + '\n');
+  }
+
+  get elapsed() {
+    return Date.now() - this.startedAt;
+  }
+
+  // ── rendering ─────────────────────────────────────────────────────────────
+
+  private emitPlain(t: Task) {
+    const paint = MARK[t.state];
+    const indent = '  '.repeat(t.depth ?? 0);
+    const tail = t.detail ? ` ${dim(t.detail)}` : '';
+    const took = t.startedAt && t.endedAt ? ` ${dim(secs(t.endedAt - t.startedAt))}` : '';
+    this.stream.write(`${indent}${paint(GLYPH[t.state] || '>')} ${t.label}${tail}${took}\n`);
+  }
+
+  private line(t: Task): string {
+    const paint = MARK[t.state];
+    const glyph = t.state === 'running' ? blue(FRAMES[this.frame % FRAMES.length] as string) : paint(GLYPH[t.state]);
+    const indent = '  '.repeat(t.depth ?? 0);
+
+    const label = t.state === 'pending' ? dim(t.label) : t.label;
+    const tags = t.tags?.length ? ' ' + t.tags.map((g) => dim(`[${g}]`)).join(' ') : '';
+
+    let timing = '';
+    if (t.state === 'running' && t.startedAt) timing = ` ${dim(secs(Date.now() - t.startedAt))}`;
+    else if (t.startedAt && t.endedAt) timing = ` ${dim(secs(t.endedAt - t.startedAt))}`;
+
+    const detail = t.detail ? ` ${dim('— ' + t.detail)}` : '';
+    return `${indent} ${glyph} ${label}${tags}${detail}${timing}`;
+  }
+
+  private render(final = false) {
+    if (!this.live) return;
+
+    const counts = this.tasks.reduce<Record<TaskState, number>>(
+      (a, t) => {
+        a[t.state]++;
+        return a;
+      },
+      { pending: 0, running: 0, done: 0, failed: 0, skipped: 0 },
+    );
+
+    const head =
+      bold(this.title) +
+      dim(
+        ` · ${counts.done}/${this.tasks.length}` +
+          (counts.failed ? ` · ${counts.failed} failed` : '') +
+          (counts.skipped ? ` · ${counts.skipped} skipped` : '') +
+          ` · ${secs(this.elapsed)}`,
+      );
+
+    /**
+     * Only the tail is kept when the list is long: a 200-step plan must not
+     * scroll the terminal on every frame. Finished work above the window is
+     * summarised in the header, which is the number people actually watch.
+     */
+    const MAX = 14;
+    const visible =
+      this.tasks.length <= MAX
+        ? this.tasks
+        : (() => {
+            const active = this.tasks.findIndex((t) => t.state === 'running' || t.state === 'pending');
+            const from = Math.max(0, Math.min(active === -1 ? this.tasks.length - MAX : active - 3, this.tasks.length - MAX));
+            return this.tasks.slice(from, from + MAX);
+          })();
+
+    const hiddenAbove = this.tasks.indexOf(visible[0] as Task);
+    const hiddenBelow = this.tasks.length - (hiddenAbove + visible.length);
+
+    const body = [
+      head,
+      ...(hiddenAbove > 0 ? [dim(`   … ${hiddenAbove} earlier`)] : []),
+      ...visible.map((t) => this.line(t)),
+      ...(hiddenBelow > 0 ? [dim(`   … ${hiddenBelow} more`)] : []),
+      ...this.notes.map((n) => `   ${n}`),
+    ];
+
+    // Repaint in place: up N lines, then clear-and-write each one.
+    const out = cursorUp(this.linesDrawn) + body.map((l) => CLEAR_LINE + l).join('\n') + '\n';
+    this.stream.write(out);
+    this.linesDrawn = final ? 0 : body.length;
+  }
+}
+
+/**
+ * A single long wait with no sub-steps — a snapshot poll, a download.
+ * Returns a function to finish it.
+ */
+export const spinner = (title: string, opts: { plain?: boolean } = {}) => {
+  const p = new Progress({ title, plain: opts.plain });
+  const id = 'only';
+  p.add({ id, label: title, state: 'running', startedAt: Date.now() });
+  p.start();
+  return {
+    detail: (text: string) => p.update(id, { detail: text }),
+    done: (label?: string) => {
+      p.update(id, { state: 'done', ...(label ? { label } : {}) });
+      p.stop();
+    },
+    fail: (label?: string) => {
+      p.update(id, { state: 'failed', ...(label ? { label } : {}) });
+      p.stop();
+    },
+  };
+};
+
+export const isLive = () => Boolean(process.stderr.isTTY) && !process.env.PPC_NO_TUI && !process.env.NO_TUI;
+
+export { magenta };

package/src/validate/index.ts

@@ -0,0 +1,123 @@
+import { green, red, yellow, dim, bold } from '../util/log.ts';
+import type { Tree } from '../model/tree.ts';
+import { validateSchema, type Finding } from './schema.ts';
+import { validateIntegrity, summarise } from './integrity.ts';
+import { validateAgainstReference } from './reference.ts';
+import type { Reference } from '../pipefy/reference.ts';
+import type { AgentsSection } from '../workspace/agents.ts';
+
+export type { Finding } from './schema.ts';
+export { summarise } from './integrity.ts';
+
+export type ValidateOpts = {
+  knownRepoIds?: number[];
+  /**
+   * The last pull, if available. Findings that are already true of the baseline
+   * are marked pre-existing and stop being blocking.
+   *
+   * This is not a convenience. Real pipes ship with `phase_jumps`
+   * pointing at phases that no longer exist — real data from a real pipe. Without
+   * this distinction, a freshly pulled, unedited workspace fails validation and
+   * the FDE is blamed for it.
+   */
+  baseline?: Tree | null;
+  /**
+   * The closed sets from the last pull. Absent means those checks are skipped
+   * — never that a value was confirmed valid.
+   */
+  reference?: Reference | null;
+  /** This repo's AI agents, read from `agents/`. Absent means unread, not absent. */
+  agents?: AgentsSection | null;
+};
+
+/**
+ * Entity plus message, deliberately *not* including the file.
+ *
+ * Renaming a phase changes the file label of every finding inside it, so a
+ * file-keyed fingerprint would report a pre-existing problem as newly
+ * introduced the moment the FDE renames something nearby.
+ */
+const fingerprint = (f: Finding) => `${f.entity}::${f.message}`;
+
+export const validateTree = (tree: Tree, opts: ValidateOpts = {}): Finding[] => {
+  const findings = [
+    ...validateSchema(tree),
+    ...validateIntegrity(tree, { knownRepoIds: opts.knownRepoIds, agents: opts.agents }),
+    ...validateAgainstReference(tree, opts.reference ?? null),
+  ];
+
+  if (!opts.baseline) return findings;
+
+  /**
+   * A multiset, not a set: if the baseline had one dangling reference of a kind
+   * and the edit introduced a second, the second must still be reported.
+   */
+  const budget = new Map<string, number>();
+  for (const f of [
+    ...validateSchema(opts.baseline),
+    ...validateIntegrity(opts.baseline, { knownRepoIds: opts.knownRepoIds, agents: opts.agents }),
+    ...validateAgainstReference(opts.baseline, opts.reference ?? null),
+  ]) {
+    const key = fingerprint(f);
+    budget.set(key, (budget.get(key) ?? 0) + 1);
+  }
+
+  return findings.map((f) => {
+    const key = fingerprint(f);
+    const left = budget.get(key) ?? 0;
+    if (left <= 0) return f;
+    budget.set(key, left - 1);
+    return { ...f, severity: 'warning' as const, preExisting: true };
+  });
+};
+
+/** Errors the edit introduced — what plan and apply actually gate on. */
+export const blockingErrors = (findings: Finding[]): Finding[] =>
+  findings.filter((f) => f.severity === 'error' && !f.preExisting);
+
+export const renderFindings = (findings: Finding[], label: string): string => {
+  const introduced = findings.filter((f) => !f.preExisting);
+  const preExisting = findings.filter((f) => f.preExisting);
+  const { errors, warnings } = summarise(introduced);
+  const lines: string[] = [];
+
+  if (!findings.length) {
+    return green(`+ ${label}: valid`);
+  }
+
+  lines.push(bold(label));
+
+  const group = (items: Finding[]) => {
+    const byFile = new Map<string, Finding[]>();
+    for (const f of items) {
+      const key = f.file ?? f.entity;
+      const arr = byFile.get(key) ?? [];
+      arr.push(f);
+      byFile.set(key, arr);
+    }
+    for (const [file, items2] of byFile) {
+      lines.push(`  ${dim(file)}`);
+      for (const f of items2) {
+        const mark = f.severity === 'error' ? red('x') : yellow('!');
+        lines.push(`    ${mark} ${f.message}`);
+        if (f.fix) lines.push(`      ${dim('-> ' + f.fix)}`);
+      }
+    }
+  };
+
+  if (introduced.length) group(introduced);
+
+  if (preExisting.length) {
+    lines.push('');
+    lines.push(dim(`  ${preExisting.length} pre-existing issue${preExisting.length === 1 ? '' : 's'} — present in the last pull, not caused by these edits:`));
+    for (const f of preExisting) lines.push(`    ${dim('·')} ${dim(`${f.entity}: ${f.message}`)}`);
+  }
+
+  lines.push('');
+  lines.push(
+    `${errors ? red(`${errors} error${errors === 1 ? '' : 's'}`) : green('0 errors')}, ` +
+      `${warnings ? yellow(`${warnings} warning${warnings === 1 ? '' : 's'}`) : '0 warnings'}` +
+      `${preExisting.length ? dim(`, ${preExisting.length} pre-existing`) : ''}`,
+  );
+  return lines.join('\n');
+};