mandrel 1.77.0 → 1.78.0

package/.agents/scripts/lib/loop-units/validate-loop-unit.js

@@ -0,0 +1,197 @@
+/**
+ * lib/loop-units/validate-loop-unit.js — loop-unit frontmatter validator.
+ *
+ * Parses a loop-unit markdown file's YAML frontmatter and AJV-validates
+ * it against `.agents/schemas/loop-unit.schema.json` (Ajv2020). Mirrors
+ * the validation pattern established by `lib/spec/loader.js` (Ajv2020 +
+ * ajv-formats + js-yaml, cached compiled validator, normalised
+ * `{ path, message }` issues).
+ *
+ * A "loop unit" is a markdown file under `.agents/workflows/loops/` whose
+ * leading `---`-fenced YAML frontmatter block defines a recurring unit of
+ * work (cadence, goal, conditional verify, round cap, exhaustion policy).
+ *
+ * Public surface:
+ *   • `parseFrontmatter(source)` → extracts and YAML-parses the leading
+ *     `---`-fenced block. Returns the parsed object (or `{}` for an empty
+ *     block). Throws `LoopUnitParseError` when the block is absent or the
+ *     YAML does not parse.
+ *   • `validateLoopUnit(filePath, opts?)` → reads the file, parses its
+ *     frontmatter, validates against the schema, and returns
+ *     `{ valid, issues, data }`. `issues` is an array of
+ *     `{ path, message }` (empty when valid). Never throws on a *validation*
+ *     failure — it reports it via `valid: false` — but does throw
+ *     `LoopUnitParseError` for an unreadable file or unparseable
+ *     frontmatter so callers can distinguish "structurally broken file"
+ *     from "schema-invalid unit".
+ *
+ * The module makes no GitHub calls and no process mutations; it is pure
+ * file I/O + schema validation. The `opts` bag accepts `{ schemaPath, fs }`
+ * so tests can point at a sandbox schema without monkey-patching globals.
+ */
+
+import {
+  existsSync as defaultExistsSync,
+  readFileSync as defaultReadFileSync,
+} from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+import yaml from 'js-yaml';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// scripts/lib/loop-units/ → scripts/lib/ → scripts/ → .agents/
+const PROJECT_AGENTS_DIR = path.resolve(__dirname, '..', '..', '..');
+export const DEFAULT_SCHEMA_PATH = path.join(
+  PROJECT_AGENTS_DIR,
+  'schemas',
+  'loop-unit.schema.json',
+);
+
+const defaultFsAdapter = Object.freeze({
+  existsSync: defaultExistsSync,
+  readFileSync: defaultReadFileSync,
+});
+
+let cachedValidator = null;
+let cachedValidatorKey = null;
+
+/**
+ * Compile (and cache) the Ajv2020 validator for the loop-unit schema.
+ * Cached by absolute schema path so tests can swap to a sandbox schema.
+ *
+ * @param {string} schemaPath
+ * @param {{ readFileSync: typeof defaultReadFileSync }} fs
+ * @returns {(data: unknown) => boolean}
+ */
+function getValidator(schemaPath, fs) {
+  if (cachedValidator && cachedValidatorKey === schemaPath) {
+    return cachedValidator;
+  }
+  const ajv = new Ajv2020({ allErrors: true, strict: false });
+  addFormats(ajv);
+  const schema = JSON.parse(fs.readFileSync(schemaPath, 'utf8'));
+  cachedValidator = ajv.compile(schema);
+  cachedValidatorKey = schemaPath;
+  return cachedValidator;
+}
+
+/**
+ * Raised when a loop-unit file cannot be read, has no YAML frontmatter
+ * block, or the frontmatter does not parse as YAML.
+ */
+export class LoopUnitParseError extends Error {
+  /**
+   * @param {string} filePath
+   * @param {string} reason
+   */
+  constructor(filePath, reason) {
+    super(`Loop unit ${filePath} could not be parsed: ${reason}`);
+    this.name = 'LoopUnitParseError';
+    this.filePath = filePath;
+    this.reason = reason;
+  }
+}
+
+// Leading `---`-fenced YAML block. Tolerates CRLF and a leading BOM. The
+// closing fence is a `---` (or `...`) on its own line.
+const FRONTMATTER_RE = /^?---\r?\n([\s\S]*?)\r?\n(?:---|\.\.\.)\s*(?:\r?\n|$)/;
+
+/**
+ * Extract and YAML-parse the leading `---`-fenced frontmatter block from a
+ * markdown source string.
+ *
+ * @param {string} source        raw file contents
+ * @param {string} [filePath]    used only for error messages
+ * @returns {object}             the parsed frontmatter object (`{}` if empty)
+ * @throws {LoopUnitParseError}  when no fence is present or the YAML fails
+ */
+export function parseFrontmatter(source, filePath = '<string>') {
+  const match = FRONTMATTER_RE.exec(source);
+  if (!match) {
+    throw new LoopUnitParseError(
+      filePath,
+      'no YAML frontmatter block (expected a leading "---" fence)',
+    );
+  }
+  let parsed;
+  try {
+    parsed = yaml.load(match[1], { filename: filePath });
+  } catch (err) {
+    throw new LoopUnitParseError(
+      filePath,
+      `frontmatter is not valid YAML: ${err.message}`,
+    );
+  }
+  if (parsed == null) return {};
+  if (typeof parsed !== 'object' || Array.isArray(parsed)) {
+    throw new LoopUnitParseError(
+      filePath,
+      'frontmatter must be a YAML mapping',
+    );
+  }
+  return parsed;
+}
+
+/**
+ * Convert Ajv's error array into a `{ path, message }` shape. For
+ * `required` errors Ajv leaves the missing property in
+ * `params.missingProperty` rather than the instance path, so we append it
+ * so the caller sees `/loop/verify` instead of `/loop` and the message
+ * names the missing field.
+ *
+ * @param {Array<{instancePath:string,message:string,keyword:string,params?:Record<string,unknown>}>} ajvErrors
+ * @returns {Array<{path:string,message:string}>}
+ */
+function normaliseAjvErrors(ajvErrors) {
+  return (ajvErrors ?? []).map((err) => {
+    let p = err.instancePath || '/';
+    let message = err.message ?? 'validation failed';
+    if (
+      err.keyword === 'required' &&
+      typeof err.params?.missingProperty === 'string'
+    ) {
+      const sep = p === '/' ? '' : '/';
+      p = `${p}${sep}${err.params.missingProperty}`;
+      message = `must have required property '${err.params.missingProperty}'`;
+    }
+    return { path: p, message };
+  });
+}
+
+/**
+ * Read, parse, and schema-validate a loop-unit markdown file.
+ *
+ * @param {string} filePath
+ * @param {{ schemaPath?: string, fs?: typeof defaultFsAdapter }} [opts]
+ * @returns {{ valid: boolean, issues: Array<{path:string,message:string}>, data: object }}
+ * @throws {LoopUnitParseError} when the file is unreadable or its
+ *   frontmatter is missing/unparseable.
+ */
+export function validateLoopUnit(filePath, opts = {}) {
+  const fs = opts.fs ?? defaultFsAdapter;
+  const schemaPath = opts.schemaPath ?? DEFAULT_SCHEMA_PATH;
+
+  if (!fs.existsSync(filePath)) {
+    throw new LoopUnitParseError(filePath, 'file does not exist');
+  }
+
+  let raw;
+  try {
+    raw = fs.readFileSync(filePath, 'utf8');
+  } catch (err) {
+    throw new LoopUnitParseError(filePath, `unreadable: ${err.message}`);
+  }
+
+  const data = parseFrontmatter(raw, filePath);
+  const validate = getValidator(schemaPath, fs);
+  const ok = validate(data);
+
+  return {
+    valid: ok,
+    issues: ok ? [] : normaliseAjvErrors(validate.errors),
+    data,
+  };
+}

package/.agents/scripts/lib/mandrel-catalog.js

@@ -140,6 +140,42 @@ export function buildCatalog(workflowsDir) {
   return catalog;
 }
 
+/**
+ * Build the loop-unit catalog from a workflows directory's `loops/`
+ * namespace. Loop units live at `.agents/workflows/loops/<name>.md` and
+ * project to the namespaced `/loops:<name>` slash command (Story #4289).
+ * They are catalogued separately from the flat top-level commands because
+ * they carry a distinct invocation form.
+ *
+ * Returns an empty array when the `loops/` subdirectory is absent (the
+ * common case before the starter loops land in a later Story) — an absent
+ * namespace is a clean "no loop units", not an error.
+ *
+ * @param {string} workflowsDir — absolute path to `.agents/workflows/`.
+ * @returns {Array<{ name: string, description: string | null, vague: boolean }>}
+ */
+export function buildLoopCatalog(workflowsDir) {
+  const loopsDir = path.join(workflowsDir, 'loops');
+  if (!fs.existsSync(loopsDir)) return [];
+  const entries = fs.readdirSync(loopsDir, { withFileTypes: true });
+  const catalog = [];
+  for (const entry of entries) {
+    if (!entry.isFile()) continue;
+    if (!entry.name.endsWith('.md')) continue;
+    if (entry.name === 'README.md') continue;
+    const filePath = path.join(loopsDir, entry.name);
+    const source = fs.readFileSync(filePath, 'utf8');
+    const description = extractDescription(source);
+    catalog.push({
+      name: entry.name.replace(/\.md$/, ''),
+      description,
+      vague: isVagueDescription(description),
+    });
+  }
+  catalog.sort((a, b) => a.name.localeCompare(b.name));
+  return catalog;
+}
+
 /**
  * Render the catalog as a plain-markdown bullet list. Kept as a
  * lightweight alternative rendering of the same catalog backend that

package/.agents/scripts/lib/orchestration/lifecycle/emit-loop-tick.js

@@ -0,0 +1,183 @@
+/**
+ * emit-loop-tick.js — Story #4287 (Epic #4284).
+ *
+ * Programmatic helper that emits a single `loop.tick` lifecycle event
+ * THROUGH the lifecycle bus so a host-driven loop (e.g. a `/loop`-style
+ * recurring command or a long-running poll) lands a per-pass record in
+ * the on-disk ledger the `/deliver` idle watchdog already scans. The
+ * record is what keeps a host loop from running silently: each round
+ * appends an inspectable `emitted` line a reconciler can read for
+ * forward-progress evidence.
+ *
+ * Distinct from `story.heartbeat` (emit-story-heartbeat.js): the
+ * heartbeat carries Story-phase info for a single in-flight Story and is
+ * always Epic-scoped (its ledger path is `epicLedgerPath(epicId)`). A
+ * host loop is not bound to a Story tier, so `loop.tick` carries a
+ * free-form `loopName`, a monotonic `round` counter, the loop's
+ * configured `cadence` label, and a per-round `status` instead. Keeping
+ * the two events separate means a loop tick never masquerades as Story
+ * progress (and vice versa).
+ *
+ * Bus path (Story acceptance: "Emitting a loop.tick event THROUGH the
+ * lifecycle bus appends a record to the per-run ledger"): this helper
+ * constructs a `Bus`, registers a `LedgerWriter` against it, and calls
+ * `bus.emit('loop.tick', payload)`. The bus validates the payload against
+ * `loop.tick.schema.json` before any listener runs, and the
+ * LedgerWriter's privileged `onEmitted` hook lands the `emitted` record
+ * on disk — exactly the same persistence path every other lifecycle
+ * event flows through. The helper does NOT bypass the bus with a direct
+ * `appendFileSync`; routing through the bus is what gives the record its
+ * schema-validated, seqId-stamped guarantee.
+ *
+ * Schema contract (loop.tick.schema.json):
+ *   { event, loopName, round, cadence, status, timestamp }
+ *
+ * The schema declares `additionalProperties: false`, so this emitter's
+ * signature is deliberately narrow: only the schema-allowed fields are
+ * accepted. `status` is one of running|done|blocked.
+ *
+ * Ledger path resolution: a caller supplies EITHER an explicit
+ * `ledgerPath` (the host-loop case — the loop owns where its ledger
+ * lives) OR an `epicId`, in which case the canonical
+ * `epicLedgerPath(epicId)` is used so an Epic-scoped loop's ticks land
+ * in the same `temp/epic-<id>/lifecycle.ndjson` the rest of the run
+ * reads. Exactly one of the two MUST be supplied.
+ */
+
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { epicLedgerPath } from '../../config/temp-paths.js';
+import { createBus } from './bus.js';
+import { createLedgerWriter } from './ledger-writer.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const SCHEMA_DIR = path.resolve(
+  __dirname,
+  '..',
+  '..',
+  '..',
+  '..',
+  'schemas',
+  'lifecycle',
+);
+
+const VALID_STATUSES = new Set(['running', 'done', 'blocked']);
+
+/**
+ * Parse `temp/epic-<id>/lifecycle.ndjson` (or any
+ * `<dir>/epic-<id>/lifecycle.ndjson`) back into `{ tempRoot, epicId }`
+ * so a LedgerWriter — which is constructed from `{ epicId, tempRoot }`
+ * rather than a raw path — can be bound to the supplied ledger path.
+ *
+ * The LedgerWriter intentionally re-derives the ledger path from its
+ * `tempRoot` + `epicId` (so it can recreate the directory if a listener
+ * moves it mid-run), so we decompose the path the caller gave us into
+ * those two parts here.
+ *
+ * @param {string} ledgerPath
+ * @returns {{ tempRoot: string, epicId: number }}
+ */
+function decomposeLedgerPath(ledgerPath) {
+  const epicDir = path.dirname(ledgerPath);
+  const tempRoot = path.dirname(epicDir);
+  const epicDirName = path.basename(epicDir);
+  const m = /^epic-(\d+)$/.exec(epicDirName);
+  if (!m) {
+    throw new Error(
+      `emitLoopTick: ledgerPath does not match <tempRoot>/epic-<id>/lifecycle.ndjson layout (got ${ledgerPath})`,
+    );
+  }
+  const epicId = Number.parseInt(m[1], 10);
+  return { tempRoot, epicId };
+}
+
+/**
+ * Emit exactly one `loop.tick` event through the lifecycle bus, landing
+ * an `emitted` (and `completed`) NDJSON record in the resolved ledger.
+ *
+ * @param {object} opts
+ * @param {string} opts.loopName     Free-form loop identifier (non-empty).
+ * @param {number} opts.round        Monotonic pass counter (integer >= 0).
+ * @param {string} opts.cadence      Configured interval label, e.g. '5m'.
+ * @param {string} [opts.status='running']
+ *                                   One of running|done|blocked.
+ * @param {string} [opts.timestamp]  ISO-8601 wall clock. Defaults to now().
+ * @param {number} [opts.epicId]     When supplied (and no `ledgerPath`),
+ *                                   the canonical `epicLedgerPath(epicId)`
+ *                                   is used for the ledger.
+ * @param {object} [opts.config]     Optional resolved config for tempRoot
+ *                                   (only consulted on the `epicId` path).
+ * @param {string} [opts.ledgerPath] Explicit ledger path (host-loop case).
+ *                                   Mutually exclusive with `epicId`.
+ * @returns {Promise<{ ledgerPath: string, payload: object, seqId: number }>}
+ */
+export async function emitLoopTick(opts) {
+  const {
+    loopName,
+    round,
+    cadence,
+    status = 'running',
+    timestamp = new Date().toISOString(),
+    epicId,
+    config,
+    ledgerPath: ledgerPathOverride,
+  } = opts ?? {};
+
+  if (typeof loopName !== 'string' || loopName.length === 0) {
+    throw new Error('emitLoopTick: loopName must be a non-empty string');
+  }
+  if (!Number.isInteger(round) || round < 0) {
+    throw new Error('emitLoopTick: round must be a non-negative integer');
+  }
+  if (typeof cadence !== 'string' || cadence.length === 0) {
+    throw new Error('emitLoopTick: cadence must be a non-empty string');
+  }
+  if (!VALID_STATUSES.has(status)) {
+    throw new Error(
+      `emitLoopTick: status "${status}" must be one of: ${[...VALID_STATUSES].join(', ')}`,
+    );
+  }
+
+  const hasEpicId = epicId !== undefined;
+  const hasLedgerPath = ledgerPathOverride !== undefined;
+  if (hasEpicId === hasLedgerPath) {
+    throw new Error('emitLoopTick: supply exactly one of epicId or ledgerPath');
+  }
+
+  let ledgerPath;
+  if (hasLedgerPath) {
+    if (
+      typeof ledgerPathOverride !== 'string' ||
+      ledgerPathOverride.length === 0
+    ) {
+      throw new Error('emitLoopTick: ledgerPath must be a non-empty string');
+    }
+    ledgerPath = ledgerPathOverride;
+  } else {
+    if (!Number.isInteger(epicId) || epicId < 1) {
+      throw new Error('emitLoopTick: epicId must be a positive integer');
+    }
+    ledgerPath = epicLedgerPath(epicId, config);
+  }
+
+  const payload = {
+    event: 'loop.tick',
+    loopName,
+    round,
+    cadence,
+    status,
+    timestamp,
+  };
+
+  // Route through the bus so the payload is schema-validated and the
+  // LedgerWriter's privileged onEmitted hook persists the record — the
+  // same path every lifecycle event flows through.
+  const { tempRoot, epicId: ledgerEpicId } = decomposeLedgerPath(ledgerPath);
+  const bus = createBus({ schemaDir: SCHEMA_DIR });
+  const writer = createLedgerWriter({ epicId: ledgerEpicId, tempRoot });
+  writer.register(bus);
+
+  const { seqId } = await bus.emit('loop.tick', payload);
+  return { ledgerPath: writer.ledgerPath, payload, seqId };
+}

package/.agents/scripts/providers/github/branch-protection.js

@@ -13,8 +13,8 @@
  * @see Story #2462 — Split GitHubProvider god class into seven composed gateways.
  */
 
+import { withTransientRetry } from './errors.js';
 import { parseApiJson } from './request-helpers.js';
-import { withTransientRetry } from './transient-retry.js';
 
 /**
  * Detect a 404 across both error surfaces:

package/.agents/scripts/providers/github/errors.js

@@ -48,6 +48,40 @@ const TRANSIENT_MESSAGES = [
 
 const PERMISSION_MESSAGES = ['unauthorized', 'forbidden', 'permission'];
 
+// Network/connectivity blips that the gh-CLI path surfaces on `err.stderr`
+// (Go HTTP errors, e.g. `dial tcp ...: i/o timeout`) and the direct `fetch`
+// path surfaces as `TypeError: fetch failed` with the real reason on
+// `err.cause` (e.g. `ETIMEDOUT`, `ENOTFOUND`). Folded in from the former
+// `transient-retry.js` predicate (Story #4298) so the single canonical
+// classifier retries the **union** of transient HTTP statuses/codes AND
+// transient network errors. The `\b50[234]\b` alternative also catches a
+// bare 502/503/504 surfaced only in an error message string (no `.status`).
+const TRANSIENT_NETWORK_RE =
+  /i\/o timeout|dial tcp|TLS handshake timeout|connection reset|connection refused|temporary failure|could not resolve host|no such host|network is unreachable|socket hang up|fetch failed|ConnectTimeoutError|UND_ERR_CONNECT_TIMEOUT|ETIMEDOUT|ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN|\b50[234]\b/i;
+
+/**
+ * True when an error looks like a retryable network/connectivity blip.
+ * Scans the union of fields both transport paths populate (`stderr`,
+ * `message`, `code`, and the nested `cause.message` / `cause.code` the
+ * `fetch` path uses). Module-private — folded in from the former
+ * `transient-retry.js` predicate (Story #4298) and consumed only by
+ * `classifyGithubError` below; its behavior is exercised through that public
+ * classifier rather than as a standalone export (keeps the dead-export gate
+ * green — nothing outside this module imports it).
+ */
+function isTransientNetworkError(err) {
+  const hay = [
+    err?.stderr,
+    err?.message,
+    err?.code,
+    err?.cause?.message,
+    err?.cause?.code,
+  ]
+    .filter(Boolean)
+    .join(' ');
+  return TRANSIENT_NETWORK_RE.test(hay);
+}
+
 function matchesAny(haystack, needles) {
   for (const n of needles) if (haystack.includes(n)) return true;
   return false;
@@ -97,19 +131,36 @@ export function classifyGithubError(err) {
   if (matchesAny(lower, FEATURE_DISABLED_MESSAGES)) return 'feature-disabled';
   if (isTransientStatus(status)) return 'transient';
   if (isTransientByCodeOrMessage(code, lower)) return 'transient';
+  // Union with the former `transient-retry.js` predicate (Story #4298):
+  // retry on network/connectivity blips the status/code checks above miss
+  // (e.g. a `dial tcp ... i/o timeout` on `err.stderr` from the gh-CLI path,
+  // or `ECONNREFUSED` / `ENETUNREACH`). Checked before the permission rule so
+  // a transient network failure never masquerades as a permanent denial.
+  if (isTransientNetworkError(err)) return 'transient';
   if (isPermissionSignal(status, lower)) return 'permission';
   return 'permanent';
 }
 
 // ---------------------------------------------------------------------------
-// Transient-retry helper (Story #2852)
+// Transient-retry helper (Story #2852; unified in Story #4298)
 // ---------------------------------------------------------------------------
 //
+// The single canonical `withTransientRetry` for the GitHub provider. Story
+// #4298 collapsed the former two divergent same-named implementations (this
+// one + the network-only one in the deleted `transient-retry.js`) into this
+// one primitive. Its default classifier (`classifyGithubError`) is the
+// **union** predicate — it retries on transient HTTP statuses/codes AND on
+// transient network/connectivity errors — so every former consumer of either
+// module keeps (or gains) its prior retry coverage with no shim.
+//
 // Mirrors the addSubIssue retry contract in `sub-issues.js` so read-path
 // callers (paginateRest, getTicket, getNativeSubIssues, …) absorb the same
 // jittered exponential backoff on transient GitHub errors instead of
 // bubbling a one-shot 502/429/ECONNRESET that kills a longer pipeline
-// (e.g. the /deliver Phase E retro).
+// (e.g. the /deliver Phase E retro). The network consumers repointed here
+// (branch-protection, labels, projects-v2-graphql) call with no opts, so
+// they adopt these defaults; their retry *classes* (the network blips) are
+// preserved via the unified classifier above.
 
 export const TRANSIENT_RETRY_DEFAULTS = Object.freeze({
   maxAttempts: 6,

package/.agents/scripts/providers/github/labels.js

@@ -16,7 +16,7 @@
  * @see Story #2462 — Split GitHubProvider god class into seven composed gateways.
  */
 
-import { withTransientRetry } from './transient-retry.js';
+import { withTransientRetry } from './errors.js';
 
 /**
  * Detect the "label already exists" signal across the surfaces `gh label

package/.agents/scripts/providers/github/projects-v2-graphql.js

@@ -7,7 +7,7 @@
  * `unavailable:true` envelopes. Wave 3 deletes the old submodules.
  */
 import { execSync } from 'node:child_process';
-import { withTransientRetry } from './transient-retry.js';
+import { withTransientRetry } from './errors.js';
 
 // Resolve an owner node id per-scope. Querying `user` and `organization`
 // together in one request makes GitHub return a NOT_FOUND error for whichever

package/.agents/scripts/run-lint.js

@@ -78,6 +78,17 @@ const tasks = [
     cmd: 'node',
     args: ['.agents/scripts/check-arch-cycles.js'],
   },
+  {
+    // Loop-unit frontmatter gate (Story #4288, Epic #4284). Validates
+    // every `.agents/workflows/loops/*.md` loop unit against
+    // `.agents/schemas/loop-unit.schema.json`. An absent/empty loops
+    // directory is a clean pass; a malformed unit (e.g. a self-paced
+    // cadence missing its required `verify`) fails the lint gate with a
+    // message naming the offending file + field.
+    name: 'loop-units',
+    cmd: 'node',
+    args: ['.agents/scripts/check-loop-units.js'],
+  },
 ];
 
 function runTask({ name, cmd, args }) {