@really-knows-ai/foundry 2.3.2 → 3.0.0

package/dist/scripts/lib/config-validators/law.js

@@ -0,0 +1,96 @@
+/**
+ * Check if a law block contains deprecated Passing:/Failing: scaffolding.
+ * @param {string[]} lines
+ * @returns {string[]} Array of error messages (empty if valid)
+ */
+function checkForDeprecatedScaffolding(lines) {
+  const errors = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line.startsWith('Passing:') || line.startsWith('Failing:')) {
+      errors.push(`Line ${i + 1}: Deprecated scaffolding (${line.split(':')[0]}:) not allowed. Prose must be plain statements without structured fields. See spec: "No structured fields within the prose — no Passing: or Failing: scaffolding."`);
+    }
+  }
+  return errors;
+}
+
+/**
+ * Check for duplicate law IDs within blocks.
+ * @param {{id: string}[]} blocks
+ * @returns {string[]} Array of error messages (empty if valid)
+ */
+function checkForDuplicateIds(blocks) {
+  const errors = [];
+  const seen = new Set();
+  for (const block of blocks) {
+    if (seen.has(block.id)) {
+      errors.push(`duplicate law id: ${block.id}`);
+    }
+    seen.add(block.id);
+  }
+  return errors;
+}
+
+/**
+ * Validate a law definition body.
+ *
+ * Laws derive their ID from the filename rather than frontmatter, so name
+ * and io are unused here but accepted for API parity with other validators.
+ *
+ * @param {object} opts
+ * @param {string} opts.name      Slugged identifier (unused; laws use filename as ID).
+ * @param {string} opts.body      Full markdown body.
+ * @param {object} [opts.io]      IO adapter (unused; laws validate body only).
+ * @returns {Promise<{ok: true} | {ok: false, errors: string[]}>}
+ */
+export async function validate({ body }) {
+  const lines = body.split('\n');
+  let errors = checkForDeprecatedScaffolding(lines);
+  
+  if (errors.length > 0) {
+    return { ok: false, errors };
+  }
+  
+  const blocks = parseLawBlocks(body);
+
+  if (blocks.length === 0) {
+    return { ok: false, errors: ['body must contain at least one law block (## <law-id>)'] };
+  }
+
+  errors = checkForDuplicateIds(blocks);
+  return errors.length ? { ok: false, errors } : { ok: true };
+}
+
+/**
+ * Parse law blocks from the body.
+ * Each block is a ## heading followed by its content until the next heading.
+ * @param {string} body
+ * @returns {{id: string, text: string}[]}
+ */
+function parseLawBlocks(body) {
+  const blocks = [];
+  const lines = body.split('\n');
+  let currentId = null;
+  let currentLines = [];
+
+  const flushBlock = () => {
+    if (currentId) {
+      blocks.push({ id: currentId, text: currentLines.join('\n') });
+    }
+  };
+
+  for (const line of lines) {
+    const heading = line.match(/^## (.+)/);
+    if (!heading && currentId) {
+      currentLines.push(line);
+    }
+    if (heading) {
+      flushBlock();
+      currentId = heading[1].trim();
+      currentLines = [];
+    }
+  }
+  flushBlock();
+
+  return blocks;
+}

package/dist/scripts/lib/config.js

@@ -0,0 +1,393 @@
+/**
+ * Structured reads of foundry/ directory contents.
+ */
+
+import { join } from 'path';
+import { parseFrontmatter } from './workfile.js';
+
+function parseDoc(text) {
+  const frontmatter = parseFrontmatter(text);
+  const body = text.replace(/^---\n.+?\n---\n?/s, '').trim();
+  return { frontmatter, body };
+}
+
+export async function getCycleDefinition(foundryDir, cycleId, io) {
+  const path = join(foundryDir, 'cycles', `${cycleId}.md`);
+  if (!(await io.exists(path))) {
+    throw new Error(`Cycle not found: ${cycleId}`);
+  }
+  const text = await io.readFile(path);
+  return parseDoc(text);
+}
+
+export async function getArtefactType(foundryDir, typeId, io) {
+  const path = join(foundryDir, 'artefacts', typeId, 'definition.md');
+  if (!(await io.exists(path))) {
+    throw new Error(`Artefact type not found: ${typeId}`);
+  }
+  const text = await io.readFile(path);
+  return parseDoc(text);
+}
+
+/**
+ * Parse law entries from a markdown file. Each `## heading` starts a new law.
+ * Each law may have an optional `validators:` block with entries containing id, command, and optional failure-means.
+ * Validators are extracted and returned separately from prose.
+ * Returns: [{id, text (prose only), validators (if present)}]
+ */
+function parseLaws(text, source) {
+  const laws = [];
+  const lines = text.split('\n');
+  let currentId = null;
+  const currentLines = [];
+
+  function flush() {
+    if (currentId) {
+      const lawText = currentLines.join('\n').trim();
+      const { prose, validators } = extractValidators(lawText, currentId);
+      const lawObj = { id: currentId, text: prose };
+      if (validators && validators.length > 0) {
+        lawObj.validators = validators;
+      }
+      laws.push(lawObj);
+    }
+  }
+
+  for (const line of lines) {
+    const heading = line.match(/^## (.+)/);
+    if (heading) {
+      flush();
+      currentId = heading[1];
+      currentLines.length = 0;
+    } else if (currentId) {
+      currentLines.push(line);
+    }
+  }
+  flush();
+  return laws;
+}
+
+/**
+ * Extract validators block from law text and return prose-only text plus parsed validators.
+ * @param {string} lawText - Full law text including optional validators block
+ * @param {string} lawId - Law ID for error messages
+ * @returns {{prose: string, validators: Array}} - Prose text and validators array
+ * @throws {Error} if validators block is malformed
+ */
+function extractValidators(lawText, lawId) {
+  // Find validators: block (must start at beginning of line, followed by indented lines)
+  // The pattern captures lines starting with spaces/tabs (indented content)
+  // Using possessive quantifier to avoid backtracking: (?:...)+ instead of (...)*.
+  const validatorBlockMatch = lawText.match(/^validators:\n((?:[ \t]+\S.*(?:\n|$))*)/m);
+  
+  if (!validatorBlockMatch || !validatorBlockMatch[1].trim()) {
+    return { prose: lawText, validators: null };
+  }
+
+  // Extract prose (everything before validators:)
+  const prose = lawText.substring(0, validatorBlockMatch.index).trim();
+  
+  // Extract and parse validators block
+  const validatorBlockText = validatorBlockMatch[1];
+  const validators = parseValidatorBlock(validatorBlockText, lawId);
+  
+  return { prose, validators };
+}
+
+/**
+ * Parse a validator entry from lines.
+ * @param {string} lawId - Law ID for error messages
+ * @param {Set} seenIds - Set of seen validator IDs to detect duplicates
+ * @param {object} validator - Validator object being built
+ * @throws {Error} if validator is invalid
+ */
+function saveValidator(validator, seenIds, lawId) {
+  validateValidator(validator, seenIds, lawId);
+  return validator;
+}
+
+/**
+ * Parse field from a line: key: value
+ * @param {string} line - Line to parse
+ * @param {string} key - Field key to match
+ * @returns {string|null} - Field value or null
+ */
+function parseField(line, key) {
+  const match = line.match(new RegExp(`^\\s*${key}:\\s*(.+)`));
+  return match ? match[1].trim() : null;
+}
+
+/**
+ * Handle a new validator entry line.
+ * @param {object} currentValidator - Current validator object or null
+ * @param {Array} validators - List to add saved validators to
+ * @param {string} line - Line to process
+ * @param {Set} seenIds - Set of seen IDs
+ * @param {string} lawId - Law ID for error messages
+ * @returns {object} - New validator object or null
+ */
+function handleValidatorEntry(currentValidator, validators, line, seenIds, lawId) {
+  const entryMatch = line.match(/^\s*-\s*id:\s*(.+)/);
+  if (entryMatch) {
+    // Save previous validator if exists
+    if (currentValidator) {
+      validators.push(saveValidator(currentValidator, seenIds, lawId));
+    }
+    return { id: entryMatch[1].trim() };
+  }
+  return null;
+}
+
+/**
+ * Process a field line in a validator entry.
+ * @param {object} validator - Current validator object
+ * @param {string} line - Line to process
+ */
+function processValidatorField(validator, line) {
+  const command = parseField(line, 'command');
+  if (command !== null) {
+    validator.command = command;
+    return;
+  }
+
+  const failureMeans = parseField(line, 'failure-means');
+  if (failureMeans !== null) {
+    validator['failure-means'] = failureMeans;
+  }
+}
+
+/**
+ * Process a single line in the validators block.
+ * @param {object} currentValidator - Current validator or null
+ * @param {string} line - Line to process
+ * @param {string} lawId - Law ID for error messages
+ * @throws {Error} if validator entry not started
+ */
+function processValidatorLine(currentValidator, line, lawId) {
+  if (!currentValidator) {
+    throw new Error(`law "${lawId}": validator entry missing required 'id'`);
+  }
+  processValidatorField(currentValidator, line);
+}
+
+/**
+ * Parse the validators block content (YAML-like format)
+ * @param {string} blockText - Text content of validators block
+ * @param {string} lawId - Law ID for error messages
+ * @returns {Array} - Array of parsed validators
+ * @throws {Error} if validators are malformed
+ */
+function parseValidatorBlock(blockText, lawId) {
+  const validators = [];
+  const lines = blockText.split('\n');
+  let currentValidator = null;
+  const seenIds = new Set();
+
+  for (const line of lines) {
+    // Skip empty lines
+    if (!line.trim()) continue;
+    
+    // Check for new validator entry (starts with -)
+    const newValidator = handleValidatorEntry(currentValidator, validators, line, seenIds, lawId);
+    if (newValidator) {
+      currentValidator = newValidator;
+      continue;
+    }
+
+    processValidatorLine(currentValidator, line, lawId);
+  }
+
+  // Save last validator
+  if (currentValidator) {
+    validators.push(saveValidator(currentValidator, seenIds, lawId));
+  }
+
+  return validators;
+}
+
+/**
+ * Validate a single validator entry.
+ * @param {object} validator - Validator object with id, command, and optional failure-means
+ * @param {Set} seenIds - Set of seen validator IDs to detect duplicates
+ * @param {string} lawId - Law ID for error messages
+ * @throws {Error} if validator is invalid
+ */
+function validateValidator(validator, seenIds, lawId) {
+  if (!validator.id) {
+    throw new Error(`law "${lawId}": validator entry missing required 'id'`);
+  }
+  if (!validator.command) {
+    throw new Error(`law "${lawId}": validator entry missing required 'command'`);
+  }
+  if (seenIds.has(validator.id)) {
+    throw new Error(`law "${lawId}": duplicate validator id '${validator.id}' in law`);
+  }
+  seenIds.add(validator.id);
+}
+
+async function collectLawsFromDir(dir, io, sourcePrefix) {
+  if (!(await io.exists(dir))) return [];
+  const files = await io.readDir(dir);
+  const mdFiles = files.filter(f => f.endsWith('.md')).sort();
+  const results = [];
+  for (const file of mdFiles) {
+    const text = await io.readFile(join(dir, file));
+    results.push(...parseLaws(text, `${sourcePrefix}/${file}`));
+  }
+  return results;
+}
+
+/**
+ * Collect all laws (global and type-specific) for a given artefact type.
+ * Returns laws with their source and validator information.
+ */
+async function collectAllLaws(foundryDir, io, { typeId } = {}) {
+  const laws = await collectLawsFromDir(join(foundryDir, 'laws'), io, 'laws');
+
+  if (typeId) {
+    const typeLawsPath = join(foundryDir, 'artefacts', typeId, 'laws.md');
+    if (await io.exists(typeLawsPath)) {
+      const text = await io.readFile(typeLawsPath);
+      laws.push(...parseLaws(text, `artefacts/${typeId}/laws.md`));
+    }
+  }
+
+  return laws;
+}
+
+export async function getLaws(foundryDir, io, { typeId } = {}) {
+  const laws = await collectAllLaws(foundryDir, io, { typeId });
+
+  // Return prose-only without source or validators
+  return laws.map(law => ({ id: law.id, text: law.text }));
+}
+
+export async function getLawsForQuench(foundryDir, io, { typeId } = {}) {
+  const laws = await collectAllLaws(foundryDir, io, { typeId });
+
+  // Return only laws that have validators
+  return laws.filter(law => law.validators && law.validators.length > 0);
+}
+
+function parseValidationEntry(line) {
+  const cmdMatch = line.match(/^Command:\s*(.+)/);
+  if (cmdMatch) return { type: 'command', value: cmdMatch[1].trim().replace(/^`|`$/g, '') };
+  const failMatch = line.match(/^Failure means:\s*(.+)/);
+  if (failMatch) return { type: 'failure', value: failMatch[1].trim() };
+  return null;
+}
+
+function buildValidationEntry(currentId, currentCommand, currentFailure) {
+  if (!currentId || !currentCommand) return null;
+  const entry = { id: currentId, command: currentCommand };
+  if (currentFailure) entry.failureMeans = currentFailure;
+  return entry;
+}
+
+function flushValidationEntry(entries, id, command, failure) {
+  const entry = buildValidationEntry(id, command, failure);
+  if (entry) entries.push(entry);
+}
+
+function applyParsedEntry(state, parsed) {
+  if (parsed?.type === 'command') state.command = parsed.value;
+  if (parsed?.type === 'failure') state.failure = parsed.value;
+}
+
+function handleValidationLine(line, state) {
+  const heading = line.match(/^## (.+)/);
+  if (heading) {
+    flushValidationEntry(state.entries, state.id, state.command, state.failure);
+    state.id = heading[1].trim();
+    state.command = null;
+    state.failure = null;
+    return;
+  }
+  if (state.id) {
+    applyParsedEntry(state, parseValidationEntry(line));
+  }
+}
+
+function internalParseValidationLines(lines) {
+  const state = { entries: [], id: null, command: null, failure: null };
+  for (const line of lines) {
+    handleValidationLine(line, state);
+  }
+  flushValidationEntry(state.entries, state.id, state.command, state.failure);
+  return state.entries;
+}
+
+/**
+ * @deprecated Use getLawsForQuench instead. Phase 2 migration of validation.md files will remove this.
+ */
+export async function getValidation(foundryDir, typeId, io) {
+  const path = join(foundryDir, 'artefacts', typeId, 'validation.md');
+  if (!(await io.exists(path))) return null;
+  const text = await io.readFile(path);
+  return internalParseValidationLines(text.split('\n'));
+}
+
+/**
+ * @deprecated Use getLawsForQuench instead. Phase 2 migration of validation.md files will remove this.
+ */
+export async function parseValidationLines(lines) {
+  return internalParseValidationLines(lines);
+}
+
+export async function getAppraisers(foundryDir, io) {
+  const dir = join(foundryDir, 'appraisers');
+  if (!(await io.exists(dir))) return [];
+  const files = await io.readDir(dir);
+  const mdFiles = files.filter(f => f.endsWith('.md')).sort();
+  const result = [];
+  for (const file of mdFiles) {
+    const text = await io.readFile(join(dir, file));
+    const { frontmatter, body } = parseDoc(text);
+    const entry = { id: frontmatter.id, personality: body };
+    if (frontmatter.model) entry.model = frontmatter.model;
+    result.push(entry);
+  }
+  return result;
+}
+
+export async function getFlow(foundryDir, flowId, io) {
+  const path = join(foundryDir, 'flows', `${flowId}.md`);
+  if (!(await io.exists(path))) {
+    throw new Error(`Flow not found: ${flowId}`);
+  }
+  const text = await io.readFile(path);
+  return parseDoc(text);
+}
+
+function buildAppraiserPool(allAppraisers, allowed) {
+  return allowed ? allAppraisers.filter(a => allowed.includes(a.id)) : allAppraisers;
+}
+
+function roundRobinSelect(pool, count) {
+  const result = [];
+  for (let i = 0; i < count; i++) {
+    result.push(pool[i % pool.length]);
+  }
+  return result;
+}
+
+function resolveAppraiserConfig(frontmatter, countOverride) {
+  const appraiserConfig = frontmatter.appraisers || {};
+  return {
+    count: countOverride || appraiserConfig.count || 3,
+    allowed: appraiserConfig.allowed || null,
+  };
+}
+
+export async function selectAppraisers(foundryDir, typeId, { io, countOverride } = {}) {
+  if (!io) throw new Error('selectAppraisers: io is required');
+
+  const { frontmatter } = await getArtefactType(foundryDir, typeId, io);
+  const { count, allowed } = resolveAppraiserConfig(frontmatter, countOverride);
+
+  const allAppraisers = await getAppraisers(foundryDir, io);
+  const pool = buildAppraiserPool(allAppraisers, allowed);
+  if (pool.length === 0) return [];
+
+  return roundRobinSelect(pool, count);
+}

package/dist/scripts/lib/failed-flow.js

@@ -0,0 +1,131 @@
+/**
+ * Failed-flow lifecycle helpers.
+ *
+ * When a tool encounters an unrecoverable error (e.g. stage_end cannot
+ * flush memory to NDJSON and the on-disk source of truth is now behind
+ * the live DB), it marks WORK.md with `status: failed` and a `reason`.
+ *
+ * Every mutating tool guards on this state via `requireNotFailed`. This
+ * includes work-branch FS writers (artefacts, feedback, workfile, stage,
+ * orchestrate), memory writers — row-level (memory_put, memory_relate,
+ * memory_unrelate) and admin (create_*, rename_*, drop_*, reset, init,
+ * vacuum, change_embedding_model) — and `validate_run`, since validation
+ * commands are project-defined subprocesses with arbitrary side effects
+ * (linters with --fix, formatters). The rule is simple: tools that mutate
+ * disk or live DB state, or run unsandboxed subprocesses that could mutate
+ * it, stay blocked while the abandoned work-branch filesystem remains the
+ * source of truth.
+ *
+ * Read-only diagnostics (workfile_get, memory_list/get/neighbours/query/search,
+ * memory_dump, memory_validate) remain available to support diagnosis before
+ * the cycle is abandoned.
+ *
+ * Recovery paths: `foundry_workfile_delete` abandons the cycle; editing
+ * WORK.md to remove the failed status after fixing the underlying issue
+ * restores normal operation. `foundry_stage_retry` provides a deterministic
+ * rollback mechanism: it discards uncommitted memory changes, clears the
+ * failed status, and resets the stage state, provided the git working tree
+ * is clean.
+ */
+import { parseFrontmatter, setFrontmatterField, writeFrontmatter } from './workfile.js';
+
+const MAX_REASON_LEN = 500;
+
+function truncateReason(reason) {
+  const s = String(reason ?? '');
+  if (s.length <= MAX_REASON_LEN) return s;
+  return s.slice(0, MAX_REASON_LEN) + '...';
+}
+
+/**
+ * @param {{exists: (p: string) => boolean, readFile: (p: string) => string}} io
+ * @returns {{reason: string} | null}
+ */
+export function readFailedStatus(io) {
+  if (!io.exists('WORK.md')) return null;
+  const text = io.readFile('WORK.md');
+  const fm = parseFrontmatter(text);
+  if (fm.status !== 'failed') return null;
+  return { reason: fm.reason === undefined ? '' : String(fm.reason) };
+}
+
+/**
+ * Idempotent: sets `status: failed` and `reason` if not already failed.
+ * Preserves the first failure reason when called multiple times - the
+ * initial diagnostic reason is more valuable than cascading failures.
+ * @param {object} io - requires exists, readFile, writeFile
+ * @param {string} reason
+ */
+export function markWorkfileFailed(io, reason) {
+  if (!io.exists('WORK.md')) {
+    throw new Error('markWorkfileFailed: WORK.md not found');
+  }
+  const text = io.readFile('WORK.md');
+  
+  // Check if already failed - preserve the first failure reason
+  const failed = readFailedStatus(io);
+  if (failed) {
+    // Already failed - skip overwrite to preserve diagnostic first reason
+    return;
+  }
+  
+  const withStatus = setFrontmatterField(text, 'status', 'failed');
+  const withReason = setFrontmatterField(withStatus, 'reason', truncateReason(reason));
+  io.writeFile('WORK.md', withReason);
+}
+
+/**
+ * Tool guard: returns `{ok:true}` when the flow is healthy, otherwise
+ * `{ok:false, error}` with a message that tells the LLM exactly how to
+ * escape (abandon the flow).
+ * @param {object} io
+ * @returns {{ok: true} | {ok: false, error: string}}
+ */
+export function requireNotFailed(io) {
+  let failed;
+  try {
+    failed = readFailedStatus(io);
+  } catch {
+    // WORK.md is corrupted (malformed YAML) or unreadable (IO error).
+    // This is a trouble signal - refuse to proceed.
+    return {
+      ok: false,
+      error:
+        `WORK.md is corrupted or unreadable. ` +
+        `No mutating tools are permitted. Use foundry_workfile_delete({confirm: true}) ` +
+        `to abandon the cycle, then back out to main and delete the work branch.`,
+    };
+  }
+  
+  if (!failed) return { ok: true };
+  const reason = failed.reason || '(no reason recorded)';
+  return {
+    ok: false,
+    error:
+      `flow is in failed state (reason: ${reason}). ` +
+      `No mutating tools are permitted. Use foundry_workfile_delete({confirm: true}) ` +
+      `to abandon the cycle, then back out to main and delete the work branch.`,
+  };
+}
+
+/**
+ * Clears the failed status from WORK.md, restoring normal operation.
+ * Idempotent: safe to call even if not failed.
+ * @param {object} io - requires exists, readFile, writeFile
+ */
+export function clearWorkfileFailed(io) {
+  if (!io.exists('WORK.md')) {
+    throw new Error('clearWorkfileFailed: WORK.md not found');
+  }
+  const text = io.readFile('WORK.md');
+  const fm = parseFrontmatter(text);
+  
+  // Remove status and reason fields
+  delete fm.status;
+  delete fm.reason;
+  
+  // Rebuild the file with cleaned frontmatter
+  const fmBlock = writeFrontmatter(fm);
+  const body = text.replace(/^---\n.+?\n---\n?/s, '');
+  io.writeFile('WORK.md', body ? `${fmBlock}\n${body}` : fmBlock);
+}