@really-knows-ai/foundry 2.3.2 → 3.0.0

package/dist/scripts/lib/assay/permissions.js

@@ -0,0 +1,52 @@
+// Extractor-level permission checks. Two scopes:
+//   1. Cycle-load: extractor.memoryWrite ⊆ cycle.memory.write
+//   2. Runtime per-row: a given entity or edge row is within the extractor's scope.
+//
+// The edge rule mirrors scripts/lib/memory/permissions.js: an edge is permitted
+// if either of its endpoint entity types is in the extractor's memoryWrite.
+
+export function checkExtractorAgainstCycle(extractor, cyclePerms) {
+  const missing = extractor.memoryWrite.filter((t) => !cyclePerms.writeTypes.has(t));
+  if (missing.length === 0) return { ok: true };
+  return {
+    ok: false,
+    error: `extractor '${extractor.name}' declares memory.write types not permitted by the cycle: ${missing.join(', ')}`,
+  };
+}
+
+export function checkEntityRowAgainstExtractor(extractor, entityType) {
+  if (extractor.memoryWrite.includes(entityType)) return { ok: true };
+  return {
+    ok: false,
+    error: `extractor '${extractor.name}': entity type '${entityType}' is not in memory.write (${extractor.memoryWrite.join(', ')})`,
+  };
+}
+
+function lookupInVocabulary(collection, typeName, extractorName, edgeType, label) {
+  if (collection?.[typeName]) return null;
+  return { ok: false, error: `extractor '${extractorName}': edge '${edgeType}' ${label} '${typeName}' not declared in project vocabulary` };
+}
+
+function validateEdgeTypesInVocabulary(extractor, edge, vocabulary) {
+  if (!vocabulary.edges?.[edge.edge_type]) {
+    return { ok: false, error: `extractor '${extractor.name}': edge type '${edge.edge_type}' not declared in project vocabulary` };
+  }
+  const fromError = lookupInVocabulary(vocabulary.entities, edge.from_type, extractor.name, edge.edge_type, 'from_type');
+  if (fromError) return fromError;
+  return lookupInVocabulary(vocabulary.entities, edge.to_type, extractor.name, edge.edge_type, 'to_type');
+}
+
+function isEdgeWritable(extractor, edge) {
+  const writable = new Set(extractor.memoryWrite);
+  return writable.has(edge.from_type) || writable.has(edge.to_type);
+}
+
+export function checkEdgeRowAgainstExtractor(extractor, edge, vocabulary) {
+  const validationError = validateEdgeTypesInVocabulary(extractor, edge, vocabulary);
+  if (validationError) return validationError;
+  if (isEdgeWritable(extractor, edge)) return { ok: true };
+  return {
+    ok: false,
+    error: `extractor '${extractor.name}': edge '${edge.edge_type}' has neither endpoint in memory.write (${extractor.memoryWrite.join(', ')})`,
+  };
+}

package/dist/scripts/lib/assay/run.js

@@ -0,0 +1,219 @@
+import { loadExtractor } from './loader.js';
+import { parseExtractorOutput } from './parse-jsonl.js';
+import { spawnWithTimeout as defaultSpawn } from './spawn-with-timeout.js';
+import {
+  checkEntityRowAgainstExtractor,
+  checkEdgeRowAgainstExtractor,
+} from './permissions.js';
+
+function spawnFailureReason(spawnResult) {
+  if (spawnResult.timedOut) return 'timedOut';
+  if (spawnResult.tooMuchOutput) return 'tooMuchOutput';
+  if (!spawnResult.ok) return 'exited';
+  return null;
+}
+
+function spawnFailureMessage(reason, ext, spawnResult) {
+  if (reason === 'timedOut') {
+    return `extractor timed out after ${ext.timeoutMs}ms`;
+  }
+  if (reason === 'tooMuchOutput') {
+    return 'extractor produced too much output'
+      + ' (stdout >50MB or stderr >1MB)';
+  }
+  const detail = spawnResult.signal
+    ? `signal ${spawnResult.signal} (exit code ${spawnResult.exitCode})`
+    : `exit code ${spawnResult.exitCode}`;
+  return `extractor exited with ${detail}`;
+}
+
+async function loadAndRunExtractor({
+  foundryDir, name, cwd, io, spawn, perExtractor,
+}) {
+  const startedAt = Date.now();
+  let ext;
+  try {
+    ext = await loadExtractor(foundryDir, name, io);
+  } catch (err) {
+    return { abort: abort(perExtractor, name,
+      `failed to load extractor: ${err.message}`) };
+  }
+
+  const spawnResult = await spawn({
+    command: ext.command,
+    cwd,
+    timeoutMs: ext.timeoutMs,
+  });
+
+  const reason = spawnFailureReason(spawnResult);
+  if (reason) {
+    const msg = spawnFailureMessage(reason, ext, spawnResult);
+    return { abort: abort(perExtractor, name, msg, spawnResult.stderr) };
+  }
+
+  let rows;
+  try {
+    rows = parseExtractorOutput(spawnResult.stdout);
+  } catch (err) {
+    return { abort: abort(perExtractor, name, err.message,
+      spawnResult.stderr) };
+  }
+
+  return { ext, rows, spawnResult, startedAt };
+}
+
+function validateEntityRow({ ext, row, vocabulary, perExtractor, name, stderr }) {
+  const r = checkEntityRowAgainstExtractor(ext, row.type);
+  if (!r.ok) return abort(perExtractor, name, r.error, stderr);
+  if (!vocabulary.entities?.[row.type]) {
+    return abort(perExtractor, name,
+      `entity type '${row.type}' not declared in project vocabulary`, stderr);
+  }
+  return null;
+}
+
+function validateEdgeRow({ row, ext, vocabulary, perExtractor, name, stderr }) {
+  const r = checkEdgeRowAgainstExtractor(ext, {
+    edge_type: row.edge_type,
+    from_type: row.from_type,
+    to_type: row.to_type,
+  }, vocabulary);
+  if (!r.ok) return abort(perExtractor, name, r.error, stderr);
+  return null;
+}
+
+function validateRows({ ext, rows, vocabulary, perExtractor, name, spawnResult }) {
+  const stderr = spawnResult.stderr;
+  for (const row of rows) {
+    const error = row.kind === 'entity'
+      ? validateEntityRow({ ext, row, vocabulary, perExtractor, name, stderr })
+      : validateEdgeRow({ row, ext, vocabulary, perExtractor, name, stderr });
+    if (error) return error;
+  }
+  return null;
+}
+
+async function upsertEntityRow(row, store, vocabulary, putEntity, writeEmbedder) {
+  await putEntity(
+    store,
+    { type: row.type, name: row.name, value: row.value },
+    vocabulary,
+    writeEmbedder ? { embedder: writeEmbedder } : undefined,
+  );
+}
+
+async function upsertEdgeRow(row, store, vocabulary, relate) {
+  await relate(store, {
+    edge_type: row.edge_type,
+    from_type: row.from_type, from_name: row.from_name,
+    to_type: row.to_type,     to_name: row.to_name,
+  }, vocabulary);
+}
+
+async function upsertRows({
+  rows, store, vocabulary, putEntity, relate,
+  writeEmbedder, perExtractor, name, spawnResult,
+}) {
+  let rowsUpserted = 0;
+  for (const row of rows) {
+    try {
+      if (row.kind === 'entity') {
+        await upsertEntityRow(row, store, vocabulary, putEntity, writeEmbedder);
+      } else {
+        await upsertEdgeRow(row, store, vocabulary, relate);
+      }
+      rowsUpserted += 1;
+    } catch (err) {
+      return { abort: abort(perExtractor, name,
+        `upsert failed: ${err.message}`, spawnResult.stderr) };
+    }
+  }
+  return { rowsUpserted };
+}
+
+async function syncStoreIfNeeded({
+  syncStore, store, io, perExtractor, name, rowsUpserted, spawnResult,
+}) {
+  if (!syncStore) return null;
+  try {
+    await syncStore({ store, io });
+  } catch (err) {
+    return abort(perExtractor, name,
+      `memory sync failed after upserting ${rowsUpserted} rows:`
+      + ` ${err.message}`, spawnResult.stderr);
+  }
+  return null;
+}
+
+async function processExtractor({
+  foundryDir, name, cwd, io, spawn, store, vocabulary,
+  putEntity, relate, writeEmbedder, syncStore, perExtractor,
+}) {
+  const loadResult = await loadAndRunExtractor({
+    foundryDir, name, cwd, io, spawn, perExtractor,
+  });
+  if (loadResult.abort) return loadResult;
+
+  const { ext, rows, spawnResult, startedAt } = loadResult;
+
+  const validationError = validateRows({
+    ext, rows, vocabulary, perExtractor, name, spawnResult,
+  });
+  if (validationError) return { abort: validationError };
+
+  const upsertResult = await upsertRows({
+    rows, store, vocabulary, putEntity, relate,
+    writeEmbedder, perExtractor, name, spawnResult,
+  });
+  if (upsertResult.abort) return upsertResult;
+
+  const syncError = await syncStoreIfNeeded({
+    syncStore, store, io, perExtractor, name,
+    rowsUpserted: upsertResult.rowsUpserted, spawnResult,
+  });
+  if (syncError) return { abort: syncError };
+
+  perExtractor.push({
+    name,
+    rowsUpserted: upsertResult.rowsUpserted,
+    durationMs: Date.now() - startedAt,
+  });
+  return { summary: perExtractor[perExtractor.length - 1] };
+}
+
+export async function runAssay({
+  foundryDir,
+  cwd,
+  io,
+  extractors,
+  store,
+  vocabulary,
+  putEntity,
+  relate,
+  writeEmbedder,
+  syncStore,
+  spawn = defaultSpawn,
+}) {
+  const perExtractor = [];
+
+  for (const name of extractors) {
+    const result = await processExtractor({
+      foundryDir, name, cwd, io, spawn, store, vocabulary,
+      putEntity, relate, writeEmbedder, syncStore, perExtractor,
+    });
+    if (result.abort) return result.abort;
+  }
+
+  return { ok: true, perExtractor };
+}
+
+function abort(perExtractor, failedExtractor, reason, stderr) {
+  return {
+    ok: false,
+    aborted: true,
+    failedExtractor,
+    reason,
+    stderr: stderr ?? '',
+    perExtractor,
+  };
+}

package/dist/scripts/lib/assay/spawn-with-timeout.js

@@ -0,0 +1,138 @@
+import { spawn } from 'node:child_process';
+import { StringDecoder } from 'node:string_decoder';
+
+// Output byte limits
+const MAX_STDOUT = 50 * 1024 * 1024; // 50MB
+const MAX_STDERR = 1 * 1024 * 1024;  // 1MB
+
+function killGroup(child, signal) {
+  // G25: Guard against undefined pid (spawn failure)
+  if (child.pid === null || child.pid === undefined) return;
+  // Negative pid targets the process group. Wrap in try/catch because
+  // the group may already be gone (ESRCH) by the time we signal.
+  try { process.kill(-child.pid, signal); } catch {}
+}
+
+function createStreamHandler(decoder, state, prop, maxBytes, onOverflow) {
+  return (b) => {
+    // G28: Check output limit
+    if (state[prop].length >= maxBytes) {
+      if (!state.tooMuchOutput) {
+        state.tooMuchOutput = true;
+        onOverflow();
+      }
+      return;
+    }
+    const decoded = decoder.write(b);
+    // Check again after decoding
+    if (state[prop].length + decoded.length >= maxBytes) {
+      const remaining = maxBytes - state[prop].length;
+      state[prop] += decoded.slice(0, remaining);
+      state.tooMuchOutput = true;
+      onOverflow();
+      return;
+    }
+    state[prop] += decoded;
+  };
+}
+
+// Runs a command (via /bin/sh -c) with a hard timeout. Never throws.
+// Returns:
+//   { ok, exitCode, signal, stdout, stderr, timedOut, tooMuchOutput }
+//
+// On timeout: sends SIGTERM to the child's process group immediately; if the
+// process is still alive 500ms later, sends SIGKILL to the group. `timedOut:
+// true` in the result.
+//
+// Output limits: stdout capped at 50MB, stderr at 1MB. On overflow, kills the
+// process group and returns `tooMuchOutput: true`.
+//
+// The child is spawned with `detached: true` so it becomes the leader of its
+// own process group. We signal the whole group (via `process.kill(-pid, ...)`)
+// to terminate all descendants in the process group tree, ensuring
+// shell-spawned subprocesses (e.g. a `sleep` launched by a shell script)
+// release the inherited stdout/stderr pipes promptly and Node can emit the
+// `close` event as soon as the group exits.
+//
+// Security: this intentionally uses a shell, matching how `foundry_validate_run`
+// expands validation commands today. Extractors are project-authored and
+// committed to the repo; they are trusted code paths, not untrusted input.
+export async function spawnWithTimeout({ command, cwd, timeoutMs, env }) {
+  return await new Promise((resolve) => {
+    const child = spawn('/bin/sh', ['-c', command], {
+      cwd,
+      env: env ?? process.env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+      detached: true,
+    });
+
+    const state = { stdout: '', stderr: '', timedOut: false, tooMuchOutput: false, settled: false, hardTimer: null };
+    const stdoutDecoder = new StringDecoder('utf-8');
+    const stderrDecoder = new StringDecoder('utf-8');
+
+    child.stdout.on('data', createStreamHandler(stdoutDecoder, state, 'stdout', MAX_STDOUT, () => killGroup(child, 'SIGKILL')));
+    child.stderr.on('data', createStreamHandler(stderrDecoder, state, 'stderr', MAX_STDERR, () => killGroup(child, 'SIGKILL')));
+
+    const softTimer = setTimeout(() => {
+      state.timedOut = true;
+      killGroup(child, 'SIGTERM');
+      // G24: Capture the hard timer so we can clear it on natural exit
+      state.hardTimer = setTimeout(() => {
+        if (!state.settled) { killGroup(child, 'SIGKILL'); }
+      }, 500);
+    }, timeoutMs);
+
+    child.on('error', makeErrorHandler(state, softTimer, resolve));
+    child.on('close', makeCloseHandler(state, softTimer, stdoutDecoder, stderrDecoder, resolve));
+  });
+}
+
+function makeErrorHandler(state, softTimer, resolve) {
+  return (err) => {
+    if (state.settled) return;
+    state.settled = true;
+    clearTimeout(softTimer);
+    // G24: Clear hard timer if it was set
+    if (state.hardTimer) clearTimeout(state.hardTimer);
+
+    // G26: Spawn error should never set timedOut to true.
+    // The error handler fires when spawn fails (ENOENT, EMFILE, etc.),
+    // which happens before the child even starts. timedOut requires
+    // the timeout to have actually elapsed AND a child to have existed.
+    const errDetail = (state.stderr.endsWith('\n') || state.stderr === '' ? '' : '\n') + `spawn error: ${err.message}`;
+    resolve({
+      ok: false,
+      exitCode: null,
+      signal: null,
+      stdout: state.stdout,
+      stderr: state.stderr + errDetail,
+      timedOut: false, // G26: Explicitly false, not inheriting timedOut flag
+      tooMuchOutput: false,
+    });
+  };
+}
+
+function makeCloseHandler(state, softTimer, stdoutDecoder, stderrDecoder, resolve) {
+  return (code, signal) => {
+    if (state.settled) return;
+    state.settled = true;
+    clearTimeout(softTimer);
+    // G24: Clear the SIGKILL timer to prevent event loop from hanging
+    if (state.hardTimer) clearTimeout(state.hardTimer);
+
+    // Flush any remaining bytes in the decoders
+    state.stdout += stdoutDecoder.end();
+    state.stderr += stderrDecoder.end();
+
+    const ok = !state.timedOut && !state.tooMuchOutput && code === 0;
+    resolve({
+      ok,
+      exitCode: code,
+      signal,
+      stdout: state.stdout,
+      stderr: state.stderr,
+      timedOut: state.timedOut,
+      tooMuchOutput: state.tooMuchOutput,
+    });
+  };
+}

package/dist/scripts/lib/attestation/attest.js

@@ -0,0 +1,111 @@
+/**
+ * Attestation builder.
+ *
+ * Verifies cycle completeness and produces the ATTEST.md content string.
+ */
+
+import path from 'node:path';
+import { load as loadYaml } from 'js-yaml';
+import { parseFrontmatter } from '../workfile.js';
+import { parseArtefactsTable } from '../artefacts.js';
+import { parseAllHistoryEntries } from '../history.js';
+import { sha256Buffer } from './hash.js';
+import { buildAttestationPayload } from './payload.js';
+import { canonicalJson } from './canonical-json.js';
+import { renderAttestedCommitMessage } from './render.js';
+
+function readWorkFiles(cwd, io) {
+  const workPath = path.join(cwd, 'WORK.md');
+  const historyPath = path.join(cwd, 'WORK.history.yaml');
+  const feedbackPath = path.join(cwd, 'WORK.feedback.yaml');
+
+  return {
+    workText: io.readFile(workPath),
+    historyText: io.fileExists(historyPath) ? io.readFile(historyPath) : '',
+    feedbackText: io.fileExists(feedbackPath) ? io.readFile(feedbackPath) : '',
+  };
+}
+
+function checkBlockedArtefacts(artefacts) {
+  const blocked = artefacts.filter(a => a.status === 'blocked');
+  if (blocked.length > 0) {
+    return `foundry_attest: cycle has blocked artefact(s): ${blocked.map(a => a.file).join(', ')}`;
+  }
+  return null;
+}
+
+function checkMissingStages(frontmatter, historyText) {
+  const entries = parseAllHistoryEntries(historyText);
+  const completed = new Set(entries.map(e => e.stage));
+  const missing = (frontmatter.stages ?? []).filter(s => !completed.has(s));
+  if (missing.length > 0) {
+    return `foundry_attest: required stage(s) not completed: ${missing.join(', ')}`;
+  }
+  return null;
+}
+
+function checkUnresolvedFeedback(feedbackText) {
+  const doc = feedbackText.trim() ? (loadYaml(feedbackText) ?? {}) : {};
+  const items = doc.items ?? [];
+  const unresolved = items.filter(item => item?.history?.[0]?.state !== 'resolved');
+  if (unresolved.length > 0) {
+    return `foundry_attest: ${unresolved.length} unresolved feedback item(s): ${unresolved.map(i => i.id).join(', ')}`;
+  }
+  return null;
+}
+
+function findCycleError(frontmatter, artefacts, historyText, feedbackText) {
+  const blockedError = checkBlockedArtefacts(artefacts);
+  if (blockedError) return blockedError;
+
+  const missingError = checkMissingStages(frontmatter, historyText);
+  if (missingError) return missingError;
+
+  return checkUnresolvedFeedback(feedbackText);
+}
+
+function computeDiffSha(execGit, baseBranch) {
+  const mergeBase = execGit(['merge-base', 'HEAD', baseBranch]).trim();
+  const diffOutput = execGit(['diff', mergeBase, 'HEAD']);
+  const diffBuf = Buffer.isBuffer(diffOutput) ? diffOutput : Buffer.from(diffOutput, 'utf8');
+  return sha256Buffer(diffBuf);
+}
+
+export async function buildAttestation({
+  cwd,
+  baseBranch,
+  goalText,
+  archiveBranch,
+  archiveTipSha,
+  io,
+  execGit,
+}) {
+  const { workText, historyText, feedbackText } = readWorkFiles(cwd, io);
+  const frontmatter = parseFrontmatter(workText);
+  const artefacts = parseArtefactsTable(workText);
+
+  const cycleError = findCycleError(frontmatter, artefacts, historyText, feedbackText);
+  if (cycleError) {
+    return { ok: false, error: cycleError };
+  }
+
+  const diffSha = computeDiffSha(execGit, baseBranch);
+
+  const payload = buildAttestationPayload({
+    cwd,
+    goalText,
+    archiveBranch,
+    archiveTipSha,
+    io,
+  });
+
+  const payloadJson = canonicalJson(payload);
+  const commitMessage = renderAttestedCommitMessage({ humanSummary: goalText, payloadJson });
+
+  const content = commitMessage.replace(
+    '-----BEGIN FOUNDRY ATTESTATION-----',
+    `diff-sha256: ${diffSha}\n\n-----BEGIN FOUNDRY ATTESTATION-----`
+  );
+
+  return { ok: true, content, diffSha };
+}

package/dist/scripts/lib/attestation/canonical-json.js

@@ -0,0 +1,109 @@
+function tryStringifyBool(value) {
+  if (typeof value === 'boolean') return value ? 'true' : 'false';
+  return undefined;
+}
+
+function tryStringifyNumber(value) {
+  if (typeof value === 'number') return JSON.stringify(value);
+  return undefined;
+}
+
+function tryStringifyString(value) {
+  if (typeof value === 'string') return JSON.stringify(value);
+  return undefined;
+}
+
+function stringifyPrimitive(value) {
+  if (value === null) return 'null';
+  if (value === undefined) return undefined;
+  const bool = tryStringifyBool(value);
+  if (bool !== undefined) return bool;
+  const num = tryStringifyNumber(value);
+  if (num !== undefined) return num;
+  return tryStringifyString(value);
+}
+
+function stringifyArray(value, seen) {
+  if (seen.has(value)) {
+    throw new TypeError('Converting circular structure to JSON');
+  }
+  seen.add(value);
+
+  const items = [];
+  for (let i = 0; i < value.length; i++) {
+    const result = stringifyCanonical(value[i], seen);
+    items.push(result === undefined ? 'null' : result);
+  }
+
+  seen.delete(value);
+  return '[' + items.join(',') + ']';
+}
+
+function isBoxedBigInt(value) {
+  return Object.prototype.toString.call(value) === '[object BigInt]';
+}
+
+function stringifyUnboxed(value) {
+  if (value instanceof Number) return JSON.stringify(value.valueOf());
+  if (value instanceof String) return JSON.stringify(value.valueOf());
+  if (value instanceof Boolean) return value.valueOf() ? 'true' : 'false';
+  return undefined;
+}
+
+function buildObjectPairs(value, seen) {
+  const keys = Object.keys(value).sort();
+  const pairs = [];
+  for (const key of keys) {
+    const val = stringifyCanonical(value[key], seen);
+    if (val !== undefined) {
+      pairs.push(JSON.stringify(key) + ':' + val);
+    }
+  }
+  return pairs;
+}
+
+function stringifyPlainObject(value, seen) {
+  if (seen.has(value)) {
+    throw new TypeError('Converting circular structure to JSON');
+  }
+  seen.add(value);
+  const pairs = buildObjectPairs(value, seen);
+  seen.delete(value);
+  return '{' + pairs.join(',') + '}';
+}
+
+function stringifyObject(value, seen) {
+  if (isBoxedBigInt(value)) {
+    throw new TypeError('Do not know how to serialise a BigInt');
+  }
+
+  const unboxed = stringifyUnboxed(value);
+  if (unboxed !== undefined) return unboxed;
+
+  if (typeof value.toJSON === 'function') {
+    return stringifyCanonical(value.toJSON(), seen);
+  }
+
+  return stringifyPlainObject(value, seen);
+}
+
+function dispatchType(value, seen) {
+  if (Array.isArray(value)) return stringifyArray(value, seen);
+  if (typeof value === 'object') return stringifyObject(value, seen);
+  return undefined;
+}
+
+function stringifyCanonical(value, seen = new Set()) {
+  if (typeof value === 'bigint') {
+    throw new TypeError('Do not know how to serialise a BigInt');
+  }
+
+  const primitive = stringifyPrimitive(value);
+  if (primitive !== undefined) return primitive;
+
+  return dispatchType(value, seen);
+}
+
+export function canonicalJson(value) {
+  return stringifyCanonical(value);
+}

package/dist/scripts/lib/attestation/hash.js

@@ -0,0 +1,17 @@
+import { createHash } from 'node:crypto';
+
+export function sha256Text(text) {
+  return createHash('sha256').update(text, 'utf8').digest('hex');
+}
+
+export function sha256Buffer(buf) {
+  return createHash('sha256').update(buf).digest('hex');
+}
+
+export function sortPaths(paths) {
+  return [...paths].sort((a, b) => {
+    if (a < b) return -1;
+    if (a > b) return 1;
+    return 0;
+  });
+}

package/dist/scripts/lib/attestation/parse.js

@@ -0,0 +1,14 @@
+const BEGIN = '-----BEGIN FOUNDRY ATTESTATION-----';
+const END = '-----END FOUNDRY ATTESTATION-----';
+
+export function extractAttestationBlock(message) {
+  const start = message.indexOf(BEGIN);
+  if (start === -1) {
+    throw new Error('attestation block not found');
+  }
+  const end = message.indexOf(END, start + BEGIN.length);
+  if (end === -1 || end <= start) {
+    throw new Error('attestation block not found');
+  }
+  return message.slice(start + BEGIN.length, end).trim();
+}