forge-workflow 0.0.3 → 0.0.5

package/scripts/lib/eval-schema.js

@@ -0,0 +1,135 @@
+/**
+ * Eval set schema validator and loader.
+ *
+ * Supports three assertion types:
+ *   - standard  : { type, check }
+ *   - hard-gate : { type, precondition, check }
+ *   - contract  : { type, producer, consumer, check }
+ */
+
+const fs = require('fs');
+
+const VALID_ASSERTION_TYPES = ['standard', 'hard-gate', 'contract'];
+
+// ── assertion-level required fields by type ────────────────────────────
+const ASSERTION_REQUIRED_FIELDS = {
+  standard: ['check'],
+  'hard-gate': ['precondition', 'check'],
+  contract: ['producer', 'consumer', 'check'],
+};
+
+// ── helpers ────────────────────────────────────────────────────────────
+
+function requireNonEmptyString(obj, field, prefix) {
+  if (typeof obj[field] !== 'string' || obj[field].length === 0) {
+    throw new Error(`${prefix}missing required field: ${field}`);
+  }
+}
+
+function validateAssertion(assertion, queryName, index) {
+  const prefix = `query "${queryName}", assertion ${index}`;
+
+  if (!assertion || typeof assertion.type !== 'string') {
+    throw new Error(`${prefix}: missing required field: type`);
+  }
+
+  if (!VALID_ASSERTION_TYPES.includes(assertion.type)) {
+    throw new Error(`${prefix}: unknown assertion type: "${assertion.type}"`);
+  }
+
+  const requiredFields = ASSERTION_REQUIRED_FIELDS[assertion.type];
+  for (const field of requiredFields) {
+    if (typeof assertion[field] !== 'string' || assertion[field].length === 0) {
+      throw new Error(`${prefix} (${assertion.type}): missing required field: ${field}`);
+    }
+  }
+}
+
+function validateQuery(query, index, seenNames) {
+  // name must exist before we can use it in error messages
+  if (typeof query.name !== 'string' || query.name.length === 0) {
+    throw new Error(`query at index ${index}: missing required field: name`);
+  }
+
+  const name = query.name;
+
+  if (seenNames.has(name)) {
+    throw new Error(`duplicate query name: "${name}"`);
+  }
+  seenNames.add(name);
+
+  requireNonEmptyString(query, 'prompt', `query "${name}": `);
+
+  // assertions: must be a non-empty array
+  if (!Array.isArray(query.assertions)) {
+    throw new Error(`query "${name}": missing required field: assertions`);
+  }
+  if (query.assertions.length === 0) {
+    throw new Error(`query "${name}": assertions must be a non-empty array`);
+  }
+
+  for (let i = 0; i < query.assertions.length; i++) {
+    validateAssertion(query.assertions[i], name, i);
+  }
+
+  // Normalise optional fields
+  if (query.setup === undefined) query.setup = null;
+  if (query.teardown === undefined) query.teardown = null;
+}
+
+// ── public API ─────────────────────────────────────────────────────────
+
+/**
+ * Validate an in-memory eval-set object.
+ * Returns the (possibly normalised) object on success, throws on failure.
+ */
+function validateEvalSet(data) {
+  if (!data || typeof data !== 'object') {
+    throw new Error('eval set must be a non-null object');
+  }
+
+  requireNonEmptyString(data, 'command', '');
+  requireNonEmptyString(data, 'description', '');
+
+  if (!Array.isArray(data.queries)) {
+    throw new Error('missing required field: queries');
+  }
+  if (data.queries.length === 0) {
+    throw new Error('queries must be a non-empty array');
+  }
+
+  const seenNames = new Set();
+  for (let i = 0; i < data.queries.length; i++) {
+    validateQuery(data.queries[i], i, seenNames);
+  }
+
+  return data;
+}
+
+/**
+ * Load an eval-set from a `.eval.json` file on disk.
+ * Returns the validated object or throws with a descriptive error.
+ */
+function loadEvalSet(filePath) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`eval set file not found: ${filePath}`);
+  }
+
+  let raw;
+  try {
+    raw = fs.readFileSync(filePath, 'utf-8');
+  } catch (err) {
+    throw new Error(`failed to read eval set file: ${err.message}`);
+  }
+
+  let data;
+  try {
+    data = JSON.parse(raw);
+  } catch (_err) {
+    throw new Error(`invalid JSON in eval set file: ${filePath}`);
+  }
+
+  return validateEvalSet(data);
+}
+
+module.exports = { loadEvalSet, validateEvalSet };

package/scripts/lib/eval-storage.js

@@ -0,0 +1,78 @@
+const fs = require('fs');
+const path = require('path');
+
+const DEFAULT_BASE_PATH = '.forge/eval-logs';
+
+/**
+ * Normalize a command name: strip leading slash, replace remaining slashes with dashes.
+ */
+function normalizeCommand(command) {
+  return command.replace(/^\//, '').replace(/\//g, '-');
+}
+
+/**
+ * Save an eval result as timestamped JSON.
+ *
+ * @param {object} result - Eval result with at minimum: { command, overall_score, results, timestamp }
+ * @param {string} [_basePath] - Directory to write to (defaults to .forge/eval-logs)
+ * @returns {string} The file path that was written
+ */
+function saveEvalResult(result, _basePath) {
+  const basePath = _basePath || DEFAULT_BASE_PATH;
+  const command = normalizeCommand(result.command);
+  const date = new Date(result.timestamp);
+
+  const yyyy = date.getUTCFullYear();
+  const mm = String(date.getUTCMonth() + 1).padStart(2, '0');
+  const dd = String(date.getUTCDate()).padStart(2, '0');
+  const hh = String(date.getUTCHours()).padStart(2, '0');
+  const min = String(date.getUTCMinutes()).padStart(2, '0');
+
+  const fileName = `${yyyy}-${mm}-${dd}-${hh}-${min}-${command}.json`;
+  const filePath = path.join(basePath, fileName);
+
+  fs.mkdirSync(basePath, { recursive: true });
+  fs.writeFileSync(filePath, JSON.stringify(result, null, 2), 'utf8');
+
+  return filePath;
+}
+
+/**
+ * Load eval history for a specific command.
+ *
+ * @param {string} command - Command name (e.g., '/status' or 'status')
+ * @param {string} [_basePath] - Directory to read from (defaults to .forge/eval-logs)
+ * @returns {object[]} Array of eval results sorted by timestamp (newest first)
+ */
+function loadEvalHistory(command, _basePath) {
+  const basePath = _basePath || DEFAULT_BASE_PATH;
+  const normalized = normalizeCommand(command);
+
+  if (!fs.existsSync(basePath)) {
+    return [];
+  }
+
+  const suffix = `-${normalized}.json`;
+  const files = fs.readdirSync(basePath).filter((f) => f.endsWith(suffix));
+
+  const results = [];
+  for (const file of files) {
+    const filePath = path.join(basePath, file);
+    try {
+      const content = fs.readFileSync(filePath, 'utf8');
+      results.push(JSON.parse(content));
+    } catch (_err) {
+      // History is best-effort context only; skip corrupted entries.
+    }
+  }
+
+  results.sort((a, b) => {
+    const ta = new Date(a.timestamp).getTime();
+    const tb = new Date(b.timestamp).getTime();
+    return tb - ta;
+  });
+
+  return results;
+}
+
+module.exports = { DEFAULT_BASE_PATH, saveEvalResult, loadEvalHistory };

package/scripts/lib/grading.js

@@ -0,0 +1,203 @@
+/**
+ * Grading orchestrator — invoke grader agent + collect results.
+ *
+ * Takes a parsed transcript (from transcript-parser.js) and an array of
+ * assertion objects, invokes the grader agent (via `claude -p` or an
+ * injectable function), parses the grader's JSON response, and computes
+ * per-query and overall scores.
+ */
+
+const { execFileSync } = require('child_process');
+
+// ---------------------------------------------------------------------------
+// buildGraderPrompt
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the prompt string sent to the command-grader agent.
+ *
+ * @param {{ messages: Array, toolCalls: Array, result: object|null }} transcript
+ * @param {Array<{ type: string, check: string, [key: string]: any }>} assertions
+ * @returns {string}
+ */
+function buildGraderPrompt(transcript, assertions) {
+  // Serialize transcript messages into readable text
+  const transcriptLines = [];
+  for (const msg of transcript.messages) {
+    if (msg.text) {
+      transcriptLines.push(msg.text);
+    }
+    for (const tc of msg.toolCalls) {
+      transcriptLines.push(`[Tool Call] ${tc.name}: ${JSON.stringify(tc.input)}`);
+    }
+  }
+  const transcriptText = transcriptLines.join('\n');
+
+  // Serialize assertions
+  const assertionsDef = JSON.stringify(assertions, null, 2);
+
+  return [
+    '## Transcript',
+    '',
+    '<transcript>',
+    transcriptText,
+    '</transcript>',
+    '',
+    '## Assertions',
+    '',
+    assertionsDef,
+    '',
+    'Grade each assertion against the transcript above. Return your response as a JSON object with a "results" array containing one entry per assertion, each with "assertion", "pass" (boolean), and "reasoning" (string).',
+  ].join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// parseGraderResponse
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse the grader's text response and extract the results array.
+ *
+ * Handles:
+ * - Raw JSON
+ * - JSON wrapped in markdown ```json code blocks
+ * - JSON embedded in leading/trailing prose
+ *
+ * @param {string} responseText
+ * @returns {Array<{ assertion: object, pass: boolean, reasoning: string }>}
+ * @throws {Error} if no valid JSON with a results array can be extracted
+ */
+function parseGraderResponse(responseText) {
+  // Strategy 1: markdown code block
+  const codeBlockMatch = responseText.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
+  if (codeBlockMatch) {
+    try {
+      const parsed = JSON.parse(codeBlockMatch[1].trim());
+      if (parsed && Array.isArray(parsed.results)) {
+        return parsed.results;
+      }
+    } catch (_e) {
+      // fall through to next strategy
+    }
+  }
+
+  // Strategy 2: find the outermost { ... } containing "results"
+  const firstBrace = responseText.indexOf('{');
+  const lastBrace = responseText.lastIndexOf('}');
+  if (firstBrace !== -1 && lastBrace > firstBrace) {
+    const candidate = responseText.slice(firstBrace, lastBrace + 1);
+    try {
+      const parsed = JSON.parse(candidate);
+      if (parsed && Array.isArray(parsed.results)) {
+        return parsed.results;
+      }
+    } catch (_e) {
+      // fall through
+    }
+  }
+
+  throw new Error('Failed to parse grader response: no valid JSON with results array found');
+}
+
+// ---------------------------------------------------------------------------
+// Default invokeGrader (calls `claude -p` with the grader agent)
+// ---------------------------------------------------------------------------
+
+/**
+ * Default grader invocation via `claude -p`.
+ *
+ * @param {string} prompt
+ * @param {{ timeout?: number }} options
+ * @returns {Promise<string>}
+ */
+async function defaultInvokeGrader(prompt, options = {}) {
+  const timeout = options.timeout || 120_000;
+  const result = execFileSync(
+    'claude',
+    ['-p', prompt, '--agent', 'command-grader', '--output-format', 'text', '--no-session-persistence'],
+    {
+      encoding: 'utf-8',
+      timeout,
+      maxBuffer: 10 * 1024 * 1024,
+    }
+  );
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// gradeTranscript
+// ---------------------------------------------------------------------------
+
+/**
+ * Orchestrate grading: build prompt, invoke grader, parse response, score.
+ *
+ * @param {{ messages: Array, toolCalls: Array, result: object|null }} transcript
+ * @param {Array<{ type: string, check: string, [key: string]: any }>} assertions
+ * @param {{ timeout?: number, _invokeGrader?: Function }} [options]
+ * @returns {Promise<{ assertions: Array<{ type: string, check: string, pass: boolean, reasoning: string }>, score: number }>}
+ */
+async function gradeTranscript(transcript, assertions, options = {}) {
+  // Empty assertions → perfect score
+  if (assertions.length === 0) {
+    return { assertions: [], score: 1.0 };
+  }
+
+  const invokeGrader = options._invokeGrader || defaultInvokeGrader;
+
+  const prompt = buildGraderPrompt(transcript, assertions);
+
+  let responseText;
+  try {
+    responseText = await invokeGrader(prompt, { timeout: options.timeout });
+  } catch (_err) {
+    // Grader invocation failed entirely — mark all as error
+    return {
+      assertions: assertions.map((a) => ({
+        ...a,
+        pass: false,
+        reasoning: 'Grader returned malformed response',
+      })),
+      score: 0,
+    };
+  }
+
+  let graderResults;
+  try {
+    graderResults = parseGraderResponse(responseText);
+  } catch (_err) {
+    // Malformed response — mark all as error
+    return {
+      assertions: assertions.map((a) => ({
+        ...a,
+        pass: false,
+        reasoning: 'Grader returned malformed response',
+      })),
+      score: 0,
+    };
+  }
+
+  // Merge grader results back onto the original assertions
+  const scoredAssertions = assertions.map((assertion, i) => {
+    const graded = graderResults[i];
+    if (graded && typeof graded.pass === 'boolean') {
+      return {
+        ...assertion,
+        pass: graded.pass,
+        reasoning: graded.reasoning || '',
+      };
+    }
+    // Missing or malformed entry for this assertion
+    return {
+      ...assertion,
+      pass: false,
+      reasoning: 'Grader returned malformed response',
+    };
+  });
+
+  const passed = scoredAssertions.filter((a) => a.pass).length;
+  const score = passed / scoredAssertions.length;
+
+  return { assertions: scoredAssertions, score };
+}
+
+module.exports = { gradeTranscript, buildGraderPrompt, parseGraderResponse };

package/scripts/lib/transcript-parser.js

@@ -0,0 +1,63 @@
+/**
+ * Parse NDJSON output from `claude -p --output-format stream-json`.
+ *
+ * Extracts assistant text content, tool calls (name + input), and final result.
+ *
+ * @param {string} ndjsonString - Raw NDJSON string (one JSON object per line)
+ * @returns {{ messages: Array, toolCalls: Array, result: object|null }}
+ */
+function parseTranscript(ndjsonString) {
+  const transcript = {
+    messages: [],
+    toolCalls: [],
+    result: null,
+  };
+
+  if (!ndjsonString || ndjsonString.trim() === '') {
+    return transcript;
+  }
+
+  const lines = ndjsonString.split('\n');
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed === '') continue;
+
+    let event;
+    try {
+      event = JSON.parse(trimmed);
+    } catch (_err) {
+      // Malformed line — skip silently
+      continue;
+    }
+
+    if (event.type === 'assistant') {
+      const message = {
+        role: 'assistant',
+        text: '',
+        toolCalls: [],
+      };
+
+      const content = event.message && event.message.content;
+      if (Array.isArray(content)) {
+        for (const block of content) {
+          if (block.type === 'text') {
+            message.text += block.text;
+          } else if (block.type === 'tool_use') {
+            const toolCall = { name: block.name, input: block.input };
+            message.toolCalls.push(toolCall);
+            transcript.toolCalls.push(toolCall);
+          }
+        }
+      }
+
+      transcript.messages.push(message);
+    } else if (event.type === 'result') {
+      transcript.result = event.result || null;
+    }
+  }
+
+  return transcript;
+}
+
+module.exports = { parseTranscript };

package/scripts/lint.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+/**
+ * Cross-platform ESLint runner for lefthook pre-push hook.
+ * Delegates to the project's package manager: <pkg> run lint
+ * Works on Windows CMD, PowerShell, macOS, Linux.
+ */
+
+const { spawnSync } = require('node:child_process');
+const fs = require('node:fs');
+
+// On Windows, package manager CLIs are .cmd files — shell: true resolves them
+const isWindows = process.platform === 'win32';
+
+// Detect package manager from lock files (same priority as forge.js and test.js)
+function detectPackageManager() {
+  if (fs.existsSync('bun.lockb') || fs.existsSync('bun.lock')) return 'bun';
+  if (fs.existsSync('pnpm-lock.yaml')) return 'pnpm';
+  if (fs.existsSync('yarn.lock')) return 'yarn';
+  return 'npm';
+}
+
+const pkgManager = detectPackageManager();
+console.log(`🔍 Running ESLint (${pkgManager} run lint)...`);
+
+const result = spawnSync(pkgManager, ['run', 'lint'], { stdio: 'inherit', shell: isWindows });
+
+if (result.error) {
+  console.error('');
+  console.error(`❌ Failed to run ${pkgManager} run lint: ${result.error.message}`);
+  console.error(`   Is '${pkgManager}' installed and on PATH?`);
+  console.error('');
+  process.exit(1);
+}
+
+if (result.status !== 0) {
+  console.error('');
+  console.error('❌ ESLint errors found. Fix them before pushing.');
+  console.error('');
+  console.error(`To see errors:  ${pkgManager} run lint`);
+  console.error(`To auto-fix:    ${pkgManager} run lint -- --fix`);
+  console.error('');
+  console.error('Fix the failing checks. Do not bypass hooks.');
+  console.error('');
+  process.exit(1);
+}
+
+console.log('✅ ESLint check passed (no errors)');