@grainulation/harvest 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 grainulation contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,102 @@
+# @grainulation/harvest
+
+**Are your decisions getting better?**
+
+Harvest is the analytics and retrospective layer for research sprints. It looks across sprints to find patterns, score predictions, and surface knowledge that's gone stale.
+
+Learn from every decision you've made.
+
+## What it does
+
+- **Cross-sprint analysis** -- claim type distributions, evidence quality, recurring themes
+- **Prediction calibration** -- score past estimates against actual outcomes
+- **Decision patterns** -- what research approaches lead to better results?
+- **Knowledge decay** -- which old claims need refreshing before they mislead you?
+- **Sprint velocity** -- how long do sprints take, where do they stall?
+- **Retrospective reports** -- dark-themed HTML reports for the team
+
+## Install
+
+```sh
+npm install @grainulation/harvest
+```
+
+Or run directly:
+
+```sh
+npx @grainulation/harvest analyze ./sprints/
+```
+
+## Usage
+
+```sh
+# Cross-sprint claim analysis
+harvest analyze ./sprints/
+
+# Score predictions against outcomes
+harvest calibrate ./sprints/
+
+# Detect decision patterns and anti-patterns
+harvest patterns ./sprints/
+
+# Find stale claims that need refreshing
+harvest decay ./sprints/ --days 60
+
+# Sprint timing and phase analysis
+harvest velocity ./sprints/
+
+# Generate a full retrospective HTML report
+harvest report ./sprints/ -o retrospective.html
+
+# All analyses in one pass
+harvest trends ./sprints/ --json
+```
+
+## Data format
+
+Harvest reads standard wheat sprint data:
+
+- `claims.json` -- array of typed claims with `id`, `type`, `evidence`, `status`, `text`, `created`, etc.
+- `compilation.json` -- compiled sprint state (optional, enriches analysis)
+- Git history on `claims.json` -- used for velocity and timing analysis
+
+Point harvest at a directory containing sprint subdirectories, or at a single sprint directory:
+
+```
+sprints/
+  sprint-alpha/
+    claims.json
+    compilation.json
+  sprint-beta/
+    claims.json
+```
+
+## Design
+
+- **Zero dependencies** -- Node built-in modules only (fs, path, child_process)
+- **Reads, never writes** -- harvest is a pure analysis tool; it won't modify your sprint data
+- **Git-aware** -- uses git log timestamps for velocity analysis when available
+- **Composable** -- each module (analyzer, calibration, patterns, decay, velocity) works independently
+
+## Claim types it understands
+
+| Type | What it means |
+|---|---|
+| `constraint` | Hard requirements, non-negotiable |
+| `factual` | Verifiable statements |
+| `estimate` | Predictions, projections, ranges |
+| `risk` | Potential failure modes |
+| `recommendation` | Proposed courses of action |
+| `feedback` | Stakeholder input |
+
+## Evidence tiers (lowest to highest)
+
+1. `stated` -- someone said it
+2. `web` -- found online
+3. `documented` -- in source code or official docs
+4. `tested` -- verified via prototype or benchmark
+5. `production` -- measured from live systems
+
+## License
+
+MIT

package/bin/harvest.js

@@ -0,0 +1,284 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const path = require('node:path');
+const fs = require('node:fs');
+
+const { analyze } = require('../lib/analyzer.js');
+const { calibrate } = require('../lib/calibration.js');
+const { detectPatterns } = require('../lib/patterns.js');
+const { checkDecay } = require('../lib/decay.js');
+const { measureVelocity } = require('../lib/velocity.js');
+const { generateReport } = require('../lib/report.js');
+const { connect: farmerConnect } = require('../lib/farmer.js');
+
+const verbose = process.argv.includes('--verbose') || process.argv.includes('-v');
+function vlog(...a) {
+  if (!verbose) return;
+  const ts = new Date().toISOString();
+  process.stderr.write(`[${ts}] harvest: ${a.join(' ')}\n`);
+}
+
+const USAGE = `
+harvest -- learn from every decision you've made
+
+Usage:
+  harvest analyze <sprints-dir>       Cross-sprint claim analysis
+  harvest calibrate <sprints-dir>     Score predictions against outcomes
+  harvest patterns <sprints-dir>      Detect decision patterns
+  harvest decay <sprints-dir>         Find claims that need refreshing
+  harvest velocity <sprints-dir>      Sprint timing and phase analysis
+  harvest report <sprints-dir> [-o <output>]  Generate retrospective HTML
+  harvest trends <sprints-dir>        All analyses in one pass
+  harvest serve [--port 9096] [--root <sprints-dir>]  Start the dashboard UI
+  harvest connect farmer [--url <url>]               Configure farmer integration
+
+Options:
+  -o, --output <path>   Output file path (default: stdout or ./retrospective.html)
+  -h, --help            Show this help
+  --json                Output as JSON instead of text
+  --days <n>            Decay threshold in days (default: 90)
+`.trim();
+
+function parseArgs(argv) {
+  const args = argv.slice(2);
+  const parsed = { command: null, dir: null, output: null, json: false, days: 90 };
+
+  if (args.length === 0 || args.includes('-h') || args.includes('--help')) {
+    console.log(USAGE);
+    process.exit(0);
+  }
+
+  parsed.command = args[0];
+  parsed.dir = (args[1] && !args[1].startsWith('-')) ? path.resolve(args[1]) : null;
+
+  for (let i = 2; i < args.length; i++) {
+    if ((args[i] === '-o' || args[i] === '--output') && args[i + 1]) {
+      parsed.output = path.resolve(args[++i]);
+    } else if (args[i] === '--json') {
+      parsed.json = true;
+    } else if (args[i] === '--days' && args[i + 1]) {
+      parsed.days = parseInt(args[++i], 10);
+    }
+  }
+
+  return parsed;
+}
+
+function loadSprintData(dir) {
+  if (!dir || !fs.existsSync(dir)) {
+    console.error(`harvest: directory not found: ${dir}`);
+    process.exit(1);
+  }
+
+  const sprints = [];
+
+  // Include root if it has claims.json
+  const directClaims = path.join(dir, 'claims.json');
+  if (fs.existsSync(directClaims)) {
+    sprints.push(loadSingleSprint(dir));
+  }
+
+  // Scan subdirectories (two levels deep to catch sprints/<name>/claims.json)
+  try {
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+      if (entry.name.startsWith('.')) continue;
+      const childDir = path.join(dir, entry.name);
+      const childClaims = path.join(childDir, 'claims.json');
+      if (fs.existsSync(childClaims)) {
+        sprints.push(loadSingleSprint(childDir));
+      }
+      // Second level
+      try {
+        const subEntries = fs.readdirSync(childDir, { withFileTypes: true });
+        for (const sub of subEntries) {
+          if (!sub.isDirectory()) continue;
+          if (sub.name.startsWith('.')) continue;
+          const subDir = path.join(childDir, sub.name);
+          const subClaims = path.join(subDir, 'claims.json');
+          if (fs.existsSync(subClaims)) {
+            sprints.push(loadSingleSprint(subDir));
+          }
+        }
+      } catch { /* skip */ }
+    }
+  } catch { /* skip */ }
+
+  if (sprints.length === 0) {
+    console.error(`harvest: no sprint data found in ${dir}`);
+    console.error('Expected claims.json in the directory or its subdirectories.');
+    process.exit(1);
+  }
+
+  return sprints;
+}
+
+function loadSingleSprint(dir) {
+  const sprint = {
+    name: path.basename(dir),
+    dir,
+    claims: [],
+    compilation: null,
+    gitLog: null,
+  };
+
+  const claimsPath = path.join(dir, 'claims.json');
+  try {
+    sprint.claims = JSON.parse(fs.readFileSync(claimsPath, 'utf8'));
+    if (!Array.isArray(sprint.claims)) {
+      // Handle { claims: [...] } wrapper
+      sprint.claims = sprint.claims.claims || [];
+    }
+  } catch (e) {
+    console.error(`harvest: could not parse ${claimsPath}: ${e.message}`);
+  }
+
+  const compilationPath = path.join(dir, 'compilation.json');
+  if (fs.existsSync(compilationPath)) {
+    try {
+      sprint.compilation = JSON.parse(fs.readFileSync(compilationPath, 'utf8'));
+    } catch (e) {
+      // skip
+    }
+  }
+
+  // Try to read git log for the sprint directory
+  try {
+    const { execSync } = require('node:child_process');
+    sprint.gitLog = execSync(
+      `git log --oneline --format="%H|%ai|%s" -- claims.json`,
+      { cwd: dir, encoding: 'utf8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }
+    ).trim().split('\n').filter(Boolean).map(line => {
+      const [hash, date, ...msg] = line.split('|');
+      return { hash, date, message: msg.join('|') };
+    });
+  } catch (e) {
+    sprint.gitLog = [];
+  }
+
+  return sprint;
+}
+
+function output(result, opts) {
+  if (opts.json) {
+    console.log(JSON.stringify(result, null, 2));
+  } else if (typeof result === 'string') {
+    console.log(result);
+  } else {
+    console.log(JSON.stringify(result, null, 2));
+  }
+}
+
+async function main() {
+  const opts = parseArgs(process.argv);
+  vlog('startup', `command=${opts.command || '(none)'}`, `dir=${opts.dir || 'none'}`);
+
+  const commands = {
+    analyze() {
+      const sprints = loadSprintData(opts.dir);
+      const result = analyze(sprints);
+      output(result, opts);
+    },
+    calibrate() {
+      const sprints = loadSprintData(opts.dir);
+      const result = calibrate(sprints);
+      output(result, opts);
+    },
+    patterns() {
+      const sprints = loadSprintData(opts.dir);
+      const result = detectPatterns(sprints);
+      output(result, opts);
+    },
+    decay() {
+      const sprints = loadSprintData(opts.dir);
+      const result = checkDecay(sprints, { thresholdDays: opts.days });
+      output(result, opts);
+    },
+    velocity() {
+      const sprints = loadSprintData(opts.dir);
+      const result = measureVelocity(sprints);
+      output(result, opts);
+    },
+    report() {
+      const sprints = loadSprintData(opts.dir);
+      const html = generateReport(sprints, {
+        analyzeFn: analyze,
+        calibrateFn: calibrate,
+        patternsFn: detectPatterns,
+        decayFn: checkDecay,
+        velocityFn: measureVelocity,
+      });
+      const outPath = opts.output || path.join(process.cwd(), 'retrospective.html');
+      fs.writeFileSync(outPath, html, 'utf8');
+      console.log(`Retrospective written to ${outPath}`);
+    },
+    trends() {
+      const sprints = loadSprintData(opts.dir);
+      const result = {
+        analysis: analyze(sprints),
+        calibration: calibrate(sprints),
+        patterns: detectPatterns(sprints),
+        decay: checkDecay(sprints, { thresholdDays: opts.days }),
+        velocity: measureVelocity(sprints),
+      };
+      output(result, opts);
+    },
+  };
+
+  if (opts.command === 'help') {
+    console.log(USAGE);
+    process.exit(0);
+  }
+
+  if (opts.command === 'connect') {
+    // Forward remaining args to farmer connect handler
+    const connectArgs = process.argv.slice(process.argv.indexOf('connect') + 1);
+    await farmerConnect(opts.dir || process.cwd(), connectArgs);
+    return;
+  }
+
+  if (opts.command === 'serve') {
+    // Launch the ESM server module
+    const { execFile } = require('node:child_process');
+    const serverPath = path.join(__dirname, '..', 'lib', 'server.js');
+    const serverArgs = [];
+    // Forward --port and --root
+    const portIdx = process.argv.indexOf('--port');
+    if (portIdx !== -1 && process.argv[portIdx + 1]) {
+      serverArgs.push('--port', process.argv[portIdx + 1]);
+    }
+    const rootIdx = process.argv.indexOf('--root');
+    if (rootIdx !== -1 && process.argv[rootIdx + 1]) {
+      serverArgs.push('--root', process.argv[rootIdx + 1]);
+    } else if (opts.dir) {
+      serverArgs.push('--root', opts.dir);
+    }
+    const child = execFile('node', [serverPath, ...serverArgs], {
+      stdio: 'inherit',
+      env: process.env,
+    });
+    child.stdout && child.stdout.pipe(process.stdout);
+    child.stderr && child.stderr.pipe(process.stderr);
+    child.on('error', (err) => {
+      console.error(`harvest: error starting server: ${err.message}`);
+      process.exit(1);
+    });
+    child.on('exit', (code) => process.exit(code || 0));
+    process.on('SIGTERM', () => child.kill('SIGTERM'));
+    process.on('SIGINT', () => child.kill('SIGINT'));
+    return;
+  }
+
+  if (!commands[opts.command]) {
+    console.error(`harvest: unknown command: ${opts.command}`);
+    console.error(`Run "harvest --help" for usage.`);
+    process.exit(1);
+  }
+
+  commands[opts.command]();
+}
+
+main();

package/lib/analyzer.js

@@ -0,0 +1,88 @@
+'use strict';
+
+/**
+ * Cross-sprint claim analysis.
+ *
+ * Looks across multiple sprints to find:
+ * - Claim type distribution (what kinds of findings dominate?)
+ * - Evidence tier distribution (how well-supported are claims?)
+ * - Cross-sprint themes (recurring topics or concerns)
+ * - Claim density per sprint (productivity signal)
+ */
+
+function analyze(sprints) {
+  const allClaims = sprints.flatMap(s => s.claims.map(c => ({ ...c, _sprint: s.name })));
+
+  const typeDistribution = countBy(allClaims, 'type');
+  const evidenceDistribution = countBy(allClaims, 'evidence');
+  const statusDistribution = countBy(allClaims, 'status');
+
+  // Per-sprint density
+  const perSprint = sprints.map(s => ({
+    name: s.name,
+    claimCount: s.claims.length,
+    types: countBy(s.claims, 'type'),
+    evidence: countBy(s.claims, 'evidence'),
+    statuses: countBy(s.claims, 'status'),
+  }));
+
+  // Find cross-sprint themes by looking at tags
+  const tagFrequency = {};
+  for (const claim of allClaims) {
+    const tags = claim.tags || [];
+    for (const tag of tags) {
+      tagFrequency[tag] = (tagFrequency[tag] || 0) + 1;
+    }
+  }
+
+  // Identify weak spots: claims with low evidence tiers
+  const weakClaims = allClaims.filter(c =>
+    c.evidence === 'stated' || c.evidence === 'web'
+  );
+
+  // Type monoculture detection: sprints dominated by a single claim type
+  const monocultures = perSprint.filter(s => {
+    const types = Object.entries(s.types);
+    if (types.length === 0) return false;
+    const max = Math.max(...types.map(([, v]) => v));
+    return max / s.claimCount > 0.7 && s.claimCount > 3;
+  }).map(s => ({
+    sprint: s.name,
+    dominantType: Object.entries(s.types).sort((a, b) => b[1] - a[1])[0][0],
+    ratio: Math.round(Math.max(...Object.values(s.types)) / s.claimCount * 100),
+  }));
+
+  return {
+    summary: {
+      totalSprints: sprints.length,
+      totalClaims: allClaims.length,
+      averageClaimsPerSprint: sprints.length > 0
+        ? Math.round(allClaims.length / sprints.length * 10) / 10
+        : 0,
+    },
+    typeDistribution,
+    evidenceDistribution,
+    statusDistribution,
+    tagFrequency,
+    weakClaims: weakClaims.map(c => ({
+      id: c.id,
+      sprint: c._sprint,
+      type: c.type,
+      evidence: c.evidence,
+      text: c.text || c.claim || c.description,
+    })),
+    monocultures,
+    perSprint,
+  };
+}
+
+function countBy(items, key) {
+  const counts = {};
+  for (const item of items) {
+    const val = item[key] || 'unknown';
+    counts[val] = (counts[val] || 0) + 1;
+  }
+  return counts;
+}
+
+module.exports = { analyze };

package/lib/calibration.js

@@ -0,0 +1,153 @@
+'use strict';
+
+/**
+ * Prediction vs outcome scoring.
+ *
+ * Compares estimate claims against calibrate claims (from wheat's /calibrate).
+ * Answers: "How often were our estimates right?"
+ *
+ * Scoring:
+ * - Each estimate claim gets matched to calibration claims by ID reference or tag overlap
+ * - Calibration claims contain actual outcomes and a confidence delta
+ * - We compute accuracy, overconfidence, and underconfidence rates
+ */
+
+const EVIDENCE_RANK = {
+  stated: 1,
+  web: 2,
+  documented: 3,
+  tested: 4,
+  production: 5,
+};
+
+function calibrate(sprints) {
+  const allClaims = sprints.flatMap(s => s.claims.map(c => ({ ...c, _sprint: s.name })));
+
+  const estimates = allClaims.filter(c => c.type === 'estimate');
+  const calibrations = allClaims.filter(c =>
+    c.id && (c.id.startsWith('cal') || c.type === 'calibration')
+  );
+
+  // Match calibrations to estimates
+  const scored = [];
+  for (const cal of calibrations) {
+    const refs = cal.references || cal.refs || [];
+    const matchedEstimates = estimates.filter(e =>
+      refs.includes(e.id) ||
+      (cal.tags && e.tags && cal.tags.some(t => e.tags.includes(t)))
+    );
+
+    for (const est of matchedEstimates) {
+      scored.push({
+        estimateId: est.id,
+        calibrationId: cal.id,
+        sprint: est._sprint,
+        estimateText: est.text || est.claim || est.description,
+        outcomeText: cal.text || cal.claim || cal.description,
+        estimateConfidence: est.confidence || null,
+        actualOutcome: cal.outcome || cal.actual || null,
+        accurate: cal.accurate ?? null,
+        delta: cal.delta ?? null,
+      });
+    }
+  }
+
+  // Unmatched estimates -- predictions with no follow-up
+  const scoredEstimateIds = new Set(scored.map(s => s.estimateId));
+  const unmatched = estimates.filter(e => !scoredEstimateIds.has(e.id));
+
+  // Compute aggregate stats
+  const accurateCount = scored.filter(s => s.accurate === true).length;
+  const inaccurateCount = scored.filter(s => s.accurate === false).length;
+  const unchecked = scored.filter(s => s.accurate === null).length;
+
+  const totalScored = accurateCount + inaccurateCount;
+  const accuracyRate = totalScored > 0
+    ? Math.round(accurateCount / totalScored * 100)
+    : null;
+
+  // Confidence calibration: group by confidence bucket
+  const buckets = { high: { total: 0, accurate: 0 }, medium: { total: 0, accurate: 0 }, low: { total: 0, accurate: 0 } };
+  for (const s of scored) {
+    const conf = s.estimateConfidence;
+    let bucket = 'medium';
+    if (typeof conf === 'number') {
+      bucket = conf >= 0.7 ? 'high' : conf >= 0.4 ? 'medium' : 'low';
+    } else if (typeof conf === 'string') {
+      bucket = conf.toLowerCase();
+    }
+    if (buckets[bucket]) {
+      buckets[bucket].total++;
+      if (s.accurate === true) buckets[bucket].accurate++;
+    }
+  }
+
+  const calibrationScore = Object.fromEntries(
+    Object.entries(buckets)
+      .filter(([, v]) => v.total > 0)
+      .map(([k, v]) => [k, Math.round(v.accurate / v.total * 100)])
+  );
+
+  return {
+    summary: {
+      totalEstimates: estimates.length,
+      totalCalibrations: calibrations.length,
+      matched: scored.length,
+      unmatched: unmatched.length,
+      accuracyRate,
+    },
+    calibrationByConfidence: calibrationScore,
+    scored: scored.map(s => ({
+      estimateId: s.estimateId,
+      calibrationId: s.calibrationId,
+      sprint: s.sprint,
+      accurate: s.accurate,
+      delta: s.delta,
+    })),
+    unmatchedEstimates: unmatched.map(e => ({
+      id: e.id,
+      sprint: e._sprint,
+      text: e.text || e.claim || e.description,
+      age: e.created ? daysSince(e.created) : null,
+    })),
+    insight: generateInsight(accuracyRate, calibrationScore, unmatched.length, estimates.length),
+  };
+}
+
+function generateInsight(accuracy, byConfidence, unmatchedCount, totalEstimates) {
+  const parts = [];
+
+  if (accuracy !== null) {
+    if (accuracy >= 80) {
+      parts.push(`Strong calibration: ${accuracy}% of scored predictions were accurate.`);
+    } else if (accuracy >= 50) {
+      parts.push(`Moderate calibration: ${accuracy}% accuracy. Room for improvement.`);
+    } else {
+      parts.push(`Weak calibration: only ${accuracy}% accuracy. Estimates may need more evidence before committing.`);
+    }
+  }
+
+  if (byConfidence.high !== undefined && byConfidence.low !== undefined) {
+    if (byConfidence.high < byConfidence.low) {
+      parts.push('Overconfidence detected: high-confidence predictions are less accurate than low-confidence ones.');
+    }
+  }
+
+  if (totalEstimates > 0 && unmatchedCount / totalEstimates > 0.5) {
+    parts.push(`${unmatchedCount} of ${totalEstimates} estimates have no calibration follow-up. Run /calibrate to close the loop.`);
+  }
+
+  return parts.length > 0 ? parts.join(' ') : 'Not enough data to generate calibration insights.';
+}
+
+function daysSince(dateStr) {
+  try {
+    const then = new Date(dateStr);
+    const now = new Date();
+    return Math.floor((now - then) / (1000 * 60 * 60 * 24));
+  } catch {
+    return null;
+  }
+}
+
+module.exports = { calibrate };