deepflow 0.1.102 → 0.1.104

package/src/eval/git-memory.test.js

@@ -0,0 +1,439 @@
+'use strict';
+
+const { describe, it, before, after } = require('node:test');
+const assert = require('node:assert');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { execSync } = require('child_process');
+
+const {
+  commitExperiment,
+  revertExperiment,
+  queryExperiments,
+  getExperimentHistory,
+  formatCommitMessage,
+  parseExperimentLine,
+} = require('./git-memory.js');
+
+/**
+ * Creates a temporary git repo with an initial commit.
+ * Returns the directory path.
+ */
+function createTempRepo() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'git-memory-test-'));
+  execSync('git init', { cwd: dir, stdio: 'pipe' });
+  execSync('git config user.email "test@test.com"', { cwd: dir, stdio: 'pipe' });
+  execSync('git config user.name "Test"', { cwd: dir, stdio: 'pipe' });
+  // Initial commit so we have a HEAD
+  fs.writeFileSync(path.join(dir, 'README.md'), '# test repo\n');
+  execSync('git add -A && git commit -m "initial commit"', { cwd: dir, stdio: 'pipe' });
+  return dir;
+}
+
+function cleanupRepo(dir) {
+  fs.rmSync(dir, { recursive: true, force: true });
+}
+
+// --- Unit tests for pure functions ---
+
+describe('formatCommitMessage', () => {
+  it('formats a complete experiment commit message', () => {
+    const msg = formatCommitMessage({
+      skillName: 'browse-fetch',
+      hypothesis: 'reduce timeout to 5s',
+      target: 'latency',
+      value: '4.2s',
+      delta: '-16',
+      status: 'pass',
+      secondaries: 'accuracy=98%',
+    });
+    assert.strictEqual(
+      msg,
+      'experiment(browse-fetch): reduce timeout to 5s | latency=4.2s delta=-16% pass | accuracy=98%'
+    );
+  });
+
+  it('handles missing secondaries as empty string', () => {
+    const msg = formatCommitMessage({
+      skillName: 'skill-a',
+      hypothesis: 'test hyp',
+      target: 'speed',
+      value: '10',
+      delta: '+5',
+      status: 'fail',
+      secondaries: undefined,
+    });
+    assert.ok(msg.endsWith('| '), `Expected message to end with "| " but got: ${msg}`);
+  });
+
+  it('handles numeric value and delta', () => {
+    const msg = formatCommitMessage({
+      skillName: 'x',
+      hypothesis: 'h',
+      target: 't',
+      value: 42,
+      delta: -3.5,
+      status: 'pass',
+      secondaries: null,
+    });
+    assert.ok(msg.includes('t=42'));
+    assert.ok(msg.includes('delta=-3.5%'));
+  });
+});
+
+describe('parseExperimentLine', () => {
+  it('parses a well-formed experiment subject', () => {
+    const subject = 'experiment(browse-fetch): reduce timeout | latency=4.2s delta=-16% pass | accuracy=98%';
+    const result = parseExperimentLine('abc123', subject);
+    assert.deepStrictEqual(result, {
+      hash: 'abc123',
+      skillName: 'browse-fetch',
+      hypothesis: 'reduce timeout',
+      target: 'latency',
+      value: '4.2s',
+      delta: -16,
+      status: 'pass',
+      secondaries: 'accuracy=98%',
+    });
+  });
+
+  it('returns null for non-experiment commits', () => {
+    assert.strictEqual(parseExperimentLine('abc', 'feat(core): add feature'), null);
+    assert.strictEqual(parseExperimentLine('abc', 'random text'), null);
+  });
+
+  it('returns null for malformed metrics section', () => {
+    const subject = 'experiment(x): hyp | bad-metrics | sec';
+    assert.strictEqual(parseExperimentLine('abc', subject), null);
+  });
+
+  it('parses positive delta', () => {
+    const subject = 'experiment(s): h | metric=100 delta=+12.5% pass | ';
+    const result = parseExperimentLine('def', subject);
+    assert.strictEqual(result.delta, 12.5);
+  });
+
+  it('handles empty secondaries', () => {
+    const subject = 'experiment(s): h | m=1 delta=0% inconclusive | ';
+    const result = parseExperimentLine('ghi', subject);
+    assert.strictEqual(result.secondaries, '');
+    assert.strictEqual(result.status, 'inconclusive');
+  });
+});
+
+// --- Integration tests with temp git repos ---
+
+describe('commitExperiment', () => {
+  let cwd;
+
+  before(() => {
+    cwd = createTempRepo();
+  });
+
+  after(() => {
+    cleanupRepo(cwd);
+  });
+
+  it('creates a commit with correctly formatted message (AC-10)', () => {
+    // Create a file change to commit
+    fs.writeFileSync(path.join(cwd, 'experiment1.txt'), 'trial 1\n');
+
+    const hash = commitExperiment({
+      cwd,
+      skillName: 'browse-fetch',
+      hypothesis: 'reduce timeout to 5s',
+      target: 'latency',
+      value: '4.2s',
+      delta: '-16',
+      status: 'pass',
+      secondaries: 'accuracy=98%',
+    });
+
+    assert.ok(typeof hash === 'string');
+    assert.ok(hash.length >= 7, `Hash should be at least 7 chars, got: ${hash}`);
+
+    // Verify commit message format
+    const subject = execSync('git log -1 --format=%s', { cwd, stdio: 'pipe' }).toString().trim();
+    assert.strictEqual(
+      subject,
+      'experiment(browse-fetch): reduce timeout to 5s | latency=4.2s delta=-16% pass | accuracy=98%'
+    );
+  });
+
+  it('stages all changes before committing', () => {
+    // Create an untracked file — commitExperiment should stage it
+    fs.writeFileSync(path.join(cwd, 'untracked.txt'), 'new file\n');
+
+    commitExperiment({
+      cwd,
+      skillName: 'test-skill',
+      hypothesis: 'auto-stage',
+      target: 'coverage',
+      value: '80',
+      delta: '+2',
+      status: 'pass',
+      secondaries: '',
+    });
+
+    // The file should be in the commit
+    const files = execSync('git diff-tree --no-commit-id --name-only -r HEAD', { cwd, stdio: 'pipe' })
+      .toString().trim();
+    assert.ok(files.includes('untracked.txt'));
+  });
+});
+
+describe('revertExperiment (AC-7)', () => {
+  let cwd;
+
+  before(() => {
+    cwd = createTempRepo();
+  });
+
+  after(() => {
+    cleanupRepo(cwd);
+  });
+
+  it('uses git revert (not reset) — preserves experiment commit in history', () => {
+    // Make an experiment commit
+    fs.writeFileSync(path.join(cwd, 'exp.txt'), 'experiment data\n');
+    commitExperiment({
+      cwd,
+      skillName: 'skill-a',
+      hypothesis: 'bad idea',
+      target: 'speed',
+      value: '10',
+      delta: '-5',
+      status: 'fail',
+      secondaries: '',
+    });
+
+    const experimentSubject = execSync('git log -1 --format=%s', { cwd, stdio: 'pipe' }).toString().trim();
+
+    // Revert
+    const revertHash = revertExperiment({ cwd });
+    assert.ok(typeof revertHash === 'string');
+
+    // Verify revert commit message contains "Revert"
+    const revertSubject = execSync('git log -1 --format=%s', { cwd, stdio: 'pipe' }).toString().trim();
+    assert.ok(revertSubject.startsWith('Revert'), `Expected revert commit to start with "Revert", got: ${revertSubject}`);
+
+    // The original experiment commit should still be in the log (not erased)
+    const fullLog = execSync('git log --format=%s', { cwd, stdio: 'pipe' }).toString();
+    assert.ok(fullLog.includes(experimentSubject), 'Original experiment commit should remain in history after revert');
+  });
+
+  it('reverted file content matches pre-experiment state', () => {
+    // Write a file, commit as experiment, then revert
+    fs.writeFileSync(path.join(cwd, 'state.txt'), 'before\n');
+    execSync('git add -A && git commit -m "baseline"', { cwd, stdio: 'pipe' });
+
+    fs.writeFileSync(path.join(cwd, 'state.txt'), 'after experiment\n');
+    commitExperiment({
+      cwd,
+      skillName: 's',
+      hypothesis: 'h',
+      target: 't',
+      value: '1',
+      delta: '0',
+      status: 'fail',
+      secondaries: '',
+    });
+
+    revertExperiment({ cwd });
+
+    const content = fs.readFileSync(path.join(cwd, 'state.txt'), 'utf-8');
+    assert.strictEqual(content, 'before\n');
+  });
+});
+
+describe('queryExperiments', () => {
+  let cwd;
+
+  before(() => {
+    cwd = createTempRepo();
+
+    // Create multiple experiment commits for different skills
+    fs.writeFileSync(path.join(cwd, 'a.txt'), '1');
+    commitExperiment({
+      cwd, skillName: 'skill-a', hypothesis: 'hyp-a1',
+      target: 'speed', value: '100', delta: '+10', status: 'pass', secondaries: 'mem=50MB',
+    });
+
+    fs.writeFileSync(path.join(cwd, 'b.txt'), '2');
+    commitExperiment({
+      cwd, skillName: 'skill-b', hypothesis: 'hyp-b1',
+      target: 'accuracy', value: '95', delta: '-2', status: 'fail', secondaries: '',
+    });
+
+    fs.writeFileSync(path.join(cwd, 'c.txt'), '3');
+    commitExperiment({
+      cwd, skillName: 'skill-a', hypothesis: 'hyp-a2',
+      target: 'speed', value: '120', delta: '+20', status: 'pass', secondaries: '',
+    });
+  });
+
+  after(() => {
+    cleanupRepo(cwd);
+  });
+
+  it('returns all experiments when no skillName filter', () => {
+    const results = queryExperiments({ cwd });
+    assert.strictEqual(results.length, 3);
+  });
+
+  it('filters by skillName', () => {
+    const results = queryExperiments({ cwd, skillName: 'skill-a' });
+    assert.strictEqual(results.length, 2);
+    for (const r of results) {
+      assert.strictEqual(r.skillName, 'skill-a');
+    }
+  });
+
+  it('returns empty array for unknown skill', () => {
+    const results = queryExperiments({ cwd, skillName: 'nonexistent' });
+    assert.deepStrictEqual(results, []);
+  });
+
+  it('parses commit messages into structured objects with correct fields', () => {
+    const results = queryExperiments({ cwd, skillName: 'skill-a' });
+    // git log returns newest first
+    const newest = results[0];
+    assert.strictEqual(newest.hypothesis, 'hyp-a2');
+    assert.strictEqual(newest.target, 'speed');
+    assert.strictEqual(newest.value, '120');
+    assert.strictEqual(newest.delta, 20);
+    assert.strictEqual(newest.status, 'pass');
+  });
+
+  it('returns empty array in repo with no experiment commits', () => {
+    const emptyRepo = createTempRepo();
+    try {
+      const results = queryExperiments({ cwd: emptyRepo, skillName: 'anything' });
+      assert.deepStrictEqual(results, []);
+    } finally {
+      cleanupRepo(emptyRepo);
+    }
+  });
+});
+
+describe('getExperimentHistory', () => {
+  let cwd;
+
+  before(() => {
+    cwd = createTempRepo();
+
+    for (let i = 1; i <= 5; i++) {
+      fs.writeFileSync(path.join(cwd, `file${i}.txt`), `${i}`);
+      commitExperiment({
+        cwd, skillName: 'perf', hypothesis: `trial-${i}`,
+        target: 'throughput', value: `${i * 100}`, delta: `+${i}`,
+        status: i % 2 === 0 ? 'fail' : 'pass', secondaries: '',
+      });
+    }
+  });
+
+  after(() => {
+    cleanupRepo(cwd);
+  });
+
+  it('returns formatted multi-line string', () => {
+    const history = getExperimentHistory({ cwd, skillName: 'perf' });
+    const lines = history.split('\n');
+    assert.strictEqual(lines.length, 5);
+    // Each line should contain the skill name and hash
+    for (const line of lines) {
+      assert.ok(line.includes('perf:'), `Line should contain "perf:": ${line}`);
+      assert.ok(line.startsWith('['), `Line should start with "[": ${line}`);
+    }
+  });
+
+  it('respects maxEntries', () => {
+    const history = getExperimentHistory({ cwd, skillName: 'perf', maxEntries: 2 });
+    const lines = history.split('\n');
+    assert.strictEqual(lines.length, 2);
+  });
+
+  it('defaults maxEntries to 20', () => {
+    // We have 5 experiments, all should appear (5 < 20)
+    const history = getExperimentHistory({ cwd, skillName: 'perf' });
+    const lines = history.split('\n');
+    assert.strictEqual(lines.length, 5);
+  });
+
+  it('returns "(no experiment history)" when no experiments match', () => {
+    const history = getExperimentHistory({ cwd, skillName: 'nonexistent' });
+    assert.strictEqual(history, '(no experiment history)');
+  });
+
+  it('returns "(no experiment history)" for empty repo', () => {
+    const emptyRepo = createTempRepo();
+    try {
+      const history = getExperimentHistory({ cwd: emptyRepo });
+      assert.strictEqual(history, '(no experiment history)');
+    } finally {
+      cleanupRepo(emptyRepo);
+    }
+  });
+});
+
+describe('round-trip: commit → query → verify', () => {
+  let cwd;
+
+  before(() => {
+    cwd = createTempRepo();
+  });
+
+  after(() => {
+    cleanupRepo(cwd);
+  });
+
+  it('parsed fields match original input', () => {
+    const input = {
+      skillName: 'round-trip-skill',
+      hypothesis: 'cache improves latency',
+      target: 'p99',
+      value: '42ms',
+      delta: '-33',
+      status: 'pass',
+      secondaries: 'p50=12ms cpu=65%',
+    };
+
+    fs.writeFileSync(path.join(cwd, 'rt.txt'), 'round-trip test\n');
+    const commitHash = commitExperiment({ cwd, ...input });
+
+    const experiments = queryExperiments({ cwd, skillName: 'round-trip-skill' });
+    assert.strictEqual(experiments.length, 1);
+
+    const parsed = experiments[0];
+    assert.strictEqual(parsed.skillName, input.skillName);
+    assert.strictEqual(parsed.hypothesis, input.hypothesis);
+    assert.strictEqual(parsed.target, input.target);
+    assert.strictEqual(parsed.value, input.value);
+    assert.strictEqual(parsed.delta, parseFloat(input.delta));
+    assert.strictEqual(parsed.status, input.status);
+    assert.strictEqual(parsed.secondaries, input.secondaries);
+    // Hash from commit should match hash from query (full vs short may differ, but short should be prefix)
+    assert.ok(
+      parsed.hash.startsWith(commitHash) || commitHash.startsWith(parsed.hash.slice(0, 7)),
+      `Hashes should be related: commit=${commitHash}, query=${parsed.hash}`
+    );
+  });
+
+  it('round-trip with numeric delta preserves sign', () => {
+    fs.writeFileSync(path.join(cwd, 'rt2.txt'), 'test\n');
+    commitExperiment({
+      cwd,
+      skillName: 'sign-test',
+      hypothesis: 'negative delta',
+      target: 'errors',
+      value: '3',
+      delta: '-50',
+      status: 'pass',
+      secondaries: '',
+    });
+
+    const [result] = queryExperiments({ cwd, skillName: 'sign-test' });
+    assert.strictEqual(result.delta, -50);
+  });
+});

package/src/eval/hypothesis.js

@@ -0,0 +1,80 @@
+'use strict';
+
+/**
+ * Hypothesis loading for df:eval.
+ *
+ * AC-11: Loop accepts --hypothesis flag; without it, reads hypotheses.md from
+ *        benchmark dir and returns the next unused hypothesis.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Parse markdown list items from hypotheses.md content.
+ * Recognises both ordered (1. ...) and unordered (- ... / * ...) list items.
+ *
+ * @param {string} content - Raw file content
+ * @returns {string[]} - Array of hypothesis strings (trimmed, non-empty)
+ */
+function parseHypothesesFile(content) {
+  return content
+    .split('\n')
+    .map((line) => line.match(/^(?:\d+\.|[-*])\s+(.+)/))
+    .filter(Boolean)
+    .map((m) => m[1].trim())
+    .filter((h) => h.length > 0);
+}
+
+/**
+ * Load the active hypothesis for an eval session.
+ *
+ * Resolution order:
+ *   1. If `flag` is a non-empty string → return it directly.
+ *   2. Otherwise read `{benchDir}/hypotheses.md` and return the first entry.
+ *      If the file is missing or contains no list items, throw an error.
+ *
+ * "Next unused" is kept simple for now: always return the first list item.
+ * Iteration tracking (marking items as used) is left to the loop's git-memory
+ * history, which records which hypotheses were already attempted.
+ *
+ * @param {object} opts
+ * @param {string} [opts.flag]     - Value of --hypothesis CLI flag (may be undefined)
+ * @param {string}  opts.benchDir  - Path to the benchmark directory
+ * @returns {string} - The hypothesis string to use
+ * @throws {Error}   - If no hypothesis can be resolved
+ */
+function loadHypothesis({ flag, benchDir }) {
+  // 1. CLI flag takes priority
+  if (flag && typeof flag === 'string' && flag.trim().length > 0) {
+    return flag.trim();
+  }
+
+  // 2. Fall back to hypotheses.md
+  const hypothesesPath = path.join(benchDir, 'hypotheses.md');
+
+  let content;
+  try {
+    content = fs.readFileSync(hypothesesPath, 'utf8');
+  } catch (err) {
+    throw new Error(
+      `No --hypothesis flag provided and could not read ${hypothesesPath}: ${err.message}`
+    );
+  }
+
+  const hypotheses = parseHypothesesFile(content);
+
+  if (hypotheses.length === 0) {
+    throw new Error(
+      `No hypotheses found in ${hypothesesPath}. Add list items (- ... or 1. ...) to define hypotheses.`
+    );
+  }
+
+  // Return the first hypothesis (loop history tracks which were attempted)
+  return hypotheses[0];
+}
+
+module.exports = {
+  loadHypothesis,
+  parseHypothesesFile,
+};

package/src/eval/hypothesis.test.js

@@ -0,0 +1,169 @@
+'use strict';
+
+const { describe, it, before, after } = require('node:test');
+const assert = require('node:assert');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const { loadHypothesis, parseHypothesesFile } = require('./hypothesis.js');
+
+// --- parseHypothesesFile ---
+
+describe('parseHypothesesFile', () => {
+  it('parses ordered list items (1. ...)', () => {
+    const content = '1. First hypothesis\n2. Second hypothesis\n3. Third one\n';
+    const result = parseHypothesesFile(content);
+    assert.deepStrictEqual(result, [
+      'First hypothesis',
+      'Second hypothesis',
+      'Third one',
+    ]);
+  });
+
+  it('parses unordered list items with dashes (- ...)', () => {
+    const content = '- Dash one\n- Dash two\n';
+    const result = parseHypothesesFile(content);
+    assert.deepStrictEqual(result, ['Dash one', 'Dash two']);
+  });
+
+  it('parses unordered list items with asterisks (* ...)', () => {
+    const content = '* Star one\n* Star two\n';
+    const result = parseHypothesesFile(content);
+    assert.deepStrictEqual(result, ['Star one', 'Star two']);
+  });
+
+  it('handles mixed ordered and unordered items', () => {
+    const content = '1. Ordered first\n- Dash second\n* Star third\n2. Ordered fourth\n';
+    const result = parseHypothesesFile(content);
+    assert.deepStrictEqual(result, [
+      'Ordered first',
+      'Dash second',
+      'Star third',
+      'Ordered fourth',
+    ]);
+  });
+
+  it('returns empty array for empty content', () => {
+    assert.deepStrictEqual(parseHypothesesFile(''), []);
+  });
+
+  it('returns empty array when content has no list items', () => {
+    const content = '# Hypotheses\n\nSome paragraph text.\nAnother line.\n';
+    const result = parseHypothesesFile(content);
+    assert.deepStrictEqual(result, []);
+  });
+
+  it('ignores non-list lines interspersed with list items', () => {
+    const content = '# Title\n\n1. Real item\nNot a list item\n- Another real item\n';
+    const result = parseHypothesesFile(content);
+    assert.deepStrictEqual(result, ['Real item', 'Another real item']);
+  });
+
+  it('trims whitespace from parsed items', () => {
+    const content = '1.   Lots of spaces   \n-   Also spaced   \n';
+    const result = parseHypothesesFile(content);
+    assert.deepStrictEqual(result, ['Lots of spaces', 'Also spaced']);
+  });
+});
+
+// --- loadHypothesis ---
+
+describe('loadHypothesis', () => {
+  let tmpDir;
+
+  before(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hypothesis-test-'));
+  });
+
+  after(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('returns flag when provided (AC-11)', () => {
+    const result = loadHypothesis({ flag: 'my hypothesis', benchDir: tmpDir });
+    assert.strictEqual(result, 'my hypothesis');
+  });
+
+  it('trims the flag value', () => {
+    const result = loadHypothesis({ flag: '  padded  ', benchDir: tmpDir });
+    assert.strictEqual(result, 'padded');
+  });
+
+  it('reads hypotheses.md when no flag is provided', () => {
+    const benchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hyp-read-'));
+    fs.writeFileSync(
+      path.join(benchDir, 'hypotheses.md'),
+      '1. First from file\n2. Second from file\n'
+    );
+    const result = loadHypothesis({ benchDir });
+    assert.strictEqual(result, 'First from file');
+    fs.rmSync(benchDir, { recursive: true, force: true });
+  });
+
+  it('ignores empty-string flag and falls back to file', () => {
+    const benchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hyp-empty-'));
+    fs.writeFileSync(
+      path.join(benchDir, 'hypotheses.md'),
+      '- Fallback hypothesis\n'
+    );
+    const result = loadHypothesis({ flag: '', benchDir });
+    assert.strictEqual(result, 'Fallback hypothesis');
+    fs.rmSync(benchDir, { recursive: true, force: true });
+  });
+
+  it('ignores whitespace-only flag and falls back to file', () => {
+    const benchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hyp-ws-'));
+    fs.writeFileSync(
+      path.join(benchDir, 'hypotheses.md'),
+      '- WS fallback\n'
+    );
+    const result = loadHypothesis({ flag: '   ', benchDir });
+    assert.strictEqual(result, 'WS fallback');
+    fs.rmSync(benchDir, { recursive: true, force: true });
+  });
+
+  it('throws when neither flag nor file available', () => {
+    const missingDir = path.join(tmpDir, 'nonexistent');
+    assert.throws(
+      () => loadHypothesis({ benchDir: missingDir }),
+      (err) => {
+        assert.ok(err instanceof Error);
+        assert.ok(err.message.includes('No --hypothesis flag provided'));
+        assert.ok(err.message.includes('hypotheses.md'));
+        return true;
+      }
+    );
+  });
+
+  it('throws when file exists but contains no list items', () => {
+    const benchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hyp-nolist-'));
+    fs.writeFileSync(
+      path.join(benchDir, 'hypotheses.md'),
+      '# Just a heading\n\nSome text but no list items.\n'
+    );
+    assert.throws(
+      () => loadHypothesis({ benchDir }),
+      (err) => {
+        assert.ok(err instanceof Error);
+        assert.ok(err.message.includes('No hypotheses found'));
+        return true;
+      }
+    );
+    fs.rmSync(benchDir, { recursive: true, force: true });
+  });
+
+  it('throws when file is empty', () => {
+    const benchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hyp-empty-file-'));
+    fs.writeFileSync(path.join(benchDir, 'hypotheses.md'), '');
+    assert.throws(
+      () => loadHypothesis({ benchDir }),
+      (err) => {
+        assert.ok(err instanceof Error);
+        assert.ok(err.message.includes('No hypotheses found'));
+        return true;
+      }
+    );
+    fs.rmSync(benchDir, { recursive: true, force: true });
+  });
+});