deepflow 0.1.102 → 0.1.104

package/src/eval/metric-collector.js

@@ -0,0 +1,163 @@
+'use strict';
+
+/**
+ * Metric collector for df:eval.
+ *
+ * Reads `.deepflow/token-history.jsonl` and `~/.claude/tool-usage.jsonl`
+ * from existing hook outputs — no new instrumentation hooks installed (AC-17).
+ *
+ * Metric source mapping (from spec doing-skill-eval.md):
+ *   cache_ratio   = cache_read_input_tokens / input_tokens
+ *   total_tokens  = sum of (input_tokens + cache_creation_input_tokens + cache_read_input_tokens + output_tokens) per entry
+ *   wall_time     = endTimestamp - startTimestamp (ms)
+ *   context_burn  = max used_percentage across entries
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const readline = require('readline');
+
+/**
+ * Parse a JSONL file, yielding one parsed object per line.
+ * Lines that are empty or fail to parse are silently skipped.
+ *
+ * @param {string} filePath
+ * @returns {Promise<object[]>}
+ */
+async function readJsonl(filePath) {
+  const entries = [];
+  let stream;
+  try {
+    stream = fs.createReadStream(filePath, { encoding: 'utf8' });
+  } catch (_) {
+    return entries;
+  }
+
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      entries.push(JSON.parse(trimmed));
+    } catch (_) {
+      // skip malformed lines
+    }
+  }
+
+  return entries;
+}
+
+/**
+ * Filter entries whose `timestamp` field falls within [startTimestamp, endTimestamp].
+ * If startTimestamp or endTimestamp is null/undefined, that bound is open.
+ *
+ * @param {object[]} entries
+ * @param {number|null} startTimestamp  — ms since epoch (inclusive)
+ * @param {number|null} endTimestamp    — ms since epoch (inclusive)
+ * @returns {object[]}
+ */
+function filterByRange(entries, startTimestamp, endTimestamp) {
+  return entries.filter((e) => {
+    const ts = new Date(e.timestamp).getTime();
+    if (isNaN(ts)) return false;
+    if (startTimestamp != null && ts < startTimestamp) return false;
+    if (endTimestamp != null && ts > endTimestamp) return false;
+    return true;
+  });
+}
+
+/**
+ * Collect evaluation metrics from existing hook output files.
+ *
+ * AC-16: reads `.deepflow/token-history.jsonl` from the fixture's execution
+ *        to compute cache_ratio (cache_read / input_tokens) and total_tokens.
+ * AC-17: metrics sourced from existing hook outputs; no new hooks installed.
+ *
+ * @param {string} deepflowDir   — path to the fixture's `.deepflow/` directory
+ * @param {number|null} startTimestamp — ms since epoch; open bound if null
+ * @param {number|null} endTimestamp   — ms since epoch; open bound if null
+ * @returns {Promise<{
+ *   cache_ratio: number,
+ *   total_tokens: number,
+ *   wall_time: number,
+ *   context_burn: number,
+ *   entry_count: number
+ * }>}
+ */
+async function collectMetrics(deepflowDir, startTimestamp = null, endTimestamp = null) {
+  const tokenHistoryPath = path.join(deepflowDir, 'token-history.jsonl');
+
+  const allEntries = await readJsonl(tokenHistoryPath);
+  const entries = filterByRange(allEntries, startTimestamp, endTimestamp);
+
+  let sumInputTokens = 0;
+  let sumCacheRead = 0;
+  let sumCacheCreation = 0;
+  let sumOutputTokens = 0;
+  let maxUsedPercentage = 0;
+
+  for (const e of entries) {
+    const inputTokens = Number(e.input_tokens) || 0;
+    const cacheRead = Number(e.cache_read_input_tokens) || 0;
+    const cacheCreation = Number(e.cache_creation_input_tokens) || 0;
+    const outputTokens = Number(e.output_tokens) || 0;
+    const usedPct = Number(e.used_percentage) || 0;
+
+    sumInputTokens += inputTokens;
+    sumCacheRead += cacheRead;
+    sumCacheCreation += cacheCreation;
+    sumOutputTokens += outputTokens;
+
+    if (usedPct > maxUsedPercentage) {
+      maxUsedPercentage = usedPct;
+    }
+  }
+
+  // cache_ratio: fraction of input tokens served from cache.
+  // Use sumInputTokens as denominator; guard against division-by-zero.
+  const cache_ratio = sumInputTokens > 0 ? sumCacheRead / sumInputTokens : 0;
+
+  // total_tokens: all tokens consumed — input (fresh + cache-creation + cache-read) + output.
+  const total_tokens = sumInputTokens + sumCacheCreation + sumCacheRead + sumOutputTokens;
+
+  // wall_time: caller-supplied timestamp delta in ms.
+  const wall_time =
+    startTimestamp != null && endTimestamp != null
+      ? endTimestamp - startTimestamp
+      : 0;
+
+  // context_burn: peak context window utilisation across entries.
+  const context_burn = maxUsedPercentage;
+
+  return {
+    cache_ratio,
+    total_tokens,
+    wall_time,
+    context_burn,
+    entry_count: entries.length,
+  };
+}
+
+/**
+ * Read tool-usage entries from `~/.claude/tool-usage.jsonl` filtered by range.
+ * Provided for orchestrator use; secondary metric source (REQ-10).
+ *
+ * @param {number|null} startTimestamp
+ * @param {number|null} endTimestamp
+ * @returns {Promise<object[]>}
+ */
+async function readToolUsage(startTimestamp = null, endTimestamp = null) {
+  const toolUsagePath = path.join(os.homedir(), '.claude', 'tool-usage.jsonl');
+  const allEntries = await readJsonl(toolUsagePath);
+  return filterByRange(allEntries, startTimestamp, endTimestamp);
+}
+
+module.exports = {
+  collectMetrics,
+  readToolUsage,
+  // exported for testing / orchestrator composition
+  readJsonl,
+  filterByRange,
+};

package/src/eval/metric-collector.test.js

@@ -0,0 +1,369 @@
+'use strict';
+
+/**
+ * Tests for src/eval/metric-collector.js — T6: wave-2 unit tests
+ *
+ * Validates metric computation from JSONL token-history data:
+ *   - cache_ratio = cache_read_input_tokens / input_tokens (AC-16)
+ *   - total_tokens sums all token fields
+ *   - context_burn picks max used_percentage
+ *   - wall_time is timestamp delta
+ *   - filterByRange filters by ISO timestamps
+ *   - readJsonl parses multi-line JSONL
+ *   - Edge cases: empty file, single entry, no entries in range
+ *
+ * Uses Node.js built-in node:test to avoid adding dependencies.
+ */
+
+const { test, describe, beforeEach, afterEach } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const os = require('os');
+
+const {
+  collectMetrics,
+  readJsonl,
+  filterByRange,
+} = require('./metric-collector');
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeTmpDir() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'metric-collector-test-'));
+}
+
+function rmrf(dir) {
+  if (fs.existsSync(dir)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/**
+ * Write a JSONL file from an array of objects.
+ */
+function writeJsonl(filePath, entries) {
+  const content = entries.map((e) => JSON.stringify(e)).join('\n') + '\n';
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, content, 'utf8');
+}
+
+/**
+ * Build a token-history entry with sensible defaults.
+ */
+function makeEntry(overrides = {}) {
+  return {
+    timestamp: '2026-03-25T10:00:00.000Z',
+    input_tokens: 1000,
+    cache_read_input_tokens: 500,
+    cache_creation_input_tokens: 200,
+    output_tokens: 300,
+    used_percentage: 50,
+    ...overrides,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// readJsonl
+// ---------------------------------------------------------------------------
+
+describe('readJsonl', () => {
+  let tmpDir;
+
+  beforeEach(() => {
+    tmpDir = makeTmpDir();
+  });
+
+  afterEach(() => {
+    rmrf(tmpDir);
+  });
+
+  test('parses multi-line JSONL correctly', async () => {
+    const filePath = path.join(tmpDir, 'data.jsonl');
+    const entries = [
+      { a: 1, b: 'hello' },
+      { a: 2, b: 'world' },
+      { a: 3, b: 'foo' },
+    ];
+    writeJsonl(filePath, entries);
+
+    const result = await readJsonl(filePath);
+    assert.equal(result.length, 3);
+    assert.deepEqual(result[0], { a: 1, b: 'hello' });
+    assert.deepEqual(result[1], { a: 2, b: 'world' });
+    assert.deepEqual(result[2], { a: 3, b: 'foo' });
+  });
+
+  test('returns empty array for empty file', async () => {
+    const filePath = path.join(tmpDir, 'empty.jsonl');
+    fs.writeFileSync(filePath, '', 'utf8');
+
+    const result = await readJsonl(filePath);
+    assert.deepEqual(result, []);
+  });
+
+  test('skips blank lines and malformed JSON', async () => {
+    const filePath = path.join(tmpDir, 'messy.jsonl');
+    const content = [
+      '{"valid": true}',
+      '',
+      'not-json',
+      '   ',
+      '{"also": "valid"}',
+    ].join('\n');
+    fs.writeFileSync(filePath, content, 'utf8');
+
+    const result = await readJsonl(filePath);
+    assert.equal(result.length, 2);
+    assert.deepEqual(result[0], { valid: true });
+    assert.deepEqual(result[1], { also: 'valid' });
+  });
+
+  test('rejects for non-existent file', async () => {
+    const filePath = path.join(tmpDir, 'nope.jsonl');
+    await assert.rejects(() => readJsonl(filePath), { code: 'ENOENT' });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// filterByRange
+// ---------------------------------------------------------------------------
+
+describe('filterByRange', () => {
+  const entries = [
+    { timestamp: '2026-03-25T10:00:00.000Z', val: 1 },
+    { timestamp: '2026-03-25T11:00:00.000Z', val: 2 },
+    { timestamp: '2026-03-25T12:00:00.000Z', val: 3 },
+    { timestamp: '2026-03-25T13:00:00.000Z', val: 4 },
+  ];
+
+  const t10 = new Date('2026-03-25T10:00:00.000Z').getTime();
+  const t11 = new Date('2026-03-25T11:00:00.000Z').getTime();
+  const t12 = new Date('2026-03-25T12:00:00.000Z').getTime();
+  const t13 = new Date('2026-03-25T13:00:00.000Z').getTime();
+
+  test('filters entries within inclusive range', () => {
+    const result = filterByRange(entries, t11, t12);
+    assert.equal(result.length, 2);
+    assert.equal(result[0].val, 2);
+    assert.equal(result[1].val, 3);
+  });
+
+  test('open start bound returns entries up to end', () => {
+    const result = filterByRange(entries, null, t11);
+    assert.equal(result.length, 2);
+    assert.equal(result[0].val, 1);
+    assert.equal(result[1].val, 2);
+  });
+
+  test('open end bound returns entries from start onward', () => {
+    const result = filterByRange(entries, t12, null);
+    assert.equal(result.length, 2);
+    assert.equal(result[0].val, 3);
+    assert.equal(result[1].val, 4);
+  });
+
+  test('both bounds null returns all entries', () => {
+    const result = filterByRange(entries, null, null);
+    assert.equal(result.length, 4);
+  });
+
+  test('no entries in range returns empty array', () => {
+    const futureStart = new Date('2030-01-01T00:00:00.000Z').getTime();
+    const futureEnd = new Date('2030-12-31T00:00:00.000Z').getTime();
+    const result = filterByRange(entries, futureStart, futureEnd);
+    assert.deepEqual(result, []);
+  });
+
+  test('entries without valid timestamp are excluded', () => {
+    const bad = [
+      { val: 1 },
+      { timestamp: 'not-a-date', val: 2 },
+      { timestamp: '2026-03-25T10:00:00.000Z', val: 3 },
+    ];
+    const result = filterByRange(bad, null, null);
+    assert.equal(result.length, 1);
+    assert.equal(result[0].val, 3);
+  });
+
+  test('exact boundary timestamps are inclusive', () => {
+    const result = filterByRange(entries, t10, t13);
+    assert.equal(result.length, 4);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// collectMetrics
+// ---------------------------------------------------------------------------
+
+describe('collectMetrics', () => {
+  let tmpDir;
+
+  beforeEach(() => {
+    tmpDir = makeTmpDir();
+  });
+
+  afterEach(() => {
+    rmrf(tmpDir);
+  });
+
+  test('computes correct cache_ratio from known data (AC-16)', async () => {
+    // Two entries: input_tokens = 1000+2000 = 3000, cache_read = 500+1500 = 2000
+    // cache_ratio = 2000 / 3000 = 0.6667
+    const entries = [
+      makeEntry({ input_tokens: 1000, cache_read_input_tokens: 500 }),
+      makeEntry({ input_tokens: 2000, cache_read_input_tokens: 1500 }),
+    ];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const result = await collectMetrics(tmpDir);
+    assert.ok(Math.abs(result.cache_ratio - 2000 / 3000) < 1e-10,
+      `Expected cache_ratio ~0.6667, got ${result.cache_ratio}`);
+  });
+
+  test('total_tokens sums all token fields correctly', async () => {
+    const entries = [
+      makeEntry({
+        input_tokens: 100,
+        cache_read_input_tokens: 200,
+        cache_creation_input_tokens: 300,
+        output_tokens: 400,
+      }),
+      makeEntry({
+        input_tokens: 50,
+        cache_read_input_tokens: 60,
+        cache_creation_input_tokens: 70,
+        output_tokens: 80,
+      }),
+    ];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const result = await collectMetrics(tmpDir);
+    // total = (100+200+300+400) + (50+60+70+80) = 1000 + 260 = 1260
+    assert.equal(result.total_tokens, 1260);
+  });
+
+  test('context_burn picks max used_percentage', async () => {
+    const entries = [
+      makeEntry({ used_percentage: 25 }),
+      makeEntry({ used_percentage: 75 }),
+      makeEntry({ used_percentage: 50 }),
+    ];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const result = await collectMetrics(tmpDir);
+    assert.equal(result.context_burn, 75);
+  });
+
+  test('wall_time is timestamp delta in ms', async () => {
+    const entries = [makeEntry()];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const start = 1000;
+    const end = 6000;
+    const result = await collectMetrics(tmpDir, start, end);
+    assert.equal(result.wall_time, 5000);
+  });
+
+  test('wall_time is 0 when start or end is null', async () => {
+    const entries = [makeEntry()];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const result1 = await collectMetrics(tmpDir, null, 6000);
+    assert.equal(result1.wall_time, 0);
+
+    const result2 = await collectMetrics(tmpDir, 1000, null);
+    assert.equal(result2.wall_time, 0);
+
+    const result3 = await collectMetrics(tmpDir);
+    assert.equal(result3.wall_time, 0);
+  });
+
+  test('entry_count reflects filtered entries', async () => {
+    const t1 = '2026-03-25T10:00:00.000Z';
+    const t2 = '2026-03-25T11:00:00.000Z';
+    const t3 = '2026-03-25T12:00:00.000Z';
+    const entries = [
+      makeEntry({ timestamp: t1 }),
+      makeEntry({ timestamp: t2 }),
+      makeEntry({ timestamp: t3 }),
+    ];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const start = new Date(t2).getTime();
+    const end = new Date(t2).getTime();
+    const result = await collectMetrics(tmpDir, start, end);
+    assert.equal(result.entry_count, 1);
+  });
+
+  test('empty file returns zero metrics', async () => {
+    fs.writeFileSync(path.join(tmpDir, 'token-history.jsonl'), '', 'utf8');
+
+    const result = await collectMetrics(tmpDir);
+    assert.equal(result.cache_ratio, 0);
+    assert.equal(result.total_tokens, 0);
+    assert.equal(result.wall_time, 0);
+    assert.equal(result.context_burn, 0);
+    assert.equal(result.entry_count, 0);
+  });
+
+  test('single entry computes metrics correctly', async () => {
+    const entries = [
+      makeEntry({
+        input_tokens: 800,
+        cache_read_input_tokens: 600,
+        cache_creation_input_tokens: 100,
+        output_tokens: 200,
+        used_percentage: 42,
+      }),
+    ];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const result = await collectMetrics(tmpDir);
+    assert.equal(result.cache_ratio, 600 / 800);
+    assert.equal(result.total_tokens, 800 + 600 + 100 + 200);
+    assert.equal(result.context_burn, 42);
+    assert.equal(result.entry_count, 1);
+  });
+
+  test('no entries in range returns zero metrics', async () => {
+    const entries = [
+      makeEntry({ timestamp: '2026-03-25T10:00:00.000Z' }),
+    ];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const futureStart = new Date('2030-01-01T00:00:00.000Z').getTime();
+    const futureEnd = new Date('2030-12-31T00:00:00.000Z').getTime();
+    const result = await collectMetrics(tmpDir, futureStart, futureEnd);
+    assert.equal(result.cache_ratio, 0);
+    assert.equal(result.total_tokens, 0);
+    assert.equal(result.context_burn, 0);
+    assert.equal(result.entry_count, 0);
+  });
+
+  test('handles missing token fields gracefully (treated as 0)', async () => {
+    const entries = [
+      { timestamp: '2026-03-25T10:00:00.000Z', input_tokens: 500 },
+    ];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const result = await collectMetrics(tmpDir);
+    // cache_read=0, cache_creation=0, output=0
+    assert.equal(result.cache_ratio, 0);
+    assert.equal(result.total_tokens, 500);
+    assert.equal(result.context_burn, 0);
+    assert.equal(result.entry_count, 1);
+  });
+
+  test('division by zero: cache_ratio is 0 when input_tokens is 0', async () => {
+    const entries = [
+      makeEntry({ input_tokens: 0, cache_read_input_tokens: 0 }),
+    ];
+    writeJsonl(path.join(tmpDir, 'token-history.jsonl'), entries);
+
+    const result = await collectMetrics(tmpDir);
+    assert.equal(result.cache_ratio, 0);
+  });
+});

package/src/eval/metric-pivot.js

@@ -0,0 +1,119 @@
+'use strict';
+
+const { queryExperiments } = require('./git-memory');
+
+/**
+ * Parses a secondaries string into a key/value map.
+ * Secondaries format: "key=value key=value ..." (space-separated, produced by formatSecondaries)
+ *
+ * @param {string} secondaries
+ * @returns {Object<string, string>}
+ */
+function parseSecondaries(secondaries) {
+  if (!secondaries) return {};
+  const result = {};
+  for (const token of secondaries.split(/\s+/)) {
+    const eqIdx = token.indexOf('=');
+    if (eqIdx === -1) continue;
+    const key = token.slice(0, eqIdx);
+    const val = token.slice(eqIdx + 1);
+    if (key) result[key] = val;
+  }
+  return result;
+}
+
+/**
+ * Surfaces previously-reverted experiments that had a positive delta on newTarget.
+ *
+ * A candidate is an experiment where:
+ *   - status === 'reverted' (the experiment was rolled back)
+ *   - The newTarget metric was either:
+ *     a) The primary target and had delta > 0, OR
+ *     b) Recorded as a secondary metric (value parsed from secondaries string)
+ *        — in this case the experiment is included as a candidate since we cannot
+ *          compute a delta without a baseline; callers should review the raw value.
+ *
+ * AC-14: After --target pivot, git log --grep="experiment:" is parsed and
+ * previously-reverted experiments with positive delta on new target are
+ * surfaced as candidates.
+ *
+ * @param {object} opts
+ * @param {string} opts.cwd        - Working directory (git repo root)
+ * @param {string} opts.skillName  - Skill being evaluated
+ * @param {string} opts.newTarget  - The new primary metric after a pivot
+ * @returns {Array<{hash, skillName, hypothesis, target, value, delta, status, secondaries, candidateValue, candidateDelta}>}
+ */
+function surfaceCandidates({ cwd, skillName, newTarget }) {
+  const experiments = queryExperiments({ cwd, skillName });
+
+  const candidates = [];
+
+  for (const exp of experiments) {
+    if (exp.status !== 'reverted') continue;
+
+    // Case A: newTarget was the primary metric for this experiment
+    if (exp.target === newTarget) {
+      if (exp.delta > 0) {
+        candidates.push({
+          ...exp,
+          candidateValue: exp.value,
+          candidateDelta: exp.delta,
+          candidateSource: 'primary',
+        });
+      }
+      continue;
+    }
+
+    // Case B: newTarget appears as a secondary metric
+    const secondaryMap = parseSecondaries(exp.secondaries);
+    if (Object.prototype.hasOwnProperty.call(secondaryMap, newTarget)) {
+      candidates.push({
+        ...exp,
+        candidateValue: secondaryMap[newTarget],
+        candidateDelta: null, // delta unknown for secondaries — only raw value available
+        candidateSource: 'secondary',
+      });
+    }
+  }
+
+  return candidates;
+}
+
+/**
+ * Formats surfaced candidates for stdout display.
+ *
+ * @param {Array} candidates - Result of surfaceCandidates()
+ * @param {string} newTarget - The new primary metric name
+ * @returns {string}
+ */
+function formatCandidates(candidates, newTarget) {
+  if (candidates.length === 0) {
+    return `No reverted experiments found with positive delta on target="${newTarget}".`;
+  }
+
+  const lines = [
+    `Reverted experiments with positive signal on "${newTarget}" (candidates for retry):`,
+    '',
+  ];
+
+  for (const c of candidates) {
+    const deltaStr =
+      c.candidateDelta !== null
+        ? `delta=+${c.candidateDelta}% (primary)`
+        : `value=${c.candidateValue} (secondary — no delta available)`;
+
+    lines.push(`  [${c.hash}] ${c.skillName}: ${c.hypothesis}`);
+    lines.push(`         ${newTarget}=${c.candidateValue}  ${deltaStr}`);
+    lines.push(`         original target: ${c.target}=${c.value} delta=${c.delta}%`);
+    lines.push('');
+  }
+
+  return lines.join('\n').trimEnd();
+}
+
+module.exports = {
+  surfaceCandidates,
+  formatCandidates,
+  // exported for testing
+  parseSecondaries,
+};