@mindrian_os/cli 1.13.0-beta.10 → 1.13.0-beta.44

package/lib/memory/query-efficiency-telemetry.test.cjs

@@ -1,370 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-/*
- * Copyright (c) 2026 Mindrian. BSL 1.1.
- *
- * Phase 88.1-16 -- query efficiency telemetry acceptance tests.
- *
- * Verifies the pure token-estimator + aggregation-math contract of the module
- * at lib/core/token-estimator.cjs. This module is consumed by the PostToolUse
- * hook at scripts/query-efficiency-telemetry.cjs (measures tokens_used per
- * query) and the /mos:scout aggregator wrapper at
- * scripts/scout-telemetry-aggregator.cjs (renders the median-ratio summary).
- *
- * Canon references:
- *   Part 6 Product-as-Venture -- the 57x efficiency claim is a VENTURE claim.
- *     Measure it or retune the copy. No third option. This module is the
- *     measurement arm.
- *   Part 8 Graph Boundary -- the estimator + aggregator are LOCAL-by-
- *     construction: pure token math + room-tree walk. No network surface.
- *     The hook persists scalar integer counts (tokens_used /
- *     tokens_naive_estimate) to ~/.mindrian/telemetry/query-efficiency.jsonl,
- *     never user artifact bytes.
- *
- * Test map (12 cases, one-to-one with the PLAN <behavior> block for Task 1):
- *   1.  estimateTokens("hello") -> 2 (5 chars / 4 = 1.25 -> ceil -> 2)
- *   2.  estimateTokens("") -> 0
- *   3.  estimateTokens(null) -> 0 (graceful on null/undefined/non-string)
- *   4.  estimateRoomTokens on valid room returns positive integer
- *   5.  estimateRoomTokens cache: second call returns same value without
- *       re-walking the filesystem (verified by assertion on a sentinel file
- *       written BETWEEN the two calls -- cached value must ignore it)
- *   6.  estimateRoomTokens on invalid room returns null (graceful)
- *   7.  clearCache() resets the session cache
- *   8.  estimateRoomTokens skips non-.md files (only .md counted)
- *   9.  ratio computation sanity: 50000 / 1000 -> 50.0
- *  10.  event-shape validator: ensures emitted JSONL has exactly the
- *       8 required fields from the <interfaces> block
- *  11.  classifyRatio: 5.2 -> 'warn'; 42.0 -> 'normal'; 10.0 boundary -> 'normal'
- *  12.  aggregateEvents(10 events with known ratios) -> median/mean match
- *       hand-computed; top5 ranked descending by ratio; count = 10
- *
- * Registered in lib/memory/run-feynman-tests.cjs.
- */
-
-const assert = require('node:assert/strict');
-const fs = require('node:fs');
-const path = require('node:path');
-const os = require('node:os');
-
-const REPO = path.resolve(__dirname, '..', '..');
-const ESTIMATOR_PATH = path.join(REPO, 'lib/core/token-estimator.cjs');
-
-// Defensive reload so repeated invocations reload fresh.
-try { delete require.cache[require.resolve(ESTIMATOR_PATH)]; } catch (_) {}
-const estimator = require(ESTIMATOR_PATH);
-
-assert.equal(typeof estimator.estimateTokens, 'function',
-  'module must export estimateTokens()');
-assert.equal(typeof estimator.estimateRoomTokens, 'function',
-  'module must export estimateRoomTokens()');
-assert.equal(typeof estimator.clearCache, 'function',
-  'module must export clearCache()');
-assert.equal(typeof estimator.aggregateEvents, 'function',
-  'module must export aggregateEvents()');
-assert.equal(typeof estimator.classifyRatio, 'function',
-  'module must export classifyRatio()');
-assert.equal(typeof estimator.validateEventShape, 'function',
-  'module must export validateEventShape()');
-
-// ---------- Fixture scaffolding ----------
-
-function mkTmpDir(prefix) {
-  const base = path.join(os.tmpdir(), 'mos-88-1-16-' + prefix + '-' + process.pid + '-' + Date.now().toString(36));
-  fs.mkdirSync(base, { recursive: true });
-  return base;
-}
-
-function rmRf(p) {
-  try { fs.rmSync(p, { recursive: true, force: true }); } catch (_) {}
-}
-
-// ---------- Test 1: estimateTokens small string ----------
-
-(function test1SmallString() {
-  const result = estimator.estimateTokens('hello');
-  assert.equal(result, 2,
-    'estimateTokens("hello") must ceil(5/4) = 2, got ' + result);
-  console.log('PASS test 1: estimateTokens("hello") -> 2');
-})();
-
-// ---------- Test 2: estimateTokens empty string ----------
-
-(function test2EmptyString() {
-  const result = estimator.estimateTokens('');
-  assert.equal(result, 0,
-    'estimateTokens("") must return 0, got ' + result);
-  console.log('PASS test 2: estimateTokens("") -> 0');
-})();
-
-// ---------- Test 3: estimateTokens graceful on null / non-string ----------
-
-(function test3GracefulNull() {
-  assert.equal(estimator.estimateTokens(null), 0,
-    'estimateTokens(null) must return 0');
-  assert.equal(estimator.estimateTokens(undefined), 0,
-    'estimateTokens(undefined) must return 0');
-  assert.equal(estimator.estimateTokens(123), 0,
-    'estimateTokens(non-string) must return 0');
-  console.log('PASS test 3: estimateTokens graceful on null/undefined/non-string');
-})();
-
-// ---------- Test 4: estimateRoomTokens returns positive integer ----------
-
-(function test4ValidRoom() {
-  estimator.clearCache();
-  const tmp = mkTmpDir('room-valid');
-  try {
-    fs.writeFileSync(path.join(tmp, 'ROOM.md'), '# Room\n'.repeat(100));
-    fs.writeFileSync(path.join(tmp, 'notes.md'), 'x'.repeat(4000));
-    const result = estimator.estimateRoomTokens(tmp);
-    assert.equal(typeof result, 'number',
-      'estimateRoomTokens must return a number on valid room');
-    assert.ok(Number.isFinite(result) && Number.isInteger(result),
-      'estimateRoomTokens must return finite integer, got ' + result);
-    assert.ok(result > 0,
-      'estimateRoomTokens on non-empty room must be > 0, got ' + result);
-    // Sanity: ~ (700 + 4000) / 4 = ~1175 tokens. Allow envelope for overhead.
-    assert.ok(result >= 1000 && result <= 1500,
-      'estimateRoomTokens must approximate chars/4; got ' + result);
-    console.log('PASS test 4: estimateRoomTokens valid room -> ' + result);
-  } finally {
-    rmRf(tmp);
-    estimator.clearCache();
-  }
-})();
-
-// ---------- Test 5: estimateRoomTokens caches per session ----------
-
-(function test5Cache() {
-  estimator.clearCache();
-  const tmp = mkTmpDir('room-cache');
-  try {
-    fs.writeFileSync(path.join(tmp, 'a.md'), 'x'.repeat(2000));
-    const first = estimator.estimateRoomTokens(tmp);
-    // Write a LARGE new artifact after first call. If the second call re-
-    // walks the fs, it will report a much larger number. If it uses the
-    // cache, it must return the same value as `first`.
-    fs.writeFileSync(path.join(tmp, 'b.md'), 'y'.repeat(100000));
-    const second = estimator.estimateRoomTokens(tmp);
-    assert.equal(second, first,
-      'second estimateRoomTokens call must return cached value; ' +
-      'got first=' + first + ' second=' + second);
-    // Clear cache and confirm it now picks up the new artifact.
-    estimator.clearCache();
-    const third = estimator.estimateRoomTokens(tmp);
-    assert.ok(third > first,
-      'after clearCache, estimate must reflect new artifact: ' +
-      'first=' + first + ' third=' + third);
-    console.log('PASS test 5: estimateRoomTokens cache hit (first=' + first +
-      ' second=' + second + ' after-clear=' + third + ')');
-  } finally {
-    rmRf(tmp);
-    estimator.clearCache();
-  }
-})();
-
-// ---------- Test 6: estimateRoomTokens invalid room -> null ----------
-
-(function test6InvalidRoom() {
-  estimator.clearCache();
-  const nonexistent = path.join(os.tmpdir(), 'mos-88-1-16-DOES-NOT-EXIST-' + Date.now());
-  const result = estimator.estimateRoomTokens(nonexistent);
-  assert.equal(result, null,
-    'estimateRoomTokens on nonexistent path must return null, got ' + result);
-  assert.equal(estimator.estimateRoomTokens(null), null,
-    'estimateRoomTokens(null) must return null');
-  assert.equal(estimator.estimateRoomTokens(''), null,
-    'estimateRoomTokens("") must return null');
-  console.log('PASS test 6: estimateRoomTokens invalid room -> null');
-})();
-
-// ---------- Test 7: clearCache resets ----------
-
-(function test7ClearCache() {
-  estimator.clearCache();
-  const tmp = mkTmpDir('room-clear');
-  try {
-    fs.writeFileSync(path.join(tmp, 'a.md'), 'x'.repeat(2000));
-    const first = estimator.estimateRoomTokens(tmp);
-    // Delete the room dir, clear cache, re-query. Without clearCache the
-    // stale value would still be returned; after clearCache, null.
-    rmRf(tmp);
-    estimator.clearCache();
-    const result = estimator.estimateRoomTokens(tmp);
-    assert.equal(result, null,
-      'after clearCache + rmdir, estimateRoomTokens must return null; got ' + result);
-    assert.ok(first > 0, 'sanity: first call was > 0 (' + first + ')');
-    console.log('PASS test 7: clearCache resets session cache');
-  } finally {
-    rmRf(tmp);
-    estimator.clearCache();
-  }
-})();
-
-// ---------- Test 8: estimateRoomTokens skips non-.md files ----------
-
-(function test8SkipsBinary() {
-  estimator.clearCache();
-  const tmp = mkTmpDir('room-skip');
-  try {
-    // Only the .md file should count. Binary .bin + .png + non-.md .txt
-    // must be ignored by the walker.
-    fs.writeFileSync(path.join(tmp, 'a.md'), 'x'.repeat(4000));
-    fs.writeFileSync(path.join(tmp, 'ignored.bin'), Buffer.alloc(100000));
-    fs.writeFileSync(path.join(tmp, 'ignored.png'), Buffer.alloc(50000));
-    fs.writeFileSync(path.join(tmp, 'ignored.txt'), 'z'.repeat(100000));
-    const result = estimator.estimateRoomTokens(tmp);
-    // Expect only the .md to contribute: 4000 chars / 4 = 1000 tokens.
-    assert.ok(result >= 900 && result <= 1100,
-      'estimateRoomTokens must count only .md; got ' + result + ' (expected ~1000)');
-    console.log('PASS test 8: estimateRoomTokens skips non-.md files (got ' + result + ')');
-  } finally {
-    rmRf(tmp);
-    estimator.clearCache();
-  }
-})();
-
-// ---------- Test 9: ratio computation sanity ----------
-
-(function test9RatioComputation() {
-  const naive = 50000;
-  const used = 1000;
-  const ratio = naive / used;
-  assert.equal(ratio, 50,
-    'ratio 50000 / 1000 must equal 50');
-  // The hook will compute this division directly. Sanity only.
-  console.log('PASS test 9: ratio computation (50000 / 1000 = 50)');
-})();
-
-// ---------- Test 10: event shape validator ----------
-
-(function test10EventShape() {
-  const good = {
-    event: 'query.served',
-    ts: '2026-04-23T00:00:00Z',
-    command: '/mos:whitespace',
-    tool: 'Read',
-    tokens_used: 100,
-    tokens_naive_estimate: 5000,
-    ratio: 50.0,
-    room_slug: 'example-room',
-  };
-  const gResult = estimator.validateEventShape(good);
-  assert.equal(gResult.valid, true,
-    'fully-specified event must validate: ' + JSON.stringify(gResult));
-  assert.deepEqual(gResult.missing, [],
-    'good event must have no missing fields');
-
-  // Missing one field -> invalid with that field listed in `missing`.
-  const badOne = Object.assign({}, good);
-  delete badOne.ratio;
-  const bResult = estimator.validateEventShape(badOne);
-  assert.equal(bResult.valid, false,
-    'event missing ratio must fail validation');
-  assert.ok(bResult.missing.indexOf('ratio') >= 0,
-    'missing array must include "ratio", got ' + JSON.stringify(bResult.missing));
-
-  // Empty object -> every field missing.
-  const empty = estimator.validateEventShape({});
-  assert.equal(empty.valid, false, '{} must fail validation');
-  assert.equal(empty.missing.length, 8,
-    'empty object must report all 8 fields missing, got ' + empty.missing.length);
-
-  // Non-object -> invalid.
-  const nullRes = estimator.validateEventShape(null);
-  assert.equal(nullRes.valid, false, 'null must fail validation');
-
-  console.log('PASS test 10: event shape validator (8 required fields)');
-})();
-
-// ---------- Test 11: classifyRatio below/at threshold ----------
-
-(function test11ClassifyRatio() {
-  assert.equal(estimator.classifyRatio(5.2), 'warn',
-    'ratio 5.2 must classify as warn (< 10x)');
-  assert.equal(estimator.classifyRatio(42.0), 'normal',
-    'ratio 42.0 must classify as normal (>= 10x)');
-  assert.equal(estimator.classifyRatio(10.0), 'normal',
-    'ratio 10.0 at boundary must classify as normal (>= 10x)');
-  assert.equal(estimator.classifyRatio(9.99), 'warn',
-    'ratio 9.99 just below boundary must classify as warn');
-  assert.equal(estimator.classifyRatio(0), 'warn',
-    'ratio 0 must classify as warn');
-  // Non-numeric -> warn (defensive: ratio was never computed correctly).
-  assert.equal(estimator.classifyRatio(null), 'warn',
-    'null ratio must classify as warn (defensive)');
-  assert.equal(estimator.classifyRatio(NaN), 'warn',
-    'NaN ratio must classify as warn (defensive)');
-  console.log('PASS test 11: classifyRatio thresholds (10x)');
-})();
-
-// ---------- Test 12: aggregateEvents median + mean + top5 ----------
-
-(function test12Aggregate() {
-  const events = [
-    { command: '/mos:whitespace', ratio: 60, tokens_used: 100, tokens_naive_estimate: 6000, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:whitespace', ratio: 55, tokens_used: 100, tokens_naive_estimate: 5500, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:status', ratio: 50, tokens_used: 100, tokens_naive_estimate: 5000, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:find-bottlenecks', ratio: 45, tokens_used: 100, tokens_naive_estimate: 4500, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:find-bottlenecks', ratio: 48, tokens_used: 100, tokens_naive_estimate: 4800, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:scout', ratio: 40, tokens_used: 100, tokens_naive_estimate: 4000, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:whitespace', ratio: 70, tokens_used: 100, tokens_naive_estimate: 7000, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:grade', ratio: 35, tokens_used: 100, tokens_naive_estimate: 3500, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:reason', ratio: 42, tokens_used: 100, tokens_naive_estimate: 4200, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/mos:persona', ratio: 38, tokens_used: 100, tokens_naive_estimate: 3800, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-  ];
-  const agg = estimator.aggregateEvents(events);
-
-  // count
-  assert.equal(agg.count, 10, 'count must equal events.length, got ' + agg.count);
-
-  // sorted ratios: [35, 38, 40, 42, 45, 48, 50, 55, 60, 70]
-  // median (even) = (45 + 48) / 2 = 46.5
-  assert.equal(agg.median, 46.5,
-    'median must be 46.5 (even-length average), got ' + agg.median);
-
-  // mean = (60+55+50+45+48+40+70+35+42+38) / 10 = 483 / 10 = 48.3
-  assert.equal(agg.mean, 48.3,
-    'mean must be 48.3, got ' + agg.mean);
-
-  // top5 by ratio: whitespace has 3 entries -> use BEST ratio per command (by convention)
-  // OR aggregate per command with averages. Either interpretation: whitespace tops.
-  // PLAN says "top-5 commands by ratio" -- commands with their best/median ratio.
-  // Our contract: per-command aggregate (max ratio), sort desc, slice 5.
-  assert.ok(Array.isArray(agg.top5), 'top5 must be array');
-  assert.equal(agg.top5.length, 5, 'top5 must have exactly 5 entries when >= 5 commands');
-
-  // Top entry must be the highest-ratio command (whitespace best: 70).
-  assert.equal(agg.top5[0].command, '/mos:whitespace',
-    'top5[0] must be /mos:whitespace (max ratio 70), got ' + agg.top5[0].command);
-  assert.equal(agg.top5[0].ratio, 70,
-    'top5[0].ratio must be 70, got ' + agg.top5[0].ratio);
-
-  // Ratios must be strictly non-increasing.
-  for (let i = 1; i < agg.top5.length; i++) {
-    assert.ok(agg.top5[i].ratio <= agg.top5[i - 1].ratio,
-      'top5 must be sorted desc by ratio at index ' + i);
-  }
-
-  // Empty events -> zero count, null median, null mean, [] top5.
-  const emptyAgg = estimator.aggregateEvents([]);
-  assert.equal(emptyAgg.count, 0, 'empty events -> count 0');
-  assert.equal(emptyAgg.median, null, 'empty events -> median null');
-  assert.equal(emptyAgg.mean, null, 'empty events -> mean null');
-  assert.deepEqual(emptyAgg.top5, [], 'empty events -> top5 []');
-
-  // Odd-length median check (sorted: 10, 20, 30 -> median 20).
-  const oddAgg = estimator.aggregateEvents([
-    { command: '/a', ratio: 10, tokens_used: 1, tokens_naive_estimate: 10, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/b', ratio: 30, tokens_used: 1, tokens_naive_estimate: 30, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-    { command: '/c', ratio: 20, tokens_used: 1, tokens_naive_estimate: 20, tool: 'Read', ts: 't', event: 'query.served', room_slug: 'r' },
-  ]);
-  assert.equal(oddAgg.median, 20, 'odd-length median must pick middle value');
-
-  console.log('PASS test 12: aggregateEvents (median=' + agg.median +
-    ', mean=' + agg.mean + ', top5[0]=' + agg.top5[0].command + '@' + agg.top5[0].ratio + ')');
-})();
-
-console.log('\nquery-efficiency-telemetry: 12/12 tests passed');