lynkr 9.7.3 → 9.9.1

package/scripts/calibrate-thresholds.js

@@ -2,11 +2,9 @@
 /**
  * Calibrate tier thresholds from telemetry.
  *
- * Phase 1.4 of the routing overhaul. Reads quality_score history from the
- * routing_telemetry table, finds where each tier's median quality drops below
- * acceptable, and writes adjusted [lo, hi] ranges to
- * data/calibrated-thresholds.json. ModelTierSelector picks the file up on
- * next start.
+ * CLI wrapper around `src/routing/calibration.js`. WS5.6 moved the core
+ * logic into a module so the same code path drives both this manual
+ * script and the in-process auto-calibration scheduler.
  *
  * Usage: node scripts/calibrate-thresholds.js [--days N] [--dry-run]
  *        npx lynkr calibrate
@@ -16,31 +14,9 @@
  *   - Exits 0 with a "skipped" message.
  */
 
-const fs = require('fs');
-const path = require('path');
+const { runCalibration } = require('../src/routing/calibration');
 
 const DEFAULT_DAYS = 7;
-const MIN_SAMPLES = 100;
-/** Quality score below which a complexity bucket is "underperforming" for its tier. */
-const QUALITY_FLOOR = {
-  SIMPLE: 55,
-  MEDIUM: 60,
-  COMPLEX: 65,
-  REASONING: 70,
-};
-
-const OUTPUT_PATH = path.join(__dirname, '../data/calibrated-thresholds.json');
-const TELEMETRY_DB_CANDIDATES = [
-  path.join(__dirname, '../.lynkr/telemetry.db'),
-  path.join(__dirname, '../data/lynkr.db'),
-];
-
-function _findDb() {
-  for (const p of TELEMETRY_DB_CANDIDATES) {
-    if (fs.existsSync(p)) return p;
-  }
-  return null;
-}
 
 function _parseArgs(argv) {
   const out = { days: DEFAULT_DAYS, dryRun: false };
@@ -52,141 +28,46 @@ function _parseArgs(argv) {
   return out;
 }
 
-const DEFAULT_RANGES = {
-  SIMPLE: [0, 25],
-  MEDIUM: [26, 50],
-  COMPLEX: [51, 75],
-  REASONING: [76, 100],
-};
-
-function _openDb(dbPath) {
-  let Database;
-  try {
-    Database = require('better-sqlite3');
-  } catch (err) {
-    console.error('better-sqlite3 not installed. Install with: npm install --save-optional better-sqlite3');
-    process.exit(2);
+function _reportSkipped(result) {
+  switch (result.reason) {
+    case 'no_db':
+      console.log('No telemetry DB found — skipping calibration.');
+      break;
+    case 'db_open_failed':
+      console.error(`Failed to open telemetry DB: ${result.error}`);
+      process.exit(2);
+      break;
+    case 'query_failed':
+      console.error(`Telemetry query failed (DB may be corrupt or schema missing): ${result.error}`);
+      break;
+    case 'insufficient_samples':
+      console.log(
+        `Only ${result.count} rows with quality_score (need ≥${result.minSamples}). Skipping.`
+      );
+      break;
+    case 'write_failed':
+      console.error(`Failed to write calibrated thresholds: ${result.error}`);
+      process.exit(2);
+      break;
+    default:
+      console.log(`Skipped: ${result.reason}`);
   }
-  return new Database(dbPath, { readonly: true, fileMustExist: true });
 }
 
-function calibrate({ days = DEFAULT_DAYS, dryRun = false } = {}) {
-  const dbPath = _findDb();
-  if (!dbPath) {
-    console.log('No telemetry DB found — skipping calibration.');
-    return { skipped: true, reason: 'no_db' };
-  }
-
-  let db;
-  try {
-    db = _openDb(dbPath);
-  } catch (err) {
-    console.error(`Failed to open telemetry DB: ${err.message}`);
-    return { skipped: true, reason: 'db_open_failed', error: err.message };
-  }
-
-  const since = Date.now() - days * 24 * 3600 * 1000;
-  let rows;
-  try {
-    rows = db
-      .prepare(
-        `SELECT tier, complexity_score AS score, quality_score AS q
-           FROM routing_telemetry
-          WHERE timestamp >= ?
-            AND quality_score IS NOT NULL
-            AND complexity_score IS NOT NULL
-            AND tier IS NOT NULL`
-      )
-      .all(since);
-  } catch (err) {
-    console.error(`Telemetry query failed (DB may be corrupt or schema missing): ${err.message}`);
-    return { skipped: true, reason: 'query_failed', error: err.message };
-  } finally {
-    try { db.close(); } catch {}
+function calibrate(opts) {
+  const result = runCalibration(opts);
+  if (result.skipped) {
+    _reportSkipped(result);
+    return result;
   }
-
-  if (!rows || rows.length < MIN_SAMPLES) {
-    console.log(`Only ${rows ? rows.length : 0} rows with quality_score in last ${days}d (need ≥${MIN_SAMPLES}). Skipping.`);
-    return { skipped: true, reason: 'insufficient_samples', count: rows ? rows.length : 0 };
-  }
-
-  // Bucket by score (0-100 in width-5 buckets) per tier, compute median quality.
-  const buckets = new Map(); // tier -> Map<bucketLowerBound, q-values[]>
-  for (const row of rows) {
-    const s = Math.max(0, Math.min(100, Math.floor(row.score)));
-    const bucket = Math.floor(s / 5) * 5;
-    if (!buckets.has(row.tier)) buckets.set(row.tier, new Map());
-    const b = buckets.get(row.tier);
-    if (!b.has(bucket)) b.set(bucket, []);
-    b.get(bucket).push(row.q);
+  if (result.dryRun) {
+    console.log(JSON.stringify(result, null, 2));
+    return result;
   }
-
-  const _median = (arr) => {
-    const s = arr.slice().sort((a, b) => a - b);
-    const m = Math.floor(s.length / 2);
-    return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
-  };
-
-  // Default ranges; will adjust per-tier upper bound if late buckets show poor quality.
-  const ranges = { ...DEFAULT_RANGES };
   const tierOrder = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
-  const stats = {};
-
-  for (const tier of tierOrder) {
-    const floor = QUALITY_FLOOR[tier];
-    const tierBuckets = buckets.get(tier);
-    if (!tierBuckets) {
-      stats[tier] = { samples: 0, adjusted: false };
-      continue;
-    }
-    const ordered = Array.from(tierBuckets.entries()).sort((a, b) => a[0] - b[0]);
-    let suggestedUpper = DEFAULT_RANGES[tier][1];
-    const buckets_summary = [];
-    for (const [lo, vals] of ordered) {
-      if (vals.length < 5) {
-        buckets_summary.push({ bucket: lo, samples: vals.length, median: null });
-        continue;
-      }
-      const med = _median(vals);
-      buckets_summary.push({ bucket: lo, samples: vals.length, median: med });
-      if (med < floor && lo + 4 < suggestedUpper) {
-        suggestedUpper = lo + 4; // shrink tier upper bound just below the failing bucket
-      }
-    }
-    if (suggestedUpper !== DEFAULT_RANGES[tier][1]) {
-      ranges[tier] = [DEFAULT_RANGES[tier][0], suggestedUpper];
-      stats[tier] = { samples: ordered.reduce((s, [, v]) => s + v.length, 0), adjusted: true, buckets: buckets_summary };
-    } else {
-      stats[tier] = { samples: ordered.reduce((s, [, v]) => s + v.length, 0), adjusted: false, buckets: buckets_summary };
-    }
-  }
-
-  // Re-stitch ranges so they don't overlap or leave gaps.
-  for (let i = 1; i < tierOrder.length; i++) {
-    const prev = ranges[tierOrder[i - 1]];
-    const cur = ranges[tierOrder[i]];
-    if (cur[0] !== prev[1] + 1) cur[0] = prev[1] + 1;
-    if (cur[0] > cur[1]) cur[1] = cur[0]; // collapsed; tier disabled in practice
-  }
-
-  const out = {
-    calibratedAt: new Date().toISOString(),
-    days,
-    sampleCount: rows.length,
-    ranges,
-    stats,
-  };
-
-  if (dryRun) {
-    console.log(JSON.stringify(out, null, 2));
-    return { ...out, dryRun: true };
-  }
-
-  fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
-  fs.writeFileSync(OUTPUT_PATH, JSON.stringify(out, null, 2));
-  console.log(`Wrote ${OUTPUT_PATH}`);
-  console.log(`Ranges: ${tierOrder.map((t) => `${t}=${ranges[t].join('-')}`).join(', ')}`);
-  return out;
+  console.log(`Wrote ${result.writtenTo}`);
+  console.log(`Ranges: ${tierOrder.map((t) => `${t}=${result.ranges[t].join('-')}`).join(', ')}`);
+  return result;
 }
 
 if (require.main === module) {

package/scripts/compact-dictionary.js

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+
+/**
+ * Compact LLM Audit Dictionary
+ *
+ * Removes redundant UPDATE entries from the dictionary file, keeping only:
+ * - One entry per hash with full content
+ * - Latest metadata (useCount, lastSeen)
+ *
+ * Usage:
+ *   node scripts/compact-dictionary.js [options]
+ *
+ * Options:
+ *   --dict-path <path>    Path to dictionary file (default: logs/llm-audit-dictionary.jsonl)
+ *   --backup              Create backup before compacting (default: true)
+ *   --dry-run             Show what would be done without making changes
+ *   --help                Show this help message
+ */
+
+const fs = require('fs');
+const path = require('path');
+const readline = require('readline');
+
+// Parse command line arguments
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const options = {
+    dictPath: 'logs/llm-audit-dictionary.jsonl',
+    backup: true,
+    dryRun: false,
+  };
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    switch (arg) {
+      case '--dict-path':
+        options.dictPath = args[++i];
+        break;
+      case '--backup':
+        options.backup = true;
+        break;
+      case '--no-backup':
+        options.backup = false;
+        break;
+      case '--dry-run':
+        options.dryRun = true;
+        break;
+      case '--help':
+        console.log(`
+Compact LLM Audit Dictionary
+
+Removes redundant UPDATE entries from the dictionary file.
+
+Usage:
+  node scripts/compact-dictionary.js [options]
+
+Options:
+  --dict-path <path>    Path to dictionary file (default: logs/llm-audit-dictionary.jsonl)
+  --backup              Create backup before compacting (default: true)
+  --no-backup           Skip creating backup
+  --dry-run             Show what would be done without making changes
+  --help                Show this help message
+
+Example:
+  node scripts/compact-dictionary.js --dict-path logs/llm-audit-dictionary.jsonl --dry-run
+        `);
+        process.exit(0);
+      default:
+        if (arg.startsWith('--')) {
+          console.error(`Unknown option: ${arg}`);
+          process.exit(1);
+        }
+    }
+  }
+
+  return options;
+}
+
+// Read and compact dictionary
+async function compactDictionary(dictPath) {
+  if (!fs.existsSync(dictPath)) {
+    throw new Error(`Dictionary file not found: ${dictPath}`);
+  }
+
+  console.log(`Reading dictionary: ${dictPath}`);
+
+  // Map: hash -> entry object
+  // For each hash, we'll keep the latest metadata merged with content
+  const entries = new Map();
+  let totalLines = 0;
+  let malformedLines = 0;
+
+  // Read all entries
+  const fileStream = fs.createReadStream(dictPath);
+  const rl = readline.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  for await (const line of rl) {
+    totalLines++;
+    if (!line.trim()) continue;
+
+    try {
+      const entry = JSON.parse(line);
+      if (!entry.hash) {
+        malformedLines++;
+        continue;
+      }
+
+      const hash = entry.hash;
+
+      // Check if we already have an entry for this hash
+      if (entries.has(hash)) {
+        const existing = entries.get(hash);
+
+        // Merge: keep content from entry that has it, use latest metadata
+        const merged = {
+          hash,
+          firstSeen: existing.firstSeen || entry.firstSeen,
+          useCount: entry.useCount || existing.useCount,
+          lastSeen: entry.lastSeen || existing.lastSeen,
+          content: existing.content || entry.content,
+        };
+
+        entries.set(hash, merged);
+      } else {
+        // First time seeing this hash
+        entries.set(hash, entry);
+      }
+    } catch (err) {
+      malformedLines++;
+      console.warn(`Skipping malformed line ${totalLines}: ${err.message}`);
+    }
+  }
+
+  const compactedCount = entries.size;
+  const removedCount = totalLines - malformedLines - compactedCount;
+
+  return {
+    entries,
+    stats: {
+      totalLines,
+      malformedLines,
+      uniqueHashes: compactedCount,
+      removedEntries: removedCount,
+    },
+  };
+}
+
+// Write compacted dictionary
+async function writeCompactedDictionary(dictPath, entries, backup = true) {
+  // Create backup if requested
+  if (backup) {
+    const backupPath = `${dictPath}.backup.${Date.now()}`;
+    console.log(`Creating backup: ${backupPath}`);
+    fs.copyFileSync(dictPath, backupPath);
+  }
+
+  // Write compacted entries
+  console.log(`Writing compacted dictionary: ${dictPath}`);
+  const lines = Array.from(entries.values()).map((entry) => JSON.stringify(entry));
+  fs.writeFileSync(dictPath, lines.join('\n') + '\n');
+}
+
+// Main
+async function main() {
+  try {
+    const options = parseArgs();
+    const dictPath = path.resolve(options.dictPath);
+
+    console.log('=== LLM Audit Dictionary Compaction ===\n');
+
+    // Read and compact
+    const { entries, stats } = await compactDictionary(dictPath);
+
+    // Report statistics
+    console.log('\nCompaction Statistics:');
+    console.log(`  Total lines in dictionary: ${stats.totalLines}`);
+    console.log(`  Malformed lines skipped: ${stats.malformedLines}`);
+    console.log(`  Unique content hashes: ${stats.uniqueHashes}`);
+    console.log(`  Redundant entries removed: ${stats.removedEntries}`);
+
+    const reductionPercent =
+      stats.totalLines > 0
+        ? ((stats.removedEntries / stats.totalLines) * 100).toFixed(1)
+        : 0;
+    console.log(`  Size reduction: ${reductionPercent}%\n`);
+
+    if (options.dryRun) {
+      console.log('DRY RUN: No changes made to dictionary file.');
+      console.log(`Would have written ${stats.uniqueHashes} entries.\n`);
+    } else {
+      // Write compacted dictionary
+      await writeCompactedDictionary(dictPath, entries, options.backup);
+      console.log('✓ Dictionary compaction complete!\n');
+    }
+  } catch (err) {
+    console.error('Error:', err.message);
+    process.exit(1);
+  }
+}
+
+main();