lynkr 9.7.2 → 9.9.0

package/scripts/audit-log-reader.js

@@ -0,0 +1,399 @@
+#!/usr/bin/env node
+
+/**
+ * LLM Audit Log Reader
+ *
+ * Utility to read and reconstruct audit log entries from deduplicated logs.
+ * Resolves hash references from the dictionary file and outputs full content.
+ *
+ * Usage:
+ *   node scripts/audit-log-reader.js [options]
+ *
+ * Options:
+ *   --log-file <path>        Path to audit log file (default: logs/llm-audit.log)
+ *   --dict-file <path>       Path to dictionary file (default: logs/llm-audit-dictionary.jsonl)
+ *   --full                   Output full restored entries (resolve all references)
+ *   --filter <type=value>    Filter by field (e.g., type=llm_request, provider=anthropic)
+ *   --correlation-id <id>    Filter by correlation ID
+ *   --last <n>               Show only last N entries
+ *   --stats                  Show deduplication statistics
+ *   --verify                 Verify all references can be resolved
+ *   --help                   Show this help message
+ *
+ * Examples:
+ *   # Show all entries with full content
+ *   node scripts/audit-log-reader.js --full
+ *
+ *   # Show only requests
+ *   node scripts/audit-log-reader.js --filter type=llm_request
+ *
+ *   # Show last 5 entries
+ *   node scripts/audit-log-reader.js --last 5 --full
+ *
+ *   # Show deduplication statistics
+ *   node scripts/audit-log-reader.js --stats
+ *
+ *   # Verify all references resolve
+ *   node scripts/audit-log-reader.js --verify
+ */
+
+const fs = require("fs");
+const path = require("path");
+const readline = require("readline");
+const { ContentDeduplicator } = require("../src/logger/deduplicator");
+
+// Default file paths
+const DEFAULT_LOG_FILE = path.join(process.cwd(), "logs", "llm-audit.log");
+const DEFAULT_DICT_FILE = path.join(process.cwd(), "logs", "llm-audit-dictionary.jsonl");
+
+/**
+ * Parse command line arguments
+ */
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const options = {
+    logFile: DEFAULT_LOG_FILE,
+    dictFile: DEFAULT_DICT_FILE,
+    full: false,
+    filter: null,
+    correlationId: null,
+    last: null,
+    stats: false,
+    verify: false,
+    help: false,
+  };
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    switch (arg) {
+      case "--log-file":
+        options.logFile = args[++i];
+        break;
+      case "--dict-file":
+        options.dictFile = args[++i];
+        break;
+      case "--full":
+        options.full = true;
+        break;
+      case "--filter":
+        const filterArg = args[++i];
+        const [key, value] = filterArg.split("=");
+        options.filter = { key, value };
+        break;
+      case "--correlation-id":
+        options.correlationId = args[++i];
+        break;
+      case "--last":
+        options.last = Number.parseInt(args[++i], 10);
+        break;
+      case "--stats":
+        options.stats = true;
+        break;
+      case "--verify":
+        options.verify = true;
+        break;
+      case "--help":
+        options.help = true;
+        break;
+      default:
+        console.error(`Unknown option: ${arg}`);
+        process.exit(1);
+    }
+  }
+
+  return options;
+}
+
+/**
+ * Show help message
+ */
+function showHelp() {
+  const helpText = `
+LLM Audit Log Reader
+
+Usage: node scripts/audit-log-reader.js [options]
+
+Options:
+  --log-file <path>        Path to audit log file (default: logs/llm-audit.log)
+  --dict-file <path>       Path to dictionary file (default: logs/llm-audit-dictionary.jsonl)
+  --full                   Output full restored entries (resolve all references)
+  --filter <type=value>    Filter by field (e.g., type=llm_request, provider=anthropic)
+  --correlation-id <id>    Filter by correlation ID
+  --last <n>               Show only last N entries
+  --stats                  Show deduplication statistics
+  --verify                 Verify all references can be resolved
+  --help                   Show this help message
+
+Examples:
+  # Show all entries with full content
+  node scripts/audit-log-reader.js --full
+
+  # Show only requests
+  node scripts/audit-log-reader.js --filter type=llm_request
+
+  # Show last 5 entries
+  node scripts/audit-log-reader.js --last 5 --full
+
+  # Show deduplication statistics
+  node scripts/audit-log-reader.js --stats
+
+  # Verify all references resolve
+  node scripts/audit-log-reader.js --verify
+`;
+  console.log(helpText);
+}
+
+/**
+ * Read and process log entries
+ */
+async function readLogEntries(options) {
+  const { logFile, dictFile, full, filter, correlationId, last } = options;
+
+  // Check if log file exists
+  if (!fs.existsSync(logFile)) {
+    console.error(`Log file not found: ${logFile}`);
+    process.exit(1);
+  }
+
+  // Initialize deduplicator if needed
+  let deduplicator = null;
+  if (full && fs.existsSync(dictFile)) {
+    deduplicator = new ContentDeduplicator(dictFile);
+  }
+
+  const entries = [];
+
+  // Read log file line by line
+  const fileStream = fs.createReadStream(logFile);
+  const rl = readline.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+
+    try {
+      let entry = JSON.parse(line);
+
+      // Apply filters
+      if (filter && entry[filter.key] !== filter.value) {
+        continue;
+      }
+      if (correlationId && entry.correlationId !== correlationId) {
+        continue;
+      }
+
+      // Restore full content if requested
+      if (full && deduplicator) {
+        entry = deduplicator.restoreEntry(entry);
+      }
+
+      entries.push(entry);
+    } catch (err) {
+      console.error("Malformed log entry:", err.message);
+    }
+  }
+
+  // Apply last N filter
+  const output = last ? entries.slice(-last) : entries;
+
+  // Output as JSONL
+  for (const entry of output) {
+    console.log(JSON.stringify(entry, null, 2));
+  }
+
+  return entries.length;
+}
+
+/**
+ * Show deduplication statistics
+ */
+async function showStats(options) {
+  const { logFile, dictFile } = options;
+
+  if (!fs.existsSync(logFile)) {
+    console.error(`Log file not found: ${logFile}`);
+    process.exit(1);
+  }
+
+  if (!fs.existsSync(dictFile)) {
+    console.log("No dictionary file found. Deduplication may not be enabled.");
+    return;
+  }
+
+  // Get file sizes
+  const logStats = fs.statSync(logFile);
+  const dictStats = fs.statSync(dictFile);
+
+  console.log("\n=== LLM Audit Log Deduplication Statistics ===\n");
+  console.log(`Log file: ${logFile}`);
+  console.log(`  Size: ${formatBytes(logStats.size)}`);
+  console.log(`  Lines: ${await countLines(logFile)}`);
+  console.log();
+  console.log(`Dictionary file: ${dictFile}`);
+  console.log(`  Size: ${formatBytes(dictStats.size)}`);
+  console.log(`  Entries: ${await countLines(dictFile)}`);
+  console.log();
+  console.log(`Total size: ${formatBytes(logStats.size + dictStats.size)}`);
+  console.log();
+
+  // Count reference occurrences in log
+  const refCount = await countReferences(logFile);
+  console.log(`Reference objects in log: ${refCount}`);
+  console.log(`Estimated space saved: ~${formatBytes(refCount * 2000)} (assuming ~2KB per deduplicated field)`);
+  console.log();
+}
+
+/**
+ * Verify all references can be resolved
+ */
+async function verifyReferences(options) {
+  const { logFile, dictFile } = options;
+
+  if (!fs.existsSync(logFile)) {
+    console.error(`Log file not found: ${logFile}`);
+    process.exit(1);
+  }
+
+  if (!fs.existsSync(dictFile)) {
+    console.log("No dictionary file found. Nothing to verify.");
+    return;
+  }
+
+  const deduplicator = new ContentDeduplicator(dictFile);
+  const fileStream = fs.createReadStream(logFile);
+  const rl = readline.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  let totalRefs = 0;
+  let unresolvedRefs = 0;
+  const unresolvedHashes = new Set();
+
+  console.log("Verifying references...\n");
+
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+
+    try {
+      const entry = JSON.parse(line);
+
+      // Check all fields for references
+      for (const [key, value] of Object.entries(entry)) {
+        if (typeof value === "object" && value !== null && value.$ref) {
+          totalRefs++;
+          const content = deduplicator.getContent(value.$ref);
+          if (content === null) {
+            unresolvedRefs++;
+            unresolvedHashes.add(value.$ref);
+            console.error(`✗ Unresolved reference: ${value.$ref} in field "${key}"`);
+          }
+        }
+      }
+    } catch (err) {
+      console.error("Malformed log entry:", err.message);
+    }
+  }
+
+  console.log("\n=== Verification Results ===\n");
+  console.log(`Total references: ${totalRefs}`);
+  console.log(`Unresolved references: ${unresolvedRefs}`);
+  console.log(`Unique unresolved hashes: ${unresolvedHashes.size}`);
+
+  if (unresolvedRefs === 0) {
+    console.log("\n✓ All references resolved successfully!");
+  } else {
+    console.log("\n✗ Some references could not be resolved. Dictionary may be incomplete.");
+    process.exit(1);
+  }
+}
+
+/**
+ * Helper: Format bytes to human-readable string
+ */
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+}
+
+/**
+ * Helper: Count lines in a file
+ */
+async function countLines(filePath) {
+  const fileStream = fs.createReadStream(filePath);
+  const rl = readline.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  let count = 0;
+  for await (const line of rl) {
+    if (line.trim()) count++;
+  }
+  return count;
+}
+
+/**
+ * Helper: Count reference objects in log
+ */
+async function countReferences(filePath) {
+  const fileStream = fs.createReadStream(filePath);
+  const rl = readline.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  let count = 0;
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    try {
+      const entry = JSON.parse(line);
+      for (const value of Object.values(entry)) {
+        if (typeof value === "object" && value !== null && value.$ref) {
+          count++;
+        }
+      }
+    } catch {
+      // Skip malformed lines
+    }
+  }
+  return count;
+}
+
+/**
+ * Main entry point
+ */
+async function main() {
+  const options = parseArgs();
+
+  if (options.help) {
+    showHelp();
+    return;
+  }
+
+  if (options.stats) {
+    await showStats(options);
+  } else if (options.verify) {
+    await verifyReferences(options);
+  } else {
+    const count = await readLogEntries(options);
+    console.error(`\n(Processed ${count} entries)`);
+  }
+}
+
+// Run if called directly
+if (require.main === module) {
+  main().catch((err) => {
+    console.error("Error:", err.message);
+    process.exit(1);
+  });
+}
+
+module.exports = {
+  readLogEntries,
+  showStats,
+  verifyReferences,
+};

package/scripts/calibrate-thresholds.js

@@ -2,11 +2,9 @@
 /**
  * Calibrate tier thresholds from telemetry.
  *
- * Phase 1.4 of the routing overhaul. Reads quality_score history from the
- * routing_telemetry table, finds where each tier's median quality drops below
- * acceptable, and writes adjusted [lo, hi] ranges to
- * data/calibrated-thresholds.json. ModelTierSelector picks the file up on
- * next start.
+ * CLI wrapper around `src/routing/calibration.js`. WS5.6 moved the core
+ * logic into a module so the same code path drives both this manual
+ * script and the in-process auto-calibration scheduler.
  *
  * Usage: node scripts/calibrate-thresholds.js [--days N] [--dry-run]
  *        npx lynkr calibrate
@@ -16,31 +14,9 @@
  *   - Exits 0 with a "skipped" message.
  */
 
-const fs = require('fs');
-const path = require('path');
+const { runCalibration } = require('../src/routing/calibration');
 
 const DEFAULT_DAYS = 7;
-const MIN_SAMPLES = 100;
-/** Quality score below which a complexity bucket is "underperforming" for its tier. */
-const QUALITY_FLOOR = {
-  SIMPLE: 55,
-  MEDIUM: 60,
-  COMPLEX: 65,
-  REASONING: 70,
-};
-
-const OUTPUT_PATH = path.join(__dirname, '../data/calibrated-thresholds.json');
-const TELEMETRY_DB_CANDIDATES = [
-  path.join(__dirname, '../.lynkr/telemetry.db'),
-  path.join(__dirname, '../data/lynkr.db'),
-];
-
-function _findDb() {
-  for (const p of TELEMETRY_DB_CANDIDATES) {
-    if (fs.existsSync(p)) return p;
-  }
-  return null;
-}
 
 function _parseArgs(argv) {
   const out = { days: DEFAULT_DAYS, dryRun: false };
@@ -52,141 +28,46 @@ function _parseArgs(argv) {
   return out;
 }
 
-const DEFAULT_RANGES = {
-  SIMPLE: [0, 25],
-  MEDIUM: [26, 50],
-  COMPLEX: [51, 75],
-  REASONING: [76, 100],
-};
-
-function _openDb(dbPath) {
-  let Database;
-  try {
-    Database = require('better-sqlite3');
-  } catch (err) {
-    console.error('better-sqlite3 not installed. Install with: npm install --save-optional better-sqlite3');
-    process.exit(2);
+function _reportSkipped(result) {
+  switch (result.reason) {
+    case 'no_db':
+      console.log('No telemetry DB found — skipping calibration.');
+      break;
+    case 'db_open_failed':
+      console.error(`Failed to open telemetry DB: ${result.error}`);
+      process.exit(2);
+      break;
+    case 'query_failed':
+      console.error(`Telemetry query failed (DB may be corrupt or schema missing): ${result.error}`);
+      break;
+    case 'insufficient_samples':
+      console.log(
+        `Only ${result.count} rows with quality_score (need ≥${result.minSamples}). Skipping.`
+      );
+      break;
+    case 'write_failed':
+      console.error(`Failed to write calibrated thresholds: ${result.error}`);
+      process.exit(2);
+      break;
+    default:
+      console.log(`Skipped: ${result.reason}`);
   }
-  return new Database(dbPath, { readonly: true, fileMustExist: true });
 }
 
-function calibrate({ days = DEFAULT_DAYS, dryRun = false } = {}) {
-  const dbPath = _findDb();
-  if (!dbPath) {
-    console.log('No telemetry DB found — skipping calibration.');
-    return { skipped: true, reason: 'no_db' };
-  }
-
-  let db;
-  try {
-    db = _openDb(dbPath);
-  } catch (err) {
-    console.error(`Failed to open telemetry DB: ${err.message}`);
-    return { skipped: true, reason: 'db_open_failed', error: err.message };
-  }
-
-  const since = Date.now() - days * 24 * 3600 * 1000;
-  let rows;
-  try {
-    rows = db
-      .prepare(
-        `SELECT tier, complexity_score AS score, quality_score AS q
-           FROM routing_telemetry
-          WHERE timestamp >= ?
-            AND quality_score IS NOT NULL
-            AND complexity_score IS NOT NULL
-            AND tier IS NOT NULL`
-      )
-      .all(since);
-  } catch (err) {
-    console.error(`Telemetry query failed (DB may be corrupt or schema missing): ${err.message}`);
-    return { skipped: true, reason: 'query_failed', error: err.message };
-  } finally {
-    try { db.close(); } catch {}
+function calibrate(opts) {
+  const result = runCalibration(opts);
+  if (result.skipped) {
+    _reportSkipped(result);
+    return result;
   }
-
-  if (!rows || rows.length < MIN_SAMPLES) {
-    console.log(`Only ${rows ? rows.length : 0} rows with quality_score in last ${days}d (need ≥${MIN_SAMPLES}). Skipping.`);
-    return { skipped: true, reason: 'insufficient_samples', count: rows ? rows.length : 0 };
-  }
-
-  // Bucket by score (0-100 in width-5 buckets) per tier, compute median quality.
-  const buckets = new Map(); // tier -> Map<bucketLowerBound, q-values[]>
-  for (const row of rows) {
-    const s = Math.max(0, Math.min(100, Math.floor(row.score)));
-    const bucket = Math.floor(s / 5) * 5;
-    if (!buckets.has(row.tier)) buckets.set(row.tier, new Map());
-    const b = buckets.get(row.tier);
-    if (!b.has(bucket)) b.set(bucket, []);
-    b.get(bucket).push(row.q);
+  if (result.dryRun) {
+    console.log(JSON.stringify(result, null, 2));
+    return result;
   }
-
-  const _median = (arr) => {
-    const s = arr.slice().sort((a, b) => a - b);
-    const m = Math.floor(s.length / 2);
-    return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
-  };
-
-  // Default ranges; will adjust per-tier upper bound if late buckets show poor quality.
-  const ranges = { ...DEFAULT_RANGES };
   const tierOrder = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
-  const stats = {};
-
-  for (const tier of tierOrder) {
-    const floor = QUALITY_FLOOR[tier];
-    const tierBuckets = buckets.get(tier);
-    if (!tierBuckets) {
-      stats[tier] = { samples: 0, adjusted: false };
-      continue;
-    }
-    const ordered = Array.from(tierBuckets.entries()).sort((a, b) => a[0] - b[0]);
-    let suggestedUpper = DEFAULT_RANGES[tier][1];
-    const buckets_summary = [];
-    for (const [lo, vals] of ordered) {
-      if (vals.length < 5) {
-        buckets_summary.push({ bucket: lo, samples: vals.length, median: null });
-        continue;
-      }
-      const med = _median(vals);
-      buckets_summary.push({ bucket: lo, samples: vals.length, median: med });
-      if (med < floor && lo + 4 < suggestedUpper) {
-        suggestedUpper = lo + 4; // shrink tier upper bound just below the failing bucket
-      }
-    }
-    if (suggestedUpper !== DEFAULT_RANGES[tier][1]) {
-      ranges[tier] = [DEFAULT_RANGES[tier][0], suggestedUpper];
-      stats[tier] = { samples: ordered.reduce((s, [, v]) => s + v.length, 0), adjusted: true, buckets: buckets_summary };
-    } else {
-      stats[tier] = { samples: ordered.reduce((s, [, v]) => s + v.length, 0), adjusted: false, buckets: buckets_summary };
-    }
-  }
-
-  // Re-stitch ranges so they don't overlap or leave gaps.
-  for (let i = 1; i < tierOrder.length; i++) {
-    const prev = ranges[tierOrder[i - 1]];
-    const cur = ranges[tierOrder[i]];
-    if (cur[0] !== prev[1] + 1) cur[0] = prev[1] + 1;
-    if (cur[0] > cur[1]) cur[1] = cur[0]; // collapsed; tier disabled in practice
-  }
-
-  const out = {
-    calibratedAt: new Date().toISOString(),
-    days,
-    sampleCount: rows.length,
-    ranges,
-    stats,
-  };
-
-  if (dryRun) {
-    console.log(JSON.stringify(out, null, 2));
-    return { ...out, dryRun: true };
-  }
-
-  fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
-  fs.writeFileSync(OUTPUT_PATH, JSON.stringify(out, null, 2));
-  console.log(`Wrote ${OUTPUT_PATH}`);
-  console.log(`Ranges: ${tierOrder.map((t) => `${t}=${ranges[t].join('-')}`).join(', ')}`);
-  return out;
+  console.log(`Wrote ${result.writtenTo}`);
+  console.log(`Ranges: ${tierOrder.map((t) => `${t}=${result.ranges[t].join('-')}`).join(', ')}`);
+  return result;
 }
 
 if (require.main === module) {