lynkr 7.2.5 → 8.0.1

package/scripts/test-deduplication.js

@@ -1,448 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Test script for deduplication functionality
- * Creates mock log entries and verifies deduplication works correctly
- */
-
-const fs = require("fs");
-const path = require("path");
-const { ContentDeduplicator } = require("../src/logger/deduplicator");
-
-// Test configuration
-const TEST_DICT_PATH = path.join(process.cwd(), "logs", "test-dictionary.jsonl");
-const TEST_LOG_PATH = path.join(process.cwd(), "logs", "test-audit.log");
-
-// Clean up test files if they exist
-function cleanup() {
-  if (fs.existsSync(TEST_DICT_PATH)) {
-    fs.unlinkSync(TEST_DICT_PATH);
-  }
-  if (fs.existsSync(TEST_LOG_PATH)) {
-    fs.unlinkSync(TEST_LOG_PATH);
-  }
-}
-
-// Test 1: Basic deduplication
-function testBasicDeduplication() {
-  console.log("\n=== Test 1: Basic Deduplication ===");
-
-  const deduplicator = new ContentDeduplicator(TEST_DICT_PATH, {
-    minSize: 50, // Lower threshold for testing
-    cacheSize: 10,
-  });
-
-  const content1 = "This is a test content that is longer than 50 characters and should be deduplicated.";
-  const content2 = "This is a test content that is longer than 50 characters and should be deduplicated.";
-  const content3 = "Short";
-
-  // First content should be stored
-  const ref1 = deduplicator.storeContent(content1);
-  console.log("✓ Stored content1:", ref1);
-
-  // Second identical content should return same reference
-  const ref2 = deduplicator.storeContent(content2);
-  console.log("✓ Stored content2 (should be same hash):", ref2);
-
-  // Verify same hash
-  if (ref1.$ref !== ref2.$ref) {
-    console.error("✗ FAIL: Different hashes for identical content!");
-    return false;
-  }
-  console.log("✓ PASS: Identical content produces same hash");
-
-  // Short content should not be deduplicated (below threshold)
-  const shouldNotDedup = deduplicator.shouldDeduplicate(content3, 50);
-  if (shouldNotDedup) {
-    console.error("✗ FAIL: Short content should not be deduplicated!");
-    return false;
-  }
-  console.log("✓ PASS: Short content not deduplicated");
-
-  return true;
-}
-
-// Test 2: Content restoration
-function testContentRestoration() {
-  console.log("\n=== Test 2: Content Restoration ===");
-
-  const deduplicator = new ContentDeduplicator(TEST_DICT_PATH, {
-    minSize: 50,
-    cacheSize: 10,
-  });
-
-  const originalContent = "This is original content that needs to be restored from the dictionary file.";
-
-  // Store and get reference
-  const ref = deduplicator.storeContent(originalContent);
-  console.log("✓ Stored content with ref:", ref.$ref);
-
-  // Retrieve content
-  const retrieved = deduplicator.getContent(ref.$ref);
-  console.log("✓ Retrieved content length:", retrieved?.length);
-
-  // Verify content matches
-  if (retrieved !== originalContent) {
-    console.error("✗ FAIL: Retrieved content doesn't match original!");
-    console.error("Expected:", originalContent);
-    console.error("Got:", retrieved);
-    return false;
-  }
-  console.log("✓ PASS: Content restored correctly");
-
-  return true;
-}
-
-// Test 3: Entry deduplication and restoration
-function testEntryProcessing() {
-  console.log("\n=== Test 3: Entry Deduplication and Restoration ===");
-
-  const deduplicator = new ContentDeduplicator(TEST_DICT_PATH, {
-    minSize: 50,
-    cacheSize: 10,
-  });
-
-  const systemPrompt = "You are a helpful AI assistant. This is a long system prompt that should be deduplicated.";
-  const userMessage = "This is a user message that is long enough to be deduplicated by the deduplication system.";
-
-  const entry = {
-    type: "llm_request",
-    correlationId: "test-123",
-    systemPrompt: systemPrompt,
-    userMessages: userMessage,
-    model: "test-model",
-  };
-
-  // Deduplicate entry
-  const deduplicated = deduplicator.deduplicateEntry(entry, ["systemPrompt", "userMessages"]);
-  console.log("✓ Deduplicated entry:", JSON.stringify(deduplicated, null, 2));
-
-  // Verify fields are now references
-  if (typeof deduplicated.systemPrompt !== "object" || !deduplicated.systemPrompt.$ref) {
-    console.error("✗ FAIL: systemPrompt was not deduplicated!");
-    return false;
-  }
-  if (typeof deduplicated.userMessages !== "object" || !deduplicated.userMessages.$ref) {
-    console.error("✗ FAIL: userMessages was not deduplicated!");
-    return false;
-  }
-  console.log("✓ PASS: Fields converted to references");
-
-  // Restore entry
-  const restored = deduplicator.restoreEntry(deduplicated);
-  console.log("✓ Restored entry keys:", Object.keys(restored));
-
-  // Verify restoration
-  if (restored.systemPrompt !== systemPrompt) {
-    console.error("✗ FAIL: systemPrompt not restored correctly!");
-    console.error("Expected:", systemPrompt);
-    console.error("Got:", restored.systemPrompt);
-    return false;
-  }
-  if (restored.userMessages !== userMessage) {
-    console.error("✗ FAIL: userMessages not restored correctly!");
-    console.error("Expected:", userMessage);
-    console.error("Got:", restored.userMessages);
-    return false;
-  }
-  console.log("✓ PASS: Entry restored correctly");
-
-  return true;
-}
-
-// Test 4: Dictionary persistence
-function testDictionaryPersistence() {
-  console.log("\n=== Test 4: Dictionary Persistence ===");
-
-  // Create first deduplicator and store content
-  const deduplicator1 = new ContentDeduplicator(TEST_DICT_PATH, {
-    minSize: 50,
-    cacheSize: 10,
-  });
-
-  const content = "This is test content for persistence verification across deduplicator instances.";
-  const ref = deduplicator1.storeContent(content);
-  console.log("✓ Stored content with first deduplicator:", ref.$ref);
-
-  // Wait for async write to complete
-  setTimeout(() => {
-    // Create second deduplicator (should load from dictionary)
-    const deduplicator2 = new ContentDeduplicator(TEST_DICT_PATH, {
-      minSize: 50,
-      cacheSize: 10,
-    });
-
-    // Try to retrieve with second deduplicator
-    const retrieved = deduplicator2.getContent(ref.$ref);
-
-    if (retrieved !== content) {
-      console.error("✗ FAIL: Content not persisted to dictionary!");
-      console.error("Expected:", content);
-      console.error("Got:", retrieved);
-      return false;
-    }
-    console.log("✓ PASS: Dictionary persisted and loaded correctly");
-
-    // Show dictionary stats
-    const stats = deduplicator2.getStats();
-    console.log("\nDeduplication Stats:");
-    console.log(`  Cache size: ${stats.cacheSize}`);
-    console.log(`  Unique blocks: ${stats.uniqueContentBlocks}`);
-    console.log(`  Total references: ${stats.totalReferences}`);
-
-    return true;
-  }, 100);
-}
-
-// Test 5: Size calculation and verification
-function testSizeCalculation() {
-  console.log("\n=== Test 5: Size Calculation ===");
-
-  const deduplicator = new ContentDeduplicator(TEST_DICT_PATH, {
-    minSize: 50,
-    cacheSize: 10,
-  });
-
-  const content = "This is a test content string that will be deduplicated and have its size calculated.";
-  const ref = deduplicator.storeContent(content);
-
-  console.log("✓ Content length:", content.length);
-  console.log("✓ Reference size field:", ref.size);
-
-  if (ref.size !== content.length) {
-    console.error("✗ FAIL: Size mismatch!");
-    return false;
-  }
-  console.log("✓ PASS: Size calculated correctly");
-
-  // Calculate space saved
-  const refSize = JSON.stringify(ref).length;
-  const originalSize = content.length;
-  const saved = originalSize - refSize;
-  const savedPercent = ((saved / originalSize) * 100).toFixed(1);
-
-  console.log(`\nSpace saved: ${saved} bytes (${savedPercent}%)`);
-  console.log(`  Original: ${originalSize} bytes`);
-  console.log(`  Reference: ${refSize} bytes`);
-
-  return true;
-}
-
-// Test 6: Content sanitization (empty User: entries removal)
-function testContentSanitization() {
-  console.log("\n=== Test 6: Content Sanitization (Empty User: Removal) ===");
-
-  const deduplicator = new ContentDeduplicator(TEST_DICT_PATH, {
-    minSize: 50,
-    cacheSize: 10,
-    sanitize: true, // Enable sanitization
-  });
-
-  // Content with multiple empty "User:" entries
-  const dirtyContent = `Claude: I'll implement...
-
-User:
-
-Claude: Now I'll implement...
-
-User:
-
-User:
-
-User:
-
-Respond with the title for the conversation and nothing else.`;
-
-  console.log("✓ Original content length:", dirtyContent.length);
-  console.log("✓ Empty 'User:' entries in original:", (dirtyContent.match(/User:\s*\n/g) || []).length);
-
-  // Store the content (should be sanitized internally)
-  const ref = deduplicator.storeContent(dirtyContent);
-  console.log("✓ Stored content with ref:", ref.$ref);
-
-  // Retrieve it back
-  const retrieved = deduplicator.getContent(ref.$ref);
-  console.log("✓ Retrieved content length:", retrieved?.length);
-
-  // Count empty "User:" entries in retrieved content
-  // Pattern: "User:" followed by newline(s) and then "Claude:" or another "User:" or end
-  const emptyUserMatches = retrieved.match(/User:\s*\n+(?=(Claude:|User:|$))/g) || [];
-  console.log("✓ Empty 'User:' entries in retrieved:", emptyUserMatches.length);
-
-  // Verify empty User: entries were removed
-  if (emptyUserMatches.length > 0) {
-    console.error("✗ FAIL: Empty User: entries not removed!");
-    console.error("Retrieved content:", retrieved);
-    return false;
-  }
-
-  // Verify content still contains Claude: entries
-  if (!retrieved.includes("Claude:")) {
-    console.error("✗ FAIL: Claude: entries were incorrectly removed!");
-    return false;
-  }
-
-  // Verify the last line is preserved
-  if (!retrieved.includes("Respond with the title")) {
-    console.error("✗ FAIL: Content was over-sanitized!");
-    return false;
-  }
-
-  console.log("✓ PASS: Empty User: entries removed, content preserved");
-
-  // Test with sanitization disabled
-  const dedupNoSanitize = new ContentDeduplicator(TEST_DICT_PATH, {
-    minSize: 50,
-    cacheSize: 10,
-    sanitize: false, // Disable sanitization
-  });
-
-  const refNoSanitize = dedupNoSanitize.storeContent(dirtyContent);
-  const retrievedNoSanitize = dedupNoSanitize.getContent(refNoSanitize.$ref);
-
-  // Should have empty User: entries when sanitization is disabled
-  const emptyUserNoSanitize = retrievedNoSanitize.match(/User:\s*\n+(?=(Claude:|User:|$))/g) || [];
-  if (emptyUserNoSanitize.length === 0) {
-    console.error("✗ FAIL: Content was sanitized even with sanitize=false!");
-    return false;
-  }
-
-  console.log("✓ PASS: Sanitization can be disabled");
-
-  return true;
-}
-
-// Test 7: Content sanitization preserves non-empty User: entries
-function testSanitizationPreservesContent() {
-  console.log("\n=== Test 7: Sanitization Preserves Non-Empty User: Entries ===");
-
-  const deduplicator = new ContentDeduplicator(TEST_DICT_PATH, {
-    minSize: 50,
-    cacheSize: 10,
-    sanitize: true,
-  });
-
-  // Content with both empty and non-empty User: entries
-  const mixedContent = `Claude: I'll help you.
-
-User: Can you explain this?
-
-Claude: Sure, here's the explanation.
-
-User:
-
-User: Another question here.
-
-Claude: Here's the answer.`;
-
-  console.log("✓ Original has both empty and non-empty User: entries");
-
-  const ref = deduplicator.storeContent(mixedContent);
-  const retrieved = deduplicator.getContent(ref.$ref);
-
-  // Check that non-empty User: entries are preserved
-  if (!retrieved.includes("User: Can you explain this?")) {
-    console.error("✗ FAIL: Non-empty User: entry was removed!");
-    return false;
-  }
-
-  if (!retrieved.includes("User: Another question here.")) {
-    console.error("✗ FAIL: Non-empty User: entry was removed!");
-    return false;
-  }
-
-  // Check that empty User: entries are removed
-  const lines = retrieved.split('\n');
-  let hasEmptyUser = false;
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i].trim();
-    if (line === 'User:' || line === 'User: ') {
-      const nextLine = i + 1 < lines.length ? lines[i + 1].trim() : '';
-      if (nextLine === '' || nextLine === 'Claude:' || nextLine === 'User:') {
-        hasEmptyUser = true;
-        break;
-      }
-    }
-  }
-
-  if (hasEmptyUser) {
-    console.error("✗ FAIL: Empty User: entries not removed from mixed content!");
-    return false;
-  }
-
-  console.log("✓ PASS: Non-empty User: entries preserved, empty ones removed");
-
-  return true;
-}
-
-// Main test runner
-async function runTests() {
-  console.log("=".repeat(60));
-  console.log("LLM Audit Log Deduplication Test Suite");
-  console.log("=".repeat(60));
-
-  // Clean up before tests
-  cleanup();
-
-  const tests = [
-    testBasicDeduplication,
-    testContentRestoration,
-    testEntryProcessing,
-    testSizeCalculation,
-    testContentSanitization,
-    testSanitizationPreservesContent,
-  ];
-
-  let passed = 0;
-  let failed = 0;
-
-  for (const test of tests) {
-    try {
-      const result = test();
-      if (result) {
-        passed++;
-      } else {
-        failed++;
-      }
-    } catch (err) {
-      console.error(`✗ Test failed with error: ${err.message}`);
-      console.error(err.stack);
-      failed++;
-    }
-  }
-
-  // Run async test separately
-  setTimeout(() => {
-    testDictionaryPersistence();
-
-    console.log("\n" + "=".repeat(60));
-    console.log("Test Results");
-    console.log("=".repeat(60));
-    console.log(`Passed: ${passed}/${passed + failed}`);
-    console.log(`Failed: ${failed}/${passed + failed}`);
-
-    if (failed === 0) {
-      console.log("\n✓ All tests passed!");
-      console.log("\nDictionary file created at:", TEST_DICT_PATH);
-      console.log("You can inspect it with: cat", TEST_DICT_PATH);
-    } else {
-      console.log("\n✗ Some tests failed!");
-      process.exit(1);
-    }
-
-    // Clean up after tests
-    console.log("\nCleaning up test files...");
-    cleanup();
-    console.log("✓ Test files removed");
-  }, 200);
-}
-
-// Run tests
-if (require.main === module) {
-  runTests().catch((err) => {
-    console.error("Test suite failed:", err);
-    process.exit(1);
-  });
-}
-
-module.exports = { runTests };

package/test/README.md

@@ -1,212 +0,0 @@
-# Test Suite Documentation
-
-All tests for the Lynkr project are consolidated in this `test/` directory.
-
-## Test Files
-
-### Unit Tests
-**File**: `routing.test.js`
-**Purpose**: Tests the hybrid routing logic in isolation
-**Run**: `DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/routing.test.js`
-**Coverage**: 10 tests
-- Routing with PREFER_OLLAMA disabled
-- Simple requests → Ollama
-- Complex requests → Cloud
-- Tool capability checks
-- Fallback configuration
-
----
-
-### Integration Tests
-**File**: `hybrid-routing-integration.test.js`
-**Purpose**: Tests configuration validation and metrics recording
-**Run**: `node --test test/hybrid-routing-integration.test.js`
-**Coverage**: 13 tests
-- Configuration validation (5 tests)
-- Metrics recording (6 tests)
-- Helper functions (2 tests)
-
----
-
-### Hybrid Routing Performance Tests
-**File**: `hybrid-routing-performance.test.js`
-**Purpose**: Measures performance overhead of hybrid routing
-**Run**: `node test/hybrid-routing-performance.test.js`
-**Key Metrics**:
-- Routing decision: <0.01ms (36.8M decisions/sec)
-- Metrics overhead: <0.01ms (43.6M ops/sec)
-- Combined overhead: <0.02ms (15.6M ops/sec)
-
----
-
-### System Performance Tests
-**File**: `performance-tests.js`
-**Purpose**: Tests system-wide performance optimizations
-**Run**: `DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-tests.js`
-**Coverage**:
-- Database indexes (100% complete)
-- Persistent prompt cache
-- Regex pattern caching (4.5x faster)
-- Lazy loading
-- HTTP connection pooling
-- Response compression
-
----
-
-### Middleware Benchmarks
-**File**: `performance-benchmark.js`
-**Purpose**: Benchmarks middleware overhead
-**Run**: `DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-benchmark.js`
-**Coverage**:
-- Metrics collection (3.4M ops/sec)
-- Circuit breakers (3.9M ops/sec)
-- Input validation (5.7M ops/sec)
-- Load shedding
-- Combined middleware stack
-
----
-
-## Running All Tests
-
-### Using npm scripts (Recommended)
-
-**Run all tests (unit + performance):**
-```bash
-npm test
-```
-
-**Run only unit/integration tests:**
-```bash
-npm run test:unit
-```
-
-**Run only performance tests:**
-```bash
-npm run test:performance
-```
-
-**Run only benchmarks:**
-```bash
-npm run test:benchmark
-```
-
-**Run quick smoke test (routing only):**
-```bash
-npm run test:quick
-```
-
-**Run everything including benchmarks:**
-```bash
-npm run test:all
-```
-
-### Manual execution (if needed)
-
-**Quick Test:**
-Run all unit and integration tests:
-```bash
-DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/
-```
-
-**Full Test Suite:**
-Run everything including performance tests:
-```bash
-# Unit + Integration tests
-DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/routing.test.js
-DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/hybrid-routing-integration.test.js
-
-# Performance tests
-node test/hybrid-routing-performance.test.js
-DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-tests.js
-DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-benchmark.js
-```
-
----
-
-## Test Organization
-
-```
-test/
-├── README.md                              ← This file
-├── routing.test.js                        ← Unit tests (10 tests)
-├── hybrid-routing-integration.test.js     ← Integration tests (13 tests)
-├── hybrid-routing-performance.test.js     ← Routing performance benchmarks
-├── performance-tests.js                   ← System performance tests
-└── performance-benchmark.js               ← Middleware benchmarks
-```
-
----
-
-## Important Notes
-
-### Not Test Files
-- `src/tests/` - This is **application code**, not tests! It provides test execution functionality as a feature.
-
-### Environment Variables
-Most tests require Databricks credentials (even though they're not used in actual API calls):
-```bash
-export DATABRICKS_API_KEY=test-key
-export DATABRICKS_API_BASE=http://test.com
-```
-
-### Test Results Summary
-
-| Test Type | Status | Count |
-|-----------|--------|-------|
-| Unit Tests | ✅ Passing | 10/10 |
-| Integration Tests | ✅ Passing | 13/13 |
-| Routing Performance | ✅ Complete | <0.02ms overhead |
-| System Performance | ✅ Complete | 100% optimizations |
-| Middleware Benchmarks | ✅ Complete | Acceptable overhead |
-
----
-
-## CI/CD Integration
-
-To run in CI/CD pipelines:
-
-```bash
-#!/bin/bash
-set -e
-
-# Run all tests using npm
-echo "Running test suite..."
-npm run test:all
-
-echo "All tests passed!"
-```
-
-Or for a faster CI pipeline (skip benchmarks):
-
-```bash
-#!/bin/bash
-set -e
-
-echo "Running tests..."
-npm test
-
-echo "Tests passed!"
-```
-
----
-
-## Adding New Tests
-
-When adding new tests, follow these conventions:
-
-1. **Unit tests**: Test individual functions in isolation
-   - Place in `test/`
-   - Use `node:test` framework
-   - Name: `feature-name.test.js`
-
-2. **Integration tests**: Test multiple components together
-   - Place in `test/`
-   - Use `node:test` framework
-   - Name: `feature-name-integration.test.js`
-
-3. **Performance tests**: Benchmark specific features
-   - Place in `test/`
-   - Use custom benchmark utilities
-   - Name: `feature-name-performance.test.js`
-
-4. **Always**: Document in this README!