@twelvehart/supermemory-runtime 1.0.0-next.0

package/scripts/migrations/test_hnsw_index.sql

@@ -0,0 +1,255 @@
+-- Test Script: test_hnsw_index.sql
+-- Description: Comprehensive testing suite for HNSW index performance
+-- Related: TASK-005 from BACKLOG.md
+-- Created: 2026-02-02
+
+-- ============================================================================
+-- TEST 1: Verify HNSW Index Creation
+-- ============================================================================
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1
+        FROM pg_indexes
+        WHERE tablename = 'memory_embeddings'
+        AND indexname = 'idx_memory_embeddings_hnsw'
+    ) THEN
+        RAISE EXCEPTION 'HNSW index idx_memory_embeddings_hnsw not found';
+    END IF;
+
+    RAISE NOTICE 'TEST 1 PASSED: HNSW index exists';
+END $$;
+
+-- ============================================================================
+-- TEST 2: Verify Index Uses HNSW Access Method
+-- ============================================================================
+DO $$
+DECLARE
+    index_method TEXT;
+BEGIN
+    SELECT am.amname INTO index_method
+    FROM pg_class c
+    JOIN pg_am am ON c.relam = am.oid
+    WHERE c.relname = 'idx_memory_embeddings_hnsw';
+
+    IF index_method != 'hnsw' THEN
+        RAISE EXCEPTION 'Index is not using HNSW access method (found: %)', index_method;
+    END IF;
+
+    RAISE NOTICE 'TEST 2 PASSED: Index uses HNSW access method';
+END $$;
+
+-- ============================================================================
+-- TEST 3: Verify HNSW Parameters (m=16, ef_construction=64)
+-- ============================================================================
+DO $$
+DECLARE
+    index_options TEXT;
+BEGIN
+    SELECT pg_get_indexdef(indexrelid, 0, true) INTO index_options
+    FROM pg_stat_user_indexes
+    WHERE indexrelname = 'idx_memory_embeddings_hnsw';
+
+    IF index_options NOT LIKE '%m=16%' THEN
+        RAISE WARNING 'Expected m=16 in index options';
+    END IF;
+
+    IF index_options NOT LIKE '%ef_construction=64%' THEN
+        RAISE WARNING 'Expected ef_construction=64 in index options';
+    END IF;
+
+    RAISE NOTICE 'TEST 3 PASSED: HNSW parameters configured (m=16, ef_construction=64)';
+    RAISE NOTICE 'Index definition: %', index_options;
+END $$;
+
+-- ============================================================================
+-- TEST 4: Verify Query Uses Index Scan
+-- ============================================================================
+-- Create a sample vector for testing
+DO $$
+DECLARE
+    explain_output TEXT := '';
+    plan_row RECORD;
+    sample_vector vector(1536);
+BEGIN
+    sample_vector := array_fill(0.1, ARRAY[1536])::vector;
+
+    FOR plan_row IN EXECUTE format(
+        'EXPLAIN (FORMAT TEXT) SELECT memory_id, 1 - (embedding <=> %L::vector) as similarity FROM memory_embeddings ORDER BY embedding <=> %L::vector LIMIT 10',
+        sample_vector::text, sample_vector::text
+    )
+    LOOP
+        explain_output := explain_output || plan_row."QUERY PLAN" || E'\n';
+    END LOOP;
+
+    IF explain_output LIKE '%Index Scan using idx_memory_embeddings_hnsw%' THEN
+        RAISE NOTICE 'TEST 4 PASSED: Query uses HNSW index scan';
+    ELSE
+        RAISE WARNING 'TEST 4 WARNING: Query may not be using HNSW index';
+        RAISE NOTICE 'Explain plan: %', explain_output;
+    END IF;
+END $$;
+
+-- ============================================================================
+-- TEST 5: Performance Benchmark (<100ms for 10K vectors)
+-- ============================================================================
+-- This test requires data in the table
+-- Run after inserting test data
+
+CREATE OR REPLACE FUNCTION run_hnsw_performance_test(
+    num_queries INTEGER DEFAULT 10
+)
+RETURNS TABLE (
+    query_num INTEGER,
+    execution_time_ms NUMERIC,
+    results_returned INTEGER,
+    status TEXT
+) AS $$
+DECLARE
+    i INTEGER;
+    start_time TIMESTAMPTZ;
+    end_time TIMESTAMPTZ;
+    exec_time NUMERIC;
+    result_count INTEGER;
+    sample_vector vector(1536);
+    row_count BIGINT;
+BEGIN
+    -- Check if table has data
+    SELECT COUNT(*) INTO row_count FROM memory_embeddings;
+
+    IF row_count = 0 THEN
+        RAISE NOTICE 'WARNING: No data in memory_embeddings table. Skipping performance test.';
+        RETURN;
+    END IF;
+
+    RAISE NOTICE 'Running % test queries on % embeddings...', num_queries, row_count;
+
+    FOR i IN 1..num_queries LOOP
+        -- Generate random test vector
+        sample_vector := (
+            SELECT array_agg(random()::REAL)::vector
+            FROM generate_series(1, 1536)
+        );
+
+        -- Measure query execution time
+        start_time := clock_timestamp();
+
+        SELECT COUNT(*) INTO result_count
+        FROM (
+            SELECT memory_id
+            FROM memory_embeddings
+            ORDER BY embedding <=> sample_vector
+            LIMIT 10
+        ) results;
+
+        end_time := clock_timestamp();
+        exec_time := EXTRACT(MILLISECONDS FROM (end_time - start_time));
+
+        RETURN QUERY SELECT
+            i AS query_num,
+            exec_time AS execution_time_ms,
+            result_count AS results_returned,
+            CASE
+                WHEN exec_time < 100 THEN 'PASS'
+                WHEN exec_time < 200 THEN 'WARNING'
+                ELSE 'FAIL'
+            END AS status;
+    END LOOP;
+
+    RETURN;
+END;
+$$ LANGUAGE plpgsql;
+
+-- ============================================================================
+-- TEST 6: Recall Accuracy Test (~99%)
+-- ============================================================================
+-- This test compares HNSW approximate results with exact results
+
+CREATE OR REPLACE FUNCTION test_hnsw_recall_accuracy(
+    num_samples INTEGER DEFAULT 5
+)
+RETURNS TABLE (
+    sample_num INTEGER,
+    recall_percentage NUMERIC,
+    status TEXT
+) AS $$
+DECLARE
+    i INTEGER;
+    sample_vector vector(1536);
+    exact_ids UUID[];
+    approx_ids UUID[];
+    matches INTEGER;
+    recall NUMERIC;
+BEGIN
+    FOR i IN 1..num_samples LOOP
+        -- Generate random test vector
+        sample_vector := (
+            SELECT array_agg(random()::REAL)::vector
+            FROM generate_series(1, 1536)
+        );
+
+        -- Get exact results (sequential scan, no index)
+        SELECT array_agg(memory_id ORDER BY distance) INTO exact_ids
+        FROM (
+            SELECT memory_id, embedding <=> sample_vector AS distance
+            FROM memory_embeddings
+            ORDER BY distance
+            LIMIT 10
+        ) exact;
+
+        -- Get approximate results (HNSW index)
+        SELECT array_agg(memory_id ORDER BY distance) INTO approx_ids
+        FROM (
+            SELECT memory_id, embedding <=> sample_vector AS distance
+            FROM memory_embeddings
+            ORDER BY distance
+            LIMIT 10
+        ) approx;
+
+        -- Calculate recall (percentage of exact results found in approximate results)
+        SELECT COUNT(*) INTO matches
+        FROM unnest(exact_ids) exact_id
+        WHERE exact_id = ANY(approx_ids);
+
+        recall := (matches::NUMERIC / COALESCE(array_length(exact_ids, 1), 1)) * 100;
+
+        RETURN QUERY SELECT
+            i AS sample_num,
+            recall AS recall_percentage,
+            CASE
+                WHEN recall >= 99 THEN 'PASS'
+                WHEN recall >= 95 THEN 'WARNING'
+                ELSE 'FAIL'
+            END AS status;
+    END LOOP;
+
+    RETURN;
+END;
+$$ LANGUAGE plpgsql;
+
+-- ============================================================================
+-- Run All Tests
+-- ============================================================================
+DO $$
+BEGIN
+    RAISE NOTICE '========================================';
+    RAISE NOTICE 'HNSW Index Test Suite';
+    RAISE NOTICE '========================================';
+    RAISE NOTICE 'Running structural tests...';
+END $$;
+
+-- Tests 1-4 run automatically above
+
+-- Note for performance tests:
+\echo ''
+\echo 'To run performance tests (requires data):'
+\echo 'SELECT * FROM run_hnsw_performance_test(10);'
+\echo ''
+\echo 'To test recall accuracy:'
+\echo 'SELECT * FROM test_hnsw_recall_accuracy(5);'
+\echo ''
+\echo 'To check current ef_search setting:'
+\echo 'SHOW hnsw.ef_search;'
+\echo ''
+\echo 'To adjust search quality:'
+\echo "SELECT set_hnsw_search_quality('balanced');"

package/scripts/pre-commit-secrets

@@ -0,0 +1,282 @@
+#!/usr/bin/env node
+
+/**
+ * Git Pre-Commit Hook: Secrets Detection
+ *
+ * Scans staged files for potential secrets and blocks commits if found.
+ * This prevents accidental secret leakage into version control.
+ *
+ * Installation:
+ *   cp scripts/pre-commit-secrets .git/hooks/pre-commit
+ *   chmod +x .git/hooks/pre-commit
+ *
+ * Or use Husky:
+ *   npx husky add .husky/pre-commit "node scripts/pre-commit-secrets"
+ */
+
+import { execSync } from 'child_process';
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+
+// ============================================================================
+// Secret Detection Patterns
+// ============================================================================
+
+const SECRET_PATTERNS = [
+  {
+    name: 'Generic API Key',
+    pattern: /(?:api[_-]?key|apikey)[=:\s]+['"]?([a-z0-9_-]{20,})/gi,
+    description: 'Looks like an API key',
+  },
+  {
+    name: 'Bearer Token',
+    pattern: /bearer\s+([a-z0-9_.-]+)/gi,
+    description: 'Bearer token detected',
+  },
+  {
+    name: 'AWS Access Key',
+    pattern: /AKIA[0-9A-Z]{16}/g,
+    description: 'AWS access key detected',
+  },
+  {
+    name: 'AWS Secret Key',
+    pattern: /aws[_-]?secret[_-]?access[_-]?key[=:\s]+['"]?([a-z0-9/+=]{40})/gi,
+    description: 'AWS secret key detected',
+  },
+  {
+    name: 'Private Key',
+    pattern: /-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----/i,
+    description: 'Private key detected',
+  },
+  {
+    name: 'Database URL with Credentials',
+    pattern: /(?:postgres|mysql|mongodb):\/\/([^:]+):([^@]+)@/gi,
+    description: 'Database URL with embedded credentials',
+  },
+  {
+    name: 'Password',
+    pattern: /(?:password|passwd|pwd)[=:\s]+['"]?([^\s'"]{8,})/gi,
+    description: 'Password detected',
+  },
+  {
+    name: 'JWT Token',
+    pattern: /eyJ[a-zA-Z0-9_-]+\.eyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+/g,
+    description: 'JWT token detected',
+  },
+  {
+    name: 'Generic Secret',
+    pattern: /secret[_-]?key[=:\s]+['"]?([a-z0-9_.-]{20,})/gi,
+    description: 'Secret key detected',
+  },
+  {
+    name: 'Anthropic API Key',
+    pattern: /sk-ant-[a-zA-Z0-9-_]{95,}/g,
+    description: 'Anthropic API key detected',
+  },
+  {
+    name: 'OpenAI API Key',
+    pattern: /sk-[a-zA-Z0-9]{48,}/g,
+    description: 'OpenAI API key detected',
+  },
+  {
+    name: 'Stripe API Key',
+    pattern: /sk_(live|test)_[a-zA-Z0-9]{24,}/g,
+    description: 'Stripe API key detected',
+  },
+  {
+    name: 'Google API Key',
+    pattern: /AIza[0-9A-Za-z_-]{35}/g,
+    description: 'Google API key detected',
+  },
+];
+
+// ============================================================================
+// Excluded Files/Patterns
+// ============================================================================
+
+const EXCLUDED_PATTERNS = [
+  // Documentation and examples
+  /\.md$/i,
+  /\.txt$/i,
+  /example/i,
+  /sample/i,
+  /template/i,
+  /\.example$/i,
+
+  // Test files (may contain fake secrets)
+  /\.test\.(ts|js)$/i,
+  /\.spec\.(ts|js)$/i,
+  /__tests__\//i,
+  /test\//i,
+  /tests\//i,
+
+  // Generated files
+  /dist\//i,
+  /build\//i,
+  /node_modules\//i,
+
+  // Lock files
+  /package-lock\.json$/i,
+  /yarn\.lock$/i,
+  /pnpm-lock\.yaml$/i,
+
+  // This file itself
+  /pre-commit-secrets$/i,
+];
+
+// Files that should ALWAYS be checked (override exclusions)
+const FORCE_CHECK_PATTERNS = [
+  /\.env$/i,
+  /\.env\.[^.]+$/i,
+];
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/**
+ * Check if a file should be excluded from scanning
+ */
+function shouldExcludeFile(filename) {
+  // Force check certain files
+  if (FORCE_CHECK_PATTERNS.some((pattern) => pattern.test(filename))) {
+    return false;
+  }
+
+  // Exclude matching patterns
+  return EXCLUDED_PATTERNS.some((pattern) => pattern.test(filename));
+}
+
+/**
+ * Get staged files from git
+ */
+function getStagedFiles() {
+  try {
+    const output = execSync('git diff --cached --name-only --diff-filter=ACM', {
+      encoding: 'utf-8',
+    });
+    return output.trim().split('\n').filter(Boolean);
+  } catch (error) {
+    console.error('Error getting staged files:', error.message);
+    return [];
+  }
+}
+
+/**
+ * Scan a file for secrets
+ */
+function scanFile(filename) {
+  const findings = [];
+
+  // Read file content
+  let content;
+  try {
+    content = readFileSync(resolve(process.cwd(), filename), 'utf-8');
+  } catch (error) {
+    console.warn(`Warning: Could not read ${filename}: ${error.message}`);
+    return findings;
+  }
+
+  // Scan with each pattern
+  for (const { name, pattern, description } of SECRET_PATTERNS) {
+    const matches = content.matchAll(pattern);
+
+    for (const match of matches) {
+      // Get line number
+      const beforeMatch = content.slice(0, match.index);
+      const lineNumber = beforeMatch.split('\n').length;
+
+      // Get line content (truncated)
+      const lines = content.split('\n');
+      const lineContent = lines[lineNumber - 1].trim().slice(0, 80);
+
+      findings.push({
+        file: filename,
+        line: lineNumber,
+        type: name,
+        description,
+        content: lineContent,
+        matched: match[0].slice(0, 40) + (match[0].length > 40 ? '...' : ''),
+      });
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * Print findings in a readable format
+ */
+function printFindings(findings) {
+  console.log('\n❌ COMMIT BLOCKED: Potential secrets detected!\n');
+
+  const groupedByFile = findings.reduce((acc, finding) => {
+    if (!acc[finding.file]) {
+      acc[finding.file] = [];
+    }
+    acc[finding.file].push(finding);
+    return acc;
+  }, {});
+
+  for (const [file, fileFindings] of Object.entries(groupedByFile)) {
+    console.log(`📄 ${file}:`);
+    for (const finding of fileFindings) {
+      console.log(`  Line ${finding.line}: ${finding.type}`);
+      console.log(`    ${finding.description}`);
+      console.log(`    "${finding.content}"`);
+      console.log('');
+    }
+  }
+
+  console.log('🔒 Security Recommendations:\n');
+  console.log('1. Remove the secret from your code');
+  console.log('2. Add the file to .gitignore if it contains secrets');
+  console.log('3. Use environment variables instead (see .env.example)');
+  console.log('4. If this is a false positive, add the pattern to the exclusion list\n');
+
+  console.log('💡 Suggested .gitignore entries:\n');
+  const uniqueFiles = [...new Set(findings.map((f) => f.file))];
+  for (const file of uniqueFiles) {
+    console.log(`  ${file}`);
+  }
+  console.log('');
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+function main() {
+  console.log('🔍 Scanning staged files for secrets...\n');
+
+  const stagedFiles = getStagedFiles();
+
+  if (stagedFiles.length === 0) {
+    console.log('✅ No files to scan\n');
+    process.exit(0);
+  }
+
+  // Filter out excluded files
+  const filesToScan = stagedFiles.filter((file) => !shouldExcludeFile(file));
+
+  console.log(`Scanning ${filesToScan.length} of ${stagedFiles.length} staged files...\n`);
+
+  // Scan each file
+  const allFindings = [];
+  for (const file of filesToScan) {
+    const findings = scanFile(file);
+    allFindings.push(...findings);
+  }
+
+  if (allFindings.length > 0) {
+    printFindings(allFindings);
+    console.log('❌ Commit rejected due to detected secrets\n');
+    process.exit(1);
+  }
+
+  console.log('✅ No secrets detected. Proceeding with commit.\n');
+  process.exit(0);
+}
+
+// Run the hook
+main();

package/scripts/run-extraction-worker.ts

@@ -0,0 +1,46 @@
+#!/usr/bin/env tsx
+/**
+ * Extraction Worker Runner
+ *
+ * Starts the extraction worker to process documents from the queue.
+ * Run with: npx tsx scripts/run-extraction-worker.ts
+ */
+
+import { createExtractionWorker } from '../src/workers/extraction.worker.js';
+import IORedis from 'ioredis';
+import dotenv from 'dotenv';
+
+// Load environment variables
+dotenv.config();
+
+// Redis connection
+const connection = new IORedis({
+  host: process.env.REDIS_HOST || 'localhost',
+  port: parseInt(process.env.REDIS_PORT || '6379', 10),
+  maxRetriesPerRequest: null,
+  enableReadyCheck: false,
+});
+
+console.log('[ExtractionWorker] Starting worker...');
+console.log(`[ExtractionWorker] Redis: ${connection.options.host}:${connection.options.port}`);
+console.log(
+  `[ExtractionWorker] Concurrency: ${process.env.BULLMQ_CONCURRENCY_EXTRACTION || 5}`
+);
+
+// Create and start worker
+const worker = createExtractionWorker(connection);
+
+// Handle shutdown
+const shutdown = async () => {
+  console.log('\n[ExtractionWorker] Shutting down gracefully...');
+  await worker.close();
+  await connection.quit();
+  console.log('[ExtractionWorker] Shutdown complete');
+  process.exit(0);
+};
+
+process.on('SIGTERM', shutdown);
+process.on('SIGINT', shutdown);
+
+console.log('[ExtractionWorker] Worker started successfully. Waiting for jobs...');
+console.log('[ExtractionWorker] Press Ctrl+C to stop');