@twelvehart/supermemory-runtime 1.0.0-next.0

package/scripts/mcp-setup.ts

@@ -0,0 +1,271 @@
+#!/usr/bin/env tsx
+import { execSync } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { resolve } from 'node:path';
+import pkg from 'pg';
+import { loadEnvFile } from '../src/config/env.js';
+import {
+  checkClaudeMcpRegistration,
+  type ClaudeMcpScope,
+} from './claude-mcp-config.js';
+
+const { Client } = pkg;
+
+function parseEnv(raw: string): Record<string, string> {
+  const env: Record<string, string> = {};
+  for (const line of raw.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const idx = trimmed.indexOf('=');
+    if (idx <= 0) continue;
+    const key = trimmed.slice(0, idx).trim();
+    const value = trimmed.slice(idx + 1).trim();
+    env[key] = value;
+  }
+  return env;
+}
+
+function ask(question: string): Promise<string> {
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  return new Promise((resolve) => {
+    rl.question(question, (answer) => {
+      rl.close();
+      resolve(answer.trim());
+    });
+  });
+}
+
+function validateScope(scope: string): ClaudeMcpScope {
+  if (scope === 'user' || scope === 'project' || scope === 'local') {
+    return scope;
+  }
+
+  throw new Error(`Invalid scope: ${scope} (expected: user, project, or local)`);
+}
+
+function formatRegistrationCommand(scope: ClaudeMcpScope, entryPoint: string): string {
+  return `claude mcp add supermemory --scope ${scope} -- node ${JSON.stringify(entryPoint)}`;
+}
+
+function formatRemovalCommand(scope: ClaudeMcpScope): string {
+  return `claude mcp remove --scope ${scope} supermemory`;
+}
+
+function commandExists(name: string): boolean {
+  try {
+    execSync(`command -v ${name}`, { stdio: 'ignore', shell: '/bin/zsh' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function askScope(): Promise<ClaudeMcpScope> {
+  return ask('\nRegister for "user", "project", or "local" scope? [user] ').then((answer) => {
+    const normalized = answer.toLowerCase();
+    if (!normalized) return 'user';
+    return validateScope(normalized);
+  });
+}
+
+function parseArgs(): {
+  envFile?: string;
+  nonInteractive: boolean;
+  registerMcp: boolean;
+  scope?: ClaudeMcpScope;
+  skipMcp: boolean;
+} {
+  const args = process.argv.slice(2);
+  let envFile: string | undefined;
+  let nonInteractive = false;
+  let registerMcp = false;
+  let scope: ClaudeMcpScope | undefined;
+  let skipMcp = false;
+
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+
+    if (arg === '--env-file') {
+      const value = args[index + 1];
+      if (!value) {
+        throw new Error('--env-file requires a value');
+      }
+      envFile = value;
+      index += 1;
+      continue;
+    }
+
+    if (arg.startsWith('--env-file=')) {
+      envFile = arg.slice('--env-file='.length);
+      continue;
+    }
+
+    if (arg === '--scope') {
+      const value = args[index + 1];
+      if (!value) {
+        throw new Error('--scope requires a value');
+      }
+      scope = validateScope(value.toLowerCase());
+      index += 1;
+      continue;
+    }
+
+    if (arg.startsWith('--scope=')) {
+      scope = validateScope(arg.slice('--scope='.length).toLowerCase());
+      continue;
+    }
+
+    if (arg === '--register-mcp') {
+      registerMcp = true;
+      continue;
+    }
+
+    if (arg === '--skip-mcp' || arg === '--skip-claude') {
+      skipMcp = true;
+      continue;
+    }
+
+    if (arg === '--non-interactive') {
+      nonInteractive = true;
+      continue;
+    }
+
+    throw new Error(`Unknown argument: ${arg}`);
+  }
+
+  return { envFile, scope, registerMcp, skipMcp, nonInteractive };
+}
+
+async function run(): Promise<void> {
+  const { envFile, nonInteractive, registerMcp, scope, skipMcp } = parseArgs();
+  console.log('Supermemory MCP Setup\n');
+
+  // Load .env if present
+  let env: Record<string, string> = {};
+  const envResolution = loadEnvFile({ cliEnvFile: envFile });
+  if (envResolution.exists && existsSync(envResolution.path)) {
+    env = parseEnv(readFileSync(envResolution.path, 'utf-8'));
+    for (const [k, v] of Object.entries(env)) {
+      if (!process.env[k]) process.env[k] = v;
+    }
+    console.log(`[OK] Using env file: ${envResolution.path}`);
+  } else if (envResolution.explicit) {
+    console.log(`[WARN] Env file not found at ${envResolution.path}; falling back to current process environment`);
+  }
+
+  // Step 1: Check for built MCP entry point
+  const entryPoint = resolve('dist/mcp/index.js');
+  if (!existsSync(entryPoint)) {
+    console.log(`Build output not found at ${entryPoint}`);
+    const answer = await ask('Run "npm run build" now? [Y/n] ');
+    if (answer === '' || answer.toLowerCase() === 'y') {
+      console.log('Building...');
+      try {
+        execSync('npm run build', { stdio: 'inherit' });
+      } catch {
+        console.error('Build failed. Fix errors and try again.');
+        process.exit(1);
+      }
+    } else {
+      console.log('Skipping build. The MCP server may not work without a build.');
+    }
+  } else {
+    console.log('[OK] Build output found');
+  }
+
+  // Step 2: Quick Postgres connectivity check
+  const databaseUrl = env.DATABASE_URL || process.env.DATABASE_URL || '';
+  if (databaseUrl) {
+    const client = new Client({ connectionString: databaseUrl });
+    try {
+      await client.connect();
+      await client.query('SELECT 1');
+      console.log('[OK] PostgreSQL connection successful');
+    } catch (error) {
+      console.log(
+        `[WARN] PostgreSQL connection failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+      console.log('       The MCP server needs Postgres at runtime. Check DATABASE_URL.');
+    } finally {
+      await client.end().catch(() => undefined);
+    }
+  } else {
+    console.log('[WARN] DATABASE_URL not set. The MCP server will need it at runtime.');
+  }
+
+  if (!commandExists('claude')) {
+    console.error(
+      '\nCould not find the "claude" CLI.\nInstall Claude Code first: https://docs.anthropic.com/en/docs/claude-code'
+    );
+    process.exit(1);
+  }
+
+  if (skipMcp) {
+    console.log('[WARN] Skipping MCP registration by request');
+    return;
+  }
+
+  let selectedScope = scope;
+  if (!selectedScope && !nonInteractive) {
+    selectedScope = await askScope();
+  }
+
+  if (!selectedScope) {
+    console.log('[WARN] Non-interactive mode requires --scope or --register-mcp to perform Claude MCP registration');
+    return;
+  }
+
+  if (nonInteractive && !registerMcp && !scope) {
+    console.log('[WARN] Non-interactive mode skipped Claude MCP registration because no explicit scope or --register-mcp flag was provided');
+    return;
+  }
+
+  const registrationCheck = checkClaudeMcpRegistration({
+    scope: selectedScope,
+    name: 'supermemory',
+    expectedCommand: 'node',
+    expectedArgs: [entryPoint],
+  });
+
+  if (registrationCheck.status === 'match') {
+    console.log(`[OK] Supermemory is already registered in ${selectedScope} scope with the expected command path`);
+    return;
+  }
+
+  const cmd = formatRegistrationCommand(selectedScope, entryPoint);
+  if (registrationCheck.status === 'mismatch') {
+    const removeCmd = formatRemovalCommand(selectedScope);
+    console.log(`[INFO] Existing ${selectedScope} scope registration does not match the current build output; repairing with: ${removeCmd} && ${cmd}`);
+    try {
+      execSync(removeCmd, { stdio: 'inherit', shell: '/bin/zsh' });
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : String(error);
+      console.error(`\nCould not remove the existing ${selectedScope} scope registration: ${msg}`);
+      process.exit(1);
+    }
+  } else {
+    console.log(`[INFO] No ${selectedScope} scope registration found; registering with: ${cmd}`);
+  }
+
+  try {
+    execSync(`claude mcp add supermemory --scope ${selectedScope} -- node ${JSON.stringify(entryPoint)}`, {
+      stdio: 'inherit',
+      shell: '/bin/zsh',
+    });
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    console.error(`\nRegistration failed: ${msg}`);
+    process.exit(1);
+  }
+
+  // Step 5: Success
+  console.log('\nSupermemory MCP server registered successfully!');
+  console.log(`Scope: ${selectedScope}`);
+  console.log('\nVerify with:  claude mcp get supermemory');
+}
+
+run().catch((error) => {
+  console.error('Setup failed:', error);
+  process.exit(1);
+});

package/scripts/migrations/001_create_pgvector_extension.sql

@@ -0,0 +1,31 @@
+-- Migration: 001_create_pgvector_extension.sql
+-- Description: Enable pgvector extension for PostgreSQL
+-- Dependencies: None (requires PostgreSQL 12+ with pgvector installed)
+-- Created: 2026-02-02
+
+-- Enable pgvector extension
+CREATE EXTENSION IF NOT EXISTS vector;
+
+-- Verify installation
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_extension WHERE extname = 'vector'
+    ) THEN
+        RAISE EXCEPTION 'pgvector extension failed to install';
+    END IF;
+END $$;
+
+-- Test basic vector operations
+DO $$
+DECLARE
+    test_distance FLOAT;
+BEGIN
+    -- Test euclidean distance
+    SELECT '[1,2,3]'::vector <-> '[4,5,6]'::vector INTO test_distance;
+
+    -- Test cosine distance
+    SELECT '[1,2,3]'::vector <=> '[4,5,6]'::vector INTO test_distance;
+
+    RAISE NOTICE 'pgvector extension verified successfully';
+END $$;

package/scripts/migrations/002_create_memory_embeddings_table.sql

@@ -0,0 +1,75 @@
+-- SUPERSEDED: This migration is superseded by the Drizzle ORM migration
+-- (drizzle/0000_dapper_the_professor.sql). The canonical memory_embeddings
+-- schema uses memory_id as the sole primary key and has no id, chunk_id,
+-- dimensions, or updated_at columns. Do NOT run this migration.
+--
+-- Migration: 002_create_memory_embeddings_table.sql
+-- Description: Create memory_embeddings table with vector support
+-- Dependencies: 001_create_pgvector_extension.sql
+-- Created: 2026-02-02
+-- Related: TASK-002 from BACKLOG.md
+
+-- Create memory_embeddings table
+-- This table stores vector embeddings for semantic search
+-- Based on the architecture research and SQLite schema
+
+CREATE TABLE IF NOT EXISTS memory_embeddings (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    chunk_id UUID NOT NULL,
+    memory_id UUID NOT NULL,
+
+    -- Vector embedding (default dimensions: 1536 for text-embedding-3-small)
+    -- Adjust dimensions based on your embedding model
+    embedding vector(1536) NOT NULL,
+
+    -- Metadata
+    model VARCHAR(255) NOT NULL,
+    dimensions INTEGER NOT NULL CHECK (dimensions > 0),
+
+    -- Timestamps
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT fk_chunk FOREIGN KEY (chunk_id)
+        REFERENCES chunks(id)
+        ON DELETE CASCADE,
+    CONSTRAINT fk_memory FOREIGN KEY (memory_id)
+        REFERENCES memories(id)
+        ON DELETE CASCADE,
+    CONSTRAINT check_dimensions_match
+        CHECK (dimensions = vector_dims(embedding))
+);
+
+-- Create standard indexes for foreign keys and lookups
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_chunk_id
+    ON memory_embeddings(chunk_id);
+
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_memory_id
+    ON memory_embeddings(memory_id);
+
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_model
+    ON memory_embeddings(model);
+
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_created_at
+    ON memory_embeddings(created_at DESC);
+
+-- Add trigger for updated_at timestamp
+CREATE OR REPLACE FUNCTION update_updated_at_column()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = NOW();
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trg_memory_embeddings_updated_at
+    BEFORE UPDATE ON memory_embeddings
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
+-- Add comment for documentation
+COMMENT ON TABLE memory_embeddings IS 'Stores vector embeddings for semantic search with pgvector support';
+COMMENT ON COLUMN memory_embeddings.embedding IS 'Vector embedding for cosine similarity search (default: 1536 dimensions for text-embedding-3-small)';
+COMMENT ON COLUMN memory_embeddings.dimensions IS 'Number of dimensions in the embedding vector';
+COMMENT ON COLUMN memory_embeddings.model IS 'Embedding model used (e.g., text-embedding-3-small, text-embedding-ada-002)';

package/scripts/migrations/003_create_hnsw_index.sql

@@ -0,0 +1,94 @@
+-- Migration: 003_create_hnsw_index.sql
+-- Description: Create HNSW index for fast vector similarity search
+-- Dependencies: 002_create_memory_embeddings_table.sql
+-- Created: 2026-02-02
+-- Related: TASK-005 from BACKLOG.md
+
+-- HNSW (Hierarchical Navigable Small World) Index Configuration
+-- Performance targets from BACKLOG.md:
+-- - Query performance < 100ms for 10K vectors
+-- - ~99% recall accuracy
+-- - Sub-100ms approximate nearest neighbor search
+
+-- Create HNSW index with optimized parameters
+-- m=16: Number of bi-directional links per node (higher = better recall, more memory)
+-- ef_construction=64: Size of dynamic candidate list during construction (higher = better quality, slower build)
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_hnsw
+    ON memory_embeddings
+    USING hnsw (embedding vector_cosine_ops)
+    WITH (
+        m = 16,
+        ef_construction = 64
+    );
+
+-- Set search-time tuning parameter
+-- ef_search controls the size of the dynamic candidate list during search
+-- Higher values = better recall but slower search
+-- Default: 40, Recommended for 99% recall: 100
+-- This can be adjusted per-session based on performance requirements
+
+-- Global setting (applies to all sessions)
+ALTER DATABASE CURRENT SET hnsw.ef_search = 100;
+
+-- Session-level setting (can be adjusted dynamically)
+-- SET hnsw.ef_search = 100;
+
+-- Add statistics for query planning
+ANALYZE memory_embeddings;
+
+-- Create a helper function for optimal search configuration
+CREATE OR REPLACE FUNCTION set_hnsw_search_quality(quality_level TEXT DEFAULT 'balanced')
+RETURNS TEXT AS $$
+BEGIN
+    CASE quality_level
+        WHEN 'fast' THEN
+            -- Fast but lower recall (~95%)
+            EXECUTE 'SET hnsw.ef_search = 40';
+            RETURN 'HNSW search quality set to FAST (ef_search=40, ~95% recall)';
+        WHEN 'balanced' THEN
+            -- Balanced performance and recall (~99%)
+            EXECUTE 'SET hnsw.ef_search = 100';
+            RETURN 'HNSW search quality set to BALANCED (ef_search=100, ~99% recall)';
+        WHEN 'accurate' THEN
+            -- Highest recall (~99.5%+) but slower
+            EXECUTE 'SET hnsw.ef_search = 200';
+            RETURN 'HNSW search quality set to ACCURATE (ef_search=200, ~99.5%+ recall)';
+        ELSE
+            RAISE EXCEPTION 'Invalid quality_level. Use: fast, balanced, or accurate';
+    END CASE;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Add comments for documentation
+COMMENT ON INDEX idx_memory_embeddings_hnsw IS 'HNSW index for fast approximate nearest neighbor search using cosine similarity';
+COMMENT ON FUNCTION set_hnsw_search_quality IS 'Helper function to adjust HNSW search quality (fast/balanced/accurate)';
+
+-- Performance validation query
+-- This query should use the HNSW index for sub-100ms performance
+CREATE OR REPLACE FUNCTION validate_hnsw_performance(
+    query_embedding vector(1536),
+    result_limit INTEGER DEFAULT 10
+)
+RETURNS TABLE (
+    memory_id UUID,
+    similarity FLOAT,
+    execution_time_ms NUMERIC
+) AS $$
+DECLARE
+    start_time TIMESTAMPTZ;
+    end_time TIMESTAMPTZ;
+BEGIN
+    start_time := clock_timestamp();
+
+    RETURN QUERY
+    SELECT
+        me.memory_id,
+        1 - (me.embedding <=> query_embedding) AS similarity,
+        EXTRACT(MILLISECONDS FROM (clock_timestamp() - start_time)) AS execution_time_ms
+    FROM memory_embeddings me
+    ORDER BY me.embedding <=> query_embedding
+    LIMIT result_limit;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION validate_hnsw_performance IS 'Validation function to measure HNSW index performance (target: <100ms for 10K vectors)';

package/scripts/migrations/004_create_memory_embeddings_standalone.sql

@@ -0,0 +1,70 @@
+-- SUPERSEDED: This migration is superseded by the Drizzle ORM migration
+-- (drizzle/0000_dapper_the_professor.sql). The canonical memory_embeddings
+-- schema uses memory_id as the sole primary key and has no id, chunk_id,
+-- dimensions, or updated_at columns. Do NOT run this migration.
+--
+-- Migration: 004_create_memory_embeddings_standalone.sql
+-- Description: Standalone memory_embeddings table for HNSW testing (no FK dependencies)
+-- Created: 2026-02-02
+-- Purpose: TASK-005 HNSW Phase 1 Testing
+
+-- Drop existing table if it has FK constraints
+DROP TABLE IF EXISTS memory_embeddings CASCADE;
+
+-- Create memory_embeddings table WITHOUT foreign key dependencies
+CREATE TABLE memory_embeddings (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    chunk_id UUID NOT NULL,
+    memory_id UUID NOT NULL,
+
+    -- Vector embedding (1536 dimensions for text-embedding-3-small)
+    embedding vector(1536) NOT NULL,
+
+    -- Metadata
+    model VARCHAR(255) NOT NULL DEFAULT 'text-embedding-3-small',
+    dimensions INTEGER NOT NULL DEFAULT 1536 CHECK (dimensions > 0),
+
+    -- Timestamps
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Dimension validation
+    CONSTRAINT check_dimensions_match
+        CHECK (dimensions = vector_dims(embedding))
+);
+
+-- Create standard indexes
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_chunk_id
+    ON memory_embeddings(chunk_id);
+
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_memory_id
+    ON memory_embeddings(memory_id);
+
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_model
+    ON memory_embeddings(model);
+
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_created_at
+    ON memory_embeddings(created_at DESC);
+
+-- Create HNSW index with optimized parameters
+CREATE INDEX IF NOT EXISTS idx_memory_embeddings_hnsw
+    ON memory_embeddings
+    USING hnsw (embedding vector_cosine_ops)
+    WITH (
+        m = 16,
+        ef_construction = 64
+    );
+
+-- Set search-time tuning parameter
+ALTER DATABASE supermemory SET hnsw.ef_search = 100;
+
+-- Add statistics for query planning
+ANALYZE memory_embeddings;
+
+-- Add comments
+COMMENT ON TABLE memory_embeddings IS 'Stores vector embeddings for semantic search with pgvector HNSW support';
+COMMENT ON COLUMN memory_embeddings.embedding IS 'Vector embedding for cosine similarity search (1536 dimensions)';
+COMMENT ON INDEX idx_memory_embeddings_hnsw IS 'HNSW index for fast approximate nearest neighbor search';
+
+-- Confirm creation
+\echo 'Memory embeddings table and HNSW index created successfully'

package/scripts/migrations/005_create_chunks_table.sql

@@ -0,0 +1,95 @@
+-- Migration 005: Create Chunks Table
+-- Purpose: Store text chunks with token tracking and metadata for embedding generation
+-- Dependencies: memories table (from Phase 1 schema)
+-- Created: February 2, 2026
+
+-- Create chunks table
+CREATE TABLE IF NOT EXISTS chunks (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+    content TEXT NOT NULL,
+    chunk_index INTEGER NOT NULL,
+    start_offset INTEGER,
+    end_offset INTEGER,
+    token_count INTEGER,
+    metadata JSONB DEFAULT '{}'::jsonb,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Ensure chunk_index is unique per memory
+    CONSTRAINT unique_chunk_per_memory UNIQUE (memory_id, chunk_index),
+
+    -- Ensure offsets are valid if provided
+    CONSTRAINT valid_offsets CHECK (
+        (start_offset IS NULL AND end_offset IS NULL) OR
+        (start_offset IS NOT NULL AND end_offset IS NOT NULL AND start_offset < end_offset)
+    ),
+
+    -- Ensure chunk_index is non-negative
+    CONSTRAINT non_negative_chunk_index CHECK (chunk_index >= 0),
+
+    -- Ensure token_count is positive if provided
+    CONSTRAINT positive_token_count CHECK (token_count IS NULL OR token_count > 0)
+);
+
+-- Create indexes for efficient querying
+
+-- Index for looking up chunks by memory
+CREATE INDEX IF NOT EXISTS idx_chunks_memory_id ON chunks(memory_id);
+
+-- Composite index for ordering chunks within a memory
+CREATE INDEX IF NOT EXISTS idx_chunks_chunk_index ON chunks(memory_id, chunk_index);
+
+-- Index for filtering by token count (useful for batch processing)
+CREATE INDEX IF NOT EXISTS idx_chunks_token_count ON chunks(token_count);
+
+-- GIN index for JSONB metadata queries
+CREATE INDEX IF NOT EXISTS idx_chunks_metadata ON chunks USING gin(metadata jsonb_path_ops);
+
+-- Add helpful comments
+COMMENT ON TABLE chunks IS 'Stores text chunks with positional and token information for embedding generation';
+COMMENT ON COLUMN chunks.id IS 'Unique identifier for the chunk';
+COMMENT ON COLUMN chunks.memory_id IS 'Reference to the parent memory this chunk belongs to';
+COMMENT ON COLUMN chunks.content IS 'The actual text content of the chunk';
+COMMENT ON COLUMN chunks.chunk_index IS 'Sequential index of this chunk within its parent memory (0-based)';
+COMMENT ON COLUMN chunks.start_offset IS 'Character offset where this chunk starts in the original content';
+COMMENT ON COLUMN chunks.end_offset IS 'Character offset where this chunk ends in the original content';
+COMMENT ON COLUMN chunks.token_count IS 'Number of tokens in this chunk (for rate limiting and cost estimation)';
+COMMENT ON COLUMN chunks.metadata IS 'Additional metadata about the chunk (e.g., chunking strategy used, overlap info)';
+COMMENT ON COLUMN chunks.created_at IS 'Timestamp when this chunk was created';
+
+-- Verification queries
+DO $$
+BEGIN
+    -- Verify table exists
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'chunks') THEN
+        RAISE NOTICE 'Table chunks created successfully';
+    ELSE
+        RAISE EXCEPTION 'Failed to create chunks table';
+    END IF;
+
+    -- Verify indexes exist
+    IF EXISTS (
+        SELECT 1 FROM pg_indexes
+        WHERE tablename = 'chunks'
+        AND indexname IN ('idx_chunks_memory_id', 'idx_chunks_chunk_index', 'idx_chunks_token_count', 'idx_chunks_metadata')
+    ) THEN
+        RAISE NOTICE 'All indexes created successfully';
+    END IF;
+
+    -- Count total indexes
+    RAISE NOTICE 'Total indexes on chunks table: %', (
+        SELECT COUNT(*) FROM pg_indexes WHERE tablename = 'chunks'
+    );
+END $$;
+
+-- Sample usage (commented out - for documentation only)
+-- INSERT INTO chunks (memory_id, content, chunk_index, start_offset, end_offset, token_count, metadata)
+-- VALUES (
+--     'memory-uuid-here',
+--     'This is a sample chunk of text.',
+--     0,
+--     0,
+--     32,
+--     8,
+--     '{"strategy": "fixed-size", "overlap": 50}'::jsonb
+-- );

package/scripts/migrations/006_create_processing_queue.sql

@@ -0,0 +1,45 @@
+-- Create processing_queue table for async job management
+-- TASK-008: Chunking Worker Implementation
+-- This table manages the processing pipeline for documents
+
+CREATE TABLE IF NOT EXISTS processing_queue (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  stage VARCHAR(30) NOT NULL DEFAULT 'extraction',
+  status VARCHAR(20) NOT NULL DEFAULT 'pending',
+  priority INTEGER DEFAULT 0,
+  error TEXT,
+  error_code VARCHAR(50),
+  attempts INTEGER DEFAULT 0,
+  max_attempts INTEGER DEFAULT 3,
+  worker_id VARCHAR(100),
+  metadata JSONB DEFAULT '{}'::jsonb,
+  created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+  started_at TIMESTAMP WITH TIME ZONE,
+  completed_at TIMESTAMP WITH TIME ZONE,
+  scheduled_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+
+  -- Constraints
+  CONSTRAINT processing_queue_stage_check
+    CHECK (stage IN ('extraction', 'embedding', 'deduplication', 'relationship', 'profile_update', 'cleanup')),
+  CONSTRAINT processing_queue_status_check
+    CHECK (status IN ('pending', 'processing', 'completed', 'failed', 'cancelled', 'retry')),
+  CONSTRAINT processing_queue_attempts_check
+    CHECK (attempts <= max_attempts)
+);
+
+-- Indexes for performance
+CREATE INDEX IF NOT EXISTS idx_processing_queue_document ON processing_queue(document_id);
+CREATE INDEX IF NOT EXISTS idx_processing_queue_status ON processing_queue(status) WHERE status IN ('pending', 'retry');
+CREATE INDEX IF NOT EXISTS idx_processing_queue_stage ON processing_queue(stage);
+CREATE INDEX IF NOT EXISTS idx_processing_queue_worker ON processing_queue(worker_id) WHERE worker_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_processing_queue_priority ON processing_queue(priority DESC, scheduled_at ASC) WHERE status IN ('pending', 'retry');
+CREATE INDEX IF NOT EXISTS idx_processing_queue_stale ON processing_queue(started_at) WHERE status = 'processing';
+CREATE INDEX IF NOT EXISTS idx_processing_queue_worker_select ON processing_queue(status, stage, priority, scheduled_at) WHERE status IN ('pending', 'retry');
+
+COMMENT ON TABLE processing_queue IS 'Manages async processing pipeline for documents';
+COMMENT ON COLUMN processing_queue.stage IS 'Processing stage: extraction, embedding, deduplication, relationship, profile_update, cleanup';
+COMMENT ON COLUMN processing_queue.status IS 'Job status: pending, processing, completed, failed, cancelled, retry';
+COMMENT ON COLUMN processing_queue.priority IS 'Higher values = higher priority';
+COMMENT ON COLUMN processing_queue.attempts IS 'Number of processing attempts';
+COMMENT ON COLUMN processing_queue.max_attempts IS 'Maximum allowed attempts before marking as failed';