@twelvehart/supermemory-runtime 1.0.0-next.0

package/drizzle/meta/_journal.json

@@ -0,0 +1,13 @@
+{
+  "version": "7",
+  "dialect": "postgresql",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "7",
+      "when": 1770055278660,
+      "tag": "0000_dapper_the_professor",
+      "breakpoints": true
+    }
+  ]
+}

package/drizzle.config.ts

@@ -0,0 +1,20 @@
+import { defineConfig } from 'drizzle-kit';
+
+// Determine database type from DATABASE_URL
+const databaseUrl = process.env.DATABASE_URL ?? './data/supermemory.db';
+const isPostgres = databaseUrl.startsWith('postgresql://') || databaseUrl.startsWith('postgres://');
+
+export default defineConfig({
+  schema: './src/db/schema/index.ts',
+  out: './drizzle',
+  dialect: isPostgres ? 'postgresql' : 'sqlite',
+  dbCredentials: isPostgres
+    ? {
+        url: databaseUrl,
+      }
+    : {
+        url: databaseUrl,
+      },
+  verbose: true,
+  strict: true,
+});

package/package.json

@@ -0,0 +1,114 @@
+{
+  "name": "@twelvehart/supermemory-runtime",
+  "version": "1.0.0-next.0",
+  "description": "A personal AI memory assistant - supermemory.ai clone",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "files": [
+    "README.md",
+    ".env.example",
+    "package-lock.json",
+    "src",
+    "scripts",
+    "drizzle",
+    "docker-compose.yml",
+    "docker-compose.prod.yml",
+    "drizzle.config.ts",
+    "tsconfig.json"
+  ],
+  "bin": {
+    "supermemory-mcp": "dist/mcp/index.js"
+  },
+  "scripts": {
+    "dev": "tsx watch src/index.ts",
+    "build": "tsc",
+    "build:install": "npm --prefix packages/install run build",
+    "start": "node dist/index.js",
+    "mcp": "node dist/mcp/index.js",
+    "mcp:dev": "tsx src/mcp/index.ts",
+    "mcp:setup": "tsx scripts/mcp-setup.ts",
+    "setup": "tsx scripts/setup.ts",
+    "setup:turnkey": "bash scripts/install.sh",
+    "stack:up": "docker compose -f docker-compose.yml -f docker-compose.prod.yml --profile production up -d api postgres redis",
+    "stack:down": "docker compose -f docker-compose.yml -f docker-compose.prod.yml down",
+    "stack:logs": "docker compose -f docker-compose.yml -f docker-compose.prod.yml logs -f api",
+    "doctor": "tsx scripts/doctor.ts",
+    "test": "vitest",
+    "test:run": "vitest run",
+    "test:coverage": "vitest run --coverage",
+    "test:ui": "vitest --ui",
+    "test:watch": "vitest --watch",
+    "db:generate": "drizzle-kit generate",
+    "db:migrate": "drizzle-kit migrate",
+    "db:studio": "drizzle-kit studio",
+    "db:push": "drizzle-kit push",
+    "db:test:phase1": "bash scripts/run-phase1-tests.sh",
+    "db:test:phase1:keep": "bash scripts/run-phase1-tests.sh --keep-db",
+    "db:test:phase1:verbose": "bash scripts/run-phase1-tests.sh --verbose",
+    "lint": "eslint src tests --ext .ts",
+    "lint:fix": "eslint src tests --ext .ts --fix",
+    "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\"",
+    "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\"",
+    "typecheck": "tsc --noEmit",
+    "typecheck:install": "node node_modules/typescript/bin/tsc -p packages/install/tsconfig.json --noEmit",
+    "validate": "npm run typecheck && npm run lint && npm run format:check && npm run test:run",
+    "pack:check:runtime": "tsx scripts/check-runtime-pack.ts",
+    "test:install": "vitest run tests/install/*.test.ts",
+    "clean": "rm -rf dist coverage",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "ai",
+    "memory",
+    "rag",
+    "embeddings",
+    "vector-search"
+  ],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@hono/node-server": "^1.13.7",
+    "@modelcontextprotocol/sdk": "^1.25.3",
+    "bcrypt": "^5.1.1",
+    "better-sqlite3": "^11.6.0",
+    "bullmq": "^5.67.2",
+    "dotenv": "^16.4.7",
+    "drizzle-orm": "^0.45.1",
+    "hono": "^4.6.14",
+    "ioredis": "^5.9.2",
+    "isomorphic-dompurify": "^2.35.0",
+    "js-yaml": "^4.1.1",
+    "openai": "^4.77.0",
+    "pdf-parse": "^1.1.1",
+    "pg": "^8.18.0",
+    "uuid": "^10.0.0",
+    "zod": "^3.24.1"
+  },
+  "devDependencies": {
+    "@types/bcrypt": "^5.0.2",
+    "@types/better-sqlite3": "^7.6.12",
+    "@types/dompurify": "^3.0.5",
+    "@types/ioredis": "^4.28.10",
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.10.5",
+    "@types/pdf-parse": "^1.1.4",
+    "@types/pg": "^8.16.0",
+    "@types/uuid": "^10.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.54.0",
+    "@typescript-eslint/parser": "^8.54.0",
+    "@vitest/coverage-v8": "^2.1.8",
+    "@vitest/ui": "^2.1.8",
+    "drizzle-kit": "^0.30.1",
+    "eslint": "^9.39.2",
+    "eslint-config-prettier": "^10.1.8",
+    "globals": "^17.3.0",
+    "prettier": "^3.8.1",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2",
+    "vitest": "^2.1.8"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}

package/scripts/add-extraction-job.ts

@@ -0,0 +1,122 @@
+#!/usr/bin/env tsx
+/**
+ * Add Extraction Job
+ *
+ * Example script to add a job to the extraction queue.
+ * Run with: npx tsx scripts/add-extraction-job.ts
+ */
+
+import { createExtractionQueue } from '../src/workers/extraction.worker.js';
+import { getDatabase } from '../src/db/index.js';
+import { documents, processingQueue } from '../src/db/schema/index.js';
+import IORedis from 'ioredis';
+import dotenv from 'dotenv';
+import { v4 as uuidv4 } from 'uuid';
+
+// Load environment variables
+dotenv.config();
+
+const DATABASE_URL = process.env.DATABASE_URL || './data/supermemory.db';
+const db = getDatabase(DATABASE_URL);
+
+// Redis connection
+const connection = new IORedis({
+  host: process.env.REDIS_HOST || 'localhost',
+  port: parseInt(process.env.REDIS_PORT || '6379', 10),
+  maxRetriesPerRequest: null,
+});
+
+async function main() {
+  console.log('[AddJob] Creating test document...');
+
+  // Create test document
+  const documentId = uuidv4();
+  const containerTag = 'test-user';
+
+  await db.insert(documents).values({
+    id: documentId,
+    content: `# Test Document
+
+This is a test document for the extraction worker.
+
+## Features
+
+- Text extraction
+- URL extraction
+- PDF extraction
+- Markdown extraction
+- Code extraction
+
+## Example Code
+
+\`\`\`javascript
+function hello() {
+  console.log('Hello, World!');
+}
+\`\`\`
+
+Visit [our website](https://example.com) for more information.
+`,
+    contentType: 'text/plain',
+    status: 'pending',
+    containerTag,
+    metadata: {
+      source: 'test-script',
+      createdBy: 'add-extraction-job.ts',
+    },
+  });
+
+  console.log(`[AddJob] Document created: ${documentId}`);
+
+  // Create processing queue entry
+  await db.insert(processingQueue).values({
+    documentId,
+    stage: 'extraction',
+    status: 'pending',
+    priority: 5,
+  });
+
+  console.log('[AddJob] Processing queue entry created');
+
+  // Add job to extraction queue
+  const queue = createExtractionQueue(connection);
+
+  const job = await queue.add(
+    'extract',
+    {
+      documentId,
+      sourceType: 'text',
+      containerTag,
+    },
+    {
+      priority: 5,
+      jobId: `extraction-${documentId}`,
+    }
+  );
+
+  console.log(`[AddJob] Job added to queue: ${job.id}`);
+  console.log('[AddJob] Waiting for job to complete...');
+
+  // Wait for completion (with timeout)
+  try {
+    const result = await job.waitUntilFinished(queue.events, 60000); // 60 second timeout
+    console.log('[AddJob] Job completed successfully!');
+    console.log(`[AddJob] Extracted ${result.extractedContent.length} characters`);
+    console.log(`[AddJob] Content type: ${result.contentType}`);
+    console.log(`[AddJob] Processing time: ${result.processingTimeMs}ms`);
+  } catch (error) {
+    console.error('[AddJob] Job failed or timed out:', error);
+  }
+
+  // Cleanup
+  await queue.close();
+  await connection.quit();
+
+  console.log('[AddJob] Done!');
+  process.exit(0);
+}
+
+main().catch((error) => {
+  console.error('[AddJob] Error:', error);
+  process.exit(1);
+});

package/scripts/benchmark-pgvector.ts

@@ -0,0 +1,122 @@
+#!/usr/bin/env tsx
+/**
+ * PgVectorStore Performance Benchmark
+ *
+ * Measures insert and search performance with various dataset sizes
+ */
+
+import { createPgVectorStore } from '../src/services/vectorstore/pgvector.js';
+import { VectorEntry } from '../src/services/vectorstore/types.js';
+
+const POSTGRES_URL = process.env.TEST_POSTGRES_URL ?? 'postgresql://supermemory:supermemory_secret@localhost:5432/supermemory';
+const DIMENSIONS = 1536;
+
+interface BenchmarkResult {
+  operation: string;
+  itemCount: number;
+  totalTimeMs: number;
+  avgTimePerItemMs: number;
+  opsPerSecond: number;
+}
+
+function formatResults(results: BenchmarkResult[]) {
+  console.log('\n╔════════════════════════════════════════════════════════════════════╗');
+  console.log('║          PgVectorStore Performance Benchmark Results              ║');
+  console.log('╚════════════════════════════════════════════════════════════════════╝\n');
+
+  for (const result of results) {
+    console.log(`Operation: ${result.operation}`);
+    console.log(`  Items: ${result.itemCount}`);
+    console.log(`  Total Time: ${result.totalTimeMs.toFixed(2)}ms`);
+    console.log(`  Avg Time/Item: ${result.avgTimePerItemMs.toFixed(4)}ms`);
+    console.log(`  Ops/Second: ${result.opsPerSecond.toFixed(2)}`);
+    console.log('');
+  }
+}
+
+async function benchmarkInsert(store: any, count: number): Promise<BenchmarkResult> {
+  const entries: VectorEntry[] = Array.from({ length: count }, (_, i) => ({
+    id: `bench-insert-${i}`,
+    embedding: new Array(DIMENSIONS).fill(0).map(() => Math.random()),
+    metadata: { index: i, benchmark: 'insert' },
+  }));
+
+  const start = performance.now();
+  await store.addBatch(entries);
+  const totalTime = performance.now() - start;
+
+  return {
+    operation: 'Batch Insert',
+    itemCount: count,
+    totalTimeMs: totalTime,
+    avgTimePerItemMs: totalTime / count,
+    opsPerSecond: (count / totalTime) * 1000,
+  };
+}
+
+async function benchmarkSearch(store: any, vectorCount: number, searchCount: number): Promise<BenchmarkResult> {
+  const queryVector = new Array(DIMENSIONS).fill(0.5);
+  const start = performance.now();
+
+  for (let i = 0; i < searchCount; i++) {
+    await store.search(queryVector, { limit: 10 });
+  }
+
+  const totalTime = performance.now() - start;
+
+  return {
+    operation: `Search (${vectorCount} vectors)`,
+    itemCount: searchCount,
+    totalTimeMs: totalTime,
+    avgTimePerItemMs: totalTime / searchCount,
+    opsPerSecond: (searchCount / totalTime) * 1000,
+  };
+}
+
+async function main() {
+  console.log('Starting PgVectorStore performance benchmarks...\n');
+
+  const store = createPgVectorStore(POSTGRES_URL, DIMENSIONS, {
+    tableName: 'benchmark_vectors',
+    hnswConfig: {
+      M: 16,
+      efConstruction: 64,
+    },
+  });
+
+  await store.initialize();
+  await store.clear();
+
+  const results: BenchmarkResult[] = [];
+
+  // Benchmark 1: Insert 1,000 vectors
+  console.log('Running benchmark: Insert 1,000 vectors...');
+  results.push(await benchmarkInsert(store, 1000));
+
+  // Benchmark 2: Search with 1,000 vectors
+  console.log('Running benchmark: Search (1,000 vectors)...');
+  results.push(await benchmarkSearch(store, 1000, 100));
+
+  // Benchmark 3: Insert 10,000 vectors
+  console.log('Running benchmark: Insert 10,000 vectors...');
+  await store.clear();
+  results.push(await benchmarkInsert(store, 10000));
+
+  // Benchmark 4: Search with 10,000 vectors
+  console.log('Running benchmark: Search (10,000 vectors)...');
+  results.push(await benchmarkSearch(store, 10000, 100));
+
+  // Clean up
+  await store.clear();
+  await store.close();
+
+  formatResults(results);
+
+  // Check performance targets
+  console.log('Performance Target Validation:');
+  console.log('  Insert < 10ms per item:', results[0]!.avgTimePerItemMs < 10 ? '✓ PASS' : '✗ FAIL');
+  console.log('  Search < 100ms (10K vectors):', results[3]!.avgTimePerItemMs < 100 ? '✓ PASS' : '✗ FAIL');
+  console.log('  Batch insert < 500ms (100 items):', results[0]!.totalTimeMs < 500 ? '✓ PASS' : '✗ FAIL');
+}
+
+main().catch(console.error);

package/scripts/bootstrap.sh

@@ -0,0 +1,209 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+log() {
+  local level="$1"
+  shift
+  printf '[%s] %s\n' "$level" "$*"
+}
+
+fail() {
+  log "FAIL" "$*"
+  exit 1
+}
+
+command_exists() {
+  command -v "$1" >/dev/null 2>&1
+}
+
+require_command() {
+  local name="$1"
+  if ! command_exists "$name"; then
+    fail "Missing required command: $name"
+  fi
+}
+
+usage() {
+  cat <<'USAGE'
+Usage: bootstrap.sh [options] [-- <install.sh args>]
+
+Options:
+  --repo-url URL         Git repository URL
+  --ref REF              Git branch/tag/sha to checkout
+  --dir DIR              Install directory (default: supermemory-clone)
+  --update-if-exists     Update an existing compatible checkout in place
+  -h, --help             Show this help
+
+Examples:
+  bootstrap.sh
+  bootstrap.sh --repo-url https://github.com/acme/supermemory-clone.git
+  bootstrap.sh --dir ./supermemory -- --non-interactive --skip-api-keys
+  bootstrap.sh --dir ./supermemory --update-if-exists -- --non-interactive --skip-mcp
+USAGE
+}
+
+repo_url="${REPO_URL:-https://github.com/ASRagab/supermemory-clone.git}"
+install_ref="${INSTALL_REF:-}"
+install_dir="${INSTALL_DIR:-supermemory-clone}"
+update_if_exists=0
+temp_clone_dir=""
+declare -a install_args=()
+
+cleanup() {
+  if [[ -n "$temp_clone_dir" && -d "$temp_clone_dir" ]]; then
+    rm -rf "$temp_clone_dir"
+  fi
+}
+
+trap cleanup EXIT
+
+run_installer() {
+  local target_dir="$1"
+  shift || true
+  (
+    cd "$target_dir"
+    bash ./scripts/install.sh "$@"
+  )
+}
+
+checkout_requested_ref() {
+  local target_dir="$1"
+
+  if [[ -z "$install_ref" ]]; then
+    return 0
+  fi
+
+  log "INFO" "Fetching requested ref: $install_ref"
+  git -C "$target_dir" fetch --depth 1 origin "$install_ref"
+  log "INFO" "Checking out ref: $install_ref"
+  git -C "$target_dir" checkout FETCH_HEAD
+}
+
+clone_into_temp_dir() {
+  local parent_dir="$1"
+  mkdir -p "$parent_dir"
+  temp_clone_dir="$(mktemp -d "${parent_dir%/}/.supermemory-bootstrap.XXXXXX")"
+
+  log "INFO" "Cloning repository: $repo_url"
+  git clone --depth 1 "$repo_url" "$temp_clone_dir"
+  checkout_requested_ref "$temp_clone_dir"
+}
+
+ensure_compatible_checkout() {
+  local target_dir="$1"
+
+  if ! git -C "$target_dir" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+    fail "--update-if-exists requires an existing git checkout: $target_dir"
+  fi
+
+  if [[ ! -f "$target_dir/scripts/install.sh" ]]; then
+    fail "--update-if-exists requires a compatible checkout with scripts/install.sh: $target_dir"
+  fi
+
+  if ! git -C "$target_dir" diff --quiet --ignore-submodules -- || ! git -C "$target_dir" diff --cached --quiet --ignore-submodules --; then
+    fail "--update-if-exists cannot run on a checkout with uncommitted tracked changes: $target_dir"
+  fi
+}
+
+update_existing_checkout() {
+  local target_dir="$1"
+  local current_branch
+  local -a effective_args=()
+
+  ensure_compatible_checkout "$target_dir"
+
+  if [[ -n "$install_ref" ]]; then
+    checkout_requested_ref "$target_dir"
+  else
+    current_branch="$(git -C "$target_dir" symbolic-ref --quiet --short HEAD || true)"
+    if [[ -z "$current_branch" ]]; then
+      fail "--update-if-exists on a detached HEAD requires --ref"
+    fi
+
+    log "INFO" "Fast-forwarding existing checkout"
+    git -C "$target_dir" pull --ff-only origin "$current_branch"
+  fi
+
+  if [[ "${#install_args[@]}" -eq 0 ]]; then
+    effective_args=(update)
+  else
+    case "${install_args[0]}" in
+      install|update|uninstall)
+        effective_args=("${install_args[@]}")
+        ;;
+      agent|api|full)
+        effective_args=(update --mode "${install_args[0]}" "${install_args[@]:1}")
+        ;;
+      *)
+        effective_args=(update "${install_args[@]}")
+        ;;
+    esac
+  fi
+
+  log "INFO" "Running installer against existing checkout"
+  run_installer "$target_dir" "${effective_args[@]}"
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --repo-url)
+      [[ $# -lt 2 ]] && fail "--repo-url requires a value"
+      repo_url="$2"
+      shift 2
+      ;;
+    --ref)
+      [[ $# -lt 2 ]] && fail "--ref requires a value"
+      install_ref="$2"
+      shift 2
+      ;;
+    --dir)
+      [[ $# -lt 2 ]] && fail "--dir requires a value"
+      install_dir="$2"
+      shift 2
+      ;;
+    --update-if-exists)
+      update_if_exists=1
+      shift
+      ;;
+    --)
+      shift
+      install_args=("$@")
+      break
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      fail "Unknown option: $1"
+      ;;
+  esac
+done
+
+require_command git
+require_command bash
+
+if [[ -e "$install_dir" ]]; then
+  if [[ "$update_if_exists" -ne 1 ]]; then
+    fail "Install directory already exists: $install_dir (use --update-if-exists to reuse a compatible checkout)"
+  fi
+else
+  if [[ "$update_if_exists" -eq 1 ]]; then
+    fail "--update-if-exists requires an existing install directory: $install_dir"
+  fi
+fi
+
+if [[ "$update_if_exists" -eq 1 ]]; then
+  update_existing_checkout "$install_dir"
+else
+  clone_into_temp_dir "$(dirname "$install_dir")"
+
+  log "INFO" "Moving installed checkout into place: $install_dir"
+  mv "$temp_clone_dir" "$install_dir"
+  temp_clone_dir=""
+
+  log "INFO" "Running turnkey installer"
+  run_installer "$install_dir" "${install_args[@]}"
+fi
+
+log "OK" "Bootstrap complete"