@soulcraft/brainy 1.5.0 → 2.0.0

package/scripts/ensure-models.js

@@ -0,0 +1,108 @@
+#!/usr/bin/env node
+/**
+ * Ensures transformer models are available for production
+ * This script handles model availability in multiple ways:
+ * 1. Check if models exist locally
+ * 2. Download from CDN if needed
+ * 3. Verify model integrity
+ */
+
+import { existsSync } from 'fs'
+import { readFile, mkdir, writeFile } from 'fs/promises'
+import { join, dirname } from 'path'
+import { createHash } from 'crypto'
+import { fileURLToPath } from 'url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const PROJECT_ROOT = join(__dirname, '..')
+
+// Model configuration
+const MODEL_CONFIG = {
+  name: 'Xenova/all-MiniLM-L6-v2',
+  files: {
+    'onnx/model.onnx': {
+      size: 90555481, // 86.3 MB
+      sha256: 'expected_hash_here' // We'd compute this from actual model
+    },
+    'tokenizer.json': {
+      size: 711661,
+      sha256: 'expected_hash_here'
+    },
+    'tokenizer_config.json': {
+      size: 366,
+      sha256: 'expected_hash_here'
+    },
+    'config.json': {
+      size: 650,
+      sha256: 'expected_hash_here'
+    }
+  }
+}
+
+// CDN URLs for model files (would be your own CDN in production)
+const CDN_BASE = 'https://cdn.soulcraft.com/models'
+
+async function ensureModels() {
+  const modelsDir = join(PROJECT_ROOT, 'models', 'Xenova', 'all-MiniLM-L6-v2')
+  
+  console.log('🔍 Checking for transformer models...')
+  
+  // Check if all model files exist
+  let missingFiles = []
+  for (const [filePath, info] of Object.entries(MODEL_CONFIG.files)) {
+    const fullPath = join(modelsDir, filePath)
+    if (!existsSync(fullPath)) {
+      missingFiles.push(filePath)
+    }
+  }
+  
+  if (missingFiles.length === 0) {
+    console.log('✅ All model files present')
+    
+    // Optionally verify integrity
+    if (process.env.VERIFY_MODELS === 'true') {
+      console.log('🔐 Verifying model integrity...')
+      // Add hash verification here
+    }
+    
+    return true
+  }
+  
+  console.log(`⚠️ Missing ${missingFiles.length} model files`)
+  
+  // In production, models should be pre-bundled
+  if (process.env.NODE_ENV === 'production' && !process.env.ALLOW_MODEL_DOWNLOAD) {
+    throw new Error(
+      'Critical: Transformer models not found in production. ' +
+      'Run "npm run download-models" during build stage.'
+    )
+  }
+  
+  // Development: offer to download
+  if (process.env.CI !== 'true') {
+    console.log('📥 Would download models from CDN in development')
+    console.log('   Run: npm run download-models')
+  }
+  
+  return false
+}
+
+// Export for use in main code
+export async function verifyModelsAvailable() {
+  try {
+    return await ensureModels()
+  } catch (error) {
+    console.error('❌ Model verification failed:', error.message)
+    return false
+  }
+}
+
+// Run if called directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  ensureModels()
+    .then(success => process.exit(success ? 0 : 1))
+    .catch(error => {
+      console.error(error)
+      process.exit(1)
+    })
+}

package/scripts/prepare-models.js

@@ -0,0 +1,387 @@
+#!/usr/bin/env node
+/**
+ * Prepare Models Script
+ * 
+ * Intelligently handles model preparation for different deployment scenarios:
+ * 1. Development: Models download automatically on first use
+ * 2. Docker/CI: Pre-download during build stage  
+ * 3. Serverless: Bundle with deployment package
+ * 4. Production: Verify models exist, fail fast if missing
+ */
+
+import { existsSync } from 'fs'
+import { readFile, mkdir, writeFile, stat } from 'fs/promises'
+import { join, dirname } from 'path'
+import { fileURLToPath } from 'url'
+import { pipeline, env } from '@huggingface/transformers'
+import { execSync } from 'child_process'
+import https from 'https'
+import { createWriteStream } from 'fs'
+import { promisify } from 'util'
+import { finished } from 'stream'
+
+const streamFinished = promisify(finished)
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+// Model configuration
+const MODEL_CONFIG = {
+  name: 'Xenova/all-MiniLM-L6-v2',
+  expectedFiles: [
+    'config.json',
+    'tokenizer.json', 
+    'tokenizer_config.json',
+    'onnx/model.onnx'
+  ],
+  fallbackUrls: {
+    // GitHub Releases (our backup)
+    github: 'https://github.com/soulcraftlabs/brainy-models/releases/download/v1.0/all-MiniLM-L6-v2.tar.gz',
+    // Future CDN
+    cdn: 'https://models.soulcraft.com/brainy/all-MiniLM-L6-v2.tar.gz'
+  }
+}
+
+class ModelPreparer {
+  constructor() {
+    this.modelsDir = join(__dirname, '..', 'models')
+    this.modelPath = join(this.modelsDir, ...MODEL_CONFIG.name.split('/'))
+  }
+
+  /**
+   * Main entry point - intelligently prepares models based on context
+   */
+  async prepare() {
+    console.log('🧠 Brainy Model Preparation')
+    console.log('===========================')
+    
+    // Detect deployment context
+    const context = this.detectContext()
+    console.log(`📍 Context: ${context}`)
+    
+    switch (context) {
+      case 'production':
+        return await this.prepareProduction()
+      case 'docker':
+        return await this.prepareDocker()
+      case 'ci':
+        return await this.prepareCI()
+      case 'development':
+        return await this.prepareDevelopment()
+      default:
+        return await this.prepareDefault()
+    }
+  }
+
+  /**
+   * Detect the deployment context
+   */
+  detectContext() {
+    // Check environment variables
+    if (process.env.NODE_ENV === 'production') return 'production'
+    if (process.env.DOCKER_BUILD === 'true') return 'docker'
+    if (process.env.CI === 'true') return 'ci'
+    if (process.env.NODE_ENV === 'development') return 'development'
+    
+    // Check for Docker build context
+    if (existsSync('/.dockerenv')) return 'docker'
+    
+    // Check for common CI indicators
+    if (process.env.GITHUB_ACTIONS || process.env.GITLAB_CI) return 'ci'
+    
+    // Default to development
+    return 'development'
+  }
+
+  /**
+   * Production: Models MUST exist, fail fast if not
+   */
+  async prepareProduction() {
+    console.log('🏭 Production mode - verifying models...')
+    
+    const modelExists = await this.verifyModels()
+    
+    if (!modelExists) {
+      console.error('❌ CRITICAL: Models not found in production!')
+      console.error('   Models must be pre-downloaded during build stage.')
+      console.error('   Run: npm run download-models')
+      process.exit(1)
+    }
+    
+    console.log('✅ Models verified for production')
+    return true
+  }
+
+  /**
+   * Docker: Download models during build stage
+   */
+  async prepareDocker() {
+    console.log('🐳 Docker build - downloading models...')
+    
+    // Check if already exists
+    if (await this.verifyModels()) {
+      console.log('✅ Models already present')
+      return true
+    }
+    
+    // Download models
+    return await this.downloadModels()
+  }
+
+  /**
+   * CI: Download models for testing
+   */
+  async prepareCI() {
+    console.log('🔧 CI environment - downloading models for tests...')
+    
+    // Check cache first
+    if (await this.checkCICache()) {
+      console.log('✅ Using cached models')
+      return true
+    }
+    
+    // Download and cache
+    const success = await this.downloadModels()
+    if (success) {
+      await this.saveCICache()
+    }
+    return success
+  }
+
+  /**
+   * Development: Optional download, will auto-download on first use
+   */
+  async prepareDevelopment() {
+    console.log('💻 Development mode')
+    
+    if (await this.verifyModels()) {
+      console.log('✅ Models already downloaded')
+      return true
+    }
+    
+    console.log('ℹ️ Models will download automatically on first use')
+    console.log('   To pre-download now: npm run download-models')
+    
+    // Ask if they want to download now
+    if (process.stdout.isTTY && !process.env.SKIP_PROMPT) {
+      const readline = await import('readline')
+      const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout
+      })
+      
+      return new Promise((resolve) => {
+        rl.question('Download models now? (y/N): ', async (answer) => {
+          rl.close()
+          if (answer.toLowerCase() === 'y') {
+            resolve(await this.downloadModels())
+          } else {
+            resolve(true)
+          }
+        })
+      })
+    }
+    
+    return true
+  }
+
+  /**
+   * Default: Try to be smart about it
+   */
+  async prepareDefault() {
+    console.log('🤖 Auto-detecting best approach...')
+    
+    if (await this.verifyModels()) {
+      console.log('✅ Models found')
+      return true
+    }
+    
+    // If running as part of install, don't download
+    if (process.env.npm_lifecycle_event === 'postinstall') {
+      console.log('ℹ️ Skipping download during install (will download on first use)')
+      return true
+    }
+    
+    // Otherwise download
+    return await this.downloadModels()
+  }
+
+  /**
+   * Verify all required model files exist
+   */
+  async verifyModels() {
+    for (const file of MODEL_CONFIG.expectedFiles) {
+      const filePath = join(this.modelPath, file)
+      if (!existsSync(filePath)) {
+        return false
+      }
+    }
+    
+    // Verify model.onnx size (should be ~87MB)
+    const modelOnnxPath = join(this.modelPath, 'onnx', 'model.onnx')
+    if (existsSync(modelOnnxPath)) {
+      const stats = await stat(modelOnnxPath)
+      const sizeMB = Math.round(stats.size / (1024 * 1024))
+      if (sizeMB < 80 || sizeMB > 100) {
+        console.warn(`⚠️ Model size unexpected: ${sizeMB}MB (expected ~87MB)`)
+        return false
+      }
+    }
+    
+    return true
+  }
+
+  /**
+   * Download models with fallback sources
+   */
+  async downloadModels() {
+    console.log('📥 Downloading transformer models...')
+    
+    // Try transformers.js first (Hugging Face)
+    try {
+      await this.downloadFromTransformers()
+      console.log('✅ Downloaded from Hugging Face')
+      return true
+    } catch (error) {
+      console.warn('⚠️ Hugging Face download failed:', error.message)
+    }
+    
+    // Try GitHub releases
+    try {
+      await this.downloadFromGitHub()
+      console.log('✅ Downloaded from GitHub')
+      return true
+    } catch (error) {
+      console.warn('⚠️ GitHub download failed:', error.message)
+    }
+    
+    // Try CDN
+    try {
+      await this.downloadFromCDN()
+      console.log('✅ Downloaded from CDN')
+      return true
+    } catch (error) {
+      console.warn('⚠️ CDN download failed:', error.message)
+    }
+    
+    console.error('❌ All download sources failed')
+    return false
+  }
+
+  /**
+   * Download using transformers.js (official Hugging Face)
+   */
+  async downloadFromTransformers() {
+    env.cacheDir = this.modelsDir
+    env.allowRemoteModels = true
+    
+    console.log('   Source: Hugging Face')
+    console.log('   Model:', MODEL_CONFIG.name)
+    
+    // Load pipeline to trigger download
+    const extractor = await pipeline('feature-extraction', MODEL_CONFIG.name)
+    
+    // Test it works
+    const test = await extractor('test', { pooling: 'mean', normalize: true })
+    console.log(`   ✓ Model test passed (dims: ${test.data.length})`)
+    
+    return true
+  }
+
+  /**
+   * Download from GitHub releases (our backup)
+   */
+  async downloadFromGitHub() {
+    const url = MODEL_CONFIG.fallbackUrls.github
+    console.log('   Source: GitHub Releases')
+    
+    // Download tar.gz
+    const tempFile = join(this.modelsDir, 'temp-model.tar.gz')
+    await this.downloadFile(url, tempFile)
+    
+    // Extract
+    await mkdir(this.modelPath, { recursive: true })
+    execSync(`tar -xzf ${tempFile} -C ${this.modelPath}`, { stdio: 'inherit' })
+    
+    // Cleanup
+    await unlink(tempFile)
+    
+    return true
+  }
+
+  /**
+   * Download from CDN (future)
+   */
+  async downloadFromCDN() {
+    const url = MODEL_CONFIG.fallbackUrls.cdn
+    console.log('   Source: Soulcraft CDN')
+    
+    // Similar to GitHub approach
+    throw new Error('CDN not yet available')
+  }
+
+  /**
+   * Download a file from URL
+   */
+  async downloadFile(url, destination) {
+    await mkdir(dirname(destination), { recursive: true })
+    
+    return new Promise((resolve, reject) => {
+      const file = createWriteStream(destination)
+      
+      https.get(url, (response) => {
+        if (response.statusCode !== 200) {
+          reject(new Error(`HTTP ${response.statusCode}`))
+          return
+        }
+        
+        response.pipe(file)
+        
+        file.on('finish', () => {
+          file.close()
+          resolve()
+        })
+      }).on('error', reject)
+    })
+  }
+
+  /**
+   * Check CI cache for models
+   */
+  async checkCICache() {
+    // GitHub Actions cache
+    if (process.env.GITHUB_ACTIONS) {
+      const cachePath = process.env.RUNNER_TEMP + '/brainy-models'
+      if (existsSync(cachePath)) {
+        // Copy from cache
+        execSync(`cp -r ${cachePath}/* ${this.modelsDir}/`, { stdio: 'inherit' })
+        return true
+      }
+    }
+    
+    return false
+  }
+
+  /**
+   * Save models to CI cache
+   */
+  async saveCICache() {
+    // GitHub Actions cache
+    if (process.env.GITHUB_ACTIONS) {
+      const cachePath = process.env.RUNNER_TEMP + '/brainy-models'
+      await mkdir(cachePath, { recursive: true })
+      execSync(`cp -r ${this.modelsDir}/* ${cachePath}/`, { stdio: 'inherit' })
+    }
+  }
+}
+
+// Run the preparer
+const preparer = new ModelPreparer()
+preparer.prepare()
+  .then(success => {
+    if (!success) {
+      process.exit(1)
+    }
+  })
+  .catch(error => {
+    console.error('❌ Fatal error:', error)
+    process.exit(1)
+  })

package/OFFLINE_MODELS.md

@@ -1,56 +0,0 @@
-# Offline Models
-
-Brainy uses Transformers.js with ONNX Runtime for **true offline operation** - no more TensorFlow.js dependency hell!
-
-## How it works
-
-Brainy automatically figures out the best approach:
-
-1. **First use**: Downloads models once (~87 MB) to local cache
-2. **Subsequent use**: Loads from cache (completely offline, zero network calls)  
-3. **Smart detection**: Automatically finds models in cache, bundled, or downloads as needed
-
-## Standard usage
-
-```bash
-npm install @soulcraft/brainy
-# Use immediately - models download automatically on first use
-```
-
-## Docker with production egress restrictions
-
-For environments where production has no internet but build does:
-
-```dockerfile
-FROM node:24-slim
-WORKDIR /app
-COPY package*.json ./
-RUN npm install @soulcraft/brainy
-RUN npm run download-models  # Download during build (when internet available)
-COPY . .
-# Production container now works completely offline
-```
-
-## Development with immediate offline
-
-If you want models available immediately for development:
-
-```bash
-npm install @soulcraft/brainy
-npm run download-models  # Optional: download now instead of on first use
-```
-
-## Key benefits vs TensorFlow.js
-
-- ✅ **95% smaller package** - 643 kB vs 12.5 MB  
-- ✅ **84% smaller models** - 87 MB vs 525 MB
-- ✅ **True offline** - Zero network calls after initial download
-- ✅ **No dependency issues** - 5 deps vs 47+, no more --legacy-peer-deps
-- ✅ **Better performance** - ONNX Runtime beats TensorFlow.js
-- ✅ **Same API** - Drop-in replacement
-
-## Philosophy
-
-**Install and use. Brainy handles the rest.**
-
-No configuration files, no environment variables, no complex setup. Brainy detects your environment and does the right thing automatically.