k0ntext 3.6.0 → 3.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "k0ntext",
-  "version": "3.6.0",
+  "version": "3.7.0",
   "description": "Unified AI Context Engineering - Intelligent context for Claude, Copilot, Cline, and more with OpenRouter-powered initialization",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/analyzer/intelligent-analyzer.ts

@@ -10,6 +10,7 @@ import path from 'path';
 import { glob } from 'glob';
 import { OpenRouterClient, createOpenRouterClient, hasOpenRouterKey } from '../embeddings/openrouter.js';
 import { AI_TOOLS, AI_TOOL_FOLDERS, type AITool } from '../db/schema.js';
+import { estimateTokens, chunkForEmbedding } from '../utils/chunking.js';
 
 /**
  * Discovery result for a file
@@ -588,12 +589,68 @@ Return ONLY valid JSON, no markdown formatting.
 
   /**
    * Generate embedding for a single text string (e.g., search query)
+   *
+   * Automatically chunks large texts (>8K tokens) to fit within API limits.
+   * For chunked texts, returns the average of all chunk embeddings.
    */
   async embedText(text: string): Promise<number[]> {
     if (!this.client) {
       throw new Error('OpenRouter client not available for embeddings');
     }
-    return this.client.embed(text);
+
+    // Check if text needs chunking (8K token limit for OpenRouter)
+    const tokenEstimate = estimateTokens(text);
+
+    if (tokenEstimate <= 8000) {
+      // Text is small enough, embed directly
+      return this.client.embed(text);
+    }
+
+    // Text is too large, chunk it and embed each chunk
+    const chunks = chunkForEmbedding(text);
+
+    if (chunks.length === 1) {
+      return this.client.embed(chunks[0]);
+    }
+
+    // Embed all chunks
+    const embeddings: number[][] = [];
+    for (const chunk of chunks) {
+      const embedding = await this.client.embed(chunk);
+      embeddings.push(embedding);
+    }
+
+    // Return the average embedding across all chunks
+    return this.averageEmbeddings(embeddings);
+  }
+
+  /**
+   * Average multiple embeddings into a single vector
+   */
+  private averageEmbeddings(embeddings: number[][]): number[] {
+    if (embeddings.length === 0) {
+      throw new Error('Cannot average empty embeddings array');
+    }
+
+    if (embeddings.length === 1) {
+      return embeddings[0];
+    }
+
+    const dimension = embeddings[0].length;
+    const averaged = new Array(dimension).fill(0);
+
+    for (const embedding of embeddings) {
+      for (let i = 0; i < dimension; i++) {
+        averaged[i] += embedding[i];
+      }
+    }
+
+    // Divide by count to get average
+    for (let i = 0; i < dimension; i++) {
+      averaged[i] /= embeddings.length;
+    }
+
+    return averaged;
   }
 
   /**

package/src/cli/commands/embeddings-refresh.ts

@@ -11,6 +11,7 @@ import { confirm } from '@inquirer/prompts';
 import { createIntelligentAnalyzer } from '../../analyzer/intelligent-analyzer.js';
 import { hasOpenRouterKey } from '../../embeddings/openrouter.js';
 import { DatabaseClient } from '../../db/client.js';
+import { estimateTokens } from '../../utils/chunking.js';
 
 /**
  * Embeddings refresh command
@@ -105,7 +106,9 @@ export const embeddingsRefreshCommand = new Command('embeddings:refresh')
         for (const item of batch) {
           if (options.verbose) {
             spinner.stop();
-            console.log(chalk.dim(`  Embedding: ${item.name}`));
+            const tokenEstimate = estimateTokens(item.content);
+            const chunkInfo = tokenEstimate > 8000 ? chalk.yellow(` (${Math.ceil(tokenEstimate / 8000)} chunks)`) : '';
+            console.log(chalk.dim(`  Embedding: ${item.name}${chunkInfo}`));
             spinner.start();
           }
 

package/src/cli/commands/migrate.ts

@@ -18,6 +18,14 @@ import { MigrationRunner } from '../../db/migrations/index.js';
  */
 export const migrateCommand = new Command('migrate')
   .description('Manage database schema migrations')
+  .action(() => {
+    // Default action: show help if no subcommand specified
+    console.log('\nAvailable subcommands:\n');
+    console.log('  k0ntext migrate status    Show migration status');
+    console.log('  k0ntext migrate up        Apply pending migrations');
+    console.log('  k0ntext migrate rollback   Rollback to a previous backup\n');
+    console.log('Run "k0ntext migrate <subcommand> --help" for more information.\n');
+  })
 
   // Status subcommand
   .command('status')

package/src/cli/repl/init/wizard.ts

@@ -8,6 +8,7 @@ import { input, confirm, select, checkbox } from '@inquirer/prompts';
 import chalk from 'chalk';
 import { ProjectType } from '../core/session.js';
 import { K0NTEXT_THEME } from '../tui/theme.js';
+import { stripBOM } from '../../../utils/encoding.js';
 
 /**
  * Wizard configuration result
@@ -66,7 +67,9 @@ export class InitWizard {
 
   constructor(projectRoot: string) {
     this.projectRoot = projectRoot;
-    this.hasExistingKey = !!process.env.OPENROUTER_API_KEY;
+    // Strip UTF-8 BOM from env var if present (Windows editors sometimes add this)
+    const cleanKey = process.env.OPENROUTER_API_KEY ? stripBOM(process.env.OPENROUTER_API_KEY) : '';
+    this.hasExistingKey = cleanKey.length > 0;
   }
 
   /**
@@ -146,7 +149,9 @@ for your specific needs.
       });
 
       if (useExisting) {
-        return process.env.OPENROUTER_API_KEY!;
+        // Strip UTF-8 BOM from env var if present (Windows editors sometimes add this)
+        const envKey = process.env.OPENROUTER_API_KEY || '';
+        return stripBOM(envKey);
       }
     }
 
@@ -158,7 +163,9 @@ for your specific needs.
       message: 'Enter your OpenRouter API key (or press Enter to skip):',
       validate: (value: string) => {
         if (!value) return true; // Allow skipping
-        if (value.startsWith('sk-or-v1-')) return true;
+        // Strip BOM before validation
+        const cleanValue = stripBOM(value);
+        if (cleanValue.startsWith('sk-or-v1-')) return true;
         return 'Invalid API key format. Should start with "sk-or-v1-"';
       }
     });
@@ -174,7 +181,8 @@ for your specific needs.
       }
     }
 
-    return apiKey || '';
+    // Strip BOM from user input before returning
+    return stripBOM(apiKey || '');
   }
 
   /**

package/src/db/migrations/files/0015_add_sync_state_version_tracking.sql

@@ -0,0 +1,18 @@
+-- Migration: 1.5.0
+-- Description: Add version tracking columns to sync_state table
+-- Breaks: false
+-- Dependencies: 1.4.0
+
+-- Add k0ntext_version column to track package version when sync occurred
+-- Note: Existing rows will have NULL for this column
+ALTER TABLE sync_state ADD COLUMN k0ntext_version TEXT;
+
+-- Add user_modified flag to track if user manually edited the synced file
+ALTER TABLE sync_state ADD COLUMN user_modified INTEGER DEFAULT 0;
+
+-- Add last_checked timestamp for version checking (ISO 8601 format)
+ALTER TABLE sync_state ADD COLUMN last_checked TEXT;
+
+-- Create indexes for efficient queries
+CREATE INDEX IF NOT EXISTS idx_sync_state_version ON sync_state(k0ntext_version);
+CREATE INDEX IF NOT EXISTS idx_sync_state_user_modified ON sync_state(user_modified);

package/src/db/schema.ts

@@ -5,7 +5,7 @@
  * Supports vector embeddings, knowledge graph, and sync state.
  */
 
-export const SCHEMA_VERSION = '1.4.0';
+export const SCHEMA_VERSION = '1.5.0';
 
 /**
  * Core database schema SQL

package/src/embeddings/openrouter.ts

@@ -1,12 +1,13 @@
 /**
  * OpenRouter Client
- * 
+ *
  * Client for OpenRouter API supporting both embeddings and chat completions.
  * Used for intelligent initialization and context understanding.
  */
 
 import { createHash } from 'crypto';
 import { K0NTEXT_MODELS, MODEL_CONFIG, getPrimaryChatModel, getEmbeddingModel } from '../config/models.js';
+import { stripBOM } from '../utils/encoding.js';
 
 /**
  * OpenRouter API endpoints
@@ -185,7 +186,8 @@ export class OpenRouterClient {
       throw new Error('OPENROUTER_API_KEY is required');
     }
 
-    this.apiKey = config.apiKey;
+    // Strip UTF-8 BOM if present (Windows editors sometimes add this to .env files)
+    this.apiKey = stripBOM(config.apiKey);
     this.embeddingModel = config.embeddingModel || DEFAULT_EMBEDDING_MODEL;
     this.chatModel = config.chatModel || DEFAULT_CHAT_MODEL;
     this.siteUrl = config.siteUrl || 'https://github.com/SireJeff/claude-context-engineering-template';
@@ -582,8 +584,11 @@ export function createOpenRouterClient(): OpenRouterClient {
     );
   }
 
+  // Strip UTF-8 BOM from API key (Windows editors sometimes add this to .env files)
+  const cleanApiKey = stripBOM(apiKey);
+
   return new OpenRouterClient({
-    apiKey,
+    apiKey: cleanApiKey,
     // Use centralized models from config, allow env override for testing
     embeddingModel: process.env.OPENROUTER_EMBEDDING_MODEL || getEmbeddingModel(),
     chatModel: process.env.OPENROUTER_CHAT_MODEL || getPrimaryChatModel()
@@ -594,7 +599,8 @@ export function createOpenRouterClient(): OpenRouterClient {
  * Check if OpenRouter API key is available
  */
 export function hasOpenRouterKey(): boolean {
-  return !!process.env.OPENROUTER_API_KEY;
+  const apiKey = process.env.OPENROUTER_API_KEY;
+  return !!apiKey && stripBOM(apiKey).length > 0;
 }
 
 /**

package/src/utils/chunking.ts

@@ -0,0 +1,152 @@
+/**
+ * Text Chunking Utility
+ *
+ * Splits large texts into chunks suitable for embedding generation.
+ * Handles token limits, word boundaries, and overlap for context preservation.
+ */
+
+/**
+ * Estimate token count for text.
+ *
+ * Uses a simple heuristic: ~4 characters per token for English text.
+ * This is approximate but works well for our use case.
+ *
+ * @param text - Text to estimate tokens for
+ * @returns Estimated token count
+ */
+export function estimateTokens(text: string): number {
+  if (!text) return 0;
+
+  // Remove whitespace for more accurate estimate
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return 0;
+
+  // Rough estimate: 1 token per 4 characters for English text
+  // This is a simplification but works well for most cases
+  return Math.ceil(trimmed.length / 4);
+}
+
+/**
+ * Split text into chunks that fit within max tokens.
+ *
+ * Tries to break at word boundaries when possible.
+ * Adds overlap between chunks to preserve context.
+ *
+ * @param text - Text to chunk
+ * @param maxTokens - Maximum tokens per chunk (default: 8000 for OpenRouter)
+ * @param overlapTokens - Number of tokens to overlap between chunks (default: 0)
+ * @returns Array of text chunks
+ */
+export function chunkText(
+  text: string,
+  maxTokens: number = 8000,
+  overlapTokens: number = 0
+): string[] {
+  // Handle empty or very short text
+  if (!text || text.trim().length === 0) {
+    return [''];
+  }
+
+  const trimmedText = text.trim();
+  const estimatedTokens = estimateTokens(trimmedText);
+
+  // If text is under the limit, return as-is
+  if (estimatedTokens <= maxTokens) {
+    return [trimmedText];
+  }
+
+  const chunks: string[] = [];
+  const maxChars = maxTokens * 4; // Convert tokens to approximate characters
+  const overlapChars = overlapTokens * 4;
+
+  let startIndex = 0;
+  let previousEndIndex = 0;
+  let loopCount = 0;
+  const maxLoops = 1000; // Safety limit to prevent infinite loops
+
+  while (startIndex < trimmedText.length && loopCount < maxLoops) {
+    loopCount++;
+
+    // Calculate end index for this chunk
+    let endIndex = Math.min(startIndex + maxChars, trimmedText.length);
+
+    // If not the last chunk, try to break at a word boundary
+    if (endIndex < trimmedText.length) {
+      // Look for word boundary near the end
+      const boundaryChars = 200; // Look back up to 200 chars
+      const searchStart = Math.max(startIndex, endIndex - boundaryChars);
+      const substring = trimmedText.slice(searchStart, endIndex);
+
+      // Try to find line break first, then space, then punctuation
+      let breakIndex = -1;
+
+      // Look for last newline in the window
+      const lastNewline = substring.lastIndexOf('\n');
+      if (lastNewline !== -1) {
+        breakIndex = searchStart + lastNewline + 1;
+      } else {
+        // Look for last space in the window
+        const lastSpace = substring.lastIndexOf(' ');
+        if (lastSpace !== -1) {
+          breakIndex = searchStart + lastSpace + 1;
+        } else {
+          // Look for sentence-ending punctuation
+          for (let i = substring.length - 1; i >= Math.max(0, substring.length - 100); i--) {
+            const char = substring[i];
+            if (char === '.' || char === '!' || char === '?') {
+              // Make sure it's actually a sentence end (followed by space or end)
+              const nextChar = substring[i + 1];
+              if (!nextChar || nextChar === ' ' || nextChar === '\n') {
+                breakIndex = searchStart + i + 1;
+                break;
+              }
+            }
+          }
+        }
+      }
+
+      // Use the break index if found, otherwise use the calculated end
+      if (breakIndex > startIndex) {
+        endIndex = breakIndex;
+      }
+    }
+
+    // Extract the chunk
+    const chunk = trimmedText.slice(startIndex, endIndex);
+    chunks.push(chunk);
+
+    // Move to next chunk, accounting for overlap
+    if (overlapChars > 0 && endIndex < trimmedText.length) {
+      // Only apply overlap if not at the end
+      startIndex = Math.max(endIndex - overlapChars, endIndex - maxChars / 2);
+
+      // Ensure we make progress
+      if (startIndex <= previousEndIndex) {
+        startIndex = endIndex;
+      }
+
+      // Also ensure we move forward at least a bit
+      if (startIndex >= endIndex) {
+        startIndex = endIndex;
+      }
+    } else {
+      startIndex = endIndex;
+    }
+
+    previousEndIndex = endIndex;
+  }
+
+  return chunks;
+}
+
+/**
+ * Chunk text specifically for embedding generation.
+ *
+ * Uses 8000 token limit (OpenRouter's limit for text-embedding-3-small).
+ *
+ * @param text - Text to chunk
+ * @returns Array of text chunks suitable for embeddings
+ */
+export function chunkForEmbedding(text: string): string[] {
+  return chunkText(text, 8000, 100); // 100 token overlap for context
+}

package/src/utils/encoding.ts

@@ -0,0 +1,33 @@
+/**
+ * Encoding Utilities
+ *
+ * Handles text encoding issues across different platforms.
+ */
+
+/**
+ * Strip UTF-8 BOM (Byte Order Mark) from a string.
+ *
+ * The UTF-8 BOM is the byte sequence EF BB BF (U+FEFF).
+ * Some Windows editors add this to the start of files,
+ * which can break environment variable parsing.
+ *
+ * @param str - String that may contain a BOM
+ * @returns String with BOM removed if present
+ */
+export function stripBOM(str: string): string {
+  // Check for BOM at position 0
+  if (str.charCodeAt(0) === 0xFEFF) {
+    return str.slice(1);
+  }
+  return str;
+}
+
+/**
+ * Detect if a string has a UTF-8 BOM.
+ *
+ * @param str - String to check
+ * @returns true if BOM is present
+ */
+export function hasBOM(str: string): boolean {
+  return str.charCodeAt(0) === 0xFEFF;
+}

package/src/utils/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Shared Utilities Module
+ *
+ * Exports all shared utility functions.
+ */
+
+export * from './encoding.js';
+export * from './chunking.js';