@crashbytes/semantic-text-toolkit 1.0.0

package/src/engine/SemanticEngine.ts

@@ -0,0 +1,225 @@
+/**
+ * Semantic Engine - Core Embedding Generation
+ * 
+ * Architectural Principles:
+ * - Lazy initialization minimizes startup overhead
+ * - Singleton pattern prevents redundant model loading
+ * - Resource management through explicit lifecycle control
+ * - Defensive error handling with semantic codes
+ */
+
+import { pipeline, FeatureExtractionPipeline } from '@xenova/transformers';
+import {
+  ModelConfig,
+  EmbeddingResult,
+  SimilarityResult,
+  Embedding,
+  BatchOptions,
+  SemanticError,
+  SemanticErrorCode,
+} from '../types';
+import { cosineSimilarity, euclideanDistance, dotProduct } from '../utils/vector';
+
+const DEFAULT_CONFIG: Required<ModelConfig> = {
+  modelName: 'Xenova/all-MiniLM-L6-v2',
+  maxLength: 512,
+  quantized: true,
+  onProgress: () => {},
+};
+
+export class SemanticEngine {
+  private model: FeatureExtractionPipeline | null = null;
+  private config: Required<ModelConfig>;
+  private initializationPromise: Promise<void> | null = null;
+
+  constructor(config: ModelConfig = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  async initialize(): Promise<void> {
+    if (this.initializationPromise) {
+      return this.initializationPromise;
+    }
+
+    if (this.model) {
+      return Promise.resolve();
+    }
+
+    this.initializationPromise = this._performInitialization();
+
+    try {
+      await this.initializationPromise;
+    } finally {
+      this.initializationPromise = null;
+    }
+  }
+
+  private async _performInitialization(): Promise<void> {
+    try {
+      this.config.onProgress({
+        status: 'downloading',
+        progress: 0,
+      });
+
+      this.model = await pipeline(
+        'feature-extraction',
+        this.config.modelName,
+        {
+          quantized: this.config.quantized,
+        }
+      );
+
+      this.config.onProgress({
+        status: 'ready',
+        progress: 100,
+      });
+    } catch (error) {
+      throw new SemanticError(
+        SemanticErrorCode.MODEL_NOT_LOADED,
+        `Failed to initialize model: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        { modelName: this.config.modelName, error }
+      );
+    }
+  }
+
+  private assertInitialized(): void {
+    if (!this.model) {
+      throw new SemanticError(
+        SemanticErrorCode.MODEL_NOT_LOADED,
+        'Model not initialized. Call initialize() first.'
+      );
+    }
+  }
+
+  async embed(text: string): Promise<EmbeddingResult> {
+    this.assertInitialized();
+
+    if (!text || typeof text !== 'string') {
+      throw new SemanticError(
+        SemanticErrorCode.INVALID_INPUT,
+        'Text must be a non-empty string',
+        { text }
+      );
+    }
+
+    const startTime = performance.now();
+
+    try {
+      const output = await this.model!(text, {
+        pooling: 'mean',
+        normalize: true,
+      });
+
+      const embedding = Array.from(output.data) as Embedding;
+      const processingTime = performance.now() - startTime;
+
+      return {
+        embedding,
+        text,
+        metadata: {
+          dimensions: embedding.length,
+          modelName: this.config.modelName,
+          processingTime,
+        },
+      };
+    } catch (error) {
+      throw new SemanticError(
+        SemanticErrorCode.EMBEDDING_FAILED,
+        `Failed to generate embedding: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        { text: text.substring(0, 100), error }
+      );
+    }
+  }
+
+  async embedBatch(
+    texts: string[],
+    options: BatchOptions = {}
+  ): Promise<EmbeddingResult[]> {
+    this.assertInitialized();
+
+    const { batchSize = 32, onProgress } = options;
+
+    if (!Array.isArray(texts) || texts.length === 0) {
+      throw new SemanticError(
+        SemanticErrorCode.INVALID_INPUT,
+        'Texts must be a non-empty array'
+      );
+    }
+
+    const results: EmbeddingResult[] = [];
+    const batches = Math.ceil(texts.length / batchSize);
+
+    for (let i = 0; i < batches; i++) {
+      const start = i * batchSize;
+      const end = Math.min(start + batchSize, texts.length);
+      const batch = texts.slice(start, end);
+
+      const batchResults = await Promise.all(
+        batch.map(text => this.embed(text))
+      );
+
+      results.push(...batchResults);
+
+      if (onProgress) {
+        onProgress(end, texts.length);
+      }
+    }
+
+    return results;
+  }
+
+  async similarity(
+    textA: string,
+    textB: string,
+    method: 'cosine' | 'euclidean' | 'dot' = 'cosine'
+  ): Promise<SimilarityResult> {
+    const startTime = performance.now();
+
+    const [resultA, resultB] = await Promise.all([
+      this.embed(textA),
+      this.embed(textB),
+    ]);
+
+    let score: number;
+    switch (method) {
+      case 'cosine':
+        score = cosineSimilarity(resultA.embedding, resultB.embedding);
+        break;
+      case 'euclidean':
+        score = -euclideanDistance(resultA.embedding, resultB.embedding);
+        break;
+      case 'dot':
+        score = dotProduct(resultA.embedding, resultB.embedding);
+        break;
+      default:
+        throw new SemanticError(
+          SemanticErrorCode.INVALID_INPUT,
+          `Unknown similarity method: ${method}`
+        );
+    }
+
+    const processingTime = performance.now() - startTime;
+
+    return {
+      score,
+      texts: [textA, textB],
+      metadata: {
+        method,
+        processingTime,
+      },
+    };
+  }
+
+  dispose(): void {
+    this.model = null;
+    this.initializationPromise = null;
+  }
+
+  isReady(): boolean {
+    return this.model !== null;
+  }
+
+  getConfig(): Required<ModelConfig> {
+    return { ...this.config };
+  }
+}

package/src/index.ts

@@ -0,0 +1,31 @@
+/**
+ * Semantic Text Toolkit
+ * Production-grade semantic text analysis
+ * 
+ * @module @crashbytes/semantic-text-toolkit
+ * @author Blackhole Software, LLC
+ * @license MIT
+ */
+
+export { SemanticEngine } from './engine/SemanticEngine';
+export { SemanticSearch, type SearchConfig, type IndexedItem } from './search/SemanticSearch';
+export { cosineSimilarity, euclideanDistance, dotProduct, magnitude, normalize, centroid, topKSimilar } from './utils/vector';
+export { type Embedding, type ModelConfig, type EmbeddingResult, type SimilarityResult, type SearchResult, type BatchOptions, type ModelLoadProgress, SemanticError, SemanticErrorCode } from './types';
+
+export async function createSemanticEngine(config?: import('./types').ModelConfig) {
+  const { SemanticEngine } = await import('./engine/SemanticEngine');
+  const engine = new SemanticEngine(config);
+  await engine.initialize();
+  return engine;
+}
+
+export async function createSemanticSearch<T = string>(
+  items: T[],
+  config?: import('./search/SemanticSearch').SearchConfig<T>
+) {
+  const engine = await createSemanticEngine();
+  const { SemanticSearch } = await import('./search/SemanticSearch');
+  const search = new SemanticSearch(engine, config);
+  await search.index(items);
+  return search;
+}

package/src/search/SemanticSearch.ts

@@ -0,0 +1,154 @@
+/**
+ * Semantic Search - Vector-Based Document Retrieval
+ * 
+ * Architectural Principles:
+ * - Pre-computed embeddings optimize retrieval latency
+ * - Configurable ranking strategies enable domain customization
+ * - Metadata filtering supports complex queries
+ * - O(n log k) complexity for top-k selection
+ */
+
+import { SemanticEngine } from '../engine/SemanticEngine';
+import {
+  Embedding,
+  SearchResult,
+  SemanticError,
+  SemanticErrorCode,
+} from '../types';
+import { topKSimilar } from '../utils/vector';
+
+export interface SearchConfig<T = string> {
+  topK?: number;
+  threshold?: number;
+  textExtractor?: (item: T) => string;
+  metadataExtractor?: (item: T) => Record<string, unknown>;
+}
+
+export interface IndexedItem<T = string> {
+  item: T;
+  embedding: Embedding;
+  metadata?: Record<string, unknown>;
+}
+
+export class SemanticSearch<T = string> {
+  private engine: SemanticEngine;
+  private indexedItems: IndexedItem<T>[] = [];
+  private config: Required<SearchConfig<T>>;
+
+  constructor(engine: SemanticEngine, config: SearchConfig<T> = {}) {
+    this.engine = engine;
+    this.config = {
+      topK: config.topK ?? 10,
+      threshold: config.threshold ?? 0,
+      textExtractor: config.textExtractor ?? ((item) => String(item)),
+      metadataExtractor: config.metadataExtractor ?? (() => ({})),
+    };
+  }
+
+  async index(items: T[], replace: boolean = false): Promise<void> {
+    if (!Array.isArray(items) || items.length === 0) {
+      throw new SemanticError(
+        SemanticErrorCode.INVALID_INPUT,
+        'Items must be a non-empty array'
+      );
+    }
+
+    if (replace) {
+      this.indexedItems = [];
+    }
+
+    const texts = items.map(this.config.textExtractor);
+    const results = await this.engine.embedBatch(texts, { batchSize: 32 });
+
+    const newIndexItems = items.map((item, idx) => ({
+      item,
+      embedding: results[idx].embedding,
+      metadata: this.config.metadataExtractor(item),
+    }));
+
+    this.indexedItems.push(...newIndexItems);
+  }
+
+  async search(
+    query: string,
+    overrideConfig?: Partial<SearchConfig<T>>
+  ): Promise<SearchResult<T>[]> {
+    if (this.indexedItems.length === 0) {
+      throw new SemanticError(
+        SemanticErrorCode.INVALID_INPUT,
+        'Index is empty. Call index() before searching.'
+      );
+    }
+
+    const config = { ...this.config, ...overrideConfig };
+    const queryResult = await this.engine.embed(query);
+    const candidateEmbeddings = this.indexedItems.map(item => item.embedding);
+    const topK = topKSimilar(queryResult.embedding, candidateEmbeddings, config.topK);
+
+    const results: SearchResult<T>[] = [];
+    let rank = 1;
+
+    for (const [idx, score] of topK) {
+      if (score < config.threshold) continue;
+
+      results.push({
+        item: this.indexedItems[idx].item,
+        score,
+        rank: rank++,
+      });
+    }
+
+    return results;
+  }
+
+  async searchWithFilter(
+    query: string,
+    filter: (metadata: Record<string, unknown>) => boolean,
+    config?: Partial<SearchConfig<T>>
+  ): Promise<SearchResult<T>[]> {
+    const originalIndex = this.indexedItems;
+    this.indexedItems = originalIndex.filter(item => filter(item.metadata ?? {}));
+
+    try {
+      return await this.search(query, config);
+    } finally {
+      this.indexedItems = originalIndex;
+    }
+  }
+
+  async findSimilar(
+    item: T,
+    config?: Partial<SearchConfig<T>>
+  ): Promise<SearchResult<T>[]> {
+    const text = this.config.textExtractor(item);
+    return this.search(text, config);
+  }
+
+  getStats(): {
+    itemCount: number;
+    dimensions: number;
+    memoryEstimate: string;
+  } {
+    const itemCount = this.indexedItems.length;
+    const dimensions = this.indexedItems[0]?.embedding.length ?? 0;
+    const totalBytes = itemCount * dimensions * 8;
+    
+    const memoryEstimate = totalBytes < 1024 * 1024
+      ? `${(totalBytes / 1024).toFixed(2)} KB`
+      : `${(totalBytes / (1024 * 1024)).toFixed(2)} MB`;
+
+    return { itemCount, dimensions, memoryEstimate };
+  }
+
+  clear(): void {
+    this.indexedItems = [];
+  }
+
+  exportIndex(): IndexedItem<T>[] {
+    return [...this.indexedItems];
+  }
+
+  importIndex(index: IndexedItem<T>[]): void {
+    this.indexedItems = [...index];
+  }
+}

package/src/types.ts

@@ -0,0 +1,73 @@
+/**
+ * Core Type Definitions
+ * 
+ * Design Philosophy:
+ * - Type safety prevents runtime failures
+ * - Semantic error codes enable precise debugging
+ * - Generic types provide flexibility without sacrificing safety
+ */
+
+export type Embedding = number[];
+
+export interface ModelConfig {
+  modelName?: string;
+  maxLength?: number;
+  quantized?: boolean;
+  onProgress?: (progress: ModelLoadProgress) => void;
+}
+
+export interface ModelLoadProgress {
+  status: 'downloading' | 'loading' | 'ready';
+  progress: number;
+  file?: string;
+}
+
+export interface EmbeddingResult {
+  embedding: Embedding;
+  text: string;
+  metadata: {
+    dimensions: number;
+    modelName: string;
+    processingTime: number;
+  };
+}
+
+export interface SimilarityResult {
+  score: number;
+  texts: [string, string];
+  metadata: {
+    method: 'cosine' | 'euclidean' | 'dot';
+    processingTime: number;
+  };
+}
+
+export interface SearchResult<T = string> {
+  item: T;
+  score: number;
+  rank: number;
+}
+
+export interface BatchOptions {
+  batchSize?: number;
+  parallel?: boolean;
+  onProgress?: (completed: number, total: number) => void;
+}
+
+export enum SemanticErrorCode {
+  MODEL_NOT_LOADED = 'MODEL_NOT_LOADED',
+  INVALID_INPUT = 'INVALID_INPUT',
+  EMBEDDING_FAILED = 'EMBEDDING_FAILED',
+  COMPUTATION_FAILED = 'COMPUTATION_FAILED',
+  DIMENSION_MISMATCH = 'DIMENSION_MISMATCH',
+}
+
+export class SemanticError extends Error {
+  constructor(
+    public code: SemanticErrorCode,
+    message: string,
+    public details?: Record<string, unknown>
+  ) {
+    super(message);
+    this.name = 'SemanticError';
+  }
+}

package/src/utils/vector.ts

@@ -0,0 +1,158 @@
+/**
+ * Vector Mathematics - Performance-Optimized Operations
+ * 
+ * Architectural Principles:
+ * - Pure functions ensure predictability
+ * - O(n) time complexity for scalability
+ * - Defensive validation at boundaries
+ * - Zero external dependencies
+ */
+
+import { Embedding, SemanticError, SemanticErrorCode } from '../types';
+
+function validateDimensions(a: Embedding, b: Embedding): void {
+  if (a.length !== b.length) {
+    throw new SemanticError(
+      SemanticErrorCode.DIMENSION_MISMATCH,
+      `Embedding dimensions must match. Got ${a.length} and ${b.length}`,
+      { dimensions: [a.length, b.length] }
+    );
+  }
+}
+
+function validateEmbedding(embedding: Embedding, name: string = 'embedding'): void {
+  if (!embedding || embedding.length === 0) {
+    throw new SemanticError(
+      SemanticErrorCode.INVALID_INPUT,
+      `${name} must be a non-empty array`,
+      { length: embedding?.length }
+    );
+  }
+}
+
+export function dotProduct(a: Embedding, b: Embedding): number {
+  validateEmbedding(a, 'first embedding');
+  validateEmbedding(b, 'second embedding');
+  validateDimensions(a, b);
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    sum += a[i] * b[i];
+  }
+  return sum;
+}
+
+export function magnitude(vector: Embedding): number {
+  validateEmbedding(vector);
+  
+  let sum = 0;
+  for (let i = 0; i < vector.length; i++) {
+    sum += vector[i] * vector[i];
+  }
+  return Math.sqrt(sum);
+}
+
+export function cosineSimilarity(a: Embedding, b: Embedding): number {
+  validateEmbedding(a, 'first embedding');
+  validateEmbedding(b, 'second embedding');
+  validateDimensions(a, b);
+
+  const dot = dotProduct(a, b);
+  const magA = magnitude(a);
+  const magB = magnitude(b);
+
+  if (magA === 0 || magB === 0) {
+    throw new SemanticError(
+      SemanticErrorCode.COMPUTATION_FAILED,
+      'Cannot compute cosine similarity with zero-magnitude vector',
+      { magnitudes: [magA, magB] }
+    );
+  }
+
+  return dot / (magA * magB);
+}
+
+export function euclideanDistance(a: Embedding, b: Embedding): number {
+  validateEmbedding(a, 'first embedding');
+  validateEmbedding(b, 'second embedding');
+  validateDimensions(a, b);
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i] - b[i];
+    sum += diff * diff;
+  }
+  return Math.sqrt(sum);
+}
+
+export function normalize(vector: Embedding): Embedding {
+  validateEmbedding(vector);
+  
+  const mag = magnitude(vector);
+  if (mag === 0) {
+    throw new SemanticError(
+      SemanticErrorCode.COMPUTATION_FAILED,
+      'Cannot normalize zero-magnitude vector'
+    );
+  }
+
+  return vector.map(v => v / mag);
+}
+
+export function centroid(embeddings: Embedding[]): Embedding {
+  if (!embeddings || embeddings.length === 0) {
+    throw new SemanticError(
+      SemanticErrorCode.INVALID_INPUT,
+      'Cannot compute centroid of empty array'
+    );
+  }
+
+  const dim = embeddings[0].length;
+  const result = new Array(dim).fill(0);
+
+  for (const embedding of embeddings) {
+    if (embedding.length !== dim) {
+      throw new SemanticError(
+        SemanticErrorCode.DIMENSION_MISMATCH,
+        'All embeddings must have same dimensions'
+      );
+    }
+    for (let i = 0; i < dim; i++) {
+      result[i] += embedding[i];
+    }
+  }
+
+  return result.map(v => v / embeddings.length);
+}
+
+export function topKSimilar(
+  query: Embedding,
+  candidates: Embedding[],
+  k: number = 10
+): Array<[number, number]> {
+  validateEmbedding(query, 'query');
+  
+  if (!candidates || candidates.length === 0) {
+    return [];
+  }
+
+  if (k <= 0) {
+    throw new SemanticError(
+      SemanticErrorCode.INVALID_INPUT,
+      'k must be positive',
+      { k }
+    );
+  }
+
+  const similarities: Array<[number, number]> = candidates.map((candidate, idx) => {
+    try {
+      return [idx, cosineSimilarity(query, candidate)];
+    } catch (error) {
+      return [idx, -Infinity];
+    }
+  });
+
+  similarities.sort((a, b) => b[1] - a[1]);
+  
+  return similarities.slice(0, Math.min(k, similarities.length));
+}

package/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "lib": ["ES2020"],
+    "moduleResolution": "node",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts"]
+}