@crashbytes/semantic-text-toolkit 1.0.0

package/dist/chunk-TPAL6DKL.mjs

@@ -0,0 +1,149 @@
+// src/types.ts
+var SemanticErrorCode = /* @__PURE__ */ ((SemanticErrorCode2) => {
+  SemanticErrorCode2["MODEL_NOT_LOADED"] = "MODEL_NOT_LOADED";
+  SemanticErrorCode2["INVALID_INPUT"] = "INVALID_INPUT";
+  SemanticErrorCode2["EMBEDDING_FAILED"] = "EMBEDDING_FAILED";
+  SemanticErrorCode2["COMPUTATION_FAILED"] = "COMPUTATION_FAILED";
+  SemanticErrorCode2["DIMENSION_MISMATCH"] = "DIMENSION_MISMATCH";
+  return SemanticErrorCode2;
+})(SemanticErrorCode || {});
+var SemanticError = class extends Error {
+  constructor(code, message, details) {
+    super(message);
+    this.code = code;
+    this.details = details;
+    this.name = "SemanticError";
+  }
+};
+
+// src/utils/vector.ts
+function validateDimensions(a, b) {
+  if (a.length !== b.length) {
+    throw new SemanticError(
+      "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
+      `Embedding dimensions must match. Got ${a.length} and ${b.length}`,
+      { dimensions: [a.length, b.length] }
+    );
+  }
+}
+function validateEmbedding(embedding, name = "embedding") {
+  if (!embedding || embedding.length === 0) {
+    throw new SemanticError(
+      "INVALID_INPUT" /* INVALID_INPUT */,
+      `${name} must be a non-empty array`,
+      { length: embedding?.length }
+    );
+  }
+}
+function dotProduct(a, b) {
+  validateEmbedding(a, "first embedding");
+  validateEmbedding(b, "second embedding");
+  validateDimensions(a, b);
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    sum += a[i] * b[i];
+  }
+  return sum;
+}
+function magnitude(vector) {
+  validateEmbedding(vector);
+  let sum = 0;
+  for (let i = 0; i < vector.length; i++) {
+    sum += vector[i] * vector[i];
+  }
+  return Math.sqrt(sum);
+}
+function cosineSimilarity(a, b) {
+  validateEmbedding(a, "first embedding");
+  validateEmbedding(b, "second embedding");
+  validateDimensions(a, b);
+  const dot = dotProduct(a, b);
+  const magA = magnitude(a);
+  const magB = magnitude(b);
+  if (magA === 0 || magB === 0) {
+    throw new SemanticError(
+      "COMPUTATION_FAILED" /* COMPUTATION_FAILED */,
+      "Cannot compute cosine similarity with zero-magnitude vector",
+      { magnitudes: [magA, magB] }
+    );
+  }
+  return dot / (magA * magB);
+}
+function euclideanDistance(a, b) {
+  validateEmbedding(a, "first embedding");
+  validateEmbedding(b, "second embedding");
+  validateDimensions(a, b);
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i] - b[i];
+    sum += diff * diff;
+  }
+  return Math.sqrt(sum);
+}
+function normalize(vector) {
+  validateEmbedding(vector);
+  const mag = magnitude(vector);
+  if (mag === 0) {
+    throw new SemanticError(
+      "COMPUTATION_FAILED" /* COMPUTATION_FAILED */,
+      "Cannot normalize zero-magnitude vector"
+    );
+  }
+  return vector.map((v) => v / mag);
+}
+function centroid(embeddings) {
+  if (!embeddings || embeddings.length === 0) {
+    throw new SemanticError(
+      "INVALID_INPUT" /* INVALID_INPUT */,
+      "Cannot compute centroid of empty array"
+    );
+  }
+  const dim = embeddings[0].length;
+  const result = new Array(dim).fill(0);
+  for (const embedding of embeddings) {
+    if (embedding.length !== dim) {
+      throw new SemanticError(
+        "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
+        "All embeddings must have same dimensions"
+      );
+    }
+    for (let i = 0; i < dim; i++) {
+      result[i] += embedding[i];
+    }
+  }
+  return result.map((v) => v / embeddings.length);
+}
+function topKSimilar(query, candidates, k = 10) {
+  validateEmbedding(query, "query");
+  if (!candidates || candidates.length === 0) {
+    return [];
+  }
+  if (k <= 0) {
+    throw new SemanticError(
+      "INVALID_INPUT" /* INVALID_INPUT */,
+      "k must be positive",
+      { k }
+    );
+  }
+  const similarities = candidates.map((candidate, idx) => {
+    try {
+      return [idx, cosineSimilarity(query, candidate)];
+    } catch (error) {
+      return [idx, -Infinity];
+    }
+  });
+  similarities.sort((a, b) => b[1] - a[1]);
+  return similarities.slice(0, Math.min(k, similarities.length));
+}
+
+export {
+  SemanticErrorCode,
+  SemanticError,
+  dotProduct,
+  magnitude,
+  cosineSimilarity,
+  euclideanDistance,
+  normalize,
+  centroid,
+  topKSimilar
+};

package/dist/chunk-XJ4PTDH6.mjs

@@ -0,0 +1,176 @@
+import {
+  SemanticError,
+  cosineSimilarity,
+  dotProduct,
+  euclideanDistance
+} from "./chunk-TPAL6DKL.mjs";
+
+// src/engine/SemanticEngine.ts
+import { pipeline } from "@xenova/transformers";
+var DEFAULT_CONFIG = {
+  modelName: "Xenova/all-MiniLM-L6-v2",
+  maxLength: 512,
+  quantized: true,
+  onProgress: () => {
+  }
+};
+var SemanticEngine = class {
+  constructor(config = {}) {
+    this.model = null;
+    this.initializationPromise = null;
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+  async initialize() {
+    if (this.initializationPromise) {
+      return this.initializationPromise;
+    }
+    if (this.model) {
+      return Promise.resolve();
+    }
+    this.initializationPromise = this._performInitialization();
+    try {
+      await this.initializationPromise;
+    } finally {
+      this.initializationPromise = null;
+    }
+  }
+  async _performInitialization() {
+    try {
+      this.config.onProgress({
+        status: "downloading",
+        progress: 0
+      });
+      this.model = await pipeline(
+        "feature-extraction",
+        this.config.modelName,
+        {
+          quantized: this.config.quantized
+        }
+      );
+      this.config.onProgress({
+        status: "ready",
+        progress: 100
+      });
+    } catch (error) {
+      throw new SemanticError(
+        "MODEL_NOT_LOADED" /* MODEL_NOT_LOADED */,
+        `Failed to initialize model: ${error instanceof Error ? error.message : "Unknown error"}`,
+        { modelName: this.config.modelName, error }
+      );
+    }
+  }
+  assertInitialized() {
+    if (!this.model) {
+      throw new SemanticError(
+        "MODEL_NOT_LOADED" /* MODEL_NOT_LOADED */,
+        "Model not initialized. Call initialize() first."
+      );
+    }
+  }
+  async embed(text) {
+    this.assertInitialized();
+    if (!text || typeof text !== "string") {
+      throw new SemanticError(
+        "INVALID_INPUT" /* INVALID_INPUT */,
+        "Text must be a non-empty string",
+        { text }
+      );
+    }
+    const startTime = performance.now();
+    try {
+      const output = await this.model(text, {
+        pooling: "mean",
+        normalize: true
+      });
+      const embedding = Array.from(output.data);
+      const processingTime = performance.now() - startTime;
+      return {
+        embedding,
+        text,
+        metadata: {
+          dimensions: embedding.length,
+          modelName: this.config.modelName,
+          processingTime
+        }
+      };
+    } catch (error) {
+      throw new SemanticError(
+        "EMBEDDING_FAILED" /* EMBEDDING_FAILED */,
+        `Failed to generate embedding: ${error instanceof Error ? error.message : "Unknown error"}`,
+        { text: text.substring(0, 100), error }
+      );
+    }
+  }
+  async embedBatch(texts, options = {}) {
+    this.assertInitialized();
+    const { batchSize = 32, onProgress } = options;
+    if (!Array.isArray(texts) || texts.length === 0) {
+      throw new SemanticError(
+        "INVALID_INPUT" /* INVALID_INPUT */,
+        "Texts must be a non-empty array"
+      );
+    }
+    const results = [];
+    const batches = Math.ceil(texts.length / batchSize);
+    for (let i = 0; i < batches; i++) {
+      const start = i * batchSize;
+      const end = Math.min(start + batchSize, texts.length);
+      const batch = texts.slice(start, end);
+      const batchResults = await Promise.all(
+        batch.map((text) => this.embed(text))
+      );
+      results.push(...batchResults);
+      if (onProgress) {
+        onProgress(end, texts.length);
+      }
+    }
+    return results;
+  }
+  async similarity(textA, textB, method = "cosine") {
+    const startTime = performance.now();
+    const [resultA, resultB] = await Promise.all([
+      this.embed(textA),
+      this.embed(textB)
+    ]);
+    let score;
+    switch (method) {
+      case "cosine":
+        score = cosineSimilarity(resultA.embedding, resultB.embedding);
+        break;
+      case "euclidean":
+        score = -euclideanDistance(resultA.embedding, resultB.embedding);
+        break;
+      case "dot":
+        score = dotProduct(resultA.embedding, resultB.embedding);
+        break;
+      default:
+        throw new SemanticError(
+          "INVALID_INPUT" /* INVALID_INPUT */,
+          `Unknown similarity method: ${method}`
+        );
+    }
+    const processingTime = performance.now() - startTime;
+    return {
+      score,
+      texts: [textA, textB],
+      metadata: {
+        method,
+        processingTime
+      }
+    };
+  }
+  dispose() {
+    this.model = null;
+    this.initializationPromise = null;
+  }
+  isReady() {
+    return this.model !== null;
+  }
+  getConfig() {
+    return { ...this.config };
+  }
+};
+
+export {
+  SemanticEngine
+};

package/dist/index.d.mts

@@ -0,0 +1,148 @@
+/**
+ * Core Type Definitions
+ *
+ * Design Philosophy:
+ * - Type safety prevents runtime failures
+ * - Semantic error codes enable precise debugging
+ * - Generic types provide flexibility without sacrificing safety
+ */
+type Embedding = number[];
+interface ModelConfig {
+    modelName?: string;
+    maxLength?: number;
+    quantized?: boolean;
+    onProgress?: (progress: ModelLoadProgress) => void;
+}
+interface ModelLoadProgress {
+    status: 'downloading' | 'loading' | 'ready';
+    progress: number;
+    file?: string;
+}
+interface EmbeddingResult {
+    embedding: Embedding;
+    text: string;
+    metadata: {
+        dimensions: number;
+        modelName: string;
+        processingTime: number;
+    };
+}
+interface SimilarityResult {
+    score: number;
+    texts: [string, string];
+    metadata: {
+        method: 'cosine' | 'euclidean' | 'dot';
+        processingTime: number;
+    };
+}
+interface SearchResult<T = string> {
+    item: T;
+    score: number;
+    rank: number;
+}
+interface BatchOptions {
+    batchSize?: number;
+    parallel?: boolean;
+    onProgress?: (completed: number, total: number) => void;
+}
+declare enum SemanticErrorCode {
+    MODEL_NOT_LOADED = "MODEL_NOT_LOADED",
+    INVALID_INPUT = "INVALID_INPUT",
+    EMBEDDING_FAILED = "EMBEDDING_FAILED",
+    COMPUTATION_FAILED = "COMPUTATION_FAILED",
+    DIMENSION_MISMATCH = "DIMENSION_MISMATCH"
+}
+declare class SemanticError extends Error {
+    code: SemanticErrorCode;
+    details?: Record<string, unknown> | undefined;
+    constructor(code: SemanticErrorCode, message: string, details?: Record<string, unknown> | undefined);
+}
+
+/**
+ * Semantic Engine - Core Embedding Generation
+ *
+ * Architectural Principles:
+ * - Lazy initialization minimizes startup overhead
+ * - Singleton pattern prevents redundant model loading
+ * - Resource management through explicit lifecycle control
+ * - Defensive error handling with semantic codes
+ */
+
+declare class SemanticEngine {
+    private model;
+    private config;
+    private initializationPromise;
+    constructor(config?: ModelConfig);
+    initialize(): Promise<void>;
+    private _performInitialization;
+    private assertInitialized;
+    embed(text: string): Promise<EmbeddingResult>;
+    embedBatch(texts: string[], options?: BatchOptions): Promise<EmbeddingResult[]>;
+    similarity(textA: string, textB: string, method?: 'cosine' | 'euclidean' | 'dot'): Promise<SimilarityResult>;
+    dispose(): void;
+    isReady(): boolean;
+    getConfig(): Required<ModelConfig>;
+}
+
+/**
+ * Semantic Search - Vector-Based Document Retrieval
+ *
+ * Architectural Principles:
+ * - Pre-computed embeddings optimize retrieval latency
+ * - Configurable ranking strategies enable domain customization
+ * - Metadata filtering supports complex queries
+ * - O(n log k) complexity for top-k selection
+ */
+
+interface SearchConfig<T = string> {
+    topK?: number;
+    threshold?: number;
+    textExtractor?: (item: T) => string;
+    metadataExtractor?: (item: T) => Record<string, unknown>;
+}
+interface IndexedItem<T = string> {
+    item: T;
+    embedding: Embedding;
+    metadata?: Record<string, unknown>;
+}
+declare class SemanticSearch<T = string> {
+    private engine;
+    private indexedItems;
+    private config;
+    constructor(engine: SemanticEngine, config?: SearchConfig<T>);
+    index(items: T[], replace?: boolean): Promise<void>;
+    search(query: string, overrideConfig?: Partial<SearchConfig<T>>): Promise<SearchResult<T>[]>;
+    searchWithFilter(query: string, filter: (metadata: Record<string, unknown>) => boolean, config?: Partial<SearchConfig<T>>): Promise<SearchResult<T>[]>;
+    findSimilar(item: T, config?: Partial<SearchConfig<T>>): Promise<SearchResult<T>[]>;
+    getStats(): {
+        itemCount: number;
+        dimensions: number;
+        memoryEstimate: string;
+    };
+    clear(): void;
+    exportIndex(): IndexedItem<T>[];
+    importIndex(index: IndexedItem<T>[]): void;
+}
+
+/**
+ * Vector Mathematics - Performance-Optimized Operations
+ *
+ * Architectural Principles:
+ * - Pure functions ensure predictability
+ * - O(n) time complexity for scalability
+ * - Defensive validation at boundaries
+ * - Zero external dependencies
+ */
+
+declare function dotProduct(a: Embedding, b: Embedding): number;
+declare function magnitude(vector: Embedding): number;
+declare function cosineSimilarity(a: Embedding, b: Embedding): number;
+declare function euclideanDistance(a: Embedding, b: Embedding): number;
+declare function normalize(vector: Embedding): Embedding;
+declare function centroid(embeddings: Embedding[]): Embedding;
+declare function topKSimilar(query: Embedding, candidates: Embedding[], k?: number): Array<[number, number]>;
+
+declare function createSemanticEngine(config?: ModelConfig): Promise<SemanticEngine>;
+declare function createSemanticSearch<T = string>(items: T[], config?: SearchConfig<T>): Promise<SemanticSearch<T>>;
+
+export { type BatchOptions, type Embedding, type EmbeddingResult, type IndexedItem, type ModelConfig, type ModelLoadProgress, type SearchConfig, type SearchResult, SemanticEngine, SemanticError, SemanticErrorCode, SemanticSearch, type SimilarityResult, centroid, cosineSimilarity, createSemanticEngine, createSemanticSearch, dotProduct, euclideanDistance, magnitude, normalize, topKSimilar };

package/dist/index.d.ts

@@ -0,0 +1,148 @@
+/**
+ * Core Type Definitions
+ *
+ * Design Philosophy:
+ * - Type safety prevents runtime failures
+ * - Semantic error codes enable precise debugging
+ * - Generic types provide flexibility without sacrificing safety
+ */
+type Embedding = number[];
+interface ModelConfig {
+    modelName?: string;
+    maxLength?: number;
+    quantized?: boolean;
+    onProgress?: (progress: ModelLoadProgress) => void;
+}
+interface ModelLoadProgress {
+    status: 'downloading' | 'loading' | 'ready';
+    progress: number;
+    file?: string;
+}
+interface EmbeddingResult {
+    embedding: Embedding;
+    text: string;
+    metadata: {
+        dimensions: number;
+        modelName: string;
+        processingTime: number;
+    };
+}
+interface SimilarityResult {
+    score: number;
+    texts: [string, string];
+    metadata: {
+        method: 'cosine' | 'euclidean' | 'dot';
+        processingTime: number;
+    };
+}
+interface SearchResult<T = string> {
+    item: T;
+    score: number;
+    rank: number;
+}
+interface BatchOptions {
+    batchSize?: number;
+    parallel?: boolean;
+    onProgress?: (completed: number, total: number) => void;
+}
+declare enum SemanticErrorCode {
+    MODEL_NOT_LOADED = "MODEL_NOT_LOADED",
+    INVALID_INPUT = "INVALID_INPUT",
+    EMBEDDING_FAILED = "EMBEDDING_FAILED",
+    COMPUTATION_FAILED = "COMPUTATION_FAILED",
+    DIMENSION_MISMATCH = "DIMENSION_MISMATCH"
+}
+declare class SemanticError extends Error {
+    code: SemanticErrorCode;
+    details?: Record<string, unknown> | undefined;
+    constructor(code: SemanticErrorCode, message: string, details?: Record<string, unknown> | undefined);
+}
+
+/**
+ * Semantic Engine - Core Embedding Generation
+ *
+ * Architectural Principles:
+ * - Lazy initialization minimizes startup overhead
+ * - Singleton pattern prevents redundant model loading
+ * - Resource management through explicit lifecycle control
+ * - Defensive error handling with semantic codes
+ */
+
+declare class SemanticEngine {
+    private model;
+    private config;
+    private initializationPromise;
+    constructor(config?: ModelConfig);
+    initialize(): Promise<void>;
+    private _performInitialization;
+    private assertInitialized;
+    embed(text: string): Promise<EmbeddingResult>;
+    embedBatch(texts: string[], options?: BatchOptions): Promise<EmbeddingResult[]>;
+    similarity(textA: string, textB: string, method?: 'cosine' | 'euclidean' | 'dot'): Promise<SimilarityResult>;
+    dispose(): void;
+    isReady(): boolean;
+    getConfig(): Required<ModelConfig>;
+}
+
+/**
+ * Semantic Search - Vector-Based Document Retrieval
+ *
+ * Architectural Principles:
+ * - Pre-computed embeddings optimize retrieval latency
+ * - Configurable ranking strategies enable domain customization
+ * - Metadata filtering supports complex queries
+ * - O(n log k) complexity for top-k selection
+ */
+
+interface SearchConfig<T = string> {
+    topK?: number;
+    threshold?: number;
+    textExtractor?: (item: T) => string;
+    metadataExtractor?: (item: T) => Record<string, unknown>;
+}
+interface IndexedItem<T = string> {
+    item: T;
+    embedding: Embedding;
+    metadata?: Record<string, unknown>;
+}
+declare class SemanticSearch<T = string> {
+    private engine;
+    private indexedItems;
+    private config;
+    constructor(engine: SemanticEngine, config?: SearchConfig<T>);
+    index(items: T[], replace?: boolean): Promise<void>;
+    search(query: string, overrideConfig?: Partial<SearchConfig<T>>): Promise<SearchResult<T>[]>;
+    searchWithFilter(query: string, filter: (metadata: Record<string, unknown>) => boolean, config?: Partial<SearchConfig<T>>): Promise<SearchResult<T>[]>;
+    findSimilar(item: T, config?: Partial<SearchConfig<T>>): Promise<SearchResult<T>[]>;
+    getStats(): {
+        itemCount: number;
+        dimensions: number;
+        memoryEstimate: string;
+    };
+    clear(): void;
+    exportIndex(): IndexedItem<T>[];
+    importIndex(index: IndexedItem<T>[]): void;
+}
+
+/**
+ * Vector Mathematics - Performance-Optimized Operations
+ *
+ * Architectural Principles:
+ * - Pure functions ensure predictability
+ * - O(n) time complexity for scalability
+ * - Defensive validation at boundaries
+ * - Zero external dependencies
+ */
+
+declare function dotProduct(a: Embedding, b: Embedding): number;
+declare function magnitude(vector: Embedding): number;
+declare function cosineSimilarity(a: Embedding, b: Embedding): number;
+declare function euclideanDistance(a: Embedding, b: Embedding): number;
+declare function normalize(vector: Embedding): Embedding;
+declare function centroid(embeddings: Embedding[]): Embedding;
+declare function topKSimilar(query: Embedding, candidates: Embedding[], k?: number): Array<[number, number]>;
+
+declare function createSemanticEngine(config?: ModelConfig): Promise<SemanticEngine>;
+declare function createSemanticSearch<T = string>(items: T[], config?: SearchConfig<T>): Promise<SemanticSearch<T>>;
+
+export { type BatchOptions, type Embedding, type EmbeddingResult, type IndexedItem, type ModelConfig, type ModelLoadProgress, type SearchConfig, type SearchResult, SemanticEngine, SemanticError, SemanticErrorCode, SemanticSearch, type SimilarityResult, centroid, cosineSimilarity, createSemanticEngine, createSemanticSearch, dotProduct, euclideanDistance, magnitude, normalize, topKSimilar };