@crashbytes/semantic-text-toolkit 1.0.0

package/dist/index.js

@@ -0,0 +1,506 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/types.ts
+var SemanticErrorCode, SemanticError;
+var init_types = __esm({
+  "src/types.ts"() {
+    "use strict";
+    SemanticErrorCode = /* @__PURE__ */ ((SemanticErrorCode2) => {
+      SemanticErrorCode2["MODEL_NOT_LOADED"] = "MODEL_NOT_LOADED";
+      SemanticErrorCode2["INVALID_INPUT"] = "INVALID_INPUT";
+      SemanticErrorCode2["EMBEDDING_FAILED"] = "EMBEDDING_FAILED";
+      SemanticErrorCode2["COMPUTATION_FAILED"] = "COMPUTATION_FAILED";
+      SemanticErrorCode2["DIMENSION_MISMATCH"] = "DIMENSION_MISMATCH";
+      return SemanticErrorCode2;
+    })(SemanticErrorCode || {});
+    SemanticError = class extends Error {
+      constructor(code, message, details) {
+        super(message);
+        this.code = code;
+        this.details = details;
+        this.name = "SemanticError";
+      }
+    };
+  }
+});
+
+// src/utils/vector.ts
+function validateDimensions(a, b) {
+  if (a.length !== b.length) {
+    throw new SemanticError(
+      "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
+      `Embedding dimensions must match. Got ${a.length} and ${b.length}`,
+      { dimensions: [a.length, b.length] }
+    );
+  }
+}
+function validateEmbedding(embedding, name = "embedding") {
+  if (!embedding || embedding.length === 0) {
+    throw new SemanticError(
+      "INVALID_INPUT" /* INVALID_INPUT */,
+      `${name} must be a non-empty array`,
+      { length: embedding?.length }
+    );
+  }
+}
+function dotProduct(a, b) {
+  validateEmbedding(a, "first embedding");
+  validateEmbedding(b, "second embedding");
+  validateDimensions(a, b);
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    sum += a[i] * b[i];
+  }
+  return sum;
+}
+function magnitude(vector) {
+  validateEmbedding(vector);
+  let sum = 0;
+  for (let i = 0; i < vector.length; i++) {
+    sum += vector[i] * vector[i];
+  }
+  return Math.sqrt(sum);
+}
+function cosineSimilarity(a, b) {
+  validateEmbedding(a, "first embedding");
+  validateEmbedding(b, "second embedding");
+  validateDimensions(a, b);
+  const dot = dotProduct(a, b);
+  const magA = magnitude(a);
+  const magB = magnitude(b);
+  if (magA === 0 || magB === 0) {
+    throw new SemanticError(
+      "COMPUTATION_FAILED" /* COMPUTATION_FAILED */,
+      "Cannot compute cosine similarity with zero-magnitude vector",
+      { magnitudes: [magA, magB] }
+    );
+  }
+  return dot / (magA * magB);
+}
+function euclideanDistance(a, b) {
+  validateEmbedding(a, "first embedding");
+  validateEmbedding(b, "second embedding");
+  validateDimensions(a, b);
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i] - b[i];
+    sum += diff * diff;
+  }
+  return Math.sqrt(sum);
+}
+function normalize(vector) {
+  validateEmbedding(vector);
+  const mag = magnitude(vector);
+  if (mag === 0) {
+    throw new SemanticError(
+      "COMPUTATION_FAILED" /* COMPUTATION_FAILED */,
+      "Cannot normalize zero-magnitude vector"
+    );
+  }
+  return vector.map((v) => v / mag);
+}
+function centroid(embeddings) {
+  if (!embeddings || embeddings.length === 0) {
+    throw new SemanticError(
+      "INVALID_INPUT" /* INVALID_INPUT */,
+      "Cannot compute centroid of empty array"
+    );
+  }
+  const dim = embeddings[0].length;
+  const result = new Array(dim).fill(0);
+  for (const embedding of embeddings) {
+    if (embedding.length !== dim) {
+      throw new SemanticError(
+        "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
+        "All embeddings must have same dimensions"
+      );
+    }
+    for (let i = 0; i < dim; i++) {
+      result[i] += embedding[i];
+    }
+  }
+  return result.map((v) => v / embeddings.length);
+}
+function topKSimilar(query, candidates, k = 10) {
+  validateEmbedding(query, "query");
+  if (!candidates || candidates.length === 0) {
+    return [];
+  }
+  if (k <= 0) {
+    throw new SemanticError(
+      "INVALID_INPUT" /* INVALID_INPUT */,
+      "k must be positive",
+      { k }
+    );
+  }
+  const similarities = candidates.map((candidate, idx) => {
+    try {
+      return [idx, cosineSimilarity(query, candidate)];
+    } catch (error) {
+      return [idx, -Infinity];
+    }
+  });
+  similarities.sort((a, b) => b[1] - a[1]);
+  return similarities.slice(0, Math.min(k, similarities.length));
+}
+var init_vector = __esm({
+  "src/utils/vector.ts"() {
+    "use strict";
+    init_types();
+  }
+});
+
+// src/engine/SemanticEngine.ts
+var SemanticEngine_exports = {};
+__export(SemanticEngine_exports, {
+  SemanticEngine: () => SemanticEngine
+});
+var import_transformers, DEFAULT_CONFIG, SemanticEngine;
+var init_SemanticEngine = __esm({
+  "src/engine/SemanticEngine.ts"() {
+    "use strict";
+    import_transformers = require("@xenova/transformers");
+    init_types();
+    init_vector();
+    DEFAULT_CONFIG = {
+      modelName: "Xenova/all-MiniLM-L6-v2",
+      maxLength: 512,
+      quantized: true,
+      onProgress: () => {
+      }
+    };
+    SemanticEngine = class {
+      constructor(config = {}) {
+        this.model = null;
+        this.initializationPromise = null;
+        this.config = { ...DEFAULT_CONFIG, ...config };
+      }
+      async initialize() {
+        if (this.initializationPromise) {
+          return this.initializationPromise;
+        }
+        if (this.model) {
+          return Promise.resolve();
+        }
+        this.initializationPromise = this._performInitialization();
+        try {
+          await this.initializationPromise;
+        } finally {
+          this.initializationPromise = null;
+        }
+      }
+      async _performInitialization() {
+        try {
+          this.config.onProgress({
+            status: "downloading",
+            progress: 0
+          });
+          this.model = await (0, import_transformers.pipeline)(
+            "feature-extraction",
+            this.config.modelName,
+            {
+              quantized: this.config.quantized
+            }
+          );
+          this.config.onProgress({
+            status: "ready",
+            progress: 100
+          });
+        } catch (error) {
+          throw new SemanticError(
+            "MODEL_NOT_LOADED" /* MODEL_NOT_LOADED */,
+            `Failed to initialize model: ${error instanceof Error ? error.message : "Unknown error"}`,
+            { modelName: this.config.modelName, error }
+          );
+        }
+      }
+      assertInitialized() {
+        if (!this.model) {
+          throw new SemanticError(
+            "MODEL_NOT_LOADED" /* MODEL_NOT_LOADED */,
+            "Model not initialized. Call initialize() first."
+          );
+        }
+      }
+      async embed(text) {
+        this.assertInitialized();
+        if (!text || typeof text !== "string") {
+          throw new SemanticError(
+            "INVALID_INPUT" /* INVALID_INPUT */,
+            "Text must be a non-empty string",
+            { text }
+          );
+        }
+        const startTime = performance.now();
+        try {
+          const output = await this.model(text, {
+            pooling: "mean",
+            normalize: true
+          });
+          const embedding = Array.from(output.data);
+          const processingTime = performance.now() - startTime;
+          return {
+            embedding,
+            text,
+            metadata: {
+              dimensions: embedding.length,
+              modelName: this.config.modelName,
+              processingTime
+            }
+          };
+        } catch (error) {
+          throw new SemanticError(
+            "EMBEDDING_FAILED" /* EMBEDDING_FAILED */,
+            `Failed to generate embedding: ${error instanceof Error ? error.message : "Unknown error"}`,
+            { text: text.substring(0, 100), error }
+          );
+        }
+      }
+      async embedBatch(texts, options = {}) {
+        this.assertInitialized();
+        const { batchSize = 32, onProgress } = options;
+        if (!Array.isArray(texts) || texts.length === 0) {
+          throw new SemanticError(
+            "INVALID_INPUT" /* INVALID_INPUT */,
+            "Texts must be a non-empty array"
+          );
+        }
+        const results = [];
+        const batches = Math.ceil(texts.length / batchSize);
+        for (let i = 0; i < batches; i++) {
+          const start = i * batchSize;
+          const end = Math.min(start + batchSize, texts.length);
+          const batch = texts.slice(start, end);
+          const batchResults = await Promise.all(
+            batch.map((text) => this.embed(text))
+          );
+          results.push(...batchResults);
+          if (onProgress) {
+            onProgress(end, texts.length);
+          }
+        }
+        return results;
+      }
+      async similarity(textA, textB, method = "cosine") {
+        const startTime = performance.now();
+        const [resultA, resultB] = await Promise.all([
+          this.embed(textA),
+          this.embed(textB)
+        ]);
+        let score;
+        switch (method) {
+          case "cosine":
+            score = cosineSimilarity(resultA.embedding, resultB.embedding);
+            break;
+          case "euclidean":
+            score = -euclideanDistance(resultA.embedding, resultB.embedding);
+            break;
+          case "dot":
+            score = dotProduct(resultA.embedding, resultB.embedding);
+            break;
+          default:
+            throw new SemanticError(
+              "INVALID_INPUT" /* INVALID_INPUT */,
+              `Unknown similarity method: ${method}`
+            );
+        }
+        const processingTime = performance.now() - startTime;
+        return {
+          score,
+          texts: [textA, textB],
+          metadata: {
+            method,
+            processingTime
+          }
+        };
+      }
+      dispose() {
+        this.model = null;
+        this.initializationPromise = null;
+      }
+      isReady() {
+        return this.model !== null;
+      }
+      getConfig() {
+        return { ...this.config };
+      }
+    };
+  }
+});
+
+// src/search/SemanticSearch.ts
+var SemanticSearch_exports = {};
+__export(SemanticSearch_exports, {
+  SemanticSearch: () => SemanticSearch
+});
+var SemanticSearch;
+var init_SemanticSearch = __esm({
+  "src/search/SemanticSearch.ts"() {
+    "use strict";
+    init_types();
+    init_vector();
+    SemanticSearch = class {
+      constructor(engine, config = {}) {
+        this.indexedItems = [];
+        this.engine = engine;
+        this.config = {
+          topK: config.topK ?? 10,
+          threshold: config.threshold ?? 0,
+          textExtractor: config.textExtractor ?? ((item) => String(item)),
+          metadataExtractor: config.metadataExtractor ?? (() => ({}))
+        };
+      }
+      async index(items, replace = false) {
+        if (!Array.isArray(items) || items.length === 0) {
+          throw new SemanticError(
+            "INVALID_INPUT" /* INVALID_INPUT */,
+            "Items must be a non-empty array"
+          );
+        }
+        if (replace) {
+          this.indexedItems = [];
+        }
+        const texts = items.map(this.config.textExtractor);
+        const results = await this.engine.embedBatch(texts, { batchSize: 32 });
+        const newIndexItems = items.map((item, idx) => ({
+          item,
+          embedding: results[idx].embedding,
+          metadata: this.config.metadataExtractor(item)
+        }));
+        this.indexedItems.push(...newIndexItems);
+      }
+      async search(query, overrideConfig) {
+        if (this.indexedItems.length === 0) {
+          throw new SemanticError(
+            "INVALID_INPUT" /* INVALID_INPUT */,
+            "Index is empty. Call index() before searching."
+          );
+        }
+        const config = { ...this.config, ...overrideConfig };
+        const queryResult = await this.engine.embed(query);
+        const candidateEmbeddings = this.indexedItems.map((item) => item.embedding);
+        const topK = topKSimilar(queryResult.embedding, candidateEmbeddings, config.topK);
+        const results = [];
+        let rank = 1;
+        for (const [idx, score] of topK) {
+          if (score < config.threshold) continue;
+          results.push({
+            item: this.indexedItems[idx].item,
+            score,
+            rank: rank++
+          });
+        }
+        return results;
+      }
+      async searchWithFilter(query, filter, config) {
+        const originalIndex = this.indexedItems;
+        this.indexedItems = originalIndex.filter((item) => filter(item.metadata ?? {}));
+        try {
+          return await this.search(query, config);
+        } finally {
+          this.indexedItems = originalIndex;
+        }
+      }
+      async findSimilar(item, config) {
+        const text = this.config.textExtractor(item);
+        return this.search(text, config);
+      }
+      getStats() {
+        const itemCount = this.indexedItems.length;
+        const dimensions = this.indexedItems[0]?.embedding.length ?? 0;
+        const totalBytes = itemCount * dimensions * 8;
+        const memoryEstimate = totalBytes < 1024 * 1024 ? `${(totalBytes / 1024).toFixed(2)} KB` : `${(totalBytes / (1024 * 1024)).toFixed(2)} MB`;
+        return { itemCount, dimensions, memoryEstimate };
+      }
+      clear() {
+        this.indexedItems = [];
+      }
+      exportIndex() {
+        return [...this.indexedItems];
+      }
+      importIndex(index) {
+        this.indexedItems = [...index];
+      }
+    };
+  }
+});
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  SemanticEngine: () => SemanticEngine,
+  SemanticError: () => SemanticError,
+  SemanticErrorCode: () => SemanticErrorCode,
+  SemanticSearch: () => SemanticSearch,
+  centroid: () => centroid,
+  cosineSimilarity: () => cosineSimilarity,
+  createSemanticEngine: () => createSemanticEngine,
+  createSemanticSearch: () => createSemanticSearch,
+  dotProduct: () => dotProduct,
+  euclideanDistance: () => euclideanDistance,
+  magnitude: () => magnitude,
+  normalize: () => normalize,
+  topKSimilar: () => topKSimilar
+});
+module.exports = __toCommonJS(index_exports);
+init_SemanticEngine();
+init_SemanticSearch();
+init_vector();
+init_types();
+async function createSemanticEngine(config) {
+  const { SemanticEngine: SemanticEngine2 } = await Promise.resolve().then(() => (init_SemanticEngine(), SemanticEngine_exports));
+  const engine = new SemanticEngine2(config);
+  await engine.initialize();
+  return engine;
+}
+async function createSemanticSearch(items, config) {
+  const engine = await createSemanticEngine();
+  const { SemanticSearch: SemanticSearch2 } = await Promise.resolve().then(() => (init_SemanticSearch(), SemanticSearch_exports));
+  const search = new SemanticSearch2(engine, config);
+  await search.index(items);
+  return search;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SemanticEngine,
+  SemanticError,
+  SemanticErrorCode,
+  SemanticSearch,
+  centroid,
+  cosineSimilarity,
+  createSemanticEngine,
+  createSemanticSearch,
+  dotProduct,
+  euclideanDistance,
+  magnitude,
+  normalize,
+  topKSimilar
+});
+/**
+ * Semantic Text Toolkit
+ * Production-grade semantic text analysis
+ * 
+ * @module @crashbytes/semantic-text-toolkit
+ * @author Blackhole Software, LLC
+ * @license MIT
+ */

package/dist/index.mjs

@@ -0,0 +1,55 @@
+import {
+  SemanticEngine
+} from "./chunk-XJ4PTDH6.mjs";
+import {
+  SemanticSearch
+} from "./chunk-ENOBULOJ.mjs";
+import {
+  SemanticError,
+  SemanticErrorCode,
+  centroid,
+  cosineSimilarity,
+  dotProduct,
+  euclideanDistance,
+  magnitude,
+  normalize,
+  topKSimilar
+} from "./chunk-TPAL6DKL.mjs";
+
+// src/index.ts
+async function createSemanticEngine(config) {
+  const { SemanticEngine: SemanticEngine2 } = await import("./SemanticEngine-3EGZZHKU.mjs");
+  const engine = new SemanticEngine2(config);
+  await engine.initialize();
+  return engine;
+}
+async function createSemanticSearch(items, config) {
+  const engine = await createSemanticEngine();
+  const { SemanticSearch: SemanticSearch2 } = await import("./SemanticSearch-CQZQEKEG.mjs");
+  const search = new SemanticSearch2(engine, config);
+  await search.index(items);
+  return search;
+}
+export {
+  SemanticEngine,
+  SemanticError,
+  SemanticErrorCode,
+  SemanticSearch,
+  centroid,
+  cosineSimilarity,
+  createSemanticEngine,
+  createSemanticSearch,
+  dotProduct,
+  euclideanDistance,
+  magnitude,
+  normalize,
+  topKSimilar
+};
+/**
+ * Semantic Text Toolkit
+ * Production-grade semantic text analysis
+ * 
+ * @module @crashbytes/semantic-text-toolkit
+ * @author Blackhole Software, LLC
+ * @license MIT
+ */

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@crashbytes/semantic-text-toolkit",
+  "version": "1.0.0",
+  "description": "Production-grade semantic text analysis with embeddings, similarity, and vector operations",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format cjs,esm --dts --clean",
+    "dev": "tsup src/index.ts --format cjs,esm --dts --watch",
+    "test": "jest --passWithNoTests",
+    "test:watch": "jest --watch",
+    "lint": "eslint src --ext .ts",
+    "prepublishOnly": "npm run build && npm test"
+  },
+  "keywords": [
+    "semantic",
+    "embeddings",
+    "text-analysis",
+    "ml",
+    "ai",
+    "vector-search",
+    "similarity",
+    "nlp"
+  ],
+  "author": "Blackhole Software, LLC",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/crashbytes/crashbytes-npmjs.git",
+    "directory": "semantic-text-toolkit"
+  },
+  "dependencies": {
+    "@xenova/transformers": "^2.17.1"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.11.19",
+    "@typescript-eslint/eslint-plugin": "^6.21.0",
+    "@typescript-eslint/parser": "^6.21.0",
+    "eslint": "^8.56.0",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.2",
+    "tsup": "^8.0.2",
+    "typescript": "^5.3.3"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  }
+}