@horribleprogram/sdk 0.1.6 → 0.1.8

package/dist/index.cjs

@@ -1,7 +1,9 @@
 "use strict";
+var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __export = (target, all) => {
   for (var name in all)
@@ -15,16 +17,32 @@ var __copyProps = (to, from, except, desc) => {
   }
   return to;
 };
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
   Botcierge: () => Botcierge,
-  query_intent: () => query_intent
+  CompanyEmbeddingManager: () => CompanyEmbeddingManager,
+  addCompany: () => addCompany,
+  closeConnection: () => closeConnection,
+  companyToText: () => companyToText,
+  parseCSV: () => parseCSV,
+  queryCompanies: () => queryCompanies,
+  query_intent: () => query_intent,
+  removeCompany: () => removeCompany,
+  seedCompanies: () => seedCompanies
 });
 module.exports = __toCommonJS(index_exports);
-var import_transformers = require("@xenova/transformers");
+var import_transformers2 = require("@xenova/transformers");
 
 // src/intents.ts
 var INTENTS_TAXONOMY = {
@@ -163,6 +181,326 @@ var INTENTS_TAXONOMY = {
   }
 };
 
+// src/companyEmbeddings.ts
+var fs = __toESM(require("fs"), 1);
+var import_mongodb = require("mongodb");
+var import_transformers = require("@xenova/transformers");
+function parseCSV(content) {
+  const lines = [];
+  let currentLine = "";
+  let insideQuote = false;
+  for (let i = 0; i < content.length; i++) {
+    const char = content[i];
+    const nextChar = content[i + 1];
+    if (char === '"') {
+      insideQuote = !insideQuote;
+    } else if (char === "\r" || char === "\n") {
+      if (!insideQuote) {
+        if (currentLine || i === 0 || content[i - 1] === "\n" || content[i - 1] === "\r") {
+          lines.push(currentLine);
+          currentLine = "";
+        }
+        if (char === "\r" && nextChar === "\n") {
+          i++;
+        }
+      } else {
+        currentLine += char;
+      }
+    } else {
+      currentLine += char;
+    }
+  }
+  if (currentLine) {
+    lines.push(currentLine);
+  }
+  if (lines.length === 0) return [];
+  const parseLine = (line) => {
+    const result = [];
+    let cell = "";
+    let inside = false;
+    for (let i = 0; i < line.length; i++) {
+      const char = line[i];
+      if (char === '"') {
+        if (inside && line[i + 1] === '"') {
+          cell += '"';
+          i++;
+        } else {
+          inside = !inside;
+        }
+      } else if (char === "," && !inside) {
+        result.push(cell.trim());
+        cell = "";
+      } else {
+        cell += char;
+      }
+    }
+    result.push(cell.trim());
+    return result;
+  };
+  const headers = parseLine(lines[0]);
+  const records = [];
+  for (let i = 1; i < lines.length; i++) {
+    const cells = parseLine(lines[i]);
+    if (cells.length === 1 && cells[0] === "") continue;
+    const record = {};
+    for (let j = 0; j < headers.length; j++) {
+      record[headers[j]] = cells[j] || "";
+    }
+    records.push(record);
+  }
+  return records;
+}
+function companyToText(company) {
+  const parts = [];
+  const keys = [
+    "company_name",
+    "website",
+    "short_description",
+    "product_description",
+    "mapped_function",
+    "mapped_industry",
+    "match_keywords",
+    "aliases",
+    "active",
+    "priority",
+    "source",
+    "zone"
+  ];
+  for (const key of keys) {
+    if (key in company) {
+      const val = String(company[key]).trim();
+      if (val) {
+        parts.push(`${key}: ${val}`);
+      }
+    }
+  }
+  for (const [key, value] of Object.entries(company)) {
+    if (keys.includes(key) || key === "_id" || key === "embedding") continue;
+    const val = String(value).trim();
+    if (val) {
+      parts.push(`${key}: ${val}`);
+    }
+  }
+  return parts.join(" | ");
+}
+function cosineSimilarity(v1, v2) {
+  let dotProduct = 0;
+  let norm1 = 0;
+  let norm2 = 0;
+  for (let i = 0; i < v1.length; i++) {
+    dotProduct += v1[i] * v2[i];
+    norm1 += v1[i] * v1[i];
+    norm2 += v2[i] * v2[i];
+  }
+  if (norm1 === 0 || norm2 === 0) return 0;
+  return dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2));
+}
+var CompanyEmbeddingManager = class {
+  client = null;
+  config;
+  extractorPromise = null;
+  constructor(config = {}) {
+    this.config = {
+      uri: config.uri || process.env.MONGODB_URI || "mongodb://localhost:27017",
+      dbName: config.dbName || process.env.MONGODB_DB_NAME || "distance_metrics",
+      collectionName: config.collectionName || "companies",
+      openAIApiKey: config.openAIApiKey || process.env.OPENAI_API_KEY || "",
+      preferOpenAI: config.preferOpenAI ?? !!(config.openAIApiKey || process.env.OPENAI_API_KEY),
+      mongoOptions: config.mongoOptions || {}
+    };
+  }
+  async getExtractor() {
+    if (!this.extractorPromise) {
+      this.extractorPromise = (async () => {
+        console.log("Loading local embedding model 'Xenova/all-MiniLM-L12-v2'...");
+        const extractor = await (0, import_transformers.pipeline)("feature-extraction", "Xenova/all-MiniLM-L12-v2");
+        console.log("Local embedding model loaded successfully.");
+        return extractor;
+      })();
+    }
+    return this.extractorPromise;
+  }
+  async connect() {
+    if (!this.client) {
+      this.client = new import_mongodb.MongoClient(this.config.uri, this.config.mongoOptions);
+      await this.client.connect();
+    }
+    return this.client;
+  }
+  async getCollection() {
+    const client = await this.connect();
+    return client.db(this.config.dbName).collection(this.config.collectionName);
+  }
+  async disconnect() {
+    if (this.client) {
+      await this.client.close();
+      this.client = null;
+    }
+  }
+  async getEmbedding(text) {
+    if (this.config.preferOpenAI && this.config.openAIApiKey) {
+      const embeddings = await this.getOpenAIBatchEmbeddings([text]);
+      return embeddings[0];
+    }
+    const extractor = await this.getExtractor();
+    const output = await extractor(text, { pooling: "mean", normalize: true });
+    return Array.from(output.data);
+  }
+  async getEmbeddings(texts) {
+    if (texts.length === 0) return [];
+    if (this.config.preferOpenAI && this.config.openAIApiKey) {
+      return this.getOpenAIBatchEmbeddings(texts);
+    }
+    const extractor = await this.getExtractor();
+    const output = await extractor(texts, { pooling: "mean", normalize: true });
+    const dim = output.dims[1];
+    const data = output.data;
+    const embeddings = [];
+    for (let i = 0; i < texts.length; i++) {
+      const startIndex = i * dim;
+      const vector = Array.from(data.subarray(startIndex, startIndex + dim));
+      embeddings.push(vector);
+    }
+    return embeddings;
+  }
+  async getOpenAIBatchEmbeddings(texts) {
+    const batchSize = 50;
+    const results = [];
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const chunk = texts.slice(i, i + batchSize);
+      const response = await fetch("https://api.openai.com/v1/embeddings", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `Bearer ${this.config.openAIApiKey}`
+        },
+        body: JSON.stringify({
+          input: chunk,
+          model: "text-embedding-3-small"
+        })
+      });
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`OpenAI API error: ${response.status} - ${errorText}`);
+      }
+      const json = await response.json();
+      results.push(...json.data.map((item) => item.embedding));
+    }
+    return results;
+  }
+  async addCompany(company) {
+    const col = await this.getCollection();
+    let active = company.active;
+    if (typeof active === "string") {
+      const activeStr = active.toLowerCase().trim();
+      active = activeStr === "true" || activeStr === "yes" || activeStr === "1";
+    }
+    const cleanCompany = {
+      ...company,
+      active: !!active
+    };
+    if (!cleanCompany.embedding) {
+      const text = companyToText(cleanCompany);
+      cleanCompany.embedding = await this.getEmbedding(text);
+    }
+    await col.updateOne(
+      { company_name: cleanCompany.company_name },
+      { $set: cleanCompany },
+      { upsert: true }
+    );
+  }
+  async removeCompany(companyName) {
+    const col = await this.getCollection();
+    const result = await col.deleteOne({
+      company_name: { $regex: new RegExp(`^${this.escapeRegExp(companyName)}$`, "i") }
+    });
+    return (result.deletedCount ?? 0) > 0;
+  }
+  async queryCompanies(prompt, limit = 10) {
+    const col = await this.getCollection();
+    const queryVector = await this.getEmbedding(prompt);
+    const allDocs = await col.find({}).toArray();
+    const results = allDocs.filter((doc) => doc.embedding && Array.isArray(doc.embedding)).map((doc) => {
+      const similarity = cosineSimilarity(queryVector, doc.embedding);
+      return {
+        ...doc,
+        similarity
+      };
+    });
+    results.sort((a, b) => b.similarity - a.similarity);
+    return results.slice(0, limit);
+  }
+  async seedFromCSV(csvPath) {
+    const csvContent = fs.readFileSync(csvPath, "utf8");
+    const records = parseCSV(csvContent);
+    if (records.length === 0) return 0;
+    const companies = records.map((rec) => {
+      const activeStr = String(rec.active).toLowerCase().trim();
+      const active = activeStr === "true" || activeStr === "yes" || activeStr === "1";
+      return {
+        company_name: rec.company_name || "",
+        website: rec.website || "",
+        short_description: rec.short_description || "",
+        product_description: rec.product_description || "",
+        mapped_function: rec.mapped_function || "",
+        mapped_industry: rec.mapped_industry || "",
+        match_keywords: rec.match_keywords || "",
+        aliases: rec.aliases || "",
+        active,
+        priority: rec.priority || "",
+        source: rec.source || "",
+        zone: rec.zone || ""
+      };
+    }).filter((c) => c.company_name);
+    const texts = companies.map((c) => companyToText(c));
+    console.log(`Generating embeddings for ${companies.length} companies...`);
+    const embeddings = await this.getEmbeddings(texts);
+    for (let i = 0; i < companies.length; i++) {
+      companies[i].embedding = embeddings[i];
+    }
+    const col = await this.getCollection();
+    await col.deleteMany({});
+    const batchSize = 100;
+    for (let i = 0; i < companies.length; i += batchSize) {
+      const batch = companies.slice(i, i + batchSize);
+      await col.insertMany(batch);
+    }
+    return companies.length;
+  }
+  escapeRegExp(string) {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  }
+};
+var defaultManager = null;
+function getDefaultManager(config) {
+  if (!defaultManager || config) {
+    defaultManager = new CompanyEmbeddingManager(config);
+  }
+  return defaultManager;
+}
+async function seedCompanies(csvPath, config) {
+  const manager = getDefaultManager(config);
+  return manager.seedFromCSV(csvPath);
+}
+async function addCompany(company, config) {
+  const manager = getDefaultManager(config);
+  return manager.addCompany(company);
+}
+async function removeCompany(companyName, config) {
+  const manager = getDefaultManager(config);
+  return manager.removeCompany(companyName);
+}
+async function queryCompanies(prompt, limit, config) {
+  const manager = getDefaultManager(config);
+  return manager.queryCompanies(prompt, limit);
+}
+async function closeConnection() {
+  if (defaultManager) {
+    await defaultManager.disconnect();
+    defaultManager = null;
+  }
+}
+
 // src/index.ts
 var extractorPromise = null;
 function getExtractor() {
@@ -170,7 +508,7 @@ function getExtractor() {
     extractorPromise = (async () => {
       console.log("Loading local embedding model 'Xenova/all-MiniLM-L12-v2'...");
       console.log("Note: On first run, this will download the model files (approx. 120MB).");
-      const extractor = await (0, import_transformers.pipeline)("feature-extraction", "Xenova/all-MiniLM-L12-v2");
+      const extractor = await (0, import_transformers2.pipeline)("feature-extraction", "Xenova/all-MiniLM-L12-v2");
       console.log("Local embedding model loaded successfully.");
       return extractor;
     })();
@@ -211,7 +549,7 @@ async function getTaxonomyEmbeddings() {
   })();
   return cachedTaxonomyEmbeddingsPromise;
 }
-function cosineSimilarity(v1, v2) {
+function cosineSimilarity2(v1, v2) {
   let dotProduct = 0;
   let norm1 = 0;
   let norm2 = 0;
@@ -234,7 +572,7 @@ var Botcierge = class {
     let bestSim = -1;
     let bestItem = null;
     for (const item of taxEmbeddings) {
-      const sim = cosineSimilarity(queryVector, item.vector);
+      const sim = cosineSimilarity2(queryVector, item.vector);
       if (sim > bestSim) {
         bestSim = sim;
         bestItem = item;
@@ -271,5 +609,13 @@ async function query_intent(utterance) {
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   Botcierge,
-  query_intent
+  CompanyEmbeddingManager,
+  addCompany,
+  closeConnection,
+  companyToText,
+  parseCSV,
+  queryCompanies,
+  query_intent,
+  removeCompany,
+  seedCompanies
 });

package/dist/index.d.cts

@@ -1,3 +1,63 @@
+import { MongoClientOptions, MongoClient, Collection } from 'mongodb';
+
+interface Company {
+    company_name: string;
+    website: string;
+    short_description: string;
+    product_description: string;
+    mapped_function: string;
+    mapped_industry: string;
+    match_keywords: string;
+    aliases: string;
+    active: boolean | string;
+    priority: string;
+    source: string;
+    zone: string;
+    embedding?: number[];
+    [key: string]: any;
+}
+interface MongoEmbeddingConfig {
+    uri?: string;
+    dbName?: string;
+    collectionName?: string;
+    openAIApiKey?: string;
+    preferOpenAI?: boolean;
+    mongoOptions?: MongoClientOptions;
+}
+declare function parseCSV(content: string): Record<string, string>[];
+declare function companyToText(company: Partial<Company>): string;
+declare class CompanyEmbeddingManager {
+    private client;
+    private config;
+    private extractorPromise;
+    constructor(config?: MongoEmbeddingConfig);
+    private getExtractor;
+    connect(): Promise<MongoClient>;
+    getCollection(): Promise<Collection<Company>>;
+    disconnect(): Promise<void>;
+    getEmbedding(text: string): Promise<number[]>;
+    getEmbeddings(texts: string[]): Promise<number[][]>;
+    private getOpenAIBatchEmbeddings;
+    addCompany(company: Omit<Company, 'embedding'> & {
+        embedding?: number[];
+    }): Promise<void>;
+    removeCompany(companyName: string): Promise<boolean>;
+    queryCompanies(prompt: string, limit?: number): Promise<(Company & {
+        similarity: number;
+    })[]>;
+    seedFromCSV(csvPath: string): Promise<number>;
+    private escapeRegExp;
+}
+declare function seedCompanies(csvPath: string, config?: MongoEmbeddingConfig): Promise<number>;
+declare function addCompany(company: Omit<Company, 'embedding'> & {
+    embedding?: number[];
+}, config?: MongoEmbeddingConfig): Promise<void>;
+declare function removeCompany(companyName: string, config?: MongoEmbeddingConfig): Promise<boolean>;
+declare function queryCompanies(prompt: string, limit?: number, config?: MongoEmbeddingConfig): Promise<(Company & {
+    similarity: number;
+})[]>;
+declare function closeConnection(): Promise<void>;
+
 interface IntentResult {
     domain: string;
     intent: string;
@@ -15,4 +75,4 @@ declare class Botcierge {
  */
 declare function query_intent(utterance: string): Promise<IntentResult>;
 
-export { Botcierge, type BotciergeConfig, type IntentResult, query_intent };
+export { Botcierge, type BotciergeConfig, type Company, CompanyEmbeddingManager, type IntentResult, type MongoEmbeddingConfig, addCompany, closeConnection, companyToText, parseCSV, queryCompanies, query_intent, removeCompany, seedCompanies };

package/dist/index.d.ts

@@ -1,3 +1,63 @@
+import { MongoClientOptions, MongoClient, Collection } from 'mongodb';
+
+interface Company {
+    company_name: string;
+    website: string;
+    short_description: string;
+    product_description: string;
+    mapped_function: string;
+    mapped_industry: string;
+    match_keywords: string;
+    aliases: string;
+    active: boolean | string;
+    priority: string;
+    source: string;
+    zone: string;
+    embedding?: number[];
+    [key: string]: any;
+}
+interface MongoEmbeddingConfig {
+    uri?: string;
+    dbName?: string;
+    collectionName?: string;
+    openAIApiKey?: string;
+    preferOpenAI?: boolean;
+    mongoOptions?: MongoClientOptions;
+}
+declare function parseCSV(content: string): Record<string, string>[];
+declare function companyToText(company: Partial<Company>): string;
+declare class CompanyEmbeddingManager {
+    private client;
+    private config;
+    private extractorPromise;
+    constructor(config?: MongoEmbeddingConfig);
+    private getExtractor;
+    connect(): Promise<MongoClient>;
+    getCollection(): Promise<Collection<Company>>;
+    disconnect(): Promise<void>;
+    getEmbedding(text: string): Promise<number[]>;
+    getEmbeddings(texts: string[]): Promise<number[][]>;
+    private getOpenAIBatchEmbeddings;
+    addCompany(company: Omit<Company, 'embedding'> & {
+        embedding?: number[];
+    }): Promise<void>;
+    removeCompany(companyName: string): Promise<boolean>;
+    queryCompanies(prompt: string, limit?: number): Promise<(Company & {
+        similarity: number;
+    })[]>;
+    seedFromCSV(csvPath: string): Promise<number>;
+    private escapeRegExp;
+}
+declare function seedCompanies(csvPath: string, config?: MongoEmbeddingConfig): Promise<number>;
+declare function addCompany(company: Omit<Company, 'embedding'> & {
+    embedding?: number[];
+}, config?: MongoEmbeddingConfig): Promise<void>;
+declare function removeCompany(companyName: string, config?: MongoEmbeddingConfig): Promise<boolean>;
+declare function queryCompanies(prompt: string, limit?: number, config?: MongoEmbeddingConfig): Promise<(Company & {
+    similarity: number;
+})[]>;
+declare function closeConnection(): Promise<void>;
+
 interface IntentResult {
     domain: string;
     intent: string;
@@ -15,4 +75,4 @@ declare class Botcierge {
  */
 declare function query_intent(utterance: string): Promise<IntentResult>;
 
-export { Botcierge, type BotciergeConfig, type IntentResult, query_intent };
+export { Botcierge, type BotciergeConfig, type Company, CompanyEmbeddingManager, type IntentResult, type MongoEmbeddingConfig, addCompany, closeConnection, companyToText, parseCSV, queryCompanies, query_intent, removeCompany, seedCompanies };

package/dist/index.js

@@ -1,8 +1,24 @@
 import {
   Botcierge,
-  query_intent
-} from "./chunk-45PYBUYK.js";
+  CompanyEmbeddingManager,
+  addCompany,
+  closeConnection,
+  companyToText,
+  parseCSV,
+  queryCompanies,
+  query_intent,
+  removeCompany,
+  seedCompanies
+} from "./chunk-UKIYZY2F.js";
 export {
   Botcierge,
-  query_intent
+  CompanyEmbeddingManager,
+  addCompany,
+  closeConnection,
+  companyToText,
+  parseCSV,
+  queryCompanies,
+  query_intent,
+  removeCompany,
+  seedCompanies
 };

package/dist/main.cjs

@@ -26,7 +26,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 var readline = __toESM(require("readline"), 1);
 
 // src/index.ts
-var import_transformers = require("@xenova/transformers");
+var import_transformers2 = require("@xenova/transformers");
 
 // src/intents.ts
 var INTENTS_TAXONOMY = {
@@ -165,6 +165,10 @@ var INTENTS_TAXONOMY = {
   }
 };
 
+// src/companyEmbeddings.ts
+var import_mongodb = require("mongodb");
+var import_transformers = require("@xenova/transformers");
+
 // src/index.ts
 var extractorPromise = null;
 function getExtractor() {
@@ -172,7 +176,7 @@ function getExtractor() {
     extractorPromise = (async () => {
       console.log("Loading local embedding model 'Xenova/all-MiniLM-L12-v2'...");
       console.log("Note: On first run, this will download the model files (approx. 120MB).");
-      const extractor = await (0, import_transformers.pipeline)("feature-extraction", "Xenova/all-MiniLM-L12-v2");
+      const extractor = await (0, import_transformers2.pipeline)("feature-extraction", "Xenova/all-MiniLM-L12-v2");
       console.log("Local embedding model loaded successfully.");
       return extractor;
     })();

package/dist/main.js

@@ -1,6 +1,6 @@
 import {
   query_intent
-} from "./chunk-45PYBUYK.js";
+} from "./chunk-UKIYZY2F.js";
 
 // src/main.ts
 import * as readline from "readline";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@horribleprogram/sdk",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "SDK for Botcierge Intent Classification",
   "type": "module",
   "main": "./dist/index.js",
@@ -44,6 +44,7 @@
     "vitest": "^4.1.6"
   },
   "dependencies": {
-    "@xenova/transformers": "^2.17.2"
+    "@xenova/transformers": "^2.17.2",
+    "mongodb": "^7.4.0"
   }
 }