doc2vec 1.1.1 → 1.3.0

package/dist/embedding-providers.js

@@ -0,0 +1,157 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TransformersEmbeddingProvider = exports.OpenAIEmbeddingProvider = void 0;
+exports.createEmbeddingProvider = createEmbeddingProvider;
+const openai_1 = require("openai");
+const transformers_1 = require("@huggingface/transformers");
+class OpenAIEmbeddingProvider {
+    constructor(params, logger) {
+        this.logger = logger.child('openai-embeddings');
+        this.model = params.model || 'text-embedding-3-large';
+        this.openai = new openai_1.OpenAI({
+            apiKey: params.api_key || process.env.OPENAI_API_KEY
+        });
+        this.logger.info(`OpenAI embedding provider initialized with model: ${this.model}`);
+    }
+    async createEmbeddings(texts) {
+        try {
+            this.logger.debug(`Creating embeddings for ${texts.length} texts using OpenAI`);
+            const response = await this.openai.embeddings.create({
+                model: this.model,
+                input: texts,
+            });
+            this.logger.debug(`Successfully created ${response.data.length} embeddings`);
+            return response.data.map(d => d.embedding);
+        }
+        catch (error) {
+            this.logger.error('Failed to create OpenAI embeddings:', error);
+            return [];
+        }
+    }
+    getDimensions() {
+        // Return dimensions based on model
+        if (this.model === 'text-embedding-3-large')
+            return 3072;
+        if (this.model === 'text-embedding-3-small')
+            return 1536;
+        if (this.model === 'text-embedding-ada-002')
+            return 1536;
+        return 1536; // Default fallback
+    }
+    getModelName() {
+        return this.model;
+    }
+}
+exports.OpenAIEmbeddingProvider = OpenAIEmbeddingProvider;
+class TransformersEmbeddingProvider {
+    constructor(params, logger) {
+        this.params = params;
+        this.dimensions = 384; // Default, will be updated after model loading
+        this.isInitialized = false;
+        this.logger = logger.child('transformers-embeddings');
+        this.modelName = params.model;
+        // Configure transformers.js environment
+        if (params.local_files_only) {
+            transformers_1.env.allowLocalModels = true;
+            transformers_1.env.allowRemoteModels = false;
+        }
+        if (params.cache_dir) {
+            transformers_1.env.cacheDir = params.cache_dir;
+        }
+        this.logger.info(`Transformers embedding provider initialized with model: ${this.modelName}`);
+    }
+    async initialize() {
+        if (this.isInitialized)
+            return;
+        try {
+            this.logger.info(`Loading transformers model: ${this.modelName}`);
+            // Load tokenizer and model
+            this.tokenizer = await transformers_1.AutoTokenizer.from_pretrained(this.modelName);
+            this.model = await transformers_1.AutoModel.from_pretrained(this.modelName);
+            // Try to determine dimensions from model config
+            if (this.model.config && this.model.config.hidden_size) {
+                this.dimensions = this.model.config.hidden_size;
+            }
+            this.isInitialized = true;
+            this.logger.info(`Transformers model loaded successfully. Dimensions: ${this.dimensions}`);
+        }
+        catch (error) {
+            this.logger.error(`Failed to load transformers model: ${this.modelName}`, error);
+            throw error;
+        }
+    }
+    async createEmbeddings(texts) {
+        await this.initialize();
+        try {
+            this.logger.debug(`Creating embeddings for ${texts.length} texts using transformers.js`);
+            const embeddings = [];
+            // Process texts in batches to avoid memory issues
+            const batchSize = Math.min(8, texts.length);
+            for (let i = 0; i < texts.length; i += batchSize) {
+                const batch = texts.slice(i, i + batchSize);
+                for (const text of batch) {
+                    // Tokenize the text
+                    const tokens = await this.tokenizer(text, {
+                        truncation: true,
+                        padding: true,
+                        max_length: 512,
+                        return_tensors: 'pt'
+                    });
+                    // Get model output
+                    const output = await this.model(tokens);
+                    // Mean pooling to get sentence embeddings
+                    let embedding;
+                    if (output.last_hidden_state) {
+                        // Mean pooling over sequence length
+                        const tensor = output.last_hidden_state;
+                        const meanPooled = tensor.mean(1); // Mean along sequence dimension
+                        embedding = Array.from(meanPooled.data);
+                    }
+                    else if (output.pooler_output) {
+                        // Use pooler output if available
+                        embedding = Array.from(output.pooler_output.data);
+                    }
+                    else {
+                        throw new Error('Unable to extract embeddings from model output');
+                    }
+                    embeddings.push(embedding);
+                }
+                // Log progress for large batches
+                if (texts.length > 10) {
+                    this.logger.debug(`Processed ${Math.min(i + batchSize, texts.length)}/${texts.length} texts`);
+                }
+            }
+            this.logger.debug(`Successfully created ${embeddings.length} embeddings`);
+            return embeddings;
+        }
+        catch (error) {
+            this.logger.error('Failed to create transformers embeddings:', error);
+            return [];
+        }
+    }
+    getDimensions() {
+        return this.dimensions;
+    }
+    getModelName() {
+        return this.modelName;
+    }
+    async cleanup() {
+        // Clean up model resources if needed
+        this.model = null;
+        this.tokenizer = null;
+        this.isInitialized = false;
+        this.logger.info('Transformers embedding provider cleaned up');
+    }
+}
+exports.TransformersEmbeddingProvider = TransformersEmbeddingProvider;
+function createEmbeddingProvider(config, logger) {
+    if (config.provider === 'openai') {
+        return new OpenAIEmbeddingProvider(config.params, logger);
+    }
+    else if (config.provider === 'transformers') {
+        return new TransformersEmbeddingProvider(config.params, logger);
+    }
+    else {
+        throw new Error(`Unsupported embedding provider: ${config.provider}`);
+    }
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils.js

@@ -0,0 +1,118 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Utils = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const path = __importStar(require("path"));
+class Utils {
+    static generateHash(content) {
+        return crypto_1.default.createHash("sha256").update(content).digest("hex");
+    }
+    static generateMetadataUUID(repo) {
+        // Simple deterministic approach - hash the repo name and convert to UUID format
+        const hash = crypto_1.default.createHash('md5').update(`metadata_${repo}`).digest('hex');
+        // Format as UUID with version bits set correctly (version 4)
+        return `${hash.substr(0, 8)}-${hash.substr(8, 4)}-4${hash.substr(13, 3)}-${hash.substr(16, 4)}-${hash.substr(20, 12)}`;
+    }
+    static getUrlPrefix(url) {
+        try {
+            const parsedUrl = new URL(url);
+            return parsedUrl.origin + parsedUrl.pathname;
+        }
+        catch (error) {
+            return url;
+        }
+    }
+    static normalizeUrl(url) {
+        try {
+            const urlObj = new URL(url);
+            urlObj.hash = '';
+            urlObj.search = '';
+            return urlObj.toString();
+        }
+        catch (error) {
+            return url;
+        }
+    }
+    static buildUrl(href, currentUrl) {
+        try {
+            return new URL(href, currentUrl).toString();
+        }
+        catch (error) {
+            console.warn(`Invalid URL found: ${href}`);
+            return '';
+        }
+    }
+    static shouldProcessUrl(url) {
+        const parsedUrl = new URL(url);
+        const pathname = parsedUrl.pathname;
+        const ext = path.extname(pathname);
+        if (!ext)
+            return true;
+        return ['.html', '.htm', '.pdf'].includes(ext.toLowerCase());
+    }
+    static isPdfUrl(url) {
+        try {
+            const parsedUrl = new URL(url);
+            const pathname = parsedUrl.pathname;
+            const ext = path.extname(pathname);
+            return ext.toLowerCase() === '.pdf';
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    static isValidUuid(str) {
+        const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+        return uuidRegex.test(str);
+    }
+    static hashToUuid(hash) {
+        const truncatedHash = hash.substring(0, 32);
+        return [
+            truncatedHash.substring(0, 8),
+            truncatedHash.substring(8, 12),
+            '5' + truncatedHash.substring(13, 16),
+            '8' + truncatedHash.substring(17, 20),
+            truncatedHash.substring(20, 32)
+        ].join('-');
+    }
+    static tokenize(text) {
+        return text.split(/(\s+)/).filter(token => token.length > 0);
+    }
+}
+exports.Utils = Utils;

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "doc2vec",
-    "version": "1.1.1",
+    "version": "1.3.0",
     "type": "commonjs",
     "description": "",
-    "main": "doc2vec.ts",
+    "main": "dist/doc2vec.js",
     "private": false,
     "publishConfig": {
         "access": "public"
@@ -11,6 +11,11 @@
     "bin": {
         "doc2vec": "dist/doc2vec.js"
     },
+    "files": [
+        "dist/",
+        "README.md",
+        "LICENSE"
+    ],
     "scripts": {
         "build": "tsc && chmod 755 dist/doc2vec.js",
         "start": "node dist/doc2vec.js",
@@ -32,6 +37,7 @@
         "js-yaml": "^4.1.0",
         "jsdom": "^26.0.0",
         "openai": "^4.20.1",
+        "pdfjs-dist": "^5.3.31",
         "puppeteer": "^24.1.1",
         "sanitize-html": "^2.11.0",
         "sqlite-vec": "0.1.7-alpha.2",

package/Dockerfile

@@ -1,29 +0,0 @@
-FROM node:20-slim
-
-WORKDIR /app
-
-RUN apt-get update && apt-get install -y \
-  git \
-  python3 \
-  make \
-  g++ \
-  sqlite3 \
-  libsqlite3-dev \
-  curl \
-  ca-certificates \
-  chromium \
-  fonts-freefont-ttf \
-  fonts-ipafont-gothic \
-  fonts-kacst \
-  fonts-liberation \
-  fonts-noto-color-emoji \
-  fonts-thai-tlwg \
-  libx11-xcb1 \
-  libxcb-dri3-0 \
-  libxcomposite1 \
-  libxdamage1 \
-  libxi6 \
-  libxrandr2 \
-  libxshmfence1 \
-  libxtst6 \
-  && apt-get clean

package/config.yaml

@@ -1,167 +0,0 @@
-# Doc2Vec Configuration
-sources:
-  # GitHub Sources
-  - type: github
-    product_name: 'istio'
-    version: 'latest'
-    repo: 'istio/istio'
-    start_date: '2025-01-01'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './istio-issues.db'
-
-  # Website Sources
-  - type: website
-    product_name: 'ambient'
-    version: 'latest'
-    url: 'https://ambientmesh.io/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './ambient.db'
-
-  - type: website
-    product_name: 'argo'
-    version: 'latest'
-    url: 'https://argo-cd.readthedocs.io/en/stable/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './argo.db'
-
-  - type: website
-    product_name: 'argo-rollouts'
-    version: 'latest'
-    url: 'https://argoproj.github.io/argo-rollouts/installation/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './argo-rollouts.db'
-
-  - type: website
-    product_name: 'argo-rollouts'
-    version: 'latest'
-    url: 'https://rollouts-plugin-trafficrouter-gatewayapi.readthedocs.io/en/latest/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './argo-rollouts.db'
-
-  - type: website
-    product_name: 'cilium'
-    version: 'latest'
-    url: 'https://docs.cilium.io/en/stable/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './cilium.db'
-
-  - type: website
-    product_name: 'gateway-api'
-    version: 'latest'
-    url: 'https://gateway-api.sigs.k8s.io/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gateway-api.db'
-
-  - type: website
-    product_name: 'gloo-mesh-core'
-    version: 'latest'
-    url: 'https://docs.solo.io/gloo-mesh/latest/'
-    sitemap_url: 'https://docs.solo.io/gloo-mesh/sitemap.xml'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gloo-mesh-core.db'
-
-  - type: website
-    product_name: 'gloo-mesh-enterprise'
-    version: 'latest'
-    url: 'https://docs.solo.io/gloo-mesh-enterprise/latest/'
-    sitemap_url: 'https://docs.solo.io/gloo-mesh-enterprise/sitemap.xml'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gloo-mesh-enterprise.db'
-
-  - type: website
-    product_name: 'gloo-edge'
-    version: 'latest'
-    url: 'https://docs.solo.io/gloo-edge/latest/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gloo-edge.db'
-
-  - type: website
-    product_name: 'gloo-gateway'
-    version: 'latest'
-    url: 'https://docs.solo.io/gateway/latest/'
-    sitemap_url: 'https://docs.solo.io/gateway/sitemap.xml'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gloo-gateway.db'
-
-  - type: website
-    product_name: 'helm'
-    version: 'latest'
-    url: 'https://helm.sh/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './helm.db'
-
-  - type: website
-    product_name: 'istio'
-    version: 'latest'
-    url: 'https://istio.io/latest/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './istio.db'
-
-  - type: website
-    product_name: 'kubernetes'
-    version: 'latest'
-    url: 'https://kubernetes.io/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './kubernetes.db'
-
-  - type: website
-    product_name: 'otel'
-    version: 'latest'
-    url: 'https://opentelemetry.io/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './otel.db'
-
-  - type: website
-    product_name: 'prometheus'
-    version: 'latest'
-    url: 'https://prometheus.io/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './prometheus.db'