doc2vec 1.1.1 → 1.2.0

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils.js

@@ -0,0 +1,118 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Utils = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const path = __importStar(require("path"));
+class Utils {
+    static generateHash(content) {
+        return crypto_1.default.createHash("sha256").update(content).digest("hex");
+    }
+    static generateMetadataUUID(repo) {
+        // Simple deterministic approach - hash the repo name and convert to UUID format
+        const hash = crypto_1.default.createHash('md5').update(`metadata_${repo}`).digest('hex');
+        // Format as UUID with version bits set correctly (version 4)
+        return `${hash.substr(0, 8)}-${hash.substr(8, 4)}-4${hash.substr(13, 3)}-${hash.substr(16, 4)}-${hash.substr(20, 12)}`;
+    }
+    static getUrlPrefix(url) {
+        try {
+            const parsedUrl = new URL(url);
+            return parsedUrl.origin + parsedUrl.pathname;
+        }
+        catch (error) {
+            return url;
+        }
+    }
+    static normalizeUrl(url) {
+        try {
+            const urlObj = new URL(url);
+            urlObj.hash = '';
+            urlObj.search = '';
+            return urlObj.toString();
+        }
+        catch (error) {
+            return url;
+        }
+    }
+    static buildUrl(href, currentUrl) {
+        try {
+            return new URL(href, currentUrl).toString();
+        }
+        catch (error) {
+            console.warn(`Invalid URL found: ${href}`);
+            return '';
+        }
+    }
+    static shouldProcessUrl(url) {
+        const parsedUrl = new URL(url);
+        const pathname = parsedUrl.pathname;
+        const ext = path.extname(pathname);
+        if (!ext)
+            return true;
+        return ['.html', '.htm', '.pdf'].includes(ext.toLowerCase());
+    }
+    static isPdfUrl(url) {
+        try {
+            const parsedUrl = new URL(url);
+            const pathname = parsedUrl.pathname;
+            const ext = path.extname(pathname);
+            return ext.toLowerCase() === '.pdf';
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    static isValidUuid(str) {
+        const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+        return uuidRegex.test(str);
+    }
+    static hashToUuid(hash) {
+        const truncatedHash = hash.substring(0, 32);
+        return [
+            truncatedHash.substring(0, 8),
+            truncatedHash.substring(8, 12),
+            '5' + truncatedHash.substring(13, 16),
+            '8' + truncatedHash.substring(17, 20),
+            truncatedHash.substring(20, 32)
+        ].join('-');
+    }
+    static tokenize(text) {
+        return text.split(/(\s+)/).filter(token => token.length > 0);
+    }
+}
+exports.Utils = Utils;

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "doc2vec",
-    "version": "1.1.1",
+    "version": "1.2.0",
     "type": "commonjs",
     "description": "",
-    "main": "doc2vec.ts",
+    "main": "dist/doc2vec.js",
     "private": false,
     "publishConfig": {
         "access": "public"
@@ -11,6 +11,11 @@
     "bin": {
         "doc2vec": "dist/doc2vec.js"
     },
+    "files": [
+        "dist/",
+        "README.md",
+        "LICENSE"
+    ],
     "scripts": {
         "build": "tsc && chmod 755 dist/doc2vec.js",
         "start": "node dist/doc2vec.js",
@@ -32,6 +37,7 @@
         "js-yaml": "^4.1.0",
         "jsdom": "^26.0.0",
         "openai": "^4.20.1",
+        "pdfjs-dist": "^5.3.31",
         "puppeteer": "^24.1.1",
         "sanitize-html": "^2.11.0",
         "sqlite-vec": "0.1.7-alpha.2",

package/Dockerfile

@@ -1,29 +0,0 @@
-FROM node:20-slim
-
-WORKDIR /app
-
-RUN apt-get update && apt-get install -y \
-  git \
-  python3 \
-  make \
-  g++ \
-  sqlite3 \
-  libsqlite3-dev \
-  curl \
-  ca-certificates \
-  chromium \
-  fonts-freefont-ttf \
-  fonts-ipafont-gothic \
-  fonts-kacst \
-  fonts-liberation \
-  fonts-noto-color-emoji \
-  fonts-thai-tlwg \
-  libx11-xcb1 \
-  libxcb-dri3-0 \
-  libxcomposite1 \
-  libxdamage1 \
-  libxi6 \
-  libxrandr2 \
-  libxshmfence1 \
-  libxtst6 \
-  && apt-get clean

package/config.yaml

@@ -1,167 +0,0 @@
-# Doc2Vec Configuration
-sources:
-  # GitHub Sources
-  - type: github
-    product_name: 'istio'
-    version: 'latest'
-    repo: 'istio/istio'
-    start_date: '2025-01-01'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './istio-issues.db'
-
-  # Website Sources
-  - type: website
-    product_name: 'ambient'
-    version: 'latest'
-    url: 'https://ambientmesh.io/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './ambient.db'
-
-  - type: website
-    product_name: 'argo'
-    version: 'latest'
-    url: 'https://argo-cd.readthedocs.io/en/stable/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './argo.db'
-
-  - type: website
-    product_name: 'argo-rollouts'
-    version: 'latest'
-    url: 'https://argoproj.github.io/argo-rollouts/installation/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './argo-rollouts.db'
-
-  - type: website
-    product_name: 'argo-rollouts'
-    version: 'latest'
-    url: 'https://rollouts-plugin-trafficrouter-gatewayapi.readthedocs.io/en/latest/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './argo-rollouts.db'
-
-  - type: website
-    product_name: 'cilium'
-    version: 'latest'
-    url: 'https://docs.cilium.io/en/stable/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './cilium.db'
-
-  - type: website
-    product_name: 'gateway-api'
-    version: 'latest'
-    url: 'https://gateway-api.sigs.k8s.io/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gateway-api.db'
-
-  - type: website
-    product_name: 'gloo-mesh-core'
-    version: 'latest'
-    url: 'https://docs.solo.io/gloo-mesh/latest/'
-    sitemap_url: 'https://docs.solo.io/gloo-mesh/sitemap.xml'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gloo-mesh-core.db'
-
-  - type: website
-    product_name: 'gloo-mesh-enterprise'
-    version: 'latest'
-    url: 'https://docs.solo.io/gloo-mesh-enterprise/latest/'
-    sitemap_url: 'https://docs.solo.io/gloo-mesh-enterprise/sitemap.xml'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gloo-mesh-enterprise.db'
-
-  - type: website
-    product_name: 'gloo-edge'
-    version: 'latest'
-    url: 'https://docs.solo.io/gloo-edge/latest/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gloo-edge.db'
-
-  - type: website
-    product_name: 'gloo-gateway'
-    version: 'latest'
-    url: 'https://docs.solo.io/gateway/latest/'
-    sitemap_url: 'https://docs.solo.io/gateway/sitemap.xml'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './gloo-gateway.db'
-
-  - type: website
-    product_name: 'helm'
-    version: 'latest'
-    url: 'https://helm.sh/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './helm.db'
-
-  - type: website
-    product_name: 'istio'
-    version: 'latest'
-    url: 'https://istio.io/latest/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './istio.db'
-
-  - type: website
-    product_name: 'kubernetes'
-    version: 'latest'
-    url: 'https://kubernetes.io/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './kubernetes.db'
-
-  - type: website
-    product_name: 'otel'
-    version: 'latest'
-    url: 'https://opentelemetry.io/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './otel.db'
-
-  - type: website
-    product_name: 'prometheus'
-    version: 'latest'
-    url: 'https://prometheus.io/docs/'
-    max_size: 1048576
-    database_config:
-      type: 'sqlite'
-      params:
-        db_path: './prometheus.db'