doc2vec 2.10.1 → 2.10.3

package/README.md

@@ -151,6 +151,16 @@ Configuration is managed through two files:
     OPENAI_API_KEY="sk-..."
     OPENAI_MODEL="text-embedding-3-large"  # Optional, defaults to text-embedding-3-large
 
+    # Optional: Override the OpenAI API base URL. Useful for pointing the
+    # OpenAI SDK at an OpenAI-compatible endpoint such as Ollama, LM Studio,
+    # llama.cpp, vLLM, or a proxy gateway. When unset, the SDK uses the
+    # default OpenAI endpoint. Equivalent to the `embedding.openai.base_url`
+    # field in config.yaml; the env var wins if both are set.
+    # Example values:
+    #   OPENAI_BASE_URL="http://localhost:11434/v1"        # Ollama
+    #   OPENAI_BASE_URL="https://gateway.example.com/v1"   # proxy / gateway
+    OPENAI_BASE_URL="http://localhost:11434/v1"
+
     # Optional: Embedding dimension size (defaults to 3072)
     EMBEDDING_DIMENSION="3072"
 
@@ -219,6 +229,7 @@ Configuration is managed through two files:
         *   `ticket_status`: (Optional) Filter tickets by status (defaults to `['new', 'open', 'pending', 'hold', 'solved']`).
         *   `ticket_priority`: (Optional) Filter tickets by priority (defaults to all priorities).
         *   `excluded_organizations`: (Optional) An array of Zendesk organization names whose tickets should be skipped. The sync will abort if any name cannot be resolved.
+        *   `include_internal_comments`: (Optional) Include non-public (internal/agent-only) comments in the indexed content (defaults to `false`, i.e. only public comments are indexed). Internal comments are labeled `(internal)` in the generated markdown.
 
         For S3 buckets (`type: 's3'`):
         *   `bucket`: The S3 bucket name.
@@ -276,6 +287,7 @@ Configuration is managed through two files:
       openai:
         api_key: '${OPENAI_API_KEY}'  # Optional, uses env var by default
         model: 'text-embedding-3-large'  # Optional, defaults to text-embedding-3-large
+        # base_url: 'http://localhost:11434/v1'  # Optional, override OpenAI API base URL for Ollama / other OpenAI-compatible endpoints. Falls back to OPENAI_BASE_URL env var.
       # For Azure OpenAI, use this instead:
       # azure:
       #   api_key: '${AZURE_OPENAI_KEY}'

package/dist/doc2vec.js

@@ -96,13 +96,17 @@ class Doc2Vec {
         else {
             const openaiApiKey = embeddingConfig.openai?.api_key || process.env.OPENAI_API_KEY;
             const openaiModel = embeddingConfig.openai?.model || process.env.OPENAI_MODEL || 'text-embedding-3-large';
+            const openaiBaseURL = embeddingConfig.openai?.base_url || process.env.OPENAI_BASE_URL;
             if (!openaiApiKey) {
                 this.logger.error('OpenAI requires api_key to be configured');
                 process.exit(1);
             }
-            this.openai = new openai_1.OpenAI({ apiKey: openaiApiKey });
+            this.openai = new openai_1.OpenAI({
+                apiKey: openaiApiKey,
+                ...(openaiBaseURL && { baseURL: openaiBaseURL }),
+            });
             this.embeddingModel = openaiModel;
-            this.logger.info(`Using OpenAI with model: ${openaiModel} (${this.embeddingDimension} dimensions)`);
+            this.logger.info(`Using OpenAI with model: ${openaiModel} (${this.embeddingDimension} dimensions)${openaiBaseURL ? ` via ${openaiBaseURL}` : ''}`);
         }
         this.contentProcessor = new content_processor_1.ContentProcessor(this.logger);
         // Initialize Postgres markdown store if configured
@@ -1245,13 +1249,16 @@ class Doc2Vec {
             if (comments && comments.length > 0) {
                 md += `## Comments\n\n`;
                 for (const comment of comments) {
-                    if (comment.public) {
-                        md += `### ${comment.author_id} - ${new Date(comment.created_at).toDateString()}\n\n`;
-                        // Handle comment body
-                        const rawBody = comment.plain_body || comment.html_body || comment.body || '';
-                        const commentBody = rawBody.replace(/ /g, " ") || '_No content._';
-                        md += `${commentBody}\n\n---\n\n`;
+                    // Skip non-public comments unless internal comments are explicitly enabled
+                    if (!comment.public && !config.include_internal_comments) {
+                        continue;
                     }
+                    const visibility = comment.public ? '' : ' (internal)';
+                    md += `### ${comment.author_id} - ${new Date(comment.created_at).toDateString()}${visibility}\n\n`;
+                    // Handle comment body
+                    const rawBody = comment.plain_body || comment.html_body || comment.body || '';
+                    const commentBody = rawBody.replace(/ /g, " ") || '_No content._';
+                    md += `${commentBody}\n\n---\n\n`;
                 }
             }
             else {
@@ -1284,10 +1291,18 @@ class Doc2Vec {
                 return;
             }
             logger.info(`Processing ticket #${ticketId}`);
-            // Fetch ticket comments
-            const commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
-            const commentsData = await fetchWithRetry(commentsUrl);
-            const comments = commentsData?.comments || [];
+            // Fetch ticket comments. Zendesk returns at most 100 comments per page
+            // (ordered oldest-first), so follow next_page to capture the newest
+            // comments on tickets with more than 100 — otherwise they're silently dropped.
+            const comments = [];
+            let commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
+            while (commentsUrl) {
+                const commentsData = await fetchWithRetry(commentsUrl);
+                comments.push(...(commentsData?.comments || []));
+                commentsUrl = commentsData?.next_page || null;
+                if (commentsUrl)
+                    await new Promise(res => setTimeout(res, 1000));
+            }
             // Generate markdown for the ticket
             const markdown = generateMarkdownForTicket(ticket, comments);
             // Chunk the markdown content

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.10.1",
+    "version": "2.10.3",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",