doc2vec 2.10.1 → 2.10.2

package/README.md

@@ -219,6 +219,7 @@ Configuration is managed through two files:
         *   `ticket_status`: (Optional) Filter tickets by status (defaults to `['new', 'open', 'pending', 'hold', 'solved']`).
         *   `ticket_priority`: (Optional) Filter tickets by priority (defaults to all priorities).
         *   `excluded_organizations`: (Optional) An array of Zendesk organization names whose tickets should be skipped. The sync will abort if any name cannot be resolved.
+        *   `include_internal_comments`: (Optional) Include non-public (internal/agent-only) comments in the indexed content (defaults to `false`, i.e. only public comments are indexed). Internal comments are labeled `(internal)` in the generated markdown.
 
         For S3 buckets (`type: 's3'`):
         *   `bucket`: The S3 bucket name.

package/dist/doc2vec.js

@@ -1245,13 +1245,16 @@ class Doc2Vec {
             if (comments && comments.length > 0) {
                 md += `## Comments\n\n`;
                 for (const comment of comments) {
-                    if (comment.public) {
-                        md += `### ${comment.author_id} - ${new Date(comment.created_at).toDateString()}\n\n`;
-                        // Handle comment body
-                        const rawBody = comment.plain_body || comment.html_body || comment.body || '';
-                        const commentBody = rawBody.replace(/ /g, " ") || '_No content._';
-                        md += `${commentBody}\n\n---\n\n`;
+                    // Skip non-public comments unless internal comments are explicitly enabled
+                    if (!comment.public && !config.include_internal_comments) {
+                        continue;
                     }
+                    const visibility = comment.public ? '' : ' (internal)';
+                    md += `### ${comment.author_id} - ${new Date(comment.created_at).toDateString()}${visibility}\n\n`;
+                    // Handle comment body
+                    const rawBody = comment.plain_body || comment.html_body || comment.body || '';
+                    const commentBody = rawBody.replace(/ /g, " ") || '_No content._';
+                    md += `${commentBody}\n\n---\n\n`;
                 }
             }
             else {
@@ -1284,10 +1287,18 @@ class Doc2Vec {
                 return;
             }
             logger.info(`Processing ticket #${ticketId}`);
-            // Fetch ticket comments
-            const commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
-            const commentsData = await fetchWithRetry(commentsUrl);
-            const comments = commentsData?.comments || [];
+            // Fetch ticket comments. Zendesk returns at most 100 comments per page
+            // (ordered oldest-first), so follow next_page to capture the newest
+            // comments on tickets with more than 100 — otherwise they're silently dropped.
+            const comments = [];
+            let commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
+            while (commentsUrl) {
+                const commentsData = await fetchWithRetry(commentsUrl);
+                comments.push(...(commentsData?.comments || []));
+                commentsUrl = commentsData?.next_page || null;
+                if (commentsUrl)
+                    await new Promise(res => setTimeout(res, 1000));
+            }
             // Generate markdown for the ticket
             const markdown = generateMarkdownForTicket(ticket, comments);
             // Chunk the markdown content

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.10.1",
+    "version": "2.10.2",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",