doc2vec 2.9.2 → 2.10.2

package/README.md

@@ -218,6 +218,8 @@ Configuration is managed through two files:
         *   `start_date`: (Optional) Only process tickets/articles updated since this date (e.g., `'2025-01-01'`).
         *   `ticket_status`: (Optional) Filter tickets by status (defaults to `['new', 'open', 'pending', 'hold', 'solved']`).
         *   `ticket_priority`: (Optional) Filter tickets by priority (defaults to all priorities).
+        *   `excluded_organizations`: (Optional) An array of Zendesk organization names whose tickets should be skipped. The sync will abort if any name cannot be resolved.
+        *   `include_internal_comments`: (Optional) Include non-public (internal/agent-only) comments in the indexed content (defaults to `false`, i.e. only public comments are indexed). Internal comments are labeled `(internal)` in the generated markdown.
 
         For S3 buckets (`type: 's3'`):
         *   `bucket`: The S3 bucket name.
@@ -359,6 +361,7 @@ Configuration is managed through two files:
         start_date: '2025-01-01'
         ticket_status: ['open', 'pending']
         ticket_priority: ['high']
+        excluded_organizations: ['Acme Corp', 'Internal Testing']
         max_size: 1048576
         database_config:
           type: 'sqlite'

package/dist/doc2vec.js

@@ -1195,6 +1195,8 @@ class Doc2Vec {
         // Status filter applied client-side (includes 'closed' by default so tickets
         // transitioning to closed get updated rather than left stale)
         const statusFilter = new Set(config.ticket_status || ['new', 'open', 'pending', 'hold', 'solved', 'closed']);
+        const excludedOrgNames = new Set((config.excluded_organizations || []).map(n => n.toLowerCase()));
+        const excludedOrgIds = new Set();
         const fetchWithRetry = async (url, retries = 3) => {
             for (let attempt = 0; attempt < retries; attempt++) {
                 try {
@@ -1243,13 +1245,16 @@ class Doc2Vec {
             if (comments && comments.length > 0) {
                 md += `## Comments\n\n`;
                 for (const comment of comments) {
-                    if (comment.public) {
-                        md += `### ${comment.author_id} - ${new Date(comment.created_at).toDateString()}\n\n`;
-                        // Handle comment body
-                        const rawBody = comment.plain_body || comment.html_body || comment.body || '';
-                        const commentBody = rawBody.replace(/&nbsp;/g, " ") || '_No content._';
-                        md += `${commentBody}\n\n---\n\n`;
+                    // Skip non-public comments unless internal comments are explicitly enabled
+                    if (!comment.public && !config.include_internal_comments) {
+                        continue;
                     }
+                    const visibility = comment.public ? '' : ' (internal)';
+                    md += `### ${comment.author_id} - ${new Date(comment.created_at).toDateString()}${visibility}\n\n`;
+                    // Handle comment body
+                    const rawBody = comment.plain_body || comment.html_body || comment.body || '';
+                    const commentBody = rawBody.replace(/&nbsp;/g, " ") || '_No content._';
+                    md += `${commentBody}\n\n---\n\n`;
                 }
             }
             else {
@@ -1271,16 +1276,29 @@ class Doc2Vec {
                 }
                 return;
             }
+            // Skip tickets belonging to excluded organizations
+            if (ticket.organization_id && excludedOrgIds.has(ticket.organization_id)) {
+                logger.debug(`Ticket #${ticketId} belongs to excluded organization ${ticket.organization_id} — skipping`);
+                return;
+            }
             // Skip tickets whose status is outside the configured filter
             if (!statusFilter.has(ticket.status)) {
                 logger.debug(`Ticket #${ticketId} has status '${ticket.status}' outside configured filter — skipping`);
                 return;
             }
             logger.info(`Processing ticket #${ticketId}`);
-            // Fetch ticket comments
-            const commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
-            const commentsData = await fetchWithRetry(commentsUrl);
-            const comments = commentsData?.comments || [];
+            // Fetch ticket comments. Zendesk returns at most 100 comments per page
+            // (ordered oldest-first), so follow next_page to capture the newest
+            // comments on tickets with more than 100 — otherwise they're silently dropped.
+            const comments = [];
+            let commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
+            while (commentsUrl) {
+                const commentsData = await fetchWithRetry(commentsUrl);
+                comments.push(...(commentsData?.comments || []));
+                commentsUrl = commentsData?.next_page || null;
+                if (commentsUrl)
+                    await new Promise(res => setTimeout(res, 1000));
+            }
             // Generate markdown for the ticket
             const markdown = generateMarkdownForTicket(ticket, comments);
             // Chunk the markdown content
@@ -1296,18 +1314,37 @@ class Doc2Vec {
             // so stale chunks from previous versions are never left behind.
             await this.processChunksForUrl(chunks, url, dbConnection, logger);
         };
+        if (excludedOrgNames.size > 0) {
+            logger.info(`Resolving ${excludedOrgNames.size} excluded organization name(s) to IDs`);
+            const resolvedNames = new Set();
+            let orgsUrl = `${baseUrl}/organizations.json?page[size]=100`;
+            while (orgsUrl) {
+                const orgsData = await fetchWithRetry(orgsUrl);
+                for (const org of orgsData?.organizations || []) {
+                    const orgName = (org.name || '').toLowerCase();
+                    if (excludedOrgNames.has(orgName)) {
+                        excludedOrgIds.add(org.id);
+                        resolvedNames.add(orgName);
+                    }
+                }
+                orgsUrl = orgsData?.meta?.has_more ? orgsData?.links?.next : null;
+            }
+            logger.info(`Excluding tickets from ${excludedOrgIds.size} organization(s): ${[...excludedOrgIds].join(', ')}`);
+            const unresolved = [...excludedOrgNames].filter(name => !resolvedNames.has(name));
+            if (unresolved.length > 0) {
+                throw new Error(`Cannot resolve excluded organization(s): ${unresolved.join(', ')}. Aborting to avoid syncing data for them.`);
+            }
+        }
         logger.info(`Fetching Zendesk tickets updated since ${lastRunDate}`);
         // Build query parameters — use the status filter for the search query
         const statusList = Array.from(statusFilter);
-        const query = `updated>${lastRunDate.split('T')[0]} status:${statusList.join(',status:')}`;
-        let nextPage = `${baseUrl}/search.json?query=${encodeURIComponent(query)}&sort_by=updated_at&sort_order=asc`;
+        const statusClause = `status:${statusList.join(',status:')}`;
+        // Zendesk /search.json caps results at 1000 (10 pages * 100). Bisect the date range
+        // into smaller windows whenever a window would exceed that cap, so we never hit page 11.
         let totalTickets = 0;
         let failedTickets = 0;
-        while (nextPage) {
-            const data = await fetchWithRetry(nextPage);
-            const tickets = data.results || [];
-            logger.info(`Processing batch of ${tickets.length} tickets`);
-            for (const ticket of tickets) {
+        const processResults = async (results) => {
+            for (const ticket of results) {
                 try {
                     await processTicket(ticket);
                     totalTickets++;
@@ -1317,11 +1354,39 @@ class Doc2Vec {
                     logger.error(`Failed to process ticket #${ticket.id}, will retry next run: ${error.message}`);
                 }
             }
-            nextPage = data.next_page || null;
-            if (nextPage) {
+        };
+        // Work queue of [start, end) date windows (ISO strings). Seed with [lastRunDate, syncStartDate).
+        const windows = [[lastRunDate, syncStartDate]];
+        const MAX_PER_WINDOW = 1000;
+        while (windows.length > 0) {
+            const [wStart, wEnd] = windows.shift();
+            const q = `updated>${wStart} updated<${wEnd} ${statusClause}`;
+            const firstUrl = `${baseUrl}/search.json?query=${encodeURIComponent(q)}&sort_by=updated_at&sort_order=asc`;
+            logger.debug(`Searching window ${wStart} .. ${wEnd}`);
+            const firstData = await fetchWithRetry(firstUrl);
+            const count = firstData?.count ?? 0;
+            if (count > MAX_PER_WINDOW) {
+                const startMs = new Date(wStart).getTime();
+                const endMs = new Date(wEnd).getTime();
+                if (endMs - startMs <= 1000) {
+                    logger.warn(`Window ${wStart} .. ${wEnd} has ${count} tickets but is already ≤1s wide — processing first 1000 only, some may be missed`);
+                }
+                else {
+                    const midIso = new Date(startMs + Math.floor((endMs - startMs) / 2)).toISOString();
+                    logger.debug(`Window has ${count} tickets (>${MAX_PER_WINDOW}) — bisecting at ${midIso}`);
+                    windows.unshift([wStart, midIso], [midIso, wEnd]);
+                    continue;
+                }
+            }
+            logger.info(`Processing window ${wStart}..${wEnd}: ${count} tickets`);
+            await processResults(firstData.results || []);
+            let nextPage = firstData.next_page || null;
+            while (nextPage) {
                 logger.debug(`Fetching next page: ${nextPage}`);
-                // Rate limiting: wait between requests
                 await new Promise(res => setTimeout(res, 1000));
+                const data = await fetchWithRetry(nextPage);
+                await processResults(data.results || []);
+                nextPage = data.next_page || null;
             }
         }
         // Only advance the watermark when all tickets succeeded

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.9.2",
+    "version": "2.10.2",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",