doc2vec 2.9.1 → 2.10.1

package/README.md

@@ -218,6 +218,7 @@ Configuration is managed through two files:
         *   `start_date`: (Optional) Only process tickets/articles updated since this date (e.g., `'2025-01-01'`).
         *   `ticket_status`: (Optional) Filter tickets by status (defaults to `['new', 'open', 'pending', 'hold', 'solved']`).
         *   `ticket_priority`: (Optional) Filter tickets by priority (defaults to all priorities).
+        *   `excluded_organizations`: (Optional) An array of Zendesk organization names whose tickets should be skipped. The sync will abort if any name cannot be resolved.
 
         For S3 buckets (`type: 's3'`):
         *   `bucket`: The S3 bucket name.
@@ -359,6 +360,7 @@ Configuration is managed through two files:
         start_date: '2025-01-01'
         ticket_status: ['open', 'pending']
         ticket_priority: ['high']
+        excluded_organizations: ['Acme Corp', 'Internal Testing']
         max_size: 1048576
         database_config:
           type: 'sqlite'

package/dist/content-processor.js

@@ -1760,9 +1760,9 @@ class ContentProcessor {
     }
     async chunkMarkdown(markdown, sourceConfig, url) {
         const logger = this.logger.child('chunker');
-        // --- Configuration ---
-        const MAX_TOKENS = 1000;
-        const MIN_TOKENS = 150; // 💡 Merges "OpenAI-compatible" sentence into the next block
+        // --- Configuration (character-based, ~4 chars ≈ 1 token) ---
+        const MAX_CHARS = 4000;
+        const MIN_CHARS = 600; // 💡 Merges short sections into the next block
         const OVERLAP_PERCENT = 0.1; // 10% overlap for large splits
         const chunks = [];
         const lines = markdown.split("\n");
@@ -1824,29 +1824,27 @@ class ContentProcessor {
         };
         /**
          * Flushes the current buffer into the chunks array.
-         * Uses sub-splitting logic if the buffer exceeds MAX_TOKENS.
+         * Uses sub-splitting logic if the buffer exceeds MAX_CHARS.
          */
         const flushBuffer = (force = false) => {
             const trimmedBuffer = buffer.trim();
             if (!trimmedBuffer)
                 return;
-            const tokenCount = utils_1.Utils.tokenize(trimmedBuffer).length;
+            const charCount = trimmedBuffer.length;
             // 💡 SEMANTIC MERGING
             // If the current section is too short (like just a title or a one-liner),
             // we don't flush yet unless it's the end of the file (force=true).
-            if (tokenCount < MIN_TOKENS && !force) {
+            if (charCount < MIN_CHARS && !force) {
                 return;
             }
             // Compute the appropriate topic hierarchy for merged content
             const topicHierarchy = computeTopicHierarchy();
-            if (tokenCount > MAX_TOKENS) {
-                // 💡 RECURSIVE OVERLAP SPLITTING
+            if (charCount > MAX_CHARS) {
+                // 💡 OVERLAP SPLITTING
                 // If the section is a massive guide, split it but keep headers on every sub-piece.
-                const tokens = utils_1.Utils.tokenize(trimmedBuffer);
-                const overlapSize = Math.floor(MAX_TOKENS * OVERLAP_PERCENT);
-                for (let i = 0; i < tokens.length; i += (MAX_TOKENS - overlapSize)) {
-                    const subTokens = tokens.slice(i, i + MAX_TOKENS);
-                    const subContent = subTokens.join("");
+                const overlapSize = Math.floor(MAX_CHARS * OVERLAP_PERCENT);
+                for (let i = 0; i < trimmedBuffer.length; i += (MAX_CHARS - overlapSize)) {
+                    const subContent = trimmedBuffer.slice(i, i + MAX_CHARS);
                     chunks.push(createDocumentChunk(subContent, topicHierarchy));
                 }
             }
@@ -1870,9 +1868,9 @@ class ContentProcessor {
                     .replace(/\[\]\(#[^)]*\)/g, "") // Remove [](#anchor) patterns  
                     .trim();
                 // Check if we should merge with previous content
-                const currentTokenCount = utils_1.Utils.tokenize(buffer.trim()).length;
-                const hasBufferContent = currentTokenCount > 0;
-                const bufferIsSmall = currentTokenCount < MIN_TOKENS;
+                const currentCharCount = buffer.trim().length;
+                const hasBufferContent = currentCharCount > 0;
+                const bufferIsSmall = currentCharCount < MIN_CHARS;
                 // Only merge if:
                 // 1. Buffer has content and is small
                 // 2. Buffer has tracked headings (we're merging sections, not just content)
@@ -1898,7 +1896,7 @@ class ContentProcessor {
             else {
                 buffer += `${line}\n`;
                 // Safety valve: if a single section is huge, flush it periodically
-                if (utils_1.Utils.tokenize(buffer).length >= MAX_TOKENS) {
+                if (buffer.length >= MAX_CHARS) {
                     flushBuffer();
                 }
             }

package/dist/doc2vec.js

@@ -1195,6 +1195,8 @@ class Doc2Vec {
         // Status filter applied client-side (includes 'closed' by default so tickets
         // transitioning to closed get updated rather than left stale)
         const statusFilter = new Set(config.ticket_status || ['new', 'open', 'pending', 'hold', 'solved', 'closed']);
+        const excludedOrgNames = new Set((config.excluded_organizations || []).map(n => n.toLowerCase()));
+        const excludedOrgIds = new Set();
         const fetchWithRetry = async (url, retries = 3) => {
             for (let attempt = 0; attempt < retries; attempt++) {
                 try {
@@ -1271,6 +1273,11 @@ class Doc2Vec {
                 }
                 return;
             }
+            // Skip tickets belonging to excluded organizations
+            if (ticket.organization_id && excludedOrgIds.has(ticket.organization_id)) {
+                logger.debug(`Ticket #${ticketId} belongs to excluded organization ${ticket.organization_id} — skipping`);
+                return;
+            }
             // Skip tickets whose status is outside the configured filter
             if (!statusFilter.has(ticket.status)) {
                 logger.debug(`Ticket #${ticketId} has status '${ticket.status}' outside configured filter — skipping`);
@@ -1296,18 +1303,37 @@ class Doc2Vec {
             // so stale chunks from previous versions are never left behind.
             await this.processChunksForUrl(chunks, url, dbConnection, logger);
         };
+        if (excludedOrgNames.size > 0) {
+            logger.info(`Resolving ${excludedOrgNames.size} excluded organization name(s) to IDs`);
+            const resolvedNames = new Set();
+            let orgsUrl = `${baseUrl}/organizations.json?page[size]=100`;
+            while (orgsUrl) {
+                const orgsData = await fetchWithRetry(orgsUrl);
+                for (const org of orgsData?.organizations || []) {
+                    const orgName = (org.name || '').toLowerCase();
+                    if (excludedOrgNames.has(orgName)) {
+                        excludedOrgIds.add(org.id);
+                        resolvedNames.add(orgName);
+                    }
+                }
+                orgsUrl = orgsData?.meta?.has_more ? orgsData?.links?.next : null;
+            }
+            logger.info(`Excluding tickets from ${excludedOrgIds.size} organization(s): ${[...excludedOrgIds].join(', ')}`);
+            const unresolved = [...excludedOrgNames].filter(name => !resolvedNames.has(name));
+            if (unresolved.length > 0) {
+                throw new Error(`Cannot resolve excluded organization(s): ${unresolved.join(', ')}. Aborting to avoid syncing data for them.`);
+            }
+        }
         logger.info(`Fetching Zendesk tickets updated since ${lastRunDate}`);
         // Build query parameters — use the status filter for the search query
         const statusList = Array.from(statusFilter);
-        const query = `updated>${lastRunDate.split('T')[0]} status:${statusList.join(',status:')}`;
-        let nextPage = `${baseUrl}/search.json?query=${encodeURIComponent(query)}&sort_by=updated_at&sort_order=asc`;
+        const statusClause = `status:${statusList.join(',status:')}`;
+        // Zendesk /search.json caps results at 1000 (10 pages * 100). Bisect the date range
+        // into smaller windows whenever a window would exceed that cap, so we never hit page 11.
         let totalTickets = 0;
         let failedTickets = 0;
-        while (nextPage) {
-            const data = await fetchWithRetry(nextPage);
-            const tickets = data.results || [];
-            logger.info(`Processing batch of ${tickets.length} tickets`);
-            for (const ticket of tickets) {
+        const processResults = async (results) => {
+            for (const ticket of results) {
                 try {
                     await processTicket(ticket);
                     totalTickets++;
@@ -1317,11 +1343,39 @@ class Doc2Vec {
                     logger.error(`Failed to process ticket #${ticket.id}, will retry next run: ${error.message}`);
                 }
             }
-            nextPage = data.next_page || null;
-            if (nextPage) {
+        };
+        // Work queue of [start, end) date windows (ISO strings). Seed with [lastRunDate, syncStartDate).
+        const windows = [[lastRunDate, syncStartDate]];
+        const MAX_PER_WINDOW = 1000;
+        while (windows.length > 0) {
+            const [wStart, wEnd] = windows.shift();
+            const q = `updated>${wStart} updated<${wEnd} ${statusClause}`;
+            const firstUrl = `${baseUrl}/search.json?query=${encodeURIComponent(q)}&sort_by=updated_at&sort_order=asc`;
+            logger.debug(`Searching window ${wStart} .. ${wEnd}`);
+            const firstData = await fetchWithRetry(firstUrl);
+            const count = firstData?.count ?? 0;
+            if (count > MAX_PER_WINDOW) {
+                const startMs = new Date(wStart).getTime();
+                const endMs = new Date(wEnd).getTime();
+                if (endMs - startMs <= 1000) {
+                    logger.warn(`Window ${wStart} .. ${wEnd} has ${count} tickets but is already ≤1s wide — processing first 1000 only, some may be missed`);
+                }
+                else {
+                    const midIso = new Date(startMs + Math.floor((endMs - startMs) / 2)).toISOString();
+                    logger.debug(`Window has ${count} tickets (>${MAX_PER_WINDOW}) — bisecting at ${midIso}`);
+                    windows.unshift([wStart, midIso], [midIso, wEnd]);
+                    continue;
+                }
+            }
+            logger.info(`Processing window ${wStart}..${wEnd}: ${count} tickets`);
+            await processResults(firstData.results || []);
+            let nextPage = firstData.next_page || null;
+            while (nextPage) {
                 logger.debug(`Fetching next page: ${nextPage}`);
-                // Rate limiting: wait between requests
                 await new Promise(res => setTimeout(res, 1000));
+                const data = await fetchWithRetry(nextPage);
+                await processResults(data.results || []);
+                nextPage = data.next_page || null;
             }
         }
         // Only advance the watermark when all tickets succeeded

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.9.1",
+    "version": "2.10.1",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",