doc2vec 2.6.0 → 2.8.0

package/README.md

@@ -176,7 +176,7 @@ Configuration is managed through two files:
     **Structure:**
 
     *   `sources`: An array of source configurations.
-        *   `type`: Either `'website'`, `'github'`, `'local_directory'`, `'code'`, or `'zendesk'`
+        *   `type`: Either `'website'`, `'github'`, `'local_directory'`, `'code'`, `'zendesk'`, or `'s3'`
         
         For websites (`type: 'website'`):
         *   `url`: The starting URL for crawling the documentation site.
@@ -219,6 +219,20 @@ Configuration is managed through two files:
         *   `ticket_status`: (Optional) Filter tickets by status (defaults to `['new', 'open', 'pending', 'hold', 'solved']`).
         *   `ticket_priority`: (Optional) Filter tickets by priority (defaults to all priorities).
 
+        For S3 buckets (`type: 's3'`):
+        *   `bucket`: The S3 bucket name.
+        *   `prefix`: (Optional) Key prefix to filter objects (e.g., `'docs/'`). Only objects under this prefix will be processed.
+        *   `region`: (Optional) AWS region (defaults to `AWS_DEFAULT_REGION` environment variable or `'us-east-1'`).
+        *   `endpoint`: (Optional) Custom S3 endpoint for S3-compatible services (MinIO, LocalStack, etc.).
+        *   `include_extensions`: (Optional) Array of file extensions to include (e.g., `['.md', '.txt', '.pdf']`). Defaults to `['.md', '.txt', '.html', '.htm', '.pdf', '.doc', '.docx']`.
+        *   `exclude_extensions`: (Optional) Array of file extensions to exclude.
+        *   `encoding`: (Optional) Text file encoding (defaults to `'utf8'`). Does not apply to binary files (PDF, DOC, DOCX).
+        *   `url_rewrite_prefix`: (Optional) URL prefix to rewrite `s3://` URLs (e.g., `'https://docs.example.com'`).
+
+        Authentication uses the AWS SDK default credential chain: environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`), `~/.aws/credentials`, IAM roles, etc.
+
+        Incremental sync tracks object `LastModified` timestamps so only new or updated objects are processed on subsequent runs. Deleted objects are automatically cleaned up.
+
         Common configuration for all types:
         *   `product_name`: A string identifying the product (used in metadata).
         *   `version`: A string identifying the product version (used in metadata).
@@ -349,6 +363,21 @@ Configuration is managed through two files:
           params:
             db_path: './zendesk-kb.db'
       
+      # S3 bucket source example
+      - type: 's3'
+        product_name: 'my-docs'
+        version: 'latest'
+        bucket: 'my-documentation-bucket'
+        prefix: 'docs/v2/'
+        region: 'us-west-2'
+        include_extensions: ['.md', '.txt', '.pdf', '.html']
+        url_rewrite_prefix: 'https://docs.example.com'
+        max_size: 1048576
+        database_config:
+          type: 'sqlite'
+          params:
+            db_path: './s3-docs.db'
+
       # Qdrant example
       - type: 'website'
         product_name: 'Istio'

package/dist/content-processor.js

@@ -1306,6 +1306,39 @@ class ContentProcessor {
             throw error;
         }
     }
+    async convertFileToMarkdown(filePath, extension, logger) {
+        const ext = extension.toLowerCase();
+        if (ext === '.pdf') {
+            return this.convertPdfToMarkdown(filePath, logger);
+        }
+        else if (ext === '.doc') {
+            return this.convertDocToMarkdown(filePath, logger);
+        }
+        else if (ext === '.docx') {
+            return this.convertDocxToMarkdown(filePath, logger);
+        }
+        else if (ext === '.html' || ext === '.htm') {
+            const content = fs.readFileSync(filePath, { encoding: 'utf8' });
+            const cleanHtml = (0, sanitize_html_1.default)(content, {
+                allowedTags: [
+                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
+                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
+                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
+                ],
+                allowedAttributes: {
+                    'a': ['href'],
+                    'pre': ['class', 'data-language'],
+                    'code': ['class', 'data-language'],
+                    'div': ['class'],
+                    'span': ['class']
+                }
+            });
+            return this.turndownService.turndown(cleanHtml);
+        }
+        else {
+            throw new Error(`Unsupported file extension for conversion: ${extension}`);
+        }
+    }
     async downloadAndConvertPdfFromUrl(url, logger) {
         logger.debug(`Downloading and converting PDF from URL: ${url}`);
         try {
@@ -1428,20 +1461,18 @@ class ContentProcessor {
                         logger.info(`Reading file: ${filePath}`);
                         let content;
                         let processedContent;
-                        if (extension === '.pdf') {
-                            // Handle PDF files
-                            logger.debug(`Processing PDF file: ${filePath}`);
-                            processedContent = await this.convertPdfToMarkdown(filePath, logger);
-                        }
-                        else if (extension === '.doc') {
-                            // Handle legacy Word DOC files
-                            logger.debug(`Processing DOC file: ${filePath}`);
-                            processedContent = await this.convertDocToMarkdown(filePath, logger);
-                        }
-                        else if (extension === '.docx') {
-                            // Handle modern Word DOCX files
-                            logger.debug(`Processing DOCX file: ${filePath}`);
-                            processedContent = await this.convertDocxToMarkdown(filePath, logger);
+                        const convertibleExtensions = ['.pdf', '.doc', '.docx', '.html', '.htm'];
+                        if (convertibleExtensions.includes(extension)) {
+                            if (extension === '.html' || extension === '.htm') {
+                                // For HTML, check raw file size before converting
+                                content = fs.readFileSync(filePath, { encoding: encoding });
+                                if (content.length > config.max_size) {
+                                    logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
+                                    skippedFiles++;
+                                    continue;
+                                }
+                            }
+                            processedContent = await this.convertFileToMarkdown(filePath, extension, logger);
                         }
                         else {
                             // Handle text-based files
@@ -1451,28 +1482,7 @@ class ContentProcessor {
                                 skippedFiles++;
                                 continue;
                             }
-                            // Convert HTML to Markdown if needed
-                            if (extension === '.html' || extension === '.htm') {
-                                logger.debug(`Converting HTML to Markdown for ${filePath}`);
-                                const cleanHtml = (0, sanitize_html_1.default)(content, {
-                                    allowedTags: [
-                                        'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
-                                        'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
-                                        'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
-                                    ],
-                                    allowedAttributes: {
-                                        'a': ['href'],
-                                        'pre': ['class', 'data-language'],
-                                        'code': ['class', 'data-language'],
-                                        'div': ['class'],
-                                        'span': ['class']
-                                    }
-                                });
-                                processedContent = this.turndownService.turndown(cleanHtml);
-                            }
-                            else {
-                                processedContent = content;
-                            }
+                            processedContent = content;
                         }
                         // Check size limit for processed content
                         if (processedContent.length > config.max_size) {

package/dist/database.js

@@ -257,8 +257,8 @@ class DatabaseManager {
         logger.info(`No saved run date found for ${repo}, using default: ${defaultDate}`);
         return defaultDate;
     }
-    static async updateLastRunDate(dbConnection, repo, logger, embeddingDimension) {
-        const now = new Date().toISOString();
+    static async updateLastRunDate(dbConnection, repo, logger, embeddingDimension, date) {
+        const now = date;
         try {
             if (dbConnection.type === 'sqlite') {
                 const metadataKey = `last_run_${repo.replace('/', '_')}`;

package/dist/doc2vec.js

@@ -53,6 +53,7 @@ const logger_1 = require("./logger");
 const utils_1 = require("./utils");
 const database_1 = require("./database");
 const content_processor_1 = require("./content-processor");
+const client_s3_1 = require("@aws-sdk/client-s3");
 const markdown_store_1 = require("./markdown-store");
 const GITHUB_TOKEN = process.env.GITHUB_PERSONAL_ACCESS_TOKEN;
 const execAsync = (0, util_1.promisify)(child_process_1.exec);
@@ -189,6 +190,9 @@ class Doc2Vec {
             else if (sourceConfig.type === 'zendesk') {
                 await this.processZendesk(sourceConfig, sourceLogger);
             }
+            else if (sourceConfig.type === 's3') {
+                await this.processS3(sourceConfig, sourceLogger);
+            }
             else {
                 sourceLogger.error(`Unknown source type: ${sourceConfig.type}`);
             }
@@ -204,6 +208,8 @@ class Doc2Vec {
         const GITHUB_API_URL = `https://api.github.com/repos/${owner}/${repoName}/issues`;
         // Initialize metadata storage if needed
         await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
+        // Capture timestamp at the start so issues updated during sync are re-processed next run
+        const syncStartDate = new Date().toISOString();
         // Get the last run date from the database
         const startDate = sourceConfig.start_date || '2025-01-01';
         const lastRunDate = await database_1.DatabaseManager.getLastRunDate(dbConnection, repo, `${startDate}T00:00:00Z`, logger);
@@ -402,7 +408,7 @@ class Doc2Vec {
             await processIssue(issues[i]);
         }
         // Update the last run date in the database after processing all issues
-        await database_1.DatabaseManager.updateLastRunDate(dbConnection, repo, logger, this.embeddingDimension);
+        await database_1.DatabaseManager.updateLastRunDate(dbConnection, repo, logger, this.embeddingDimension, syncStartDate);
         logger.info(`Successfully processed ${issues.length} issues`);
     }
     async processGithubRepo(config, parentLogger) {
@@ -637,6 +643,191 @@ class Doc2Vec {
         }
         logger.info(`Finished processing local directory: ${config.path}`);
     }
+    async processS3(config, parentLogger) {
+        const logger = parentLogger.child('process');
+        logger.info(`Starting processing for S3 bucket: ${config.bucket}${config.prefix ? ` (prefix: ${config.prefix})` : ''}`);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger, this.embeddingDimension);
+        await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
+        const processedFiles = new Set();
+        // Capture timestamp at the start so objects modified during sync are re-processed next run
+        const syncStartTimestamp = Date.now();
+        const s3Client = new client_s3_1.S3Client({
+            region: config.region || process.env.AWS_DEFAULT_REGION || 'us-east-1',
+            ...(config.endpoint ? { endpoint: config.endpoint, forcePathStyle: true } : {})
+        });
+        // Metadata keys for incremental sync
+        const sanitizedPrefix = (config.prefix || '').replace(/[^a-zA-Z0-9]+/g, '_');
+        const bucketKey = `${config.bucket}_${sanitizedPrefix}`;
+        const lastSyncKey = `s3_last_sync_${bucketKey}`;
+        const fileListKey = `s3_filelist_${bucketKey}`;
+        const lastSyncValue = await database_1.DatabaseManager.getMetadataValue(dbConnection, lastSyncKey, '0', logger);
+        const lastSyncTimestamp = parseInt(lastSyncValue || '0', 10);
+        const includeExtensions = config.include_extensions || ['.md', '.txt', '.html', '.htm', '.pdf', '.doc', '.docx'];
+        const excludeExtensions = config.exclude_extensions || [];
+        const encoding = config.encoding || 'utf8';
+        // List all matching objects
+        logger.section('S3 OBJECT LISTING');
+        const allObjects = [];
+        let continuationToken;
+        do {
+            const response = await s3Client.send(new client_s3_1.ListObjectsV2Command({
+                Bucket: config.bucket,
+                Prefix: config.prefix || undefined,
+                ContinuationToken: continuationToken,
+            }));
+            for (const obj of response.Contents || []) {
+                if (!obj.Key || obj.Key.endsWith('/'))
+                    continue; // skip folder markers
+                const extension = path.extname(obj.Key).toLowerCase();
+                if (excludeExtensions.includes(extension))
+                    continue;
+                if (includeExtensions.length > 0 && !includeExtensions.includes(extension))
+                    continue;
+                allObjects.push({
+                    key: obj.Key,
+                    lastModified: obj.LastModified,
+                    size: obj.Size || 0,
+                });
+            }
+            continuationToken = response.NextContinuationToken;
+        } while (continuationToken);
+        logger.info(`Found ${allObjects.length} matching objects in S3 bucket`);
+        // Process objects
+        logger.section('S3 FILE PROCESSING AND EMBEDDING');
+        let processedCount = 0;
+        let skippedCount = 0;
+        for (const obj of allObjects) {
+            processedFiles.add(obj.key);
+            // Incremental check: skip if object hasn't been modified since last sync
+            if (lastSyncTimestamp > 0 && obj.lastModified.getTime() <= lastSyncTimestamp) {
+                logger.debug(`Skipping unchanged object: ${obj.key}`);
+                skippedCount++;
+                continue;
+            }
+            // Size check
+            if (obj.size > config.max_size) {
+                logger.warn(`Object ${obj.key} (${obj.size} bytes) exceeds max_size (${config.max_size}). Skipping.`);
+                skippedCount++;
+                continue;
+            }
+            logger.info(`Processing S3 object: ${obj.key} (${obj.size} bytes)`);
+            try {
+                const getResponse = await s3Client.send(new client_s3_1.GetObjectCommand({
+                    Bucket: config.bucket,
+                    Key: obj.key,
+                }));
+                const extension = path.extname(obj.key).toLowerCase();
+                let content;
+                const binaryExtensions = ['.pdf', '.doc', '.docx'];
+                if (binaryExtensions.includes(extension)) {
+                    // Binary files: write to temp file and convert
+                    const bodyBytes = await getResponse.Body.transformToByteArray();
+                    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 's3-doc2vec-'));
+                    const tempFilePath = path.join(tempDir, path.basename(obj.key));
+                    try {
+                        fs.writeFileSync(tempFilePath, buffer_1.Buffer.from(bodyBytes));
+                        content = await this.contentProcessor.convertFileToMarkdown(tempFilePath, extension, logger);
+                    }
+                    finally {
+                        // Cleanup temp files
+                        try {
+                            fs.unlinkSync(tempFilePath);
+                        }
+                        catch { }
+                        try {
+                            fs.rmdirSync(tempDir);
+                        }
+                        catch { }
+                    }
+                }
+                else if (extension === '.html' || extension === '.htm') {
+                    // HTML files: write to temp, convert via contentProcessor
+                    const bodyString = await getResponse.Body.transformToString(encoding);
+                    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 's3-doc2vec-'));
+                    const tempFilePath = path.join(tempDir, path.basename(obj.key));
+                    try {
+                        fs.writeFileSync(tempFilePath, bodyString, { encoding });
+                        content = await this.contentProcessor.convertFileToMarkdown(tempFilePath, extension, logger);
+                    }
+                    finally {
+                        try {
+                            fs.unlinkSync(tempFilePath);
+                        }
+                        catch { }
+                        try {
+                            fs.rmdirSync(tempDir);
+                        }
+                        catch { }
+                    }
+                }
+                else {
+                    // Text files: read directly as string
+                    content = await getResponse.Body.transformToString(encoding);
+                }
+                if (content.length > config.max_size) {
+                    logger.warn(`Processed content for ${obj.key} (${content.length} chars) exceeds max_size. Skipping.`);
+                    skippedCount++;
+                    continue;
+                }
+                // Generate URL
+                let fileUrl;
+                if (config.url_rewrite_prefix) {
+                    const prefix = config.url_rewrite_prefix.endsWith('/')
+                        ? config.url_rewrite_prefix.slice(0, -1)
+                        : config.url_rewrite_prefix;
+                    const relativePath = config.prefix
+                        ? obj.key.substring(config.prefix.length).replace(/^\//, '')
+                        : obj.key;
+                    fileUrl = `${prefix}/${relativePath}`;
+                }
+                else {
+                    fileUrl = `s3://${config.bucket}/${obj.key}`;
+                }
+                const chunks = await this.contentProcessor.chunkMarkdown(content, config, fileUrl);
+                logger.info(`Created ${chunks.length} chunks for ${obj.key}`);
+                await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
+                processedCount++;
+            }
+            catch (error) {
+                logger.error(`Error processing S3 object ${obj.key}:`, error);
+            }
+        }
+        logger.info(`Processed: ${processedCount}, Skipped: ${skippedCount}`);
+        // Cleanup: remove chunks for deleted objects
+        logger.section('CLEANUP');
+        const previousListValue = await database_1.DatabaseManager.getMetadataValue(dbConnection, fileListKey, '[]', logger);
+        const previousList = previousListValue ? JSON.parse(previousListValue) : [];
+        const deletedKeys = previousList.filter(key => !processedFiles.has(key));
+        if (deletedKeys.length > 0) {
+            logger.info(`Removing chunks for ${deletedKeys.length} deleted objects`);
+            for (const deletedKey of deletedKeys) {
+                let fileUrl;
+                if (config.url_rewrite_prefix) {
+                    const prefix = config.url_rewrite_prefix.endsWith('/')
+                        ? config.url_rewrite_prefix.slice(0, -1)
+                        : config.url_rewrite_prefix;
+                    const relativePath = config.prefix
+                        ? deletedKey.substring(config.prefix.length).replace(/^\//, '')
+                        : deletedKey;
+                    fileUrl = `${prefix}/${relativePath}`;
+                }
+                else {
+                    fileUrl = `s3://${config.bucket}/${deletedKey}`;
+                }
+                if (dbConnection.type === 'sqlite') {
+                    database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, fileUrl, logger);
+                }
+                else if (dbConnection.type === 'qdrant') {
+                    await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, fileUrl, logger);
+                }
+            }
+        }
+        // Persist sync state
+        const currentKeys = Array.from(processedFiles);
+        await database_1.DatabaseManager.setMetadataValue(dbConnection, fileListKey, JSON.stringify(currentKeys), logger, this.embeddingDimension);
+        await database_1.DatabaseManager.setMetadataValue(dbConnection, lastSyncKey, `${syncStartTimestamp}`, logger, this.embeddingDimension);
+        logger.info(`Finished processing S3 bucket: ${config.bucket}`);
+    }
     async processCodeSource(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for code source (${config.source})`);
@@ -974,9 +1165,14 @@ class Doc2Vec {
     async fetchAndProcessZendeskTickets(config, dbConnection, logger) {
         const baseUrl = `https://${config.zendesk_subdomain}.zendesk.com/api/v2`;
         const auth = buffer_1.Buffer.from(`${config.email}/token:${config.api_token}`).toString('base64');
+        // Capture timestamp at the start so tickets updated during sync are re-processed next run
+        const syncStartDate = new Date().toISOString();
         // Get the last run date from the database
         const startDate = config.start_date || `${new Date().getFullYear()}-01-01`;
         const lastRunDate = await database_1.DatabaseManager.getLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, `${startDate}T00:00:00Z`, logger);
+        // Status filter applied client-side (includes 'closed' by default so tickets
+        // transitioning to closed get updated rather than left stale)
+        const statusFilter = new Set(config.ticket_status || ['new', 'open', 'pending', 'hold', 'solved', 'closed']);
         const fetchWithRetry = async (url, retries = 3) => {
             for (let attempt = 0; attempt < retries; attempt++) {
                 try {
@@ -986,15 +1182,19 @@ class Doc2Vec {
                             'Content-Type': 'application/json',
                         },
                     });
-                    if (response.status === 429) {
-                        const retryAfter = parseInt(response.headers['retry-after'] || '60');
-                        logger.warn(`Rate limited, waiting ${retryAfter}s before retry`);
-                        await new Promise(res => setTimeout(res, retryAfter * 1000));
-                        continue;
-                    }
                     return response.data;
                 }
                 catch (error) {
+                    // 403 is a permissions error — retrying won't help
+                    if (error.response?.status === 403)
+                        throw error;
+                    if (error.response?.status === 429) {
+                        const retryAfter = parseInt(error.response.headers?.['retry-after'] || '60', 10);
+                        logger.warn(`Rate limited by Zendesk, waiting ${retryAfter}s before retry`);
+                        await new Promise(res => setTimeout(res, retryAfter * 1000));
+                        attempt--; // Don't burn a retry on rate-limit waits
+                        continue;
+                    }
                     logger.error(`Zendesk API error (attempt ${attempt + 1}):`, error.message);
                     if (attempt === retries - 1)
                         throw error;
@@ -1038,6 +1238,22 @@ class Doc2Vec {
         const processTicket = async (ticket) => {
             const ticketId = ticket.id;
             const url = `https://${config.zendesk_subdomain}.zendesk.com/agent/tickets/${ticketId}`;
+            // Deleted tickets — remove their chunks and stop
+            if (ticket.status === 'deleted') {
+                logger.info(`Ticket #${ticketId} was deleted in Zendesk — removing its chunks`);
+                if (dbConnection.type === 'sqlite') {
+                    database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                }
+                else {
+                    await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                }
+                return;
+            }
+            // Skip tickets whose status is outside the configured filter
+            if (!statusFilter.has(ticket.status)) {
+                logger.debug(`Ticket #${ticketId} has status '${ticket.status}' outside configured filter — skipping`);
+                return;
+            }
             logger.info(`Processing ticket #${ticketId}`);
             // Fetch ticket comments
             const commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
@@ -1053,86 +1269,49 @@ class Doc2Vec {
             };
             const chunks = await this.contentProcessor.chunkMarkdown(markdown, ticketConfig, url);
             logger.info(`Ticket #${ticketId}: Created ${chunks.length} chunks`);
-            // Process and store each chunk
-            for (const chunk of chunks) {
-                const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                if (dbConnection.type === 'sqlite') {
-                    const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                    const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                    if (existing && existing.hash === chunkHash) {
-                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                        continue;
-                    }
-                    const embeddings = await this.createEmbeddings([chunk.content]);
-                    if (embeddings.length) {
-                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
-                        logger.debug(`Stored chunk ${chunkId} in SQLite`);
-                    }
-                    else {
-                        logger.error(`Embedding failed for chunk: ${chunkId}`);
-                    }
-                }
-                else if (dbConnection.type === 'qdrant') {
-                    try {
-                        let pointId;
-                        try {
-                            pointId = chunk.metadata.chunk_id;
-                            if (!utils_1.Utils.isValidUuid(pointId)) {
-                                pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                            }
-                        }
-                        catch (e) {
-                            pointId = crypto_1.default.randomUUID();
-                        }
-                        const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                            ids: [pointId],
-                            with_payload: true,
-                            with_vector: false,
-                        });
-                        if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                            logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            continue;
-                        }
-                        const embeddings = await this.createEmbeddings([chunk.content]);
-                        if (embeddings.length) {
-                            await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
-                            logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                        }
-                        else {
-                            logger.error(`Embedding failed for chunk: ${chunkId}`);
-                        }
-                    }
-                    catch (error) {
-                        logger.error(`Error processing chunk in Qdrant:`, error);
-                    }
-                }
-            }
+            // Use processChunksForUrl which performs a URL-level diff:
+            // deletes all existing chunks for this URL before reinserting,
+            // so stale chunks from previous versions are never left behind.
+            await this.processChunksForUrl(chunks, url, dbConnection, logger);
         };
         logger.info(`Fetching Zendesk tickets updated since ${lastRunDate}`);
-        // Build query parameters
-        const statusFilter = config.ticket_status || ['new', 'open', 'pending', 'hold', 'solved'];
-        const query = `updated>${lastRunDate.split('T')[0]} status:${statusFilter.join(',status:')}`;
+        // Build query parameters — use the status filter for the search query
+        const statusList = Array.from(statusFilter);
+        const query = `updated>${lastRunDate.split('T')[0]} status:${statusList.join(',status:')}`;
         let nextPage = `${baseUrl}/search.json?query=${encodeURIComponent(query)}&sort_by=updated_at&sort_order=asc`;
         let totalTickets = 0;
+        let failedTickets = 0;
         while (nextPage) {
             const data = await fetchWithRetry(nextPage);
             const tickets = data.results || [];
             logger.info(`Processing batch of ${tickets.length} tickets`);
             for (const ticket of tickets) {
-                await processTicket(ticket);
-                totalTickets++;
+                try {
+                    await processTicket(ticket);
+                    totalTickets++;
+                }
+                catch (error) {
+                    failedTickets++;
+                    logger.error(`Failed to process ticket #${ticket.id}, will retry next run: ${error.message}`);
+                }
             }
-            nextPage = data.next_page;
+            nextPage = data.next_page || null;
             if (nextPage) {
                 logger.debug(`Fetching next page: ${nextPage}`);
                 // Rate limiting: wait between requests
                 await new Promise(res => setTimeout(res, 1000));
             }
         }
-        // Update the last run date in the database
-        await database_1.DatabaseManager.updateLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, logger, this.embeddingDimension);
-        logger.info(`Successfully processed ${totalTickets} tickets`);
+        // Only advance the watermark when all tickets succeeded
+        if (failedTickets === 0) {
+            await database_1.DatabaseManager.updateLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, logger, this.embeddingDimension, syncStartDate);
+            logger.info(`Successfully processed ${totalTickets} tickets`);
+        }
+        else {
+            logger.warn(`Run completed with ${failedTickets} ticket failure(s). ` +
+                `Watermark NOT advanced — failed tickets will be retried next run. ` +
+                `Successfully processed: ${totalTickets}.`);
+        }
     }
     async fetchAndProcessZendeskArticles(config, dbConnection, logger) {
         const baseUrl = `https://${config.zendesk_subdomain}.zendesk.com/api/v2/help_center`;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.6.0",
+    "version": "2.8.0",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",
@@ -29,6 +29,7 @@
     "author": "",
     "license": "ISC",
     "dependencies": {
+        "@aws-sdk/client-s3": "^3.1004.0",
         "@chonkiejs/core": "^0.0.7",
         "@mozilla/readability": "^0.4.4",
         "@qdrant/js-client-rest": "^1.13.0",