doc2vec 2.7.0 → 2.9.0

package/README.md

@@ -229,6 +229,8 @@ Configuration is managed through two files:
         *   `encoding`: (Optional) Text file encoding (defaults to `'utf8'`). Does not apply to binary files (PDF, DOC, DOCX).
         *   `url_rewrite_prefix`: (Optional) URL prefix to rewrite `s3://` URLs (e.g., `'https://docs.example.com'`).
 
+        **S3 user metadata resolution:** The `product_name` and `version` fields support a `metadata(...)` syntax to dynamically resolve values from S3 object user metadata. For example, `product_name: 'metadata(x-amz-meta-product-name)'` will set `product_name` to the value of the `x-amz-meta-product-name` user metadata on each S3 object. If the metadata key doesn't exist on an object, an empty string is used. Literal values (without the `metadata(...)` wrapper) work as before.
+
         Authentication uses the AWS SDK default credential chain: environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`), `~/.aws/credentials`, IAM roles, etc.
 
         Incremental sync tracks object `LastModified` timestamps so only new or updated objects are processed on subsequent runs. Deleted objects are automatically cleaned up.
@@ -365,7 +367,7 @@ Configuration is managed through two files:
       
       # S3 bucket source example
       - type: 's3'
-        product_name: 'my-docs'
+        product_name: 'metadata(x-amz-meta-product-name)'
         version: 'latest'
         bucket: 'my-documentation-bucket'
         prefix: 'docs/v2/'

package/dist/doc2vec.js

@@ -783,7 +783,14 @@ class Doc2Vec {
                 else {
                     fileUrl = `s3://${config.bucket}/${obj.key}`;
                 }
-                const chunks = await this.contentProcessor.chunkMarkdown(content, config, fileUrl);
+                // Resolve metadata(...) references in product_name and version
+                const s3Meta = getResponse.Metadata || {};
+                const resolvedConfig = {
+                    ...config,
+                    product_name: this.resolveS3MetadataValue(config.product_name, s3Meta),
+                    version: this.resolveS3MetadataValue(config.version, s3Meta),
+                };
+                const chunks = await this.contentProcessor.chunkMarkdown(content, resolvedConfig, fileUrl);
                 logger.info(`Created ${chunks.length} chunks for ${obj.key}`);
                 await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
                 processedCount++;
@@ -828,6 +835,21 @@ class Doc2Vec {
         await database_1.DatabaseManager.setMetadataValue(dbConnection, lastSyncKey, `${syncStartTimestamp}`, logger, this.embeddingDimension);
         logger.info(`Finished processing S3 bucket: ${config.bucket}`);
     }
+    /**
+     * Resolves a config value that may use the metadata(...) syntax.
+     * e.g. "metadata(x-amz-meta-product-name)" looks up "product-name" in the S3 object's user metadata.
+     * Returns the original value if no metadata(...) pattern is found.
+     * Returns empty string if the referenced metadata key doesn't exist on the object.
+     */
+    resolveS3MetadataValue(configValue, s3Metadata) {
+        const match = configValue.match(/^metadata\((.+)\)$/);
+        if (!match)
+            return configValue;
+        const metaKey = match[1];
+        // AWS SDK returns user metadata keys without the x-amz-meta- prefix
+        const lookupKey = metaKey.replace(/^x-amz-meta-/, '');
+        return s3Metadata[lookupKey] ?? '';
+    }
     async processCodeSource(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for code source (${config.source})`);
@@ -1170,6 +1192,9 @@ class Doc2Vec {
         // Get the last run date from the database
         const startDate = config.start_date || `${new Date().getFullYear()}-01-01`;
         const lastRunDate = await database_1.DatabaseManager.getLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, `${startDate}T00:00:00Z`, logger);
+        // Status filter applied client-side (includes 'closed' by default so tickets
+        // transitioning to closed get updated rather than left stale)
+        const statusFilter = new Set(config.ticket_status || ['new', 'open', 'pending', 'hold', 'solved', 'closed']);
         const fetchWithRetry = async (url, retries = 3) => {
             for (let attempt = 0; attempt < retries; attempt++) {
                 try {
@@ -1179,15 +1204,19 @@ class Doc2Vec {
                             'Content-Type': 'application/json',
                         },
                     });
-                    if (response.status === 429) {
-                        const retryAfter = parseInt(response.headers['retry-after'] || '60');
-                        logger.warn(`Rate limited, waiting ${retryAfter}s before retry`);
-                        await new Promise(res => setTimeout(res, retryAfter * 1000));
-                        continue;
-                    }
                     return response.data;
                 }
                 catch (error) {
+                    // 403 is a permissions error — retrying won't help
+                    if (error.response?.status === 403)
+                        throw error;
+                    if (error.response?.status === 429) {
+                        const retryAfter = parseInt(error.response.headers?.['retry-after'] || '60', 10);
+                        logger.warn(`Rate limited by Zendesk, waiting ${retryAfter}s before retry`);
+                        await new Promise(res => setTimeout(res, retryAfter * 1000));
+                        attempt--; // Don't burn a retry on rate-limit waits
+                        continue;
+                    }
                     logger.error(`Zendesk API error (attempt ${attempt + 1}):`, error.message);
                     if (attempt === retries - 1)
                         throw error;
@@ -1231,6 +1260,22 @@ class Doc2Vec {
         const processTicket = async (ticket) => {
             const ticketId = ticket.id;
             const url = `https://${config.zendesk_subdomain}.zendesk.com/agent/tickets/${ticketId}`;
+            // Deleted tickets — remove their chunks and stop
+            if (ticket.status === 'deleted') {
+                logger.info(`Ticket #${ticketId} was deleted in Zendesk — removing its chunks`);
+                if (dbConnection.type === 'sqlite') {
+                    database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                }
+                else {
+                    await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                }
+                return;
+            }
+            // Skip tickets whose status is outside the configured filter
+            if (!statusFilter.has(ticket.status)) {
+                logger.debug(`Ticket #${ticketId} has status '${ticket.status}' outside configured filter — skipping`);
+                return;
+            }
             logger.info(`Processing ticket #${ticketId}`);
             // Fetch ticket comments
             const commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
@@ -1246,86 +1291,49 @@ class Doc2Vec {
             };
             const chunks = await this.contentProcessor.chunkMarkdown(markdown, ticketConfig, url);
             logger.info(`Ticket #${ticketId}: Created ${chunks.length} chunks`);
-            // Process and store each chunk
-            for (const chunk of chunks) {
-                const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                if (dbConnection.type === 'sqlite') {
-                    const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                    const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                    if (existing && existing.hash === chunkHash) {
-                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                        continue;
-                    }
-                    const embeddings = await this.createEmbeddings([chunk.content]);
-                    if (embeddings.length) {
-                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
-                        logger.debug(`Stored chunk ${chunkId} in SQLite`);
-                    }
-                    else {
-                        logger.error(`Embedding failed for chunk: ${chunkId}`);
-                    }
-                }
-                else if (dbConnection.type === 'qdrant') {
-                    try {
-                        let pointId;
-                        try {
-                            pointId = chunk.metadata.chunk_id;
-                            if (!utils_1.Utils.isValidUuid(pointId)) {
-                                pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                            }
-                        }
-                        catch (e) {
-                            pointId = crypto_1.default.randomUUID();
-                        }
-                        const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                            ids: [pointId],
-                            with_payload: true,
-                            with_vector: false,
-                        });
-                        if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                            logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            continue;
-                        }
-                        const embeddings = await this.createEmbeddings([chunk.content]);
-                        if (embeddings.length) {
-                            await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
-                            logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                        }
-                        else {
-                            logger.error(`Embedding failed for chunk: ${chunkId}`);
-                        }
-                    }
-                    catch (error) {
-                        logger.error(`Error processing chunk in Qdrant:`, error);
-                    }
-                }
-            }
+            // Use processChunksForUrl which performs a URL-level diff:
+            // deletes all existing chunks for this URL before reinserting,
+            // so stale chunks from previous versions are never left behind.
+            await this.processChunksForUrl(chunks, url, dbConnection, logger);
         };
         logger.info(`Fetching Zendesk tickets updated since ${lastRunDate}`);
-        // Build query parameters
-        const statusFilter = config.ticket_status || ['new', 'open', 'pending', 'hold', 'solved'];
-        const query = `updated>${lastRunDate.split('T')[0]} status:${statusFilter.join(',status:')}`;
+        // Build query parameters — use the status filter for the search query
+        const statusList = Array.from(statusFilter);
+        const query = `updated>${lastRunDate.split('T')[0]} status:${statusList.join(',status:')}`;
         let nextPage = `${baseUrl}/search.json?query=${encodeURIComponent(query)}&sort_by=updated_at&sort_order=asc`;
         let totalTickets = 0;
+        let failedTickets = 0;
         while (nextPage) {
             const data = await fetchWithRetry(nextPage);
             const tickets = data.results || [];
             logger.info(`Processing batch of ${tickets.length} tickets`);
             for (const ticket of tickets) {
-                await processTicket(ticket);
-                totalTickets++;
+                try {
+                    await processTicket(ticket);
+                    totalTickets++;
+                }
+                catch (error) {
+                    failedTickets++;
+                    logger.error(`Failed to process ticket #${ticket.id}, will retry next run: ${error.message}`);
+                }
             }
-            nextPage = data.next_page;
+            nextPage = data.next_page || null;
             if (nextPage) {
                 logger.debug(`Fetching next page: ${nextPage}`);
                 // Rate limiting: wait between requests
                 await new Promise(res => setTimeout(res, 1000));
             }
         }
-        // Update the last run date in the database
-        await database_1.DatabaseManager.updateLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, logger, this.embeddingDimension, syncStartDate);
-        logger.info(`Successfully processed ${totalTickets} tickets`);
+        // Only advance the watermark when all tickets succeeded
+        if (failedTickets === 0) {
+            await database_1.DatabaseManager.updateLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, logger, this.embeddingDimension, syncStartDate);
+            logger.info(`Successfully processed ${totalTickets} tickets`);
+        }
+        else {
+            logger.warn(`Run completed with ${failedTickets} ticket failure(s). ` +
+                `Watermark NOT advanced — failed tickets will be retried next run. ` +
+                `Successfully processed: ${totalTickets}.`);
+        }
     }
     async fetchAndProcessZendeskArticles(config, dbConnection, logger) {
         const baseUrl = `https://${config.zendesk_subdomain}.zendesk.com/api/v2/help_center`;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.7.0",
+    "version": "2.9.0",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",