@toxplanet/pegasus-sdk 1.1.10 → 1.1.12

package/config/environment.acc.js

@@ -1,6 +1,7 @@
 module.exports = {
   environment: 'acc',
   region: 'us-east-1',
+  awsAccountId: '605134466764',
   secretName: 'arn:aws:secretsmanager:us-east-1:605134466764:secret:rds!cluster-9b502dde-5e2a-49db-b2c5-9801141ee40b-gkHbLm',
   openSearchEndpoint: 'https://1pbu0yqr197lq07hfcjh.us-east-1.aoss.amazonaws.com',
   openSearchIndex: 'chemicals',
@@ -19,6 +20,7 @@ module.exports = {
       rejectUnauthorized: false
     }
   },
+  bulkFailureQueueName: 'pegasus-bulk-failures-acc',
   indexRoutes: {
     chemicals: ['chemicals*'],
     documents: ['documents*'],

package/config/environment.dev.js

@@ -1,6 +1,7 @@
 module.exports = {
   environment: 'dev',
   region: 'us-east-1',
+  awsAccountId: '292931567094',
   secretName: 'arn:aws:secretsmanager:us-east-1:292931567094:secret:rds!cluster-b851c3ce-58cc-41cd-aeae-05cc7f5e031a-ZYSjiI',
   openSearchEndpoint: 'https://war8lk73nzswquk8dcz1.us-east-1.aoss.amazonaws.com',
   openSearchIndex: 'chemicals',
@@ -19,6 +20,7 @@ module.exports = {
       rejectUnauthorized: false
     }
   },
+  bulkFailureQueueName: 'pegasus-bulk-failures-dev',
   indexRoutes: {
     chemicals: ['chemicals*'],
     documents: ['documents*'],

package/config/environment.prod.js

@@ -1,6 +1,7 @@
 module.exports = {
   environment: 'prod',
   region: 'us-east-1',
+  awsAccountId: '147997144422',
   secretName: 'rds!cluster-25483b3f-3758-43ed-9548-26c91de16c2d',
   openSearchEndpoint: 'https://odusb11s00j5hyy5r6.us-east-1.aoss.amazonaws.com',
   openSearchIndex: 'chemicals',
@@ -19,6 +20,7 @@ module.exports = {
       rejectUnauthorized: true
     }
   },
+  bulkFailureQueueName: 'pegasus-bulk-failures-prod',
   indexRoutes: {
     chemicals: ['chemicals*'],
     documents: ['documents*'],

package/config/environment.qa.js

@@ -1,6 +1,7 @@
 module.exports = {
   environment: 'qa',
   region: 'us-east-1',
+  awsAccountId: '147997144422',
   secretName: 'arn:aws:secretsmanager:us-east-1:147997144422:secret:rds!cluster-25483b3f-3758-43ed-9548-26c91de16c2d-oYjysU',
   openSearchEndpoint: 'https://odusb11s00j5hyy5r6.us-east-1.aoss.amazonaws.com',
   openSearchIndex: 'chemicals',
@@ -19,6 +20,7 @@ module.exports = {
       rejectUnauthorized: true
     }
   },
+  bulkFailureQueueName: 'pegasus-bulk-failures-qa',
   indexRoutes: {
     chemicals: ['chemicals*'],
     documents: ['documents*'],

package/lib/chemicals.js

@@ -1,6 +1,7 @@
 const { logError, logInfo } = require('@toxplanet/tphelper/logging');
 const { getDrizzle, schema } = require('./db');
 const { eq, sql, and, inArray, arrayContains } = require('drizzle-orm');
+const { SQSClient, SendMessageCommand } = require('@aws-sdk/client-sqs');
 
 const SEARCH_BOOST_EXACT_PRIMARY = 100;
 const SEARCH_BOOST_PREFIX_PRIMARY = 50;
@@ -17,6 +18,7 @@ class ChemicalsService {
   constructor(connection) {
     this.connection = connection;
     this.db = null;
+    this.sqsClient = null;
   }
 
   getDb() {
@@ -26,6 +28,46 @@ class ChemicalsService {
     return this.db;
   }
 
+  async sendToRetryQueue(operation, error) {
+    try {
+      const queueName = this.connection.config.bulkFailureQueueName || 'pegasus-bulk-failures';
+      const queueUrl = this.connection.config.bulkFailureQueueUrl || 
+        `https://sqs.${this.connection.region}.amazonaws.com/${this.connection.config.awsAccountId}/${queueName}`;
+      
+      if (!this.sqsClient) {
+        this.sqsClient = new SQSClient({ region: this.connection.region });
+      }
+
+      const message = {
+        operation_type: operation.type || 'bulk',
+        timestamp: new Date().toISOString(),
+        environment: this.connection.environment,
+        error_message: error.message,
+        error_type: error.name,
+        bulk_params: operation.params,
+        cdi_documents: operation.cdiDocuments
+      };
+
+      const command = new SendMessageCommand({
+        QueueUrl: queueUrl,
+        MessageBody: JSON.stringify(message),
+        MessageAttributes: {
+          Operation: { StringValue: operation.type || 'bulk', DataType: 'String' },
+          Environment: { StringValue: this.connection.environment, DataType: 'String' },
+          ErrorType: { StringValue: error.name || 'Unknown', DataType: 'String' },
+          Timestamp: { StringValue: new Date().toISOString(), DataType: 'String' }
+        }
+      });
+
+      await this.sqsClient.send(command);
+      logInfo('pegasus-sdk', `[sendToRetryQueue] Successfully sent bulk operation to SQS for retry: ${queueName}`);
+      return true;
+    } catch (sqsError) {
+      logError('pegasus-sdk', 'sendToRetryQueue', 'Failed to send to SQS retry queue', sqsError);
+      return false;
+    }
+  }
+
   async bulkIndexFielded(documents) {
     try {
       logInfo('pegasus-sdk', `[bulkIndexFielded] Starting bulk index with ${documents?.length || 0} documents`);
@@ -46,6 +88,13 @@ class ChemicalsService {
         logInfo('pegasus-sdk', `[bulkIndexFielded] Processing document ${i}: source_id=${doc.source_id}, chemical_name=${doc.chemical_name}`);
         
         try {
+          const parseDate = (dateValue) => {
+            if (!dateValue) return new Date();
+            if (dateValue instanceof Date) return dateValue;
+            if (typeof dateValue === 'string') return new Date(dateValue);
+            return new Date();
+          };
+
           const chemical = {
             sourceId: doc.source_id || doc._id,
             chemicalName: doc.chemical_name || doc.name,
@@ -53,8 +102,8 @@ class ChemicalsService {
             chemicalIdentifiers: doc.chemical_identifiers || {},
             chemicalSynonyms: doc.chemical_synonyms || [],
             chemicalCategories: doc.chemical_categories || [],
-            createdAt: doc.created_at || new Date(),
-            updatedAt: doc.updated_at || new Date(),
+            createdAt: parseDate(doc.created_at),
+            updatedAt: parseDate(doc.updated_at),
             ...(doc.imported_at && { importedAt: doc.imported_at }),
             ...(doc.chemical_id && { chemicalId: doc.chemical_id })
           };
@@ -75,7 +124,10 @@ class ChemicalsService {
                 updatedAt: new Date()
               }
             })
-            .returning();
+            .returning({
+              chemicalId: schema.chemicals.chemicalId,
+              sourceId: schema.chemicals.sourceId
+            });
 
           logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully: ${result?.chemicalId || 'no ID returned'}`);
           results.push({ index: i, success: true, result });
@@ -690,8 +742,8 @@ class ChemicalsService {
                 chemical_identifiers: doc.chemical_identifiers || {},
                 chemical_synonyms: doc.chemical_synonyms || [],
                 chemical_categories: doc.chemical_categories || [],
-                created_at: doc.chemical_created_at,
-                updated_at: doc.chemical_updated_at
+                created_at: doc.chemical_created_at ? (typeof doc.chemical_created_at === 'string' ? new Date(doc.chemical_created_at) : doc.chemical_created_at) : new Date(),
+                updated_at: doc.chemical_updated_at ? (typeof doc.chemical_updated_at === 'string' ? new Date(doc.chemical_updated_at) : doc.chemical_updated_at) : new Date()
               };
               cdiDocuments.push(cdiDoc);
               logInfo('pegasus-sdk', `[ChemicalsService.bulk] Extracted CDI doc: ${JSON.stringify({ source_id: cdiDoc.source_id, chemical_name: cdiDoc.chemical_name })}`);
@@ -737,6 +789,43 @@ class ChemicalsService {
           };
         } catch (error) {
           logError('pegasus-sdk', 'ChemicalsService.bulk', 'Fatal error during bulk indexing', error);
+          
+          const isTimeoutError = error.message && (
+            error.message.includes('timeout') || 
+            error.message.includes('Connection terminated') ||
+            error.code === 'ECONNREFUSED' ||
+            error.code === 'ETIMEDOUT'
+          );
+          
+          if (isTimeoutError || error.message?.includes('Connection')) {
+            logInfo('pegasus-sdk', `[ChemicalsService.bulk] Database timeout detected, attempting to queue for retry`);
+            
+            const queueSuccess = await this.sendToRetryQueue({
+              type: 'bulk',
+              params,
+              cdiDocuments
+            }, error);
+            
+            if (queueSuccess) {
+              logInfo('pegasus-sdk', `[ChemicalsService.bulk] Successfully queued ${cdiDocuments.length} CDI documents for retry in SQS`);
+              return {
+                took: 1,
+                errors: false,
+                items: cdiDocuments.map(doc => ({
+                  index: {
+                    _index: 'chemical_data_index',
+                    _id: doc.source_id,
+                    status: 202,
+                    result: 'accepted_for_retry',
+                    _notes: 'Operation queued for retry due to database timeout'
+                  }
+                }))
+              };
+            } else {
+              logError('pegasus-sdk', 'ChemicalsService.bulk', 'Failed to queue operation for retry', error);
+            }
+          }
+          
           throw error;
         }
       },

package/lib/db/schema.js

@@ -1,4 +1,4 @@
-const { pgTable, uuid, text, jsonb, timestamp, index, uniqueIndex } = require('drizzle-orm/pg-core');
+const { pgTable, uuid, text, jsonb, timestamp, index } = require('drizzle-orm/pg-core');
 const { sql } = require('drizzle-orm');
 
 const chemicals = pgTable('chemicals', {
@@ -14,12 +14,11 @@ const chemicals = pgTable('chemicals', {
   importedAt: timestamp('imported_at', { withTimezone: true }).defaultNow()
 }, (table) => {
   return {
-    sourceIdIdx: uniqueIndex('idx_chemicals_source_id').on(table.sourceId),
     nameIdx: index('idx_chemicals_name').on(table.chemicalName),
     createdAtIdx: index('idx_chemicals_created_at').on(table.createdAt),
     updatedAtIdx: index('idx_chemicals_updated_at').on(table.updatedAt),
-    identifiersGinIdx: index('idx_chemicals_identifiers_gin').on(table.chemicalIdentifiers),
-    synonymsGinIdx: index('idx_chemicals_synonyms_gin').on(table.chemicalSynonyms)
+    identifiersGinIdx: index('idx_chemicals_identifiers_gin').using('gin').on(table.chemicalIdentifiers),
+    synonymsGinIdx: index('idx_chemicals_synonyms_gin').using('gin').on(table.chemicalSynonyms)
   };
 });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toxplanet/pegasus-sdk",
-  "version": "1.1.10",
+  "version": "1.1.12",
   "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
   "main": "index.js",
   "type": "commonjs",
@@ -29,6 +29,7 @@
     "drizzle-orm": "^0.30.0",
     "@opensearch-project/opensearch": "^2.5.0",
     "@aws-sdk/client-secrets-manager": "^3.490.0",
+    "@aws-sdk/client-sqs": "^3.490.0",
     "@aws-sdk/credential-providers": "^3.490.0"
   },
   "engines": {