@toxplanet/pegasus-sdk 1.1.12 → 1.1.14

package/config/environment.acc.js

@@ -2,6 +2,7 @@ module.exports = {
   environment: 'acc',
   region: 'us-east-1',
   awsAccountId: '605134466764',
+  sourceService: 'pegasus-sdk',
   secretName: 'arn:aws:secretsmanager:us-east-1:605134466764:secret:rds!cluster-9b502dde-5e2a-49db-b2c5-9801141ee40b-gkHbLm',
   openSearchEndpoint: 'https://1pbu0yqr197lq07hfcjh.us-east-1.aoss.amazonaws.com',
   openSearchIndex: 'chemicals',
@@ -20,7 +21,6 @@ module.exports = {
       rejectUnauthorized: false
     }
   },
-  bulkFailureQueueName: 'pegasus-bulk-failures-acc',
   indexRoutes: {
     chemicals: ['chemicals*'],
     documents: ['documents*'],

package/config/environment.dev.js

@@ -2,6 +2,7 @@ module.exports = {
   environment: 'dev',
   region: 'us-east-1',
   awsAccountId: '292931567094',
+  sourceService: 'pegasus-sdk',
   secretName: 'arn:aws:secretsmanager:us-east-1:292931567094:secret:rds!cluster-b851c3ce-58cc-41cd-aeae-05cc7f5e031a-ZYSjiI',
   openSearchEndpoint: 'https://war8lk73nzswquk8dcz1.us-east-1.aoss.amazonaws.com',
   openSearchIndex: 'chemicals',
@@ -20,7 +21,6 @@ module.exports = {
       rejectUnauthorized: false
     }
   },
-  bulkFailureQueueName: 'pegasus-bulk-failures-dev',
   indexRoutes: {
     chemicals: ['chemicals*'],
     documents: ['documents*'],

package/config/environment.prod.js

@@ -2,6 +2,7 @@ module.exports = {
   environment: 'prod',
   region: 'us-east-1',
   awsAccountId: '147997144422',
+  sourceService: 'pegasus-sdk',
   secretName: 'rds!cluster-25483b3f-3758-43ed-9548-26c91de16c2d',
   openSearchEndpoint: 'https://odusb11s00j5hyy5r6.us-east-1.aoss.amazonaws.com',
   openSearchIndex: 'chemicals',
@@ -20,7 +21,6 @@ module.exports = {
       rejectUnauthorized: true
     }
   },
-  bulkFailureQueueName: 'pegasus-bulk-failures-prod',
   indexRoutes: {
     chemicals: ['chemicals*'],
     documents: ['documents*'],

package/config/environment.qa.js

@@ -2,6 +2,7 @@ module.exports = {
   environment: 'qa',
   region: 'us-east-1',
   awsAccountId: '147997144422',
+  sourceService: 'pegasus-sdk',
   secretName: 'arn:aws:secretsmanager:us-east-1:147997144422:secret:rds!cluster-25483b3f-3758-43ed-9548-26c91de16c2d-oYjysU',
   openSearchEndpoint: 'https://odusb11s00j5hyy5r6.us-east-1.aoss.amazonaws.com',
   openSearchIndex: 'chemicals',
@@ -20,7 +21,6 @@ module.exports = {
       rejectUnauthorized: true
     }
   },
-  bulkFailureQueueName: 'pegasus-bulk-failures-qa',
   indexRoutes: {
     chemicals: ['chemicals*'],
     documents: ['documents*'],

package/lib/chemicals.js

@@ -28,50 +28,81 @@ class ChemicalsService {
     return this.db;
   }
 
-  async sendToRetryQueue(operation, error) {
+  async sendSqlWriteFailure({ sql, parameters, error, retryCount, failedAt }) {
     try {
-      const queueName = this.connection.config.bulkFailureQueueName || 'pegasus-bulk-failures';
-      const queueUrl = this.connection.config.bulkFailureQueueUrl || 
-        `https://sqs.${this.connection.region}.amazonaws.com/${this.connection.config.awsAccountId}/${queueName}`;
-      
+      const region = process.env.AWS_REGION || this.connection.region;
+      const { awsAccountId, environment } = this.connection.config;
+      const defaultQueueUrl = awsAccountId
+        ? `https://sqs.${region}.amazonaws.com/${awsAccountId}/cr-pegasus-failed-items-${environment}`
+        : null;
+      const queueUrl = process.env.SQS_FAILED_ITEMS_QUEUE || defaultQueueUrl;
+
+      if (!queueUrl) {
+        logError('pegasus-sdk', 'sendSqlWriteFailure', 'No SQS queue URL available: set SQS_FAILED_ITEMS_QUEUE or provide awsAccountId in config');
+        return false;
+      }
+
+      logInfo('pegasus-sdk', `[sendSqlWriteFailure] Using queue: ${queueUrl}${process.env.SQS_FAILED_ITEMS_QUEUE ? ' (from env)' : ' (default)'}`);
+
       if (!this.sqsClient) {
-        this.sqsClient = new SQSClient({ region: this.connection.region });
+        this.sqsClient = new SQSClient({ region });
       }
 
       const message = {
-        operation_type: operation.type || 'bulk',
-        timestamp: new Date().toISOString(),
-        environment: this.connection.environment,
-        error_message: error.message,
-        error_type: error.name,
-        bulk_params: operation.params,
-        cdi_documents: operation.cdiDocuments
+        MessageType: 'SqlWriteFailure',
+        SourceService: this.connection.config.sourceService || 'pegasus-sdk',
+        Timestamp: (failedAt || new Date()).toISOString(),
+        Sql: sql,
+        Parameters: parameters,
+        OriginalError: error.message,
+        RetryCount: retryCount
       };
 
       const command = new SendMessageCommand({
         QueueUrl: queueUrl,
-        MessageBody: JSON.stringify(message),
-        MessageAttributes: {
-          Operation: { StringValue: operation.type || 'bulk', DataType: 'String' },
-          Environment: { StringValue: this.connection.environment, DataType: 'String' },
-          ErrorType: { StringValue: error.name || 'Unknown', DataType: 'String' },
-          Timestamp: { StringValue: new Date().toISOString(), DataType: 'String' }
-        }
+        MessageBody: JSON.stringify(message)
       });
 
-      await this.sqsClient.send(command);
-      logInfo('pegasus-sdk', `[sendToRetryQueue] Successfully sent bulk operation to SQS for retry: ${queueName}`);
+      const response = await this.sqsClient.send(command);
+      logInfo('pegasus-sdk', `[sendSqlWriteFailure] SqlWriteFailure posted to SQS: MessageId=${response.MessageId}, RetryCount=${retryCount}`);
       return true;
     } catch (sqsError) {
-      logError('pegasus-sdk', 'sendToRetryQueue', 'Failed to send to SQS retry queue', sqsError);
+      logError('pegasus-sdk', 'sendSqlWriteFailure', 'Failed to post SqlWriteFailure to SQS', sqsError);
       return false;
     }
   }
 
+  _buildChemicalUpsertSql(chemical) {
+    const sql = [
+      'INSERT INTO chemicals (source_id, chemical_name, chemical_meta, chemical_identifiers, chemical_synonyms, chemical_categories, created_at, updated_at)',
+      'VALUES (@source_id, @chemical_name, @chemical_meta, @chemical_identifiers, @chemical_synonyms, @chemical_categories, @created_at, @updated_at)',
+      'ON CONFLICT (source_id) DO UPDATE SET',
+      '  chemical_name = @chemical_name,',
+      '  chemical_meta = @chemical_meta,',
+      '  chemical_identifiers = @chemical_identifiers,',
+      '  chemical_synonyms = @chemical_synonyms,',
+      '  chemical_categories = @chemical_categories,',
+      '  updated_at = @updated_at'
+    ].join('\n');
+
+    const parameters = {
+      '@source_id': chemical.sourceId,
+      '@chemical_name': chemical.chemicalName,
+      '@chemical_meta': chemical.chemicalMeta,
+      '@chemical_identifiers': chemical.chemicalIdentifiers,
+      '@chemical_synonyms': chemical.chemicalSynonyms,
+      '@chemical_categories': chemical.chemicalCategories,
+      '@created_at': chemical.createdAt instanceof Date ? chemical.createdAt.toISOString() : chemical.createdAt,
+      '@updated_at': chemical.updatedAt instanceof Date ? chemical.updatedAt.toISOString() : chemical.updatedAt
+    };
+
+    return { sql, parameters };
+  }
+
   async bulkIndexFielded(documents) {
     try {
       logInfo('pegasus-sdk', `[bulkIndexFielded] Starting bulk index with ${documents?.length || 0} documents`);
-      
+
       if (!documents || documents.length === 0) {
         logInfo('pegasus-sdk', `[bulkIndexFielded] No documents provided, returning empty result`);
         return { indexed: 0, errors: [], results: [] };
@@ -86,32 +117,37 @@ class ChemicalsService {
       for (let i = 0; i < documents.length; i++) {
         const doc = documents[i];
         logInfo('pegasus-sdk', `[bulkIndexFielded] Processing document ${i}: source_id=${doc.source_id}, chemical_name=${doc.chemical_name}`);
-        
-        try {
-          const parseDate = (dateValue) => {
-            if (!dateValue) return new Date();
-            if (dateValue instanceof Date) return dateValue;
-            if (typeof dateValue === 'string') return new Date(dateValue);
-            return new Date();
-          };
 
-          const chemical = {
-            sourceId: doc.source_id || doc._id,
-            chemicalName: doc.chemical_name || doc.name,
-            chemicalMeta: doc.chemical_meta || {},
-            chemicalIdentifiers: doc.chemical_identifiers || {},
-            chemicalSynonyms: doc.chemical_synonyms || [],
-            chemicalCategories: doc.chemical_categories || [],
-            createdAt: parseDate(doc.created_at),
-            updatedAt: parseDate(doc.updated_at),
-            ...(doc.imported_at && { importedAt: doc.imported_at }),
-            ...(doc.chemical_id && { chemicalId: doc.chemical_id })
-          };
+        const parseDate = (dateValue) => {
+          if (!dateValue) return new Date();
+          if (dateValue instanceof Date) return dateValue;
+          if (typeof dateValue === 'string') return new Date(dateValue);
+          return new Date();
+        };
+
+        const chemical = {
+          sourceId: doc.source_id || doc._id,
+          chemicalName: doc.chemical_name || doc.name,
+          chemicalMeta: doc.chemical_meta || {},
+          chemicalIdentifiers: doc.chemical_identifiers || {},
+          chemicalSynonyms: doc.chemical_synonyms || [],
+          chemicalCategories: doc.chemical_categories || [],
+          createdAt: parseDate(doc.created_at),
+          updatedAt: parseDate(doc.updated_at),
+          ...(doc.imported_at && { importedAt: doc.imported_at }),
+          ...(doc.chemical_id && { chemicalId: doc.chemical_id })
+        };
 
-          logInfo('pegasus-sdk', `[bulkIndexFielded] Prepared chemical object: sourceId=${chemical.sourceId}, chemicalName=${chemical.chemicalName}`);
+        logInfo('pegasus-sdk', `[bulkIndexFielded] Prepared chemical object: sourceId=${chemical.sourceId}, chemicalName=${chemical.chemicalName}`);
 
-          const [result] = await db
-            .insert(schema.chemicals)
+        const isConnectionError = (err) =>
+          err.message?.toLowerCase().includes('timeout') ||
+          err.message?.toLowerCase().includes('connection') ||
+          err.code === 'ECONNREFUSED' ||
+          err.code === 'ETIMEDOUT';
+
+        const attemptUpsert = () =>
+          db.insert(schema.chemicals)
             .values(chemical)
             .onConflictDoUpdate({
               target: schema.chemicals.sourceId,
@@ -129,18 +165,56 @@ class ChemicalsService {
               sourceId: schema.chemicals.sourceId
             });
 
+        let lastError = null;
+        let retryCount = 0;
+        const failedAt = new Date();
+
+        try {
+          const [result] = await attemptUpsert();
           logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully: ${result?.chemicalId || 'no ID returned'}`);
           results.push({ index: i, success: true, result });
-        } catch (err) {
-          logError('pegasus-sdk', 'bulkIndexFielded', `Error indexing document ${i}`, err);
-          results.push({ index: i, success: false, error: err.message });
-          errors.push({ document: doc, error: err.message });
+          continue;
+        } catch (firstErr) {
+          lastError = firstErr;
+          if (!isConnectionError(firstErr)) {
+            logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} first attempt failed (${firstErr.message}), retrying once`);
+            try {
+              const [result] = await attemptUpsert();
+              logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully on retry: ${result?.chemicalId || 'no ID returned'}`);
+              results.push({ index: i, success: true, result });
+              continue;
+            } catch (retryErr) {
+              lastError = retryErr;
+              retryCount = 1;
+            }
+          }
+        }
+
+        logError('pegasus-sdk', 'bulkIndexFielded', `Document ${i} failed after ${retryCount} local retries (source_id=${chemical.sourceId})`, lastError);
+
+        const { sql: failureSql, parameters: failureParams } = this._buildChemicalUpsertSql(chemical);
+        const queued = await this.sendSqlWriteFailure({
+          sql: failureSql,
+          parameters: failureParams,
+          error: lastError,
+          retryCount,
+          failedAt
+        });
+
+        if (queued) {
+          logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} (source_id=${chemical.sourceId}) queued for repair via SQS`);
+        } else {
+          logError('pegasus-sdk', 'bulkIndexFielded', `Document ${i} (source_id=${chemical.sourceId}) failed and could not be queued — data loss risk`, lastError);
         }
+
+        results.push({ index: i, success: false, error: lastError.message, queued });
+        errors.push({ document: doc, error: lastError.message, queued });
       }
 
       const successCount = results.filter(r => r.success).length;
-      logInfo('pegasus-sdk', `[bulkIndexFielded] Bulk index complete: ${successCount}/${documents.length} succeeded, ${errors.length} errors`);
-      
+      const queuedCount = results.filter(r => !r.success && r.queued).length;
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Bulk index complete: ${successCount}/${documents.length} succeeded, ${queuedCount} queued for repair, ${errors.length - queuedCount} unhandled errors`);
+
       return { indexed: successCount, errors, results };
     } catch (error) {
       logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFielded', error);
@@ -789,43 +863,6 @@ class ChemicalsService {
           };
         } catch (error) {
           logError('pegasus-sdk', 'ChemicalsService.bulk', 'Fatal error during bulk indexing', error);
-          
-          const isTimeoutError = error.message && (
-            error.message.includes('timeout') || 
-            error.message.includes('Connection terminated') ||
-            error.code === 'ECONNREFUSED' ||
-            error.code === 'ETIMEDOUT'
-          );
-          
-          if (isTimeoutError || error.message?.includes('Connection')) {
-            logInfo('pegasus-sdk', `[ChemicalsService.bulk] Database timeout detected, attempting to queue for retry`);
-            
-            const queueSuccess = await this.sendToRetryQueue({
-              type: 'bulk',
-              params,
-              cdiDocuments
-            }, error);
-            
-            if (queueSuccess) {
-              logInfo('pegasus-sdk', `[ChemicalsService.bulk] Successfully queued ${cdiDocuments.length} CDI documents for retry in SQS`);
-              return {
-                took: 1,
-                errors: false,
-                items: cdiDocuments.map(doc => ({
-                  index: {
-                    _index: 'chemical_data_index',
-                    _id: doc.source_id,
-                    status: 202,
-                    result: 'accepted_for_retry',
-                    _notes: 'Operation queued for retry due to database timeout'
-                  }
-                }))
-              };
-            } else {
-              logError('pegasus-sdk', 'ChemicalsService.bulk', 'Failed to queue operation for retry', error);
-            }
-          }
-          
           throw error;
         }
       },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toxplanet/pegasus-sdk",
-  "version": "1.1.12",
+  "version": "1.1.14",
   "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
   "main": "index.js",
   "type": "commonjs",