@toxplanet/pegasus-sdk 1.1.8 → 1.1.10

package/lib/chemicals.js

@@ -1,4 +1,4 @@
-const { logError } = require('@toxplanet/tphelper/logging');
+const { logError, logInfo } = require('@toxplanet/tphelper/logging');
 const { getDrizzle, schema } = require('./db');
 const { eq, sql, and, inArray, arrayContains } = require('drizzle-orm');
 
@@ -28,7 +28,10 @@ class ChemicalsService {
 
   async bulkIndexFielded(documents) {
     try {
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Starting bulk index with ${documents?.length || 0} documents`);
+      
       if (!documents || documents.length === 0) {
+        logInfo('pegasus-sdk', `[bulkIndexFielded] No documents provided, returning empty result`);
         return { indexed: 0, errors: [], results: [] };
       }
 
@@ -36,8 +39,12 @@ class ChemicalsService {
       const results = [];
       const errors = [];
 
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Database connection established`);
+
       for (let i = 0; i < documents.length; i++) {
         const doc = documents[i];
+        logInfo('pegasus-sdk', `[bulkIndexFielded] Processing document ${i}: source_id=${doc.source_id}, chemical_name=${doc.chemical_name}`);
+        
         try {
           const chemical = {
             sourceId: doc.source_id || doc._id,
@@ -52,6 +59,8 @@ class ChemicalsService {
             ...(doc.chemical_id && { chemicalId: doc.chemical_id })
           };
 
+          logInfo('pegasus-sdk', `[bulkIndexFielded] Prepared chemical object: sourceId=${chemical.sourceId}, chemicalName=${chemical.chemicalName}`);
+
           const [result] = await db
             .insert(schema.chemicals)
             .values(chemical)
@@ -68,14 +77,19 @@ class ChemicalsService {
             })
             .returning();
 
+          logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully: ${result?.chemicalId || 'no ID returned'}`);
           results.push({ index: i, success: true, result });
         } catch (err) {
+          logError('pegasus-sdk', 'bulkIndexFielded', `Error indexing document ${i}`, err);
           results.push({ index: i, success: false, error: err.message });
           errors.push({ document: doc, error: err.message });
         }
       }
 
-      return { indexed: results.filter(r => r.success).length, errors, results };
+      const successCount = results.filter(r => r.success).length;
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Bulk index complete: ${successCount}/${documents.length} succeeded, ${errors.length} errors`);
+      
+      return { indexed: successCount, errors, results };
     } catch (error) {
       logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFielded', error);
       throw error;
@@ -645,16 +659,31 @@ class ChemicalsService {
       
       bulk: async (params) => {
         const operations = params.body || params.operations;
+        
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Starting bulk operation with ${operations?.length || 0} total operations`);
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Params index: ${params.index}`);
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Operations array type: ${Array.isArray(operations) ? 'array' : typeof operations}`);
+        
         const cdiDocuments = [];
+        let cdiOpCount = 0;
+        let otherOpCount = 0;
         
         for (let i = 0; i < operations.length; i++) {
           const op = operations[i];
+          const isIndexOp = !!(op.index || op.create);
+          const indexName = op.index?._index || op.create?._index || op.delete?._index || op.update?._index;
+          
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] Op[${i}]: action=${Object.keys(op)[0] || 'unknown'}, index=${indexName}`);
+          
           if ((op.index || op.create) &&
               (op.index?._index === 'chemical_data_index' || op.create?._index === 'chemical_data_index')) {
             const doc = operations[i + 1];
             const sourceId = op.index?._id || op.create?._id;
+            
+            logInfo('pegasus-sdk', `[ChemicalsService.bulk] Found CDI entry: sourceId=${sourceId}, hasDoc=${!!doc}`);
+            
             if (doc && sourceId) {
-              cdiDocuments.push({
+              const cdiDoc = {
                 source_id: sourceId,
                 chemical_name: doc.chemical_primary_name || (doc.chemical_names && doc.chemical_names[0]) || null,
                 chemical_meta: doc.chemical_meta || {},
@@ -663,31 +692,53 @@ class ChemicalsService {
                 chemical_categories: doc.chemical_categories || [],
                 created_at: doc.chemical_created_at,
                 updated_at: doc.chemical_updated_at
-              });
+              };
+              cdiDocuments.push(cdiDoc);
+              logInfo('pegasus-sdk', `[ChemicalsService.bulk] Extracted CDI doc: ${JSON.stringify({ source_id: cdiDoc.source_id, chemical_name: cdiDoc.chemical_name })}`);
               i++;
+              cdiOpCount++;
+            } else {
+              logInfo('pegasus-sdk', `[ChemicalsService.bulk] CDI entry incomplete: sourceId=${sourceId}, doc=${!!doc}`);
             }
+          } else {
+            otherOpCount++;
           }
         }
         
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Scan complete: ${cdiOpCount} CDI docs found, ${otherOpCount} other operations skipped`);
+        
         if (cdiDocuments.length === 0) {
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] No CDI documents to index, returning empty no-op response`);
           return { took: 0, errors: false, items: [] };
         }
         
-        const result = await this.bulkIndexFielded(cdiDocuments);
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Calling bulkIndexFielded with ${cdiDocuments.length} CDI documents`);
         
-        return {
-          took: 1,
-          errors: result.errors.length > 0,
-          items: result.results.map((res, idx) => ({
-            index: {
-              _index: 'chemical_data_index',
-              _id: cdiDocuments[idx].source_id,
-              status: res.success ? 200 : 400,
-              result: res.success ? 'created' : 'error',
-              ...(res.success ? {} : { error: { type: 'mapper_parsing_exception', reason: res.error } })
-            }
-          }))
-        };
+        try {
+          const result = await this.bulkIndexFielded(cdiDocuments);
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] bulkIndexFielded returned: indexed=${result.indexed}, errors=${result.errors.length}`);
+          
+          if (result.errors.length > 0) {
+            logError('pegasus-sdk', 'ChemicalsService.bulk', 'Errors during bulk indexing', result.errors);
+          }
+          
+          return {
+            took: 1,
+            errors: result.errors.length > 0,
+            items: result.results.map((res, idx) => ({
+              index: {
+                _index: 'chemical_data_index',
+                _id: cdiDocuments[idx].source_id,
+                status: res.success ? 200 : 400,
+                result: res.success ? 'created' : 'error',
+                ...(res.success ? {} : { error: { type: 'mapper_parsing_exception', reason: res.error } })
+              }
+            }))
+          };
+        } catch (error) {
+          logError('pegasus-sdk', 'ChemicalsService.bulk', 'Fatal error during bulk indexing', error);
+          throw error;
+        }
       },
     
       get: async (params) => {

package/lib/elasticsearch.js

@@ -1,3 +1,5 @@
+const { logInfo, logError } = require('@toxplanet/tphelper/logging');
+
 class ElasticsearchService {
   constructor(connection) {
     this.connection = connection;
@@ -65,25 +67,42 @@ class ElasticsearchService {
 
   async bulk(params) {
     const operations = params.body || params.operations;
+    
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] Starting bulk routing with ${operations?.length || 0} operations`);
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] params.index=${params.index}`);
+    
     if (!Array.isArray(operations) || operations.length === 0) {
       throw new Error('Bulk operations must be a non-empty array');
     }
 
     let indexName = params.index;
     let cdiIndexName = null;
+    const foundIndices = [];
+    
     for (const op of operations) {
       const name = op?.index?._index   || op?.index?.index   ||
                    op?.create?._index  || op?.create?.index  ||
                    op?.delete?._index  || op?.delete?.index  ||
                    op?.update?._index  || op?.update?.index;
       if (!name) continue;
+      
+      if (!foundIndices.includes(name)) foundIndices.push(name);
+      
       if (name === 'chemical_data_index') { cdiIndexName = name; break; }
       if (!indexName) indexName = name;
     }
+    
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] Found indices in operations: ${foundIndices.join(', ')}`);
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] cdiIndexName=${cdiIndexName}, fallback indexName=${indexName}`);
+    
     indexName = cdiIndexName || indexName;
     if (!indexName) throw new Error('Could not determine index from bulk operations');
 
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] Routing to index: ${indexName}`);
+    
     const handler = this.getRouteHandler(indexName);
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] Handler found: ${!!handler}, has bulk method: ${!!(handler && handler.bulk)}`);
+    
     if (handler && handler.bulk) return await handler.bulk(params);
     throw new Error(`No handler registered for index: ${indexName}`);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toxplanet/pegasus-sdk",
-  "version": "1.1.8",
+  "version": "1.1.10",
   "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
   "main": "index.js",
   "type": "commonjs",