@toxplanet/pegasus-sdk 1.1.7 → 1.1.9

package/config/environment.acc.js

@@ -20,7 +20,7 @@ module.exports = {
     }
   },
   indexRoutes: {
-    chemicals: ['*'],
+    chemicals: ['chemicals*'],
     documents: ['documents*'],
     search: [/^(chemicals|substances|search)/]
   }

package/config/environment.dev.js

@@ -19,9 +19,9 @@ module.exports = {
       rejectUnauthorized: false
     }
   },
-indexRoutes: {
-  chemicals: ['chemical_index*', 'synonym_lookup_index'],
-  documents: ['document_nones_index*'],
-  search: [/^(chemical_index|document_nones_index|search)/]
-}
+  indexRoutes: {
+    chemicals: ['chemicals*'],
+    documents: ['documents*'],
+    search: [/^(chemicals|substances|search)/]
+  }
 };

package/config/environment.qa.js

@@ -20,7 +20,7 @@ module.exports = {
     }
   },
   indexRoutes: {
-    chemicals: ['*'],
+    chemicals: ['chemicals*'],
     documents: ['documents*'],
     search: [/^(chemicals|substances|search)/]
   }

package/lib/chemicals.js

@@ -28,7 +28,10 @@ class ChemicalsService {
 
   async bulkIndexFielded(documents) {
     try {
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Starting bulk index with ${documents?.length || 0} documents`);
+      
       if (!documents || documents.length === 0) {
+        logInfo('pegasus-sdk', `[bulkIndexFielded] No documents provided, returning empty result`);
         return { indexed: 0, errors: [], results: [] };
       }
 
@@ -36,8 +39,12 @@ class ChemicalsService {
       const results = [];
       const errors = [];
 
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Database connection established`);
+
       for (let i = 0; i < documents.length; i++) {
         const doc = documents[i];
+        logInfo('pegasus-sdk', `[bulkIndexFielded] Processing document ${i}: source_id=${doc.source_id}, chemical_name=${doc.chemical_name}`);
+        
         try {
           const chemical = {
             sourceId: doc.source_id || doc._id,
@@ -52,6 +59,8 @@ class ChemicalsService {
             ...(doc.chemical_id && { chemicalId: doc.chemical_id })
           };
 
+          logInfo('pegasus-sdk', `[bulkIndexFielded] Prepared chemical object: sourceId=${chemical.sourceId}, chemicalName=${chemical.chemicalName}`);
+
           const [result] = await db
             .insert(schema.chemicals)
             .values(chemical)
@@ -68,14 +77,19 @@ class ChemicalsService {
             })
             .returning();
 
+          logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully: ${result?.chemicalId || 'no ID returned'}`);
           results.push({ index: i, success: true, result });
         } catch (err) {
+          logError('pegasus-sdk', 'bulkIndexFielded', `Error indexing document ${i}`, err);
           results.push({ index: i, success: false, error: err.message });
           errors.push({ document: doc, error: err.message });
         }
       }
 
-      return { indexed: results.filter(r => r.success).length, errors, results };
+      const successCount = results.filter(r => r.success).length;
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Bulk index complete: ${successCount}/${documents.length} succeeded, ${errors.length} errors`);
+      
+      return { indexed: successCount, errors, results };
     } catch (error) {
       logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFielded', error);
       throw error;
@@ -628,121 +642,209 @@ class ChemicalsService {
     }
   }
 
-  registerElasticsearchHandlers(elasticsearchService) {
-    const indexPatterns = this.connection.config.indexRoutes?.chemicals || ['chemicals*'];
-    
-    indexPatterns.forEach(pattern => {
-      elasticsearchService.registerIndexRoute(pattern, {
-        index: async (params) => {
-          const chemical = params.body;
-          const result = await this.createChemical(chemical);
-          
-          return {
-            _index: params.index,
-            _id: result.chemicalId,
-            _version: 1,
-            result: 'created',
-            _source: result
-          };
-        },
+  _buildEsHandlers() {
+    return {
+      index: async (params) => {
+        const chemical = params.body;
+        const result = await this.createChemical(chemical);
         
-        bulk: async (params) => {
-          const operations = params.body || params.operations;
-          const documents = [];
-          
-          for (let i = 0; i < operations.length; i += 2) {
-            const action = operations[i];
-            const document = operations[i + 1];
-            
-            if (action.index || action.create) {
-              documents.push(document);
-            }
-          }
+        return {
+          _index: params.index,
+          _id: result.chemicalId,
+          _version: 1,
+          result: 'created',
+          _source: result
+        };
+      },
+      
+      bulk: async (params) => {
+        const { logInfo, logError } = require('@toxplanet/tphelper/logging');
+        const operations = params.body || params.operations;
+        
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Starting bulk operation with ${operations?.length || 0} total operations`);
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Params index: ${params.index}`);
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Operations array type: ${Array.isArray(operations) ? 'array' : typeof operations}`);
+        
+        const cdiDocuments = [];
+        let cdiOpCount = 0;
+        let otherOpCount = 0;
+        
+        for (let i = 0; i < operations.length; i++) {
+          const op = operations[i];
+          const isIndexOp = !!(op.index || op.create);
+          const indexName = op.index?._index || op.create?._index || op.delete?._index || op.update?._index;
           
-          const result = await this.bulkIndexFielded(documents);
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] Op[${i}]: action=${Object.keys(op)[0] || 'unknown'}, index=${indexName}`);
           
-          const items = result.results.map((res, idx) => {
-            if (res.success) {
-              return {
-                index: {
-                  _index: params.index || 'chemicals',
-                  _id: documents[idx].source_id || documents[idx]._id,
-                  status: 201,
-                  result: 'created'
-                }
+          if ((op.index || op.create) &&
+              (op.index?._index === 'chemical_data_index' || op.create?._index === 'chemical_data_index')) {
+            const doc = operations[i + 1];
+            const sourceId = op.index?._id || op.create?._id;
+            
+            logInfo('pegasus-sdk', `[ChemicalsService.bulk] Found CDI entry: sourceId=${sourceId}, hasDoc=${!!doc}`);
+            
+            if (doc && sourceId) {
+              const cdiDoc = {
+                source_id: sourceId,
+                chemical_name: doc.chemical_primary_name || (doc.chemical_names && doc.chemical_names[0]) || null,
+                chemical_meta: doc.chemical_meta || {},
+                chemical_identifiers: doc.chemical_identifiers || {},
+                chemical_synonyms: doc.chemical_synonyms || [],
+                chemical_categories: doc.chemical_categories || [],
+                created_at: doc.chemical_created_at,
+                updated_at: doc.chemical_updated_at
               };
+              cdiDocuments.push(cdiDoc);
+              logInfo('pegasus-sdk', `[ChemicalsService.bulk] Extracted CDI doc: ${JSON.stringify({ source_id: cdiDoc.source_id, chemical_name: cdiDoc.chemical_name })}`);
+              i++;
+              cdiOpCount++;
             } else {
-              return {
-                index: {
-                  _index: params.index || 'chemicals',
-                  _id: documents[idx].source_id || documents[idx]._id,
-                  status: 400,
-                  error: {
-                    type: 'mapper_parsing_exception',
-                    reason: res.error
-                  }
-                }
-              };
+              logInfo('pegasus-sdk', `[ChemicalsService.bulk] CDI entry incomplete: sourceId=${sourceId}, doc=${!!doc}`);
             }
-          });
-          
-          return {
-            took: 1,
-            errors: result.errors.length > 0,
-            items
-          };
-        },
-      
-        get: async (params) => {
-          const result = await this.getChemicalById(params.id);
+          } else {
+            otherOpCount++;
+          }
+        }
+        
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Scan complete: ${cdiOpCount} CDI docs found, ${otherOpCount} other operations skipped`);
+        
+        if (cdiDocuments.length === 0) {
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] No CDI documents to index, returning empty no-op response`);
+          return { took: 0, errors: false, items: [] };
+        }
+        
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Calling bulkIndexFielded with ${cdiDocuments.length} CDI documents`);
+        
+        try {
+          const result = await this.bulkIndexFielded(cdiDocuments);
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] bulkIndexFielded returned: indexed=${result.indexed}, errors=${result.errors.length}`);
           
-          if (!result) {
-            return {
-              _index: params.index,
-              _id: params.id,
-              found: false
-            };
+          if (result.errors.length > 0) {
+            logError('pegasus-sdk', 'ChemicalsService.bulk', 'Errors during bulk indexing', result.errors);
           }
           
           return {
-            _index: params.index,
-            _id: result.chemicalId,
-            _version: 1,
-            found: true,
-            _source: result
+            took: 1,
+            errors: result.errors.length > 0,
+            items: result.results.map((res, idx) => ({
+              index: {
+                _index: 'chemical_data_index',
+                _id: cdiDocuments[idx].source_id,
+                status: res.success ? 200 : 400,
+                result: res.success ? 'created' : 'error',
+                ...(res.success ? {} : { error: { type: 'mapper_parsing_exception', reason: res.error } })
+              }
+            }))
           };
-        },
+        } catch (error) {
+          logError('pegasus-sdk', 'ChemicalsService.bulk', 'Fatal error during bulk indexing', error);
+          throw error;
+        }
+      },
+    
+      get: async (params) => {
+        const result = await this.getChemicalBySourceId(params.id);
         
-        update: async (params) => {
-          const result = await this.updateChemical(params.id, params.body);
-          
+        if (!result) {
           return {
             _index: params.index,
             _id: params.id,
-            _version: 2,
-            result: result ? 'updated' : 'noop',
-            _source: result
+            found: false
           };
-        },
+        }
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          _version: 1,
+          found: true,
+          _source: result
+        };
+      },
+      
+      update: async (params) => {
+        const result = await this.updateChemical(params.id, params.body);
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          _version: 2,
+          result: result ? 'updated' : 'noop',
+          _source: result
+        };
+      },
+      
+      delete: async (params) => {
+        if (params.index === 'synonym_lookup_index') {
+          return { _index: params.index, _id: params.id, result: 'not_found' };
+        }
+        const result = await this.deleteBySourceId(params.id);
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          result: result ? 'deleted' : 'not_found'
+        };
+      },
+
+      deleteByQuery: async (params) => {
+        const sourceId = params.body?.query?.term?.chemical_set_identifier
+                      || params.body?.query?.term?.source_id;
+        if (!sourceId) {
+          return { deleted: 0, failures: [] };
+        }
+        const result = await this.deleteBySourceId(sourceId);
+        return {
+          deleted: result ? 1 : 0,
+          failures: []
+        };
+      },
+      
+      search: async (params) => {
+        let searchTerm = '';
+        let limit = params.body?.size || 10;
         
-        delete: async (params) => {
-          const result = await this.deleteChemical(params.id);
+        if (params.index === 'synonym_lookup_index') {
+          const query = params.body?.query;
+          searchTerm = query?.match?.chemical_name || 
+                       query?.term?.chemical_name || 
+                       query?.query_string?.query || '';
+          const searchResults = await this.searchBySynonym(searchTerm, limit);
           
           return {
-            _index: params.index,
-            _id: params.id,
-            _version: 1,
-            result: result ? 'deleted' : 'not_found'
+            took: 1,
+            timed_out: false,
+            _shards: {
+              total: 1,
+              successful: 1,
+              skipped: 0,
+              failed: 0
+            },
+            hits: {
+              total: {
+                value: searchResults.results.length,
+                relation: 'eq'
+              },
+              max_score: searchResults.results[0]?.score || 0,
+              hits: searchResults.results.map(result => ({
+                _index: params.index,
+                _id: result.id,
+                _score: result.score,
+                _source: {
+                  postgres_id: result.id,
+                  chemical_name: result.name,
+                  cas_numbers: result.cas,
+                  identifier_values: result.identifiers,
+                  synonyms: result.synonyms
+                }
+              }))
+            }
           };
-        },
-        
-        search: async (params) => {
+        } else {
           const query = params.body?.query;
-          const searchTerm = query?.match?.chemical_name || 
-                            query?.term?.chemical_name || 
-                            query?.query_string?.query || '';
-          const limit = params.body?.size || 10;
-          
+          searchTerm = query?.match?.chemical_name || 
+                       query?.term?.chemical_name || 
+                       query?.query_string?.query || '';
           const searchResults = await this.searchByName(searchTerm, limit);
           
           return {
@@ -774,12 +876,25 @@ class ChemicalsService {
               }))
             }
           };
-        },
-        
-        count: async (params) => {
-          return await this.countAll();
         }
-      });
+      },
+      
+      count: async (params) => {
+        if (params.index === 'synonym_lookup_index') {
+          return await this.getTotalSynonymCount();
+        }
+        return await this.countAll();
+      }
+    };
+  }
+
+  registerElasticsearchHandlers(elasticsearchService) {
+    const configurablePatterns = this.connection.config.indexRoutes?.chemicals || ['chemicals*'];
+    const legacyPatterns = ['synonym_lookup_index', 'chemical_data_index', 'chemical_converter_index'];
+    const allPatterns = [...new Set([...configurablePatterns, ...legacyPatterns])];
+    const handlers = this._buildEsHandlers();
+    allPatterns.forEach(pattern => {
+      elasticsearchService.registerIndexRoute(pattern, handlers);
     });
   }
 }

package/lib/db/index.js

@@ -1,18 +1,18 @@
-const { drizzle } = require('drizzle-orm/node-postgres');
-const { logInfo } = require('@toxplanet/tphelper/logging');
-const schema = require('./schema');
-
-const logger = {
-  logQuery(query, params) {
-    logInfo('pegasus-sdk', `[SQL] ${query}${params?.length ? ` -- params: ${JSON.stringify(params)}` : ''}`);
-  }
-};
-
-function getDrizzle(pgPool) {
-  return drizzle(pgPool, { schema, logger });
-}
-
-module.exports = {
-  getDrizzle,
-  schema
-};
+const { drizzle } = require('drizzle-orm/node-postgres');
+const { logInfo } = require('@toxplanet/tphelper/logging');
+const schema = require('./schema');
+
+const logger = {
+  logQuery(query, params) {
+    logInfo('pegasus-sdk', `[SQL] ${query}${params?.length ? ` -- params: ${JSON.stringify(params)}` : ''}`);
+  }
+};
+
+function getDrizzle(pgPool) {
+  return drizzle(pgPool, { schema, logger });
+}
+
+module.exports = {
+  getDrizzle,
+  schema
+};

package/lib/elasticsearch.js

@@ -64,22 +64,45 @@ class ElasticsearchService {
   }
 
   async bulk(params) {
+    const { logInfo, logError } = require('@toxplanet/tphelper/logging');
     const operations = params.body || params.operations;
+    
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] Starting bulk routing with ${operations?.length || 0} operations`);
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] params.index=${params.index}`);
+    
     if (!Array.isArray(operations) || operations.length === 0) {
       throw new Error('Bulk operations must be a non-empty array');
     }
 
-    const firstOp = operations[0];
-    const indexName = firstOp?.index?._index || firstOp?.index?.index || firstOp?.create?._index || firstOp?.create?.index || params.index;
+    let indexName = params.index;
+    let cdiIndexName = null;
+    const foundIndices = [];
     
-    if (!indexName) {
-      throw new Error('Could not determine index from bulk operations');
+    for (const op of operations) {
+      const name = op?.index?._index   || op?.index?.index   ||
+                   op?.create?._index  || op?.create?.index  ||
+                   op?.delete?._index  || op?.delete?.index  ||
+                   op?.update?._index  || op?.update?.index;
+      if (!name) continue;
+      
+      if (!foundIndices.includes(name)) foundIndices.push(name);
+      
+      if (name === 'chemical_data_index') { cdiIndexName = name; break; }
+      if (!indexName) indexName = name;
     }
+    
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] Found indices in operations: ${foundIndices.join(', ')}`);
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] cdiIndexName=${cdiIndexName}, fallback indexName=${indexName}`);
+    
+    indexName = cdiIndexName || indexName;
+    if (!indexName) throw new Error('Could not determine index from bulk operations');
 
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] Routing to index: ${indexName}`);
+    
     const handler = this.getRouteHandler(indexName);
-    if (handler && handler.bulk) {
-      return await handler.bulk(params);
-    }
+    logInfo('pegasus-sdk', `[ElasticsearchService.bulk] Handler found: ${!!handler}, has bulk method: ${!!(handler && handler.bulk)}`);
+    
+    if (handler && handler.bulk) return await handler.bulk(params);
     throw new Error(`No handler registered for index: ${indexName}`);
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toxplanet/pegasus-sdk",
-  "version": "1.1.7",
+  "version": "1.1.9",
   "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
   "main": "index.js",
   "type": "commonjs",