@toxplanet/pegasus-sdk 1.1.6 → 1.1.8

package/config/environment.acc.js

@@ -20,7 +20,7 @@ module.exports = {
     }
   },
   indexRoutes: {
-    chemicals: ['*'],
+    chemicals: ['chemicals*'],
     documents: ['documents*'],
     search: [/^(chemicals|substances|search)/]
   }

package/config/environment.dev.js

@@ -19,9 +19,9 @@ module.exports = {
       rejectUnauthorized: false
     }
   },
-indexRoutes: {
-  chemicals: ['chemical_index*', 'synonym_lookup_index'],
-  documents: ['document_nones_index*'],
-  search: [/^(chemical_index|document_nones_index|search)/]
-}
+  indexRoutes: {
+    chemicals: ['chemicals*'],
+    documents: ['documents*'],
+    search: [/^(chemicals|substances|search)/]
+  }
 };

package/config/environment.qa.js

@@ -20,7 +20,7 @@ module.exports = {
     }
   },
   indexRoutes: {
-    chemicals: ['*'],
+    chemicals: ['chemicals*'],
     documents: ['documents*'],
     search: [/^(chemicals|substances|search)/]
   }

package/lib/chemicals.js

@@ -628,121 +628,171 @@ class ChemicalsService {
     }
   }
 
-  registerElasticsearchHandlers(elasticsearchService) {
-    const indexPatterns = this.connection.config.indexRoutes?.chemicals || ['chemicals*'];
-    
-    indexPatterns.forEach(pattern => {
-      elasticsearchService.registerIndexRoute(pattern, {
-        index: async (params) => {
-          const chemical = params.body;
-          const result = await this.createChemical(chemical);
-          
-          return {
-            _index: params.index,
-            _id: result.chemicalId,
-            _version: 1,
-            result: 'created',
-            _source: result
-          };
-        },
+  _buildEsHandlers() {
+    return {
+      index: async (params) => {
+        const chemical = params.body;
+        const result = await this.createChemical(chemical);
         
-        bulk: async (params) => {
-          const operations = params.body || params.operations;
-          const documents = [];
-          
-          for (let i = 0; i < operations.length; i += 2) {
-            const action = operations[i];
-            const document = operations[i + 1];
-            
-            if (action.index || action.create) {
-              documents.push(document);
+        return {
+          _index: params.index,
+          _id: result.chemicalId,
+          _version: 1,
+          result: 'created',
+          _source: result
+        };
+      },
+      
+      bulk: async (params) => {
+        const operations = params.body || params.operations;
+        const cdiDocuments = [];
+        
+        for (let i = 0; i < operations.length; i++) {
+          const op = operations[i];
+          if ((op.index || op.create) &&
+              (op.index?._index === 'chemical_data_index' || op.create?._index === 'chemical_data_index')) {
+            const doc = operations[i + 1];
+            const sourceId = op.index?._id || op.create?._id;
+            if (doc && sourceId) {
+              cdiDocuments.push({
+                source_id: sourceId,
+                chemical_name: doc.chemical_primary_name || (doc.chemical_names && doc.chemical_names[0]) || null,
+                chemical_meta: doc.chemical_meta || {},
+                chemical_identifiers: doc.chemical_identifiers || {},
+                chemical_synonyms: doc.chemical_synonyms || [],
+                chemical_categories: doc.chemical_categories || [],
+                created_at: doc.chemical_created_at,
+                updated_at: doc.chemical_updated_at
+              });
+              i++;
             }
           }
-          
-          const result = await this.bulkIndexFielded(documents);
-          
-          const items = result.results.map((res, idx) => {
-            if (res.success) {
-              return {
-                index: {
-                  _index: params.index || 'chemicals',
-                  _id: documents[idx].source_id || documents[idx]._id,
-                  status: 201,
-                  result: 'created'
-                }
-              };
-            } else {
-              return {
-                index: {
-                  _index: params.index || 'chemicals',
-                  _id: documents[idx].source_id || documents[idx]._id,
-                  status: 400,
-                  error: {
-                    type: 'mapper_parsing_exception',
-                    reason: res.error
-                  }
-                }
-              };
+        }
+        
+        if (cdiDocuments.length === 0) {
+          return { took: 0, errors: false, items: [] };
+        }
+        
+        const result = await this.bulkIndexFielded(cdiDocuments);
+        
+        return {
+          took: 1,
+          errors: result.errors.length > 0,
+          items: result.results.map((res, idx) => ({
+            index: {
+              _index: 'chemical_data_index',
+              _id: cdiDocuments[idx].source_id,
+              status: res.success ? 200 : 400,
+              result: res.success ? 'created' : 'error',
+              ...(res.success ? {} : { error: { type: 'mapper_parsing_exception', reason: res.error } })
             }
-          });
-          
-          return {
-            took: 1,
-            errors: result.errors.length > 0,
-            items
-          };
-        },
-      
-        get: async (params) => {
-          const result = await this.getChemicalById(params.id);
-          
-          if (!result) {
-            return {
-              _index: params.index,
-              _id: params.id,
-              found: false
-            };
-          }
-          
-          return {
-            _index: params.index,
-            _id: result.chemicalId,
-            _version: 1,
-            found: true,
-            _source: result
-          };
-        },
+          }))
+        };
+      },
+    
+      get: async (params) => {
+        const result = await this.getChemicalBySourceId(params.id);
         
-        update: async (params) => {
-          const result = await this.updateChemical(params.id, params.body);
-          
+        if (!result) {
           return {
             _index: params.index,
             _id: params.id,
-            _version: 2,
-            result: result ? 'updated' : 'noop',
-            _source: result
+            found: false
           };
-        },
+        }
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          _version: 1,
+          found: true,
+          _source: result
+        };
+      },
+      
+      update: async (params) => {
+        const result = await this.updateChemical(params.id, params.body);
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          _version: 2,
+          result: result ? 'updated' : 'noop',
+          _source: result
+        };
+      },
+      
+      delete: async (params) => {
+        if (params.index === 'synonym_lookup_index') {
+          return { _index: params.index, _id: params.id, result: 'not_found' };
+        }
+        const result = await this.deleteBySourceId(params.id);
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          result: result ? 'deleted' : 'not_found'
+        };
+      },
+
+      deleteByQuery: async (params) => {
+        const sourceId = params.body?.query?.term?.chemical_set_identifier
+                      || params.body?.query?.term?.source_id;
+        if (!sourceId) {
+          return { deleted: 0, failures: [] };
+        }
+        const result = await this.deleteBySourceId(sourceId);
+        return {
+          deleted: result ? 1 : 0,
+          failures: []
+        };
+      },
+      
+      search: async (params) => {
+        let searchTerm = '';
+        let limit = params.body?.size || 10;
         
-        delete: async (params) => {
-          const result = await this.deleteChemical(params.id);
+        if (params.index === 'synonym_lookup_index') {
+          const query = params.body?.query;
+          searchTerm = query?.match?.chemical_name || 
+                       query?.term?.chemical_name || 
+                       query?.query_string?.query || '';
+          const searchResults = await this.searchBySynonym(searchTerm, limit);
           
           return {
-            _index: params.index,
-            _id: params.id,
-            _version: 1,
-            result: result ? 'deleted' : 'not_found'
+            took: 1,
+            timed_out: false,
+            _shards: {
+              total: 1,
+              successful: 1,
+              skipped: 0,
+              failed: 0
+            },
+            hits: {
+              total: {
+                value: searchResults.results.length,
+                relation: 'eq'
+              },
+              max_score: searchResults.results[0]?.score || 0,
+              hits: searchResults.results.map(result => ({
+                _index: params.index,
+                _id: result.id,
+                _score: result.score,
+                _source: {
+                  postgres_id: result.id,
+                  chemical_name: result.name,
+                  cas_numbers: result.cas,
+                  identifier_values: result.identifiers,
+                  synonyms: result.synonyms
+                }
+              }))
+            }
           };
-        },
-        
-        search: async (params) => {
+        } else {
           const query = params.body?.query;
-          const searchTerm = query?.match?.chemical_name || 
-                            query?.term?.chemical_name || 
-                            query?.query_string?.query || '';
-          const limit = params.body?.size || 10;
-          
+          searchTerm = query?.match?.chemical_name || 
+                       query?.term?.chemical_name || 
+                       query?.query_string?.query || '';
           const searchResults = await this.searchByName(searchTerm, limit);
           
           return {
@@ -774,12 +824,25 @@ class ChemicalsService {
               }))
             }
           };
-        },
-        
-        count: async (params) => {
-          return await this.countAll();
         }
-      });
+      },
+      
+      count: async (params) => {
+        if (params.index === 'synonym_lookup_index') {
+          return await this.getTotalSynonymCount();
+        }
+        return await this.countAll();
+      }
+    };
+  }
+
+  registerElasticsearchHandlers(elasticsearchService) {
+    const configurablePatterns = this.connection.config.indexRoutes?.chemicals || ['chemicals*'];
+    const legacyPatterns = ['synonym_lookup_index', 'chemical_data_index', 'chemical_converter_index'];
+    const allPatterns = [...new Set([...configurablePatterns, ...legacyPatterns])];
+    const handlers = this._buildEsHandlers();
+    allPatterns.forEach(pattern => {
+      elasticsearchService.registerIndexRoute(pattern, handlers);
     });
   }
 }

package/lib/connection.js

@@ -195,7 +195,11 @@ class PegasusConnection {
 
   async query(sql, params) {
     const pool = this.getPostgresClient();
-    return pool.query(sql, params);
+    const start = Date.now();
+    logInfo('pegasus-sdk', `[SQL] ${sql}${params ? ` -- params: ${JSON.stringify(params)}` : ''}`);
+    const result = await pool.query(sql, params);
+    logInfo('pegasus-sdk', `[SQL] rowCount: ${result.rowCount} duration: ${Date.now() - start}ms`);
+    return result;
   }
 
   async getClient() {

package/lib/db/index.js

@@ -1,11 +1,18 @@
-const { drizzle } = require('drizzle-orm/node-postgres');
-const schema = require('./schema');
-
-function getDrizzle(pgPool) {
-  return drizzle(pgPool, { schema });
-}
-
-module.exports = {
-  getDrizzle,
-  schema
-};
+const { drizzle } = require('drizzle-orm/node-postgres');
+const { logInfo } = require('@toxplanet/tphelper/logging');
+const schema = require('./schema');
+
+const logger = {
+  logQuery(query, params) {
+    logInfo('pegasus-sdk', `[SQL] ${query}${params?.length ? ` -- params: ${JSON.stringify(params)}` : ''}`);
+  }
+};
+
+function getDrizzle(pgPool) {
+  return drizzle(pgPool, { schema, logger });
+}
+
+module.exports = {
+  getDrizzle,
+  schema
+};

package/lib/elasticsearch.js

@@ -69,17 +69,22 @@ class ElasticsearchService {
       throw new Error('Bulk operations must be a non-empty array');
     }
 
-    const firstOp = operations[0];
-    const indexName = firstOp?.index?._index || firstOp?.index?.index || firstOp?.create?._index || firstOp?.create?.index || params.index;
-    
-    if (!indexName) {
-      throw new Error('Could not determine index from bulk operations');
+    let indexName = params.index;
+    let cdiIndexName = null;
+    for (const op of operations) {
+      const name = op?.index?._index   || op?.index?.index   ||
+                   op?.create?._index  || op?.create?.index  ||
+                   op?.delete?._index  || op?.delete?.index  ||
+                   op?.update?._index  || op?.update?.index;
+      if (!name) continue;
+      if (name === 'chemical_data_index') { cdiIndexName = name; break; }
+      if (!indexName) indexName = name;
     }
+    indexName = cdiIndexName || indexName;
+    if (!indexName) throw new Error('Could not determine index from bulk operations');
 
     const handler = this.getRouteHandler(indexName);
-    if (handler && handler.bulk) {
-      return await handler.bulk(params);
-    }
+    if (handler && handler.bulk) return await handler.bulk(params);
     throw new Error(`No handler registered for index: ${indexName}`);
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toxplanet/pegasus-sdk",
-  "version": "1.1.6",
+  "version": "1.1.8",
   "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
   "main": "index.js",
   "type": "commonjs",