@toxplanet/pegasus-sdk 1.1.1 → 1.1.3

package/config/environment.dev.js

@@ -19,9 +19,9 @@ module.exports = {
       rejectUnauthorized: false
     }
   },
-  indexRoutes: {
-    chemicals: ['chemicals*'],
-    documents: ['documents*'],
-    search: [/^(chemicals|substances|search)/]
-  }
+indexRoutes: {
+  chemicals: ['chemical_index*'],
+  documents: ['document_nones_index*'],
+  search: [/^(chemical_index|document_nones_index|search)/]
+}
 };

package/lib/chemicals.js

@@ -1,6 +1,17 @@
 const { logError } = require('@toxplanet/tphelper/logging');
 const { getDrizzle, schema } = require('./db');
-const { eq } = require('drizzle-orm');
+const { eq, sql, and, inArray, arrayContains } = require('drizzle-orm');
+
+const SEARCH_BOOST_EXACT_PRIMARY = 100;
+const SEARCH_BOOST_PREFIX_PRIMARY = 50;
+const SEARCH_BOOST_EXACT_SECONDARY = 30;
+const SEARCH_BOOST_PREFIX_SECONDARY = 10;
+
+const ALLOWED_IDENTIFIER_TYPES = new Set(['CAS', 'SMILES', 'InChI', 'InChIKey', 'PubChem', 'DTXSID', 'EINECS', 'EC']);
+
+function escapeLikePattern(value) {
+  return value.replace(/[%_\\]/g, '\\$&');
+}
 
 class ChemicalsService {
   constructor(connection) {
@@ -15,11 +26,91 @@ class ChemicalsService {
     return this.db;
   }
 
-  async bulkIndexFielded(documents) {}
+  async bulkIndexFielded(documents) {
+    try {
+      if (!documents || documents.length === 0) {
+        return { indexed: 0, errors: [], results: [] };
+      }
 
-  async bulkIndexFulltext(documents) {}
+      const db = this.getDb();
+      const results = [];
+      const errors = [];
+
+      for (let i = 0; i < documents.length; i++) {
+        const doc = documents[i];
+        try {
+          const chemical = {
+            sourceId: doc.source_id || doc._id,
+            chemicalName: doc.chemical_name || doc.name,
+            chemicalMeta: doc.chemical_meta || {},
+            chemicalIdentifiers: doc.chemical_identifiers || {},
+            chemicalSynonyms: doc.chemical_synonyms || [],
+            chemicalCategories: doc.chemical_categories || [],
+            createdAt: doc.created_at || new Date(),
+            updatedAt: doc.updated_at || new Date(),
+            ...(doc.imported_at && { importedAt: doc.imported_at }),
+            ...(doc.chemical_id && { chemicalId: doc.chemical_id })
+          };
+
+          const [result] = await db
+            .insert(schema.chemicals)
+            .values(chemical)
+            .onConflictDoUpdate({
+              target: schema.chemicals.sourceId,
+              set: {
+                chemicalName: chemical.chemicalName,
+                chemicalMeta: chemical.chemicalMeta,
+                chemicalIdentifiers: chemical.chemicalIdentifiers,
+                chemicalSynonyms: chemical.chemicalSynonyms,
+                chemicalCategories: chemical.chemicalCategories,
+                updatedAt: new Date()
+              }
+            })
+            .returning();
+
+          results.push({ index: i, success: true, result });
+        } catch (err) {
+          results.push({ index: i, success: false, error: err.message });
+          errors.push({ document: doc, error: err.message });
+        }
+      }
 
-  async bulkIndexSubstances(substances) {}
+      return { indexed: results.filter(r => r.success).length, errors, results };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFielded', error);
+      throw error;
+    }
+  }
+
+  async bulkIndexFulltext(documents) {
+    try {
+      return { acknowledged: true, count: documents?.length || 0 };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFulltext', error);
+      throw error;
+    }
+  }
+
+  async bulkIndexSubstances(substances) {
+    try {
+      const documents = substances.map(substance => ({
+        source_id: substance.substance_id || substance.id,
+        chemical_name: substance.name || substance.substance_name,
+        chemical_meta: substance.meta || {},
+        chemical_identifiers: substance.identifiers || {},
+        chemical_synonyms: substance.synonyms || [],
+        chemical_categories: substance.categories || substance.substance_types || [],
+        created_at: substance.created_at,
+        updated_at: substance.updated_at,
+        imported_at: substance.imported_at
+      }));
+
+      return await this.bulkIndexFielded(documents);
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexSubstances', error);
+      throw error;
+    }
+  }
 
   async createChemical(chemical) {
     try {
@@ -48,7 +139,30 @@ class ChemicalsService {
     }
   }
 
-  async updateChemical(chemicalId, updates) {}
+  async updateChemical(chemicalId, updates) {
+    try {
+      const db = this.getDb();
+      
+      const updateData = {};
+      if (updates.chemical_name) updateData.chemicalName = updates.chemical_name;
+      if (updates.chemical_meta) updateData.chemicalMeta = updates.chemical_meta;
+      if (updates.chemical_identifiers) updateData.chemicalIdentifiers = updates.chemical_identifiers;
+      if (updates.chemical_synonyms) updateData.chemicalSynonyms = updates.chemical_synonyms;
+      if (updates.chemical_categories) updateData.chemicalCategories = updates.chemical_categories;
+      updateData.updatedAt = new Date();
+
+      const [result] = await db
+        .update(schema.chemicals)
+        .set(updateData)
+        .where(eq(schema.chemicals.chemicalId, chemicalId))
+        .returning();
+
+      return result || null;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'updateChemical', error);
+      throw error;
+    }
+  }
 
   async deleteChemical(chemicalId) {
     try {
@@ -66,13 +180,92 @@ class ChemicalsService {
     }
   }
 
-  async deleteBySourceId(sourceId) {}
+  async deleteBySourceId(sourceId) {
+    try {
+      const db = this.getDb();
+      
+      const [deleted] = await db
+        .delete(schema.chemicals)
+        .where(eq(schema.chemicals.sourceId, sourceId))
+        .returning();
 
-  async deleteCollection(collectionName) {}
+      return deleted || null;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'deleteBySourceId', error);
+      throw error;
+    }
+  }
 
-  async updateCollectionProperty(collectionName, propertyPath, newValue) {}
+  async deleteCollection(collectionName) {
+    try {
+      const db = this.getDb();
+      
+      const deleted = await db
+        .delete(schema.chemicals)
+        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]))
+        .returning();
 
-  async bulkUpdateProperty(filter, propertyPath, newValue) {}
+      return { deletedCount: deleted.length, deleted };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'deleteCollection', error);
+      throw error;
+    }
+  }
+
+  async updateCollectionProperty(collectionName, propertyPath, newValue) {
+    try {
+      const db = this.getDb();
+      const pathArray = propertyPath.split('.');
+      const valueJson = JSON.stringify(newValue);
+
+      const results = await db
+        .update(schema.chemicals)
+        .set({
+          chemicalMeta: sql`jsonb_set(${schema.chemicals.chemicalMeta}, ${pathArray}::text[], ${valueJson}::jsonb)`,
+          updatedAt: new Date()
+        })
+        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]))
+        .returning();
+
+      return { updatedCount: results.length, updated: results };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'updateCollectionProperty', error);
+      throw error;
+    }
+  }
+
+  async bulkUpdateProperty(filter, propertyPath, newValue) {
+    try {
+      const db = this.getDb();
+      
+      let whereCondition = sql`1=1`;
+      
+      if (filter.chemicalIds && filter.chemicalIds.length > 0) {
+        whereCondition = inArray(schema.chemicals.chemicalId, filter.chemicalIds);
+      } else if (filter.sourceIds && filter.sourceIds.length > 0) {
+        whereCondition = inArray(schema.chemicals.sourceId, filter.sourceIds);
+      } else if (filter.category) {
+        whereCondition = arrayContains(schema.chemicals.chemicalCategories, [filter.category]);
+      }
+
+      const pathArray = propertyPath.split('.');
+      const valueJson = JSON.stringify(newValue);
+
+      const results = await db
+        .update(schema.chemicals)
+        .set({
+          chemicalMeta: sql`jsonb_set(COALESCE(${schema.chemicals.chemicalMeta}, '{}'), ${pathArray}::text[], ${valueJson}::jsonb)`,
+          updatedAt: new Date()
+        })
+        .where(whereCondition)
+        .returning();
+
+      return { updatedCount: results.length, updated: results };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'bulkUpdateProperty', error);
+      throw error;
+    }
+  }
 
   async getChemicalById(chemicalId) {
     try {
@@ -91,25 +284,184 @@ class ChemicalsService {
     }
   }
 
-  async getChemicalBySourceId(sourceId) {}
+  async getChemicalBySourceId(sourceId) {
+    try {
+      const db = this.getDb();
+      
+      const [result] = await db
+        .select()
+        .from(schema.chemicals)
+        .where(eq(schema.chemicals.sourceId, sourceId))
+        .limit(1);
 
-  async getChemicalsByCAS(casNumber) {}
+      return result || null;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalBySourceId', error);
+      throw error;
+    }
+  }
 
-  async getChemicalsByIdentifier(identifierType, identifierValue) {}
+  async getChemicalsByCAS(casNumber) {
+    try {
+      const db = this.getDb();
+      
+      const results = await db
+        .select()
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}->>'CAS' = ${casNumber} OR ${schema.chemicals.chemicalIdentifiers}->'CAS' ? ${casNumber}`);
 
-  async countByCollection(collectionName) {}
+      return results;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalsByCAS', error);
+      throw error;
+    }
+  }
 
-  async countByIdentifier(identifierValue) {}
+  async getChemicalsByIdentifier(identifierType, identifierValue) {
+    try {
+      if (!ALLOWED_IDENTIFIER_TYPES.has(identifierType)) {
+        throw new Error(`Invalid identifier type: ${identifierType}`);
+      }
 
-  async countByCAS(casNumber) {}
+      const db = this.getDb();
 
-  async getTotalSynonymCount() {}
+      const results = await db
+        .select()
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}->>${identifierType} = ${identifierValue} OR ${schema.chemicals.chemicalIdentifiers}->${identifierType} ? ${identifierValue}`);
 
-  async getSynonymCount(synonymTerm) {}
+      return results;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalsByIdentifier', error);
+      throw error;
+    }
+  }
+
+  async countByCollection(collectionName) {
+    try {
+      const db = this.getDb();
+      
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]));
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countByCollection', error);
+      throw error;
+    }
+  }
+
+  async countByIdentifier(identifierValue) {
+    try {
+      const db = this.getDb();
+
+      const searchPattern = `%${escapeLikePattern(identifierValue)}%`;
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}::text LIKE ${searchPattern}`);
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countByIdentifier', error);
+      throw error;
+    }
+  }
+
+  async countByCAS(casNumber) {
+    try {
+      const db = this.getDb();
+      
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}->>'CAS' = ${casNumber} OR ${schema.chemicals.chemicalIdentifiers}->'CAS' ? ${casNumber}`);
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countByCAS', error);
+      throw error;
+    }
+  }
+
+  async getTotalSynonymCount() {
+    try {
+      const db = this.getDb();
+      
+      const result = await db
+        .select({ count: sql`sum(array_length(${schema.chemicals.chemicalSynonyms}, 1))::int` })
+        .from(schema.chemicals);
+
+      return { count: result[0].count || 0 };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getTotalSynonymCount', error);
+      throw error;
+    }
+  }
+
+  async getSynonymCount(synonymTerm) {
+    try {
+      const db = this.getDb();
+      
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(arrayContains(schema.chemicals.chemicalSynonyms, [synonymTerm]));
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getSynonymCount', error);
+      throw error;
+    }
+  }
 
-  async convertIdentifier(fromIdentifier, toIdentifierType) {}
+  async convertIdentifier(fromIdentifier, toIdentifierType) {
+    try {
+      const db = this.getDb();
+
+      const searchPattern = `%${escapeLikePattern(fromIdentifier)}%`;
+      const chemicals = await db
+        .select()
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}::text LIKE ${searchPattern}`);
+
+      if (chemicals.length === 0) {
+        return null;
+      }
+
+      const chemical = chemicals[0];
+      const identifiers = chemical.chemicalIdentifiers || {};
+      const toIdentifier = identifiers[toIdentifierType];
+
+      return {
+        fromIdentifier,
+        toIdentifierType,
+        toIdentifier,
+        chemicalId: chemical.chemicalId,
+        chemicalName: chemical.chemicalName
+      };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'convertIdentifier', error);
+      throw error;
+    }
+  }
 
-  async convertIdentifiersBatch(fromIdentifiers, toIdentifierType) {}
+  async convertIdentifiersBatch(fromIdentifiers, toIdentifierType) {
+    try {
+      const conversions = await Promise.all(
+        fromIdentifiers.map(fromIdentifier => 
+          this.convertIdentifier(fromIdentifier, toIdentifierType)
+        )
+      );
+
+      return conversions.filter(conversion => conversion !== null);
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'convertIdentifiersBatch', error);
+      throw error;
+    }
+  }
 
   /**
    * Search for chemicals by name using OpenSearch
@@ -133,13 +485,10 @@ class ChemicalsService {
           query: {
             bool: {
               should: [
-                // Prioritize exact name matches
-                { term: { 'chemical_name.keyword': { value: searchTerm, boost: 100, case_insensitive: true } } },
-                // Then prefix matches
-                { prefix: { 'chemical_name.keyword': { value: searchTerm, boost: 50, case_insensitive: true } } },
-                // Include synonym matches as secondary
-                { term: { 'synonyms.keyword': { value: searchTerm, boost: 30, case_insensitive: true } } },
-                { prefix: { 'synonyms.keyword': { value: searchTerm, boost: 10, case_insensitive: true } } }
+                { term: { 'chemical_name.keyword': { value: searchTerm, boost: SEARCH_BOOST_EXACT_PRIMARY, case_insensitive: true } } },
+                { prefix: { 'chemical_name.keyword': { value: searchTerm, boost: SEARCH_BOOST_PREFIX_PRIMARY, case_insensitive: true } } },
+                { term: { 'synonyms.keyword': { value: searchTerm, boost: SEARCH_BOOST_EXACT_SECONDARY, case_insensitive: true } } },
+                { prefix: { 'synonyms.keyword': { value: searchTerm, boost: SEARCH_BOOST_PREFIX_SECONDARY, case_insensitive: true } } }
               ],
               minimum_should_match: 1
             }
@@ -187,13 +536,10 @@ class ChemicalsService {
           query: {
             bool: {
               should: [
-                // Prioritize exact synonym matches
-                { term: { 'synonyms.keyword': { value: synonymTerm, boost: 100, case_insensitive: true } } },
-                // Then prefix matches
-                { prefix: { 'synonyms.keyword': { value: synonymTerm, boost: 50, case_insensitive: true } } },
-                // Include name matches as secondary
-                { term: { 'chemical_name.keyword': { value: synonymTerm, boost: 30, case_insensitive: true } } },
-                { prefix: { 'chemical_name.keyword': { value: synonymTerm, boost: 10, case_insensitive: true } } }
+                { term: { 'synonyms.keyword': { value: synonymTerm, boost: SEARCH_BOOST_EXACT_PRIMARY, case_insensitive: true } } },
+                { prefix: { 'synonyms.keyword': { value: synonymTerm, boost: SEARCH_BOOST_PREFIX_PRIMARY, case_insensitive: true } } },
+                { term: { 'chemical_name.keyword': { value: synonymTerm, boost: SEARCH_BOOST_EXACT_SECONDARY, case_insensitive: true } } },
+                { prefix: { 'chemical_name.keyword': { value: synonymTerm, boost: SEARCH_BOOST_PREFIX_SECONDARY, case_insensitive: true } } }
               ],
               minimum_should_match: 1
             }
@@ -222,7 +568,9 @@ class ChemicalsService {
   async countAll() {
     try {
       const db = this.getDb();
-      const result = await db.select({ count: sql`count(*)::int` }).from(schema.chemicals);
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals);
       return { count: result[0].count };
     } catch (error) {
       logError('pegasus-sdk', 'ChemicalsService', 'countAll', error);
@@ -230,9 +578,55 @@ class ChemicalsService {
     }
   }
 
-  async findChemicalsWithoutDocuments(collectionName, searchTerm, pageSize) {}
+  async findChemicalsWithoutDocuments(collectionName, searchTerm, pageSize = 100) {
+    try {
+      const db = this.getDb();
+      
+      let whereConditions = [];
+      
+      if (collectionName) {
+        whereConditions.push(arrayContains(schema.chemicals.chemicalCategories, [collectionName]));
+      }
+      
+      if (searchTerm) {
+        const searchPattern = `%${escapeLikePattern(searchTerm)}%`;
+        whereConditions.push(sql`${schema.chemicals.chemicalName} ILIKE ${searchPattern}`);
+      }
+      
+      const whereClause = whereConditions.length > 0 ? and(...whereConditions) : undefined;
+      
+      const results = await db
+        .select()
+        .from(schema.chemicals)
+        .where(whereClause)
+        .limit(pageSize);
 
-  async countChemicalsWithoutDocuments(collectionName) {}
+      return results;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'findChemicalsWithoutDocuments', error);
+      throw error;
+    }
+  }
+
+  async countChemicalsWithoutDocuments(collectionName) {
+    try {
+      const db = this.getDb();
+      
+      const whereClause = collectionName 
+        ? arrayContains(schema.chemicals.chemicalCategories, [collectionName])
+        : undefined;
+      
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(whereClause);
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countChemicalsWithoutDocuments', error);
+      throw error;
+    }
+  }
 
   registerElasticsearchHandlers(elasticsearchService) {
     const indexPatterns = this.connection.config.indexRoutes?.chemicals || ['chemicals*'];
@@ -240,8 +634,16 @@ class ChemicalsService {
     indexPatterns.forEach(pattern => {
       elasticsearchService.registerIndexRoute(pattern, {
         index: async (params) => {
-        const chemical = params.body;
-          return await this.createChemical(chemical);
+          const chemical = params.body;
+          const result = await this.createChemical(chemical);
+          
+          return {
+            _index: params.index,
+            _id: result.chemicalId,
+            _version: 1,
+            result: 'created',
+            _source: result
+          };
         },
         
         bulk: async (params) => {
@@ -257,19 +659,81 @@ class ChemicalsService {
             }
           }
           
-          return await this.bulkIndexFielded(documents);
+          const result = await this.bulkIndexFielded(documents);
+          
+          const items = result.results.map((res, idx) => {
+            if (res.success) {
+              return {
+                index: {
+                  _index: params.index || 'chemicals',
+                  _id: documents[idx].source_id || documents[idx]._id,
+                  status: 201,
+                  result: 'created'
+                }
+              };
+            } else {
+              return {
+                index: {
+                  _index: params.index || 'chemicals',
+                  _id: documents[idx].source_id || documents[idx]._id,
+                  status: 400,
+                  error: {
+                    type: 'mapper_parsing_exception',
+                    reason: res.error
+                  }
+                }
+              };
+            }
+          });
+          
+          return {
+            took: 1,
+            errors: result.errors.length > 0,
+            items
+          };
         },
         
         get: async (params) => {
-          return await this.getChemicalById(params.id);
+          const result = await this.getChemicalById(params.id);
+          
+          if (!result) {
+            return {
+              _index: params.index,
+              _id: params.id,
+              found: false
+            };
+          }
+          
+          return {
+            _index: params.index,
+            _id: result.chemicalId,
+            _version: 1,
+            found: true,
+            _source: result
+          };
         },
         
         update: async (params) => {
-          return await this.updateChemical(params.id, params.body);
+          const result = await this.updateChemical(params.id, params.body);
+          
+          return {
+            _index: params.index,
+            _id: params.id,
+            _version: 2,
+            result: result ? 'updated' : 'noop',
+            _source: result
+          };
         },
         
         delete: async (params) => {
-          return await this.deleteChemical(params.id);
+          const result = await this.deleteChemical(params.id);
+          
+          return {
+            _index: params.index,
+            _id: params.id,
+            _version: 1,
+            result: result ? 'deleted' : 'not_found'
+          };
         },
         
         search: async (params) => {
@@ -279,7 +743,37 @@ class ChemicalsService {
                             query?.query_string?.query || '';
           const limit = params.body?.size || 10;
           
-          return await this.searchByName(searchTerm, limit);
+          const searchResults = await this.searchByName(searchTerm, limit);
+          
+          return {
+            took: 1,
+            timed_out: false,
+            _shards: {
+              total: 1,
+              successful: 1,
+              skipped: 0,
+              failed: 0
+            },
+            hits: {
+              total: {
+                value: searchResults.results.length,
+                relation: 'eq'
+              },
+              max_score: searchResults.results[0]?.score || 0,
+              hits: searchResults.results.map(result => ({
+                _index: params.index,
+                _id: result.id,
+                _score: result.score,
+                _source: {
+                  postgres_id: result.id,
+                  chemical_name: result.name,
+                  cas_numbers: result.cas,
+                  identifier_values: result.identifiers,
+                  synonyms: result.synonyms
+                }
+              }))
+            }
+          };
         },
         
         count: async (params) => {
@@ -290,4 +784,4 @@ class ChemicalsService {
   }
 }
 
-module.exports = ChemicalsService;
+module.exports = ChemicalsService;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toxplanet/pegasus-sdk",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
   "main": "index.js",
   "type": "commonjs",