@toxplanet/pegasus-sdk 1.1.17 → 1.1.18

package/lib/chemicals.js

@@ -1,1062 +1,1088 @@
-const { logError, logInfo } = require('@toxplanet/tphelper/logging');
-const { getDrizzle, schema } = require('./db');
-const { eq, sql, and, inArray, arrayContains } = require('drizzle-orm');
-const { SQSClient, SendMessageCommand } = require('@aws-sdk/client-sqs');
-
-const SEARCH_BOOST_EXACT_PRIMARY = 100;
-const SEARCH_BOOST_PREFIX_PRIMARY = 50;
-const SEARCH_BOOST_EXACT_SECONDARY = 30;
-const SEARCH_BOOST_PREFIX_SECONDARY = 10;
-
-const ALLOWED_IDENTIFIER_TYPES = new Set(['CAS', 'SMILES', 'InChI', 'InChIKey', 'PubChem', 'DTXSID', 'EINECS', 'EC']);
-
-function escapeLikePattern(value) {
-  return value.replace(/[%_\\]/g, '\\$&');
-}
-
-class ChemicalsService {
-  constructor(connection) {
-    this.connection = connection;
-    this.db = null;
-    this.sqsClient = null;
-  }
-
-  getDb() {
-    if (!this.db) {
-      this.db = getDrizzle(this.connection.pgPool);
-    }
-    return this.db;
-  }
-
-  async sendSqlWriteFailure({ sql, parameters, error, retryCount, failedAt }) {
-    try {
-      const region = process.env.AWS_REGION || this.connection.region;
-      const { awsAccountId, environment } = this.connection.config;
-      const defaultQueueUrl = awsAccountId
-        ? `https://sqs.${region}.amazonaws.com/${awsAccountId}/cr-pegasus-failed-items-${environment}`
-        : null;
-      const queueUrl = process.env.SQS_FAILED_ITEMS_QUEUE || defaultQueueUrl;
-
-      if (!queueUrl) {
-        logError('pegasus-sdk', 'sendSqlWriteFailure', 'No SQS queue URL available: set SQS_FAILED_ITEMS_QUEUE or provide awsAccountId in config');
-        return false;
-      }
-
-      logInfo('pegasus-sdk', `[sendSqlWriteFailure] Using queue: ${queueUrl}${process.env.SQS_FAILED_ITEMS_QUEUE ? ' (from env)' : ' (default)'}`);
-
-      if (!this.sqsClient) {
-        this.sqsClient = new SQSClient({ region });
-      }
-
-      const message = {
-        MessageType: 'SqlWriteFailure',
-        SourceService: this.connection.config.sourceService || 'pegasus-sdk',
-        Timestamp: (failedAt || new Date()).toISOString(),
-        Sql: sql,
-        Parameters: parameters,
-        OriginalError: error.message,
-        RetryCount: retryCount
-      };
-
-      const command = new SendMessageCommand({
-        QueueUrl: queueUrl,
-        MessageBody: JSON.stringify(message)
-      });
-
-      const response = await this.sqsClient.send(command);
-      logInfo('pegasus-sdk', `[sendSqlWriteFailure] SqlWriteFailure posted to SQS: MessageId=${response.MessageId}, RetryCount=${retryCount}`);
-      return true;
-    } catch (sqsError) {
-      logError('pegasus-sdk', 'sendSqlWriteFailure', 'Failed to post SqlWriteFailure to SQS', sqsError);
-      return false;
-    }
-  }
-
-  _buildChemicalUpsertSql(chemical) {
-    const sql = [
-      'INSERT INTO chemicals (source_id, chemical_name, chemical_meta, chemical_identifiers, chemical_synonyms, chemical_categories, created_at, updated_at)',
-      'VALUES (@source_id, @chemical_name, @chemical_meta::jsonb, @chemical_identifiers::jsonb, @chemical_synonyms, @chemical_categories, @created_at, @updated_at)',
-      'ON CONFLICT (source_id) DO UPDATE SET',
-      '  chemical_name = @chemical_name,',
-      '  chemical_meta = @chemical_meta::jsonb,',
-      '  chemical_identifiers = @chemical_identifiers::jsonb,',
-      '  chemical_synonyms = @chemical_synonyms,',
-      '  chemical_categories = @chemical_categories,',
-      '  updated_at = @updated_at'
-    ].join('\n');
-
-    const serializeDate = (d) => d instanceof Date ? d.toISOString() : d;
-
-    const parameters = {
-      '@source_id': chemical.sourceId,
-      '@chemical_name': chemical.chemicalName,
-      '@chemical_meta': JSON.stringify(chemical.chemicalMeta ?? {}),
-      '@chemical_identifiers': JSON.stringify(chemical.chemicalIdentifiers ?? {}),
-      '@chemical_synonyms': JSON.stringify(chemical.chemicalSynonyms ?? []),
-      '@chemical_categories': JSON.stringify(chemical.chemicalCategories ?? []),
-      '@created_at': serializeDate(chemical.createdAt),
-      '@updated_at': serializeDate(chemical.updatedAt)
-    };
-
-    return { sql, parameters };
-  }
-
-  _buildDebugSql(chemical) {
-    const esc = (s) => `'${String(s ?? '').replace(/'/g, "''")}'`;
-    const escJson = (v) => `'${JSON.stringify(v ?? {}).replace(/'/g, "''")}'`;
-    const escArr = (arr) => {
-      if (!Array.isArray(arr) || arr.length === 0) return `ARRAY[]::text[]`;
-      return `ARRAY[${arr.map(s => esc(s)).join(', ')}]`;
-    };
-    const escDate = (d) => esc(d instanceof Date ? d.toISOString() : (d ?? new Date().toISOString()));
-
-    return [
-      `INSERT INTO chemicals`,
-      `  (source_id, chemical_name, chemical_meta, chemical_identifiers, chemical_synonyms, chemical_categories, created_at, updated_at)`,
-      `VALUES (`,
-      `  ${esc(chemical.sourceId)},`,
-      `  ${esc(chemical.chemicalName)},`,
-      `  ${escJson(chemical.chemicalMeta)}::jsonb,`,
-      `  ${escJson(chemical.chemicalIdentifiers)}::jsonb,`,
-      `  ${escArr(chemical.chemicalSynonyms)},`,
-      `  ${escArr(chemical.chemicalCategories)},`,
-      `  ${escDate(chemical.createdAt)},`,
-      `  ${escDate(chemical.updatedAt)}`,
-      `)`,
-      `ON CONFLICT (source_id) DO UPDATE SET`,
-      `  chemical_name       = ${esc(chemical.chemicalName)},`,
-      `  chemical_meta       = ${escJson(chemical.chemicalMeta)}::jsonb,`,
-      `  chemical_identifiers = ${escJson(chemical.chemicalIdentifiers)}::jsonb,`,
-      `  chemical_synonyms   = ${escArr(chemical.chemicalSynonyms)},`,
-      `  chemical_categories = ${escArr(chemical.chemicalCategories)},`,
-      `  updated_at          = NOW();`
-    ].join('\n');
-  }
-
-  async bulkIndexFielded(documents) {
-    try {
-      logInfo('pegasus-sdk', `[bulkIndexFielded] Starting bulk index with ${documents?.length || 0} documents`);
-
-      if (!documents || documents.length === 0) {
-        logInfo('pegasus-sdk', `[bulkIndexFielded] No documents provided, returning empty result`);
-        return { indexed: 0, errors: [], results: [] };
-      }
-
-      const db = this.getDb();
-      const results = [];
-      const errors = [];
-
-      logInfo('pegasus-sdk', `[bulkIndexFielded] Database connection established`);
-
-      for (let i = 0; i < documents.length; i++) {
-        const doc = documents[i];
-        logInfo('pegasus-sdk', `[bulkIndexFielded] Processing document ${i}: source_id=${doc.source_id}, chemical_name=${doc.chemical_name}`);
-
-        const parseDate = (dateValue) => {
-          if (!dateValue) return new Date();
-          if (dateValue instanceof Date) return dateValue;
-          if (typeof dateValue === 'string') return new Date(dateValue);
-          return new Date();
-        };
-
-        const chemical = {
-          sourceId: doc.source_id || doc._id,
-          chemicalName: doc.chemical_name || doc.name,
-          chemicalMeta: doc.chemical_meta || {},
-          chemicalIdentifiers: doc.chemical_identifiers || {},
-          chemicalSynonyms: doc.chemical_synonyms || [],
-          chemicalCategories: doc.chemical_categories || [],
-          createdAt: parseDate(doc.created_at),
-          updatedAt: parseDate(doc.updated_at),
-          ...(doc.imported_at && { importedAt: doc.imported_at }),
-          ...(doc.chemical_id && { chemicalId: doc.chemical_id })
-        };
-
-        logInfo('pegasus-sdk', `[bulkIndexFielded] Prepared chemical object: sourceId=${chemical.sourceId}, chemicalName=${chemical.chemicalName}`);
-        logInfo('pegasus-sdk', `[bulkIndexFielded] DEBUG SQL for document ${i}:\n${this._buildDebugSql(chemical)}`);
-
-        const isConnectionError = (err) =>
-          err.message?.toLowerCase().includes('timeout') ||
-          err.message?.toLowerCase().includes('connection') ||
-          err.code === 'ECONNREFUSED' ||
-          err.code === 'ETIMEDOUT';
-
-        const attemptUpsert = () =>
-          db.insert(schema.chemicals)
-            .values(chemical)
-            .onConflictDoUpdate({
-              target: schema.chemicals.sourceId,
-              set: {
-                chemicalName: chemical.chemicalName,
-                chemicalMeta: chemical.chemicalMeta,
-                chemicalIdentifiers: chemical.chemicalIdentifiers,
-                chemicalSynonyms: chemical.chemicalSynonyms,
-                chemicalCategories: chemical.chemicalCategories,
-                updatedAt: new Date()
-              }
-            })
-            .returning({
-              chemicalId: schema.chemicals.chemicalId,
-              sourceId: schema.chemicals.sourceId
-            });
-
-        let lastError = null;
-        let retryCount = 0;
-        const failedAt = new Date();
-
-        try {
-          const [result] = await attemptUpsert();
-          logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully: ${result?.chemicalId || 'no ID returned'}`);
-          results.push({ index: i, success: true, result });
-          continue;
-        } catch (firstErr) {
-          lastError = firstErr;
-          if (!isConnectionError(firstErr)) {
-            logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} first attempt failed (${firstErr.message}), retrying once`);
-            try {
-              const [result] = await attemptUpsert();
-              logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully on retry: ${result?.chemicalId || 'no ID returned'}`);
-              results.push({ index: i, success: true, result });
-              continue;
-            } catch (retryErr) {
-              lastError = retryErr;
-              retryCount = 1;
-            }
-          }
-        }
-
-        logError('pegasus-sdk', 'bulkIndexFielded', `Document ${i} failed after ${retryCount} local retries (source_id=${chemical.sourceId})`, lastError);
-
-        const { sql: failureSql, parameters: failureParams } = this._buildChemicalUpsertSql(chemical);
-        const queued = await this.sendSqlWriteFailure({
-          sql: failureSql,
-          parameters: failureParams,
-          error: lastError,
-          retryCount,
-          failedAt
-        });
-
-        if (queued) {
-          logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} (source_id=${chemical.sourceId}) queued for repair via SQS`);
-        } else {
-          logError('pegasus-sdk', 'bulkIndexFielded', `Document ${i} (source_id=${chemical.sourceId}) failed and could not be queued — data loss risk`, lastError);
-        }
-
-        results.push({ index: i, success: false, error: lastError.message, queued });
-        errors.push({ document: doc, error: lastError.message, queued });
-      }
-
-      const successCount = results.filter(r => r.success).length;
-      const queuedCount = results.filter(r => !r.success && r.queued).length;
-      logInfo('pegasus-sdk', `[bulkIndexFielded] Bulk index complete: ${successCount}/${documents.length} succeeded, ${queuedCount} queued for repair, ${errors.length - queuedCount} unhandled errors`);
-
-      return { indexed: successCount, errors, results };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFielded', error);
-      throw error;
-    }
-  }
-
-  async bulkIndexFulltext(documents) {
-    try {
-      return { acknowledged: true, count: documents?.length || 0 };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFulltext', error);
-      throw error;
-    }
-  }
-
-  async bulkIndexSubstances(substances) {
-    try {
-      const documents = substances.map(substance => ({
-        source_id: substance.substance_id || substance.id,
-        chemical_name: substance.name || substance.substance_name,
-        chemical_meta: substance.meta || {},
-        chemical_identifiers: substance.identifiers || {},
-        chemical_synonyms: substance.synonyms || [],
-        chemical_categories: substance.categories || substance.substance_types || [],
-        created_at: substance.created_at,
-        updated_at: substance.updated_at,
-        imported_at: substance.imported_at
-      }));
-
-      return await this.bulkIndexFielded(documents);
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexSubstances', error);
-      throw error;
-    }
-  }
-
-  async createChemical(chemical) {
-    try {
-      const db = this.getDb();
-      
-      const [result] = await db
-        .insert(schema.chemicals)
-        .values({
-          sourceId: chemical.source_id,
-          chemicalName: chemical.chemical_name,
-          chemicalMeta: chemical.chemical_meta,
-          chemicalIdentifiers: chemical.chemical_identifiers,
-          chemicalSynonyms: chemical.chemical_synonyms,
-          chemicalCategories: chemical.chemical_categories,
-          createdAt: chemical.created_at || new Date(),
-          updatedAt: chemical.updated_at || new Date(),
-          ...(chemical.imported_at && { importedAt: chemical.imported_at }),
-          ...(chemical.chemical_id && { chemicalId: chemical.chemical_id })
-        })
-        .returning();
-
-      return result;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'createChemical', error);
-      throw error;
-    }
-  }
-
-  async updateChemical(chemicalId, updates) {
-    try {
-      const db = this.getDb();
-      
-      const updateData = {};
-      if (updates.chemical_name) updateData.chemicalName = updates.chemical_name;
-      if (updates.chemical_meta) updateData.chemicalMeta = updates.chemical_meta;
-      if (updates.chemical_identifiers) updateData.chemicalIdentifiers = updates.chemical_identifiers;
-      if (updates.chemical_synonyms) updateData.chemicalSynonyms = updates.chemical_synonyms;
-      if (updates.chemical_categories) updateData.chemicalCategories = updates.chemical_categories;
-      updateData.updatedAt = new Date();
-
-      const [result] = await db
-        .update(schema.chemicals)
-        .set(updateData)
-        .where(eq(schema.chemicals.chemicalId, chemicalId))
-        .returning();
-
-      return result || null;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'updateChemical', error);
-      throw error;
-    }
-  }
-
-  async deleteChemical(chemicalId) {
-    try {
-      const db = this.getDb();
-      
-      const [deleted] = await db
-        .delete(schema.chemicals)
-        .where(eq(schema.chemicals.chemicalId, chemicalId))
-        .returning();
-
-      return deleted || null;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'deleteChemical', error);
-      throw error;
-    }
-  }
-
-  async deleteBySourceId(sourceId) {
-    try {
-      const db = this.getDb();
-      
-      const [deleted] = await db
-        .delete(schema.chemicals)
-        .where(eq(schema.chemicals.sourceId, sourceId))
-        .returning();
-
-      return deleted || null;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'deleteBySourceId', error);
-      throw error;
-    }
-  }
-
-  async deleteCollection(collectionName) {
-    try {
-      const db = this.getDb();
-      
-      const deleted = await db
-        .delete(schema.chemicals)
-        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]))
-        .returning();
-
-      return { deletedCount: deleted.length, deleted };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'deleteCollection', error);
-      throw error;
-    }
-  }
-
-  async updateCollectionProperty(collectionName, propertyPath, newValue) {
-    try {
-      const db = this.getDb();
-      const pathArray = propertyPath.split('.');
-      const valueJson = JSON.stringify(newValue);
-
-      const results = await db
-        .update(schema.chemicals)
-        .set({
-          chemicalMeta: sql`jsonb_set(${schema.chemicals.chemicalMeta}, ${pathArray}::text[], ${valueJson}::jsonb)`,
-          updatedAt: new Date()
-        })
-        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]))
-        .returning();
-
-      return { updatedCount: results.length, updated: results };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'updateCollectionProperty', error);
-      throw error;
-    }
-  }
-
-  async bulkUpdateProperty(filter, propertyPath, newValue) {
-    try {
-      const db = this.getDb();
-      
-      let whereCondition = sql`1=1`;
-      
-      if (filter.chemicalIds && filter.chemicalIds.length > 0) {
-        whereCondition = inArray(schema.chemicals.chemicalId, filter.chemicalIds);
-      } else if (filter.sourceIds && filter.sourceIds.length > 0) {
-        whereCondition = inArray(schema.chemicals.sourceId, filter.sourceIds);
-      } else if (filter.category) {
-        whereCondition = arrayContains(schema.chemicals.chemicalCategories, [filter.category]);
-      }
-
-      const pathArray = propertyPath.split('.');
-      const valueJson = JSON.stringify(newValue);
-
-      const results = await db
-        .update(schema.chemicals)
-        .set({
-          chemicalMeta: sql`jsonb_set(COALESCE(${schema.chemicals.chemicalMeta}, '{}'), ${pathArray}::text[], ${valueJson}::jsonb)`,
-          updatedAt: new Date()
-        })
-        .where(whereCondition)
-        .returning();
-
-      return { updatedCount: results.length, updated: results };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'bulkUpdateProperty', error);
-      throw error;
-    }
-  }
-
-  async getChemicalById(chemicalId) {
-    try {
-      const db = this.getDb();
-      
-      const [result] = await db
-        .select()
-        .from(schema.chemicals)
-        .where(eq(schema.chemicals.chemicalId, chemicalId))
-        .limit(1);
-
-      return result || null;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalById', error);
-      throw error;
-    }
-  }
-
-  async getChemicalBySourceId(sourceId) {
-    try {
-      const db = this.getDb();
-      
-      const [result] = await db
-        .select()
-        .from(schema.chemicals)
-        .where(eq(schema.chemicals.sourceId, sourceId))
-        .limit(1);
-
-      return result || null;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalBySourceId', error);
-      throw error;
-    }
-  }
-
-  async getChemicalsByCAS(casNumber) {
-    try {
-      const db = this.getDb();
-      
-      const results = await db
-        .select()
-        .from(schema.chemicals)
-        .where(sql`${schema.chemicals.chemicalIdentifiers}->>'CAS' = ${casNumber} OR ${schema.chemicals.chemicalIdentifiers}->'CAS' ? ${casNumber}`);
-
-      return results;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalsByCAS', error);
-      throw error;
-    }
-  }
-
-  async getChemicalsByIdentifier(identifierType, identifierValue) {
-    try {
-      if (!ALLOWED_IDENTIFIER_TYPES.has(identifierType)) {
-        throw new Error(`Invalid identifier type: ${identifierType}`);
-      }
-
-      const db = this.getDb();
-
-      const results = await db
-        .select()
-        .from(schema.chemicals)
-        .where(sql`${schema.chemicals.chemicalIdentifiers}->>${identifierType} = ${identifierValue} OR ${schema.chemicals.chemicalIdentifiers}->${identifierType} ? ${identifierValue}`);
-
-      return results;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalsByIdentifier', error);
-      throw error;
-    }
-  }
-
-  async countByCollection(collectionName) {
-    try {
-      const db = this.getDb();
-      
-      const result = await db
-        .select({ count: sql`count(*)::int` })
-        .from(schema.chemicals)
-        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]));
-
-      return { count: result[0].count };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'countByCollection', error);
-      throw error;
-    }
-  }
-
-  async countByIdentifier(identifierValue) {
-    try {
-      const db = this.getDb();
-
-      const searchPattern = `%${escapeLikePattern(identifierValue)}%`;
-      const result = await db
-        .select({ count: sql`count(*)::int` })
-        .from(schema.chemicals)
-        .where(sql`${schema.chemicals.chemicalIdentifiers}::text LIKE ${searchPattern}`);
-
-      return { count: result[0].count };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'countByIdentifier', error);
-      throw error;
-    }
-  }
-
-  async countByCAS(casNumber) {
-    try {
-      const db = this.getDb();
-      
-      const result = await db
-        .select({ count: sql`count(*)::int` })
-        .from(schema.chemicals)
-        .where(sql`${schema.chemicals.chemicalIdentifiers}->>'CAS' = ${casNumber} OR ${schema.chemicals.chemicalIdentifiers}->'CAS' ? ${casNumber}`);
-
-      return { count: result[0].count };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'countByCAS', error);
-      throw error;
-    }
-  }
-
-  async getTotalSynonymCount() {
-    try {
-      const db = this.getDb();
-      
-      const result = await db
-        .select({ count: sql`sum(array_length(${schema.chemicals.chemicalSynonyms}, 1))::int` })
-        .from(schema.chemicals);
-
-      return { count: result[0].count || 0 };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'getTotalSynonymCount', error);
-      throw error;
-    }
-  }
-
-  async getSynonymCount(synonymTerm) {
-    try {
-      const db = this.getDb();
-      
-      const result = await db
-        .select({ count: sql`count(*)::int` })
-        .from(schema.chemicals)
-        .where(arrayContains(schema.chemicals.chemicalSynonyms, [synonymTerm]));
-
-      return { count: result[0].count };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'getSynonymCount', error);
-      throw error;
-    }
-  }
-
-  async convertIdentifier(fromIdentifier, toIdentifierType) {
-    try {
-      const db = this.getDb();
-
-      const searchPattern = `%${escapeLikePattern(fromIdentifier)}%`;
-      const chemicals = await db
-        .select()
-        .from(schema.chemicals)
-        .where(sql`${schema.chemicals.chemicalIdentifiers}::text LIKE ${searchPattern}`);
-
-      if (chemicals.length === 0) {
-        return null;
-      }
-
-      const chemical = chemicals[0];
-      const identifiers = chemical.chemicalIdentifiers || {};
-      const toIdentifier = identifiers[toIdentifierType];
-
-      return {
-        fromIdentifier,
-        toIdentifierType,
-        toIdentifier,
-        chemicalId: chemical.chemicalId,
-        chemicalName: chemical.chemicalName
-      };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'convertIdentifier', error);
-      throw error;
-    }
-  }
-
-  async convertIdentifiersBatch(fromIdentifiers, toIdentifierType) {
-    try {
-      const conversions = await Promise.all(
-        fromIdentifiers.map(fromIdentifier => 
-          this.convertIdentifier(fromIdentifier, toIdentifierType)
-        )
-      );
-
-      return conversions.filter(conversion => conversion !== null);
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'convertIdentifiersBatch', error);
-      throw error;
-    }
-  }
-
-  /**
-   * Search for chemicals by name using OpenSearch
-   * @param {string} searchTerm - Name to search for
-   * @param {number} limit - Maximum number of results (default: 10)
-   * @returns {Promise<Object>} Search results
-   */
-  async searchByName(searchTerm, limit = 10) {
-    if (!searchTerm) {
-      return { results: [] };
-    }
-
-    try {
-      const opensearchClient = this.connection.getOpenSearchClient();
-      const indexName = this.connection.getOpenSearchIndex();
-      
-      const response = await opensearchClient.search({
-        index: indexName,
-        body: {
-          size: limit,
-          query: {
-            bool: {
-              should: [
-                { term: { 'chemical_name.keyword': { value: searchTerm, boost: SEARCH_BOOST_EXACT_PRIMARY, case_insensitive: true } } },
-                { prefix: { 'chemical_name.keyword': { value: searchTerm, boost: SEARCH_BOOST_PREFIX_PRIMARY, case_insensitive: true } } },
-                { term: { 'synonyms.keyword': { value: searchTerm, boost: SEARCH_BOOST_EXACT_SECONDARY, case_insensitive: true } } },
-                { prefix: { 'synonyms.keyword': { value: searchTerm, boost: SEARCH_BOOST_PREFIX_SECONDARY, case_insensitive: true } } }
-              ],
-              minimum_should_match: 1
-            }
-          },
-          _source: ['postgres_id', 'chemical_name', 'cas_numbers', 'identifier_values', 'synonyms']
-        }
-      });
-
-      const hits = response.body?.hits?.hits || [];
-      const results = hits.map((hit) => ({
-        id: hit._source.postgres_id,
-        name: hit._source.chemical_name,
-        cas: hit._source.cas_numbers || [],
-        identifiers: hit._source.identifier_values || [],
-        synonyms: hit._source.synonyms || [],
-        score: hit._score
-      }));
-
-      return { results };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'searchByName', error);
-      throw error;
-    }
-  }
-
-  /**
-   * Search for chemicals by synonym using OpenSearch
-   * @param {string} synonymTerm - Synonym to search for
-   * @param {number} limit - Maximum number of results (default: 10)
-   * @returns {Promise<Object>} Search results
-   */
-  async searchBySynonym(synonymTerm, limit = 10) {
-    if (!synonymTerm) {
-      return { results: [] };
-    }
-
-    try {
-      const opensearchClient = this.connection.getOpenSearchClient();
-      const indexName = this.connection.getOpenSearchIndex();
-      
-      const response = await opensearchClient.search({
-        index: indexName,
-        body: {
-          size: limit,
-          query: {
-            bool: {
-              should: [
-                { term: { 'synonyms.keyword': { value: synonymTerm, boost: SEARCH_BOOST_EXACT_PRIMARY, case_insensitive: true } } },
-                { prefix: { 'synonyms.keyword': { value: synonymTerm, boost: SEARCH_BOOST_PREFIX_PRIMARY, case_insensitive: true } } },
-                { term: { 'chemical_name.keyword': { value: synonymTerm, boost: SEARCH_BOOST_EXACT_SECONDARY, case_insensitive: true } } },
-                { prefix: { 'chemical_name.keyword': { value: synonymTerm, boost: SEARCH_BOOST_PREFIX_SECONDARY, case_insensitive: true } } }
-              ],
-              minimum_should_match: 1
-            }
-          },
-          _source: ['postgres_id', 'chemical_name', 'cas_numbers', 'identifier_values', 'synonyms']
-        }
-      });
-
-      const hits = response.body?.hits?.hits || [];
-      const results = hits.map((hit) => ({
-        id: hit._source.postgres_id,
-        name: hit._source.chemical_name,
-        cas: hit._source.cas_numbers || [],
-        identifiers: hit._source.identifier_values || [],
-        synonyms: hit._source.synonyms || [],
-        score: hit._score
-      }));
-
-      return { results };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'searchBySynonym', error);
-      throw error;
-    }
-  }
-
-  async countAll() {
-    try {
-      const db = this.getDb();
-      const result = await db
-        .select({ count: sql`count(*)::int` })
-        .from(schema.chemicals);
-      return { count: result[0].count };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'countAll', error);
-      throw error;
-    }
-  }
-
-  async findChemicalsWithoutDocuments(collectionName, searchTerm, pageSize = 100) {
-    try {
-      const db = this.getDb();
-      
-      let whereConditions = [];
-      
-      if (collectionName) {
-        whereConditions.push(arrayContains(schema.chemicals.chemicalCategories, [collectionName]));
-      }
-      
-      if (searchTerm) {
-        const searchPattern = `%${escapeLikePattern(searchTerm)}%`;
-        whereConditions.push(sql`${schema.chemicals.chemicalName} ILIKE ${searchPattern}`);
-      }
-      
-      const whereClause = whereConditions.length > 0 ? and(...whereConditions) : undefined;
-      
-      const results = await db
-        .select()
-        .from(schema.chemicals)
-        .where(whereClause)
-        .limit(pageSize);
-
-      return results;
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'findChemicalsWithoutDocuments', error);
-      throw error;
-    }
-  }
-
-  async countChemicalsWithoutDocuments(collectionName) {
-    try {
-      const db = this.getDb();
-      
-      const whereClause = collectionName 
-        ? arrayContains(schema.chemicals.chemicalCategories, [collectionName])
-        : undefined;
-      
-      const result = await db
-        .select({ count: sql`count(*)::int` })
-        .from(schema.chemicals)
-        .where(whereClause);
-
-      return { count: result[0].count };
-    } catch (error) {
-      logError('pegasus-sdk', 'ChemicalsService', 'countChemicalsWithoutDocuments', error);
-      throw error;
-    }
-  }
-
-  _buildEsHandlers() {
-    return {
-      index: async (params) => {
-        const chemical = params.body;
-        const result = await this.createChemical(chemical);
-        
-        return {
-          _index: params.index,
-          _id: result.chemicalId,
-          _version: 1,
-          result: 'created',
-          _source: result
-        };
-      },
-      
-      bulk: async (params) => {
-        const operations = params.body || params.operations;
-        
-        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Starting bulk operation with ${operations?.length || 0} total operations`);
-        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Params index: ${params.index}`);
-        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Operations array type: ${Array.isArray(operations) ? 'array' : typeof operations}`);
-        
-        const cdiDocuments = [];
-        let cdiOpCount = 0;
-        let otherOpCount = 0;
-        
-        for (let i = 0; i < operations.length; i++) {
-          const op = operations[i];
-          const isIndexOp = !!(op.index || op.create);
-          const indexName = op.index?._index || op.create?._index || op.delete?._index || op.update?._index;
-          
-          logInfo('pegasus-sdk', `[ChemicalsService.bulk] Op[${i}]: action=${Object.keys(op)[0] || 'unknown'}, index=${indexName}`);
-          
-          if ((op.index || op.create) &&
-              (op.index?._index === 'chemical_data_index' || op.create?._index === 'chemical_data_index')) {
-            const doc = operations[i + 1];
-            const sourceId = op.index?._id || op.create?._id;
-            
-            logInfo('pegasus-sdk', `[ChemicalsService.bulk] Found CDI entry: sourceId=${sourceId}, hasDoc=${!!doc}`);
-            
-            if (doc && sourceId) {
-              const cdiDoc = {
-                source_id: sourceId,
-                chemical_name: doc.chemical_primary_name || (doc.chemical_names && doc.chemical_names[0]) || null,
-                chemical_meta: doc.chemical_meta || {},
-                chemical_identifiers: doc.chemical_identifiers || {},
-                chemical_synonyms: doc.chemical_synonyms || [],
-                chemical_categories: doc.chemical_categories || [],
-                created_at: doc.chemical_created_at ? (typeof doc.chemical_created_at === 'string' ? new Date(doc.chemical_created_at) : doc.chemical_created_at) : new Date(),
-                updated_at: doc.chemical_updated_at ? (typeof doc.chemical_updated_at === 'string' ? new Date(doc.chemical_updated_at) : doc.chemical_updated_at) : new Date()
-              };
-              cdiDocuments.push(cdiDoc);
-              logInfo('pegasus-sdk', `[ChemicalsService.bulk] Extracted CDI doc: ${JSON.stringify({ source_id: cdiDoc.source_id, chemical_name: cdiDoc.chemical_name })}`);
-              i++;
-              cdiOpCount++;
-            } else {
-              logInfo('pegasus-sdk', `[ChemicalsService.bulk] CDI entry incomplete: sourceId=${sourceId}, doc=${!!doc}`);
-            }
-          } else {
-            otherOpCount++;
-          }
-        }
-        
-        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Scan complete: ${cdiOpCount} CDI docs found, ${otherOpCount} other operations skipped`);
-        
-        if (cdiDocuments.length === 0) {
-          logInfo('pegasus-sdk', `[ChemicalsService.bulk] No CDI documents to index, returning empty no-op response`);
-          return { took: 0, errors: false, items: [] };
-        }
-        
-        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Calling bulkIndexFielded with ${cdiDocuments.length} CDI documents`);
-        
-        try {
-          const result = await this.bulkIndexFielded(cdiDocuments);
-          logInfo('pegasus-sdk', `[ChemicalsService.bulk] bulkIndexFielded returned: indexed=${result.indexed}, errors=${result.errors.length}`);
-          
-          if (result.errors.length > 0) {
-            logError('pegasus-sdk', 'ChemicalsService.bulk', 'Errors during bulk indexing', result.errors);
-          }
-          
-          return {
-            took: 1,
-            errors: result.errors.length > 0,
-            items: result.results.map((res, idx) => ({
-              index: {
-                _index: 'chemical_data_index',
-                _id: cdiDocuments[idx].source_id,
-                status: res.success ? 200 : 400,
-                result: res.success ? 'created' : 'error',
-                ...(res.success ? {} : { error: { type: 'mapper_parsing_exception', reason: res.error } })
-              }
-            }))
-          };
-        } catch (error) {
-          logError('pegasus-sdk', 'ChemicalsService.bulk', 'Fatal error during bulk indexing', error);
-          throw error;
-        }
-      },
-    
-      get: async (params) => {
-        const result = await this.getChemicalBySourceId(params.id);
-        
-        if (!result) {
-          return {
-            _index: params.index,
-            _id: params.id,
-            found: false
-          };
-        }
-        
-        return {
-          _index: params.index,
-          _id: params.id,
-          _version: 1,
-          found: true,
-          _source: result
-        };
-      },
-      
-      update: async (params) => {
-        const result = await this.updateChemical(params.id, params.body);
-        
-        return {
-          _index: params.index,
-          _id: params.id,
-          _version: 2,
-          result: result ? 'updated' : 'noop',
-          _source: result
-        };
-      },
-      
-      delete: async (params) => {
-        if (params.index === 'synonym_lookup_index') {
-          return { _index: params.index, _id: params.id, result: 'not_found' };
-        }
-        const result = await this.deleteBySourceId(params.id);
-        
-        return {
-          _index: params.index,
-          _id: params.id,
-          result: result ? 'deleted' : 'not_found'
-        };
-      },
-
-      deleteByQuery: async (params) => {
-        const sourceId = params.body?.query?.term?.chemical_set_identifier
-                      || params.body?.query?.term?.source_id;
-        if (!sourceId) {
-          return { deleted: 0, failures: [] };
-        }
-        const result = await this.deleteBySourceId(sourceId);
-        return {
-          deleted: result ? 1 : 0,
-          failures: []
-        };
-      },
-      
-      search: async (params) => {
-        let searchTerm = '';
-        let limit = params.body?.size || 10;
-        
-        if (params.index === 'synonym_lookup_index') {
-          const query = params.body?.query;
-          searchTerm = query?.match?.chemical_name || 
-                       query?.term?.chemical_name || 
-                       query?.query_string?.query || '';
-          const searchResults = await this.searchBySynonym(searchTerm, limit);
-          
-          return {
-            took: 1,
-            timed_out: false,
-            _shards: {
-              total: 1,
-              successful: 1,
-              skipped: 0,
-              failed: 0
-            },
-            hits: {
-              total: {
-                value: searchResults.results.length,
-                relation: 'eq'
-              },
-              max_score: searchResults.results[0]?.score || 0,
-              hits: searchResults.results.map(result => ({
-                _index: params.index,
-                _id: result.id,
-                _score: result.score,
-                _source: {
-                  postgres_id: result.id,
-                  chemical_name: result.name,
-                  cas_numbers: result.cas,
-                  identifier_values: result.identifiers,
-                  synonyms: result.synonyms
-                }
-              }))
-            }
-          };
-        } else {
-          const query = params.body?.query;
-          searchTerm = query?.match?.chemical_name || 
-                       query?.term?.chemical_name || 
-                       query?.query_string?.query || '';
-          const searchResults = await this.searchByName(searchTerm, limit);
-          
-          return {
-            took: 1,
-            timed_out: false,
-            _shards: {
-              total: 1,
-              successful: 1,
-              skipped: 0,
-              failed: 0
-            },
-            hits: {
-              total: {
-                value: searchResults.results.length,
-                relation: 'eq'
-              },
-              max_score: searchResults.results[0]?.score || 0,
-              hits: searchResults.results.map(result => ({
-                _index: params.index,
-                _id: result.id,
-                _score: result.score,
-                _source: {
-                  postgres_id: result.id,
-                  chemical_name: result.name,
-                  cas_numbers: result.cas,
-                  identifier_values: result.identifiers,
-                  synonyms: result.synonyms
-                }
-              }))
-            }
-          };
-        }
-      },
-      
-      count: async (params) => {
-        if (params.index === 'synonym_lookup_index') {
-          return await this.getTotalSynonymCount();
-        }
-        return await this.countAll();
-      }
-    };
-  }
-
-  registerElasticsearchHandlers(elasticsearchService) {
-    const configurablePatterns = this.connection.config.indexRoutes?.chemicals || ['chemicals*'];
-    const legacyPatterns = ['synonym_lookup_index', 'chemical_data_index', 'chemical_converter_index'];
-    const allPatterns = [...new Set([...configurablePatterns, ...legacyPatterns])];
-    const handlers = this._buildEsHandlers();
-    allPatterns.forEach(pattern => {
-      elasticsearchService.registerIndexRoute(pattern, handlers);
-    });
-  }
-}
-
+const { logError, logInfo } = require('@toxplanet/tphelper/logging');
+const { getDrizzle, schema } = require('./db');
+const { eq, sql, and, inArray, arrayContains } = require('drizzle-orm');
+const { SQSClient, SendMessageCommand } = require('@aws-sdk/client-sqs');
+
+const SEARCH_BOOST_EXACT_PRIMARY = 100;
+const SEARCH_BOOST_PREFIX_PRIMARY = 50;
+const SEARCH_BOOST_EXACT_SECONDARY = 30;
+const SEARCH_BOOST_PREFIX_SECONDARY = 10;
+
+const ALLOWED_IDENTIFIER_TYPES = new Set(['CAS', 'SMILES', 'InChI', 'InChIKey', 'PubChem', 'DTXSID', 'EINECS', 'EC']);
+
+function escapeLikePattern(value) {
+  return value.replace(/[%_\\]/g, '\\$&');
+}
+
+class ChemicalsService {
+  constructor(connection) {
+    this.connection = connection;
+    this.db = null;
+    this.sqsClient = null;
+  }
+
+  getDb() {
+    if (!this.db) {
+      this.db = getDrizzle(this.connection.pgPool);
+    }
+    return this.db;
+  }
+
+  async sendSqlWriteFailure({ sql, parameters, error, retryCount, failedAt }) {
+    try {
+      const region = process.env.AWS_REGION || this.connection.region;
+      const { awsAccountId, environment } = this.connection.config;
+      const defaultQueueUrl = awsAccountId
+        ? `https://sqs.${region}.amazonaws.com/${awsAccountId}/cr-pegasus-failed-items-${environment}`
+        : null;
+      const queueUrl = process.env.SQS_FAILED_ITEMS_QUEUE || defaultQueueUrl;
+
+      if (!queueUrl) {
+        logError('pegasus-sdk', 'sendSqlWriteFailure', 'No SQS queue URL available: set SQS_FAILED_ITEMS_QUEUE or provide awsAccountId in config');
+        return false;
+      }
+
+      logInfo('pegasus-sdk', `[sendSqlWriteFailure] Using queue: ${queueUrl}${process.env.SQS_FAILED_ITEMS_QUEUE ? ' (from env)' : ' (default)'}`);
+
+      if (!this.sqsClient) {
+        this.sqsClient = new SQSClient({ region });
+      }
+
+      const message = {
+        MessageType: 'SqlWriteFailure',
+        SourceService: this.connection.config.sourceService || 'pegasus-sdk',
+        Timestamp: (failedAt || new Date()).toISOString(),
+        Sql: sql,
+        Parameters: parameters,
+        OriginalError: error.message,
+        RetryCount: retryCount
+      };
+
+      const command = new SendMessageCommand({
+        QueueUrl: queueUrl,
+        MessageBody: JSON.stringify(message)
+      });
+
+      const response = await this.sqsClient.send(command);
+      logInfo('pegasus-sdk', `[sendSqlWriteFailure] SqlWriteFailure posted to SQS: MessageId=${response.MessageId}, RetryCount=${retryCount}`);
+      return true;
+    } catch (sqsError) {
+      logError('pegasus-sdk', 'sendSqlWriteFailure', 'Failed to post SqlWriteFailure to SQS', sqsError);
+      return false;
+    }
+  }
+
+  _buildChemicalUpsertSql(chemical) {
+    const sql = [
+      'INSERT INTO chemicals (source_id, chemical_name, chemical_meta, chemical_identifiers, chemical_synonyms, chemical_categories, created_at, updated_at)',
+      'VALUES (@source_id, @chemical_name, @chemical_meta::jsonb, @chemical_identifiers::jsonb, @chemical_synonyms, @chemical_categories, @created_at, @updated_at)',
+      'ON CONFLICT (source_id) DO UPDATE SET',
+      '  chemical_name = @chemical_name,',
+      '  chemical_meta = @chemical_meta::jsonb,',
+      '  chemical_identifiers = @chemical_identifiers::jsonb,',
+      '  chemical_synonyms = @chemical_synonyms,',
+      '  chemical_categories = @chemical_categories,',
+      '  updated_at = @updated_at'
+    ].join('\n');
+
+    const serializeDate = (d) => d instanceof Date ? d.toISOString() : d;
+
+    const parameters = {
+      '@source_id': chemical.sourceId,
+      '@chemical_name': chemical.chemicalName,
+      '@chemical_meta': JSON.stringify(chemical.chemicalMeta ?? {}),
+      '@chemical_identifiers': JSON.stringify(chemical.chemicalIdentifiers ?? {}),
+      '@chemical_synonyms': JSON.stringify(chemical.chemicalSynonyms ?? []),
+      '@chemical_categories': JSON.stringify(chemical.chemicalCategories ?? []),
+      '@created_at': serializeDate(chemical.createdAt),
+      '@updated_at': serializeDate(chemical.updatedAt)
+    };
+
+    return { sql, parameters };
+  }
+
+  _buildDebugSql(chemical) {
+    const esc = (s) => `'${String(s ?? '').replace(/'/g, "''")}'`;
+    const escJson = (v) => `'${JSON.stringify(v ?? {}).replace(/'/g, "''")}'`;
+    const escArr = (arr) => {
+      if (!Array.isArray(arr) || arr.length === 0) return `ARRAY[]::text[]`;
+      return `ARRAY[${arr.map(s => esc(s)).join(', ')}]`;
+    };
+    const escDate = (d) => esc(d instanceof Date ? d.toISOString() : (d ?? new Date().toISOString()));
+
+    return [
+      `INSERT INTO chemicals`,
+      `  (source_id, chemical_name, chemical_meta, chemical_identifiers, chemical_synonyms, chemical_categories, created_at, updated_at)`,
+      `VALUES (`,
+      `  ${esc(chemical.sourceId)},`,
+      `  ${esc(chemical.chemicalName)},`,
+      `  ${escJson(chemical.chemicalMeta)}::jsonb,`,
+      `  ${escJson(chemical.chemicalIdentifiers)}::jsonb,`,
+      `  ${escArr(chemical.chemicalSynonyms)},`,
+      `  ${escArr(chemical.chemicalCategories)},`,
+      `  ${escDate(chemical.createdAt)},`,
+      `  ${escDate(chemical.updatedAt)}`,
+      `)`,
+      `ON CONFLICT (source_id) DO UPDATE SET`,
+      `  chemical_name       = ${esc(chemical.chemicalName)},`,
+      `  chemical_meta       = ${escJson(chemical.chemicalMeta)}::jsonb,`,
+      `  chemical_identifiers = ${escJson(chemical.chemicalIdentifiers)}::jsonb,`,
+      `  chemical_synonyms   = ${escArr(chemical.chemicalSynonyms)},`,
+      `  chemical_categories = ${escArr(chemical.chemicalCategories)},`,
+      `  updated_at          = NOW();`
+    ].join('\n');
+  }
+
+  async bulkIndexFielded(documents) {
+    try {
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Starting bulk index with ${documents?.length || 0} documents`);
+
+      if (!documents || documents.length === 0) {
+        logInfo('pegasus-sdk', `[bulkIndexFielded] No documents provided, returning empty result`);
+        return { indexed: 0, errors: [], results: [] };
+      }
+
+      // Proactively validate the connection before any real query fires.
+      // If idle too long, this reconnects first so the real query never faces
+      // the full connectionTimeoutMillis wait on a stale pool.
+      const reconnected = await this.connection.ensureConnected();
+      if (reconnected) {
+        this.db = null; // force getDb() to bind to the fresh pool
+      }
+
+      const db = this.getDb();
+      const results = [];
+      const errors = [];
+
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Database connection established`);
+
+      for (let i = 0; i < documents.length; i++) {
+        const doc = documents[i];
+        logInfo('pegasus-sdk', `[bulkIndexFielded] Processing document ${i}: source_id=${doc.source_id}, chemical_name=${doc.chemical_name}`);
+
+        const parseDate = (dateValue) => {
+          if (!dateValue) return new Date();
+          if (dateValue instanceof Date) return dateValue;
+          if (typeof dateValue === 'string') return new Date(dateValue);
+          return new Date();
+        };
+
+        const chemical = {
+          sourceId: doc.source_id || doc._id,
+          chemicalName: doc.chemical_name || doc.name,
+          chemicalMeta: doc.chemical_meta || {},
+          chemicalIdentifiers: doc.chemical_identifiers || {},
+          chemicalSynonyms: doc.chemical_synonyms || [],
+          chemicalCategories: doc.chemical_categories || [],
+          createdAt: parseDate(doc.created_at),
+          updatedAt: parseDate(doc.updated_at),
+          ...(doc.imported_at && { importedAt: doc.imported_at }),
+          ...(doc.chemical_id && { chemicalId: doc.chemical_id })
+        };
+
+        logInfo('pegasus-sdk', `[bulkIndexFielded] Prepared chemical object: sourceId=${chemical.sourceId}, chemicalName=${chemical.chemicalName}`);
+        logInfo('pegasus-sdk', `[bulkIndexFielded] DEBUG SQL for document ${i}:\n${this._buildDebugSql(chemical)}`);
+
+        const isConnectionError = (err) =>
+          err.message?.toLowerCase().includes('timeout') ||
+          err.message?.toLowerCase().includes('connection') ||
+          err.code === 'ECONNREFUSED' ||
+          err.code === 'ETIMEDOUT';
+
+        const attemptUpsert = () =>
+          db.insert(schema.chemicals)
+            .values(chemical)
+            .onConflictDoUpdate({
+              target: schema.chemicals.sourceId,
+              set: {
+                chemicalName: chemical.chemicalName,
+                chemicalMeta: chemical.chemicalMeta,
+                chemicalIdentifiers: chemical.chemicalIdentifiers,
+                chemicalSynonyms: chemical.chemicalSynonyms,
+                chemicalCategories: chemical.chemicalCategories,
+                updatedAt: new Date()
+              }
+            })
+            .returning({
+              chemicalId: schema.chemicals.chemicalId,
+              sourceId: schema.chemicals.sourceId
+            });
+
+        let lastError = null;
+        let retryCount = 0;
+        const failedAt = new Date();
+
+        try {
+          const [result] = await attemptUpsert();
+          logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully: ${result?.chemicalId || 'no ID returned'}`);
+          this.connection.recordActivity();
+          results.push({ index: i, success: true, result });
+          continue;
+        } catch (firstErr) {
+          lastError = firstErr;
+
+          if (isConnectionError(firstErr)) {
+            // Stale pool — rebuild the connection and try once more before queuing
+            logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} connection error (${firstErr.message}), reconnecting pool and retrying`);
+            try {
+              await this.connection.reconnect();
+              this.db = null; // force getDb() to bind to the new pool
+              const [result] = await attemptUpsert();
+              logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully after reconnect: ${result?.chemicalId || 'no ID returned'}`);
+              this.connection.recordActivity();
+              results.push({ index: i, success: true, result });
+              continue;
+            } catch (reconnectErr) {
+              lastError = reconnectErr;
+              retryCount = 1;
+            }
+          } else {
+            logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} first attempt failed (${firstErr.message}), retrying once`);
+            try {
+              const [result] = await attemptUpsert();
+              logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully on retry: ${result?.chemicalId || 'no ID returned'}`);
+              this.connection.recordActivity();
+              results.push({ index: i, success: true, result });
+              continue;
+            } catch (retryErr) {
+              lastError = retryErr;
+              retryCount = 1;
+            }
+          }
+        }
+
+        logError('pegasus-sdk', 'bulkIndexFielded', `Document ${i} failed after ${retryCount} local retries (source_id=${chemical.sourceId})`, lastError);
+
+        const { sql: failureSql, parameters: failureParams } = this._buildChemicalUpsertSql(chemical);
+        const queued = await this.sendSqlWriteFailure({
+          sql: failureSql,
+          parameters: failureParams,
+          error: lastError,
+          retryCount,
+          failedAt
+        });
+
+        if (queued) {
+          logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} (source_id=${chemical.sourceId}) queued for repair via SQS`);
+        } else {
+          logError('pegasus-sdk', 'bulkIndexFielded', `Document ${i} (source_id=${chemical.sourceId}) failed and could not be queued — data loss risk`, lastError);
+        }
+
+        results.push({ index: i, success: false, error: lastError.message, queued });
+        errors.push({ document: doc, error: lastError.message, queued });
+      }
+
+      const successCount = results.filter(r => r.success).length;
+      const queuedCount = results.filter(r => !r.success && r.queued).length;
+      logInfo('pegasus-sdk', `[bulkIndexFielded] Bulk index complete: ${successCount}/${documents.length} succeeded, ${queuedCount} queued for repair, ${errors.length - queuedCount} unhandled errors`);
+
+      return { indexed: successCount, errors, results };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFielded', error);
+      throw error;
+    }
+  }
+
+  async bulkIndexFulltext(documents) {
+    try {
+      return { acknowledged: true, count: documents?.length || 0 };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexFulltext', error);
+      throw error;
+    }
+  }
+
+  async bulkIndexSubstances(substances) {
+    try {
+      const documents = substances.map(substance => ({
+        source_id: substance.substance_id || substance.id,
+        chemical_name: substance.name || substance.substance_name,
+        chemical_meta: substance.meta || {},
+        chemical_identifiers: substance.identifiers || {},
+        chemical_synonyms: substance.synonyms || [],
+        chemical_categories: substance.categories || substance.substance_types || [],
+        created_at: substance.created_at,
+        updated_at: substance.updated_at,
+        imported_at: substance.imported_at
+      }));
+
+      return await this.bulkIndexFielded(documents);
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'bulkIndexSubstances', error);
+      throw error;
+    }
+  }
+
+  async createChemical(chemical) {
+    try {
+      const db = this.getDb();
+      
+      const [result] = await db
+        .insert(schema.chemicals)
+        .values({
+          sourceId: chemical.source_id,
+          chemicalName: chemical.chemical_name,
+          chemicalMeta: chemical.chemical_meta,
+          chemicalIdentifiers: chemical.chemical_identifiers,
+          chemicalSynonyms: chemical.chemical_synonyms,
+          chemicalCategories: chemical.chemical_categories,
+          createdAt: chemical.created_at || new Date(),
+          updatedAt: chemical.updated_at || new Date(),
+          ...(chemical.imported_at && { importedAt: chemical.imported_at }),
+          ...(chemical.chemical_id && { chemicalId: chemical.chemical_id })
+        })
+        .returning();
+
+      return result;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'createChemical', error);
+      throw error;
+    }
+  }
+
+  async updateChemical(chemicalId, updates) {
+    try {
+      const db = this.getDb();
+      
+      const updateData = {};
+      if (updates.chemical_name) updateData.chemicalName = updates.chemical_name;
+      if (updates.chemical_meta) updateData.chemicalMeta = updates.chemical_meta;
+      if (updates.chemical_identifiers) updateData.chemicalIdentifiers = updates.chemical_identifiers;
+      if (updates.chemical_synonyms) updateData.chemicalSynonyms = updates.chemical_synonyms;
+      if (updates.chemical_categories) updateData.chemicalCategories = updates.chemical_categories;
+      updateData.updatedAt = new Date();
+
+      const [result] = await db
+        .update(schema.chemicals)
+        .set(updateData)
+        .where(eq(schema.chemicals.chemicalId, chemicalId))
+        .returning();
+
+      return result || null;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'updateChemical', error);
+      throw error;
+    }
+  }
+
+  async deleteChemical(chemicalId) {
+    try {
+      const db = this.getDb();
+      
+      const [deleted] = await db
+        .delete(schema.chemicals)
+        .where(eq(schema.chemicals.chemicalId, chemicalId))
+        .returning();
+
+      return deleted || null;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'deleteChemical', error);
+      throw error;
+    }
+  }
+
+  async deleteBySourceId(sourceId) {
+    try {
+      const db = this.getDb();
+      
+      const [deleted] = await db
+        .delete(schema.chemicals)
+        .where(eq(schema.chemicals.sourceId, sourceId))
+        .returning();
+
+      return deleted || null;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'deleteBySourceId', error);
+      throw error;
+    }
+  }
+
+  async deleteCollection(collectionName) {
+    try {
+      const db = this.getDb();
+      
+      const deleted = await db
+        .delete(schema.chemicals)
+        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]))
+        .returning();
+
+      return { deletedCount: deleted.length, deleted };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'deleteCollection', error);
+      throw error;
+    }
+  }
+
+  async updateCollectionProperty(collectionName, propertyPath, newValue) {
+    try {
+      const db = this.getDb();
+      const pathArray = propertyPath.split('.');
+      const valueJson = JSON.stringify(newValue);
+
+      const results = await db
+        .update(schema.chemicals)
+        .set({
+          chemicalMeta: sql`jsonb_set(${schema.chemicals.chemicalMeta}, ${pathArray}::text[], ${valueJson}::jsonb)`,
+          updatedAt: new Date()
+        })
+        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]))
+        .returning();
+
+      return { updatedCount: results.length, updated: results };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'updateCollectionProperty', error);
+      throw error;
+    }
+  }
+
+  async bulkUpdateProperty(filter, propertyPath, newValue) {
+    try {
+      const db = this.getDb();
+      
+      let whereCondition = sql`1=1`;
+      
+      if (filter.chemicalIds && filter.chemicalIds.length > 0) {
+        whereCondition = inArray(schema.chemicals.chemicalId, filter.chemicalIds);
+      } else if (filter.sourceIds && filter.sourceIds.length > 0) {
+        whereCondition = inArray(schema.chemicals.sourceId, filter.sourceIds);
+      } else if (filter.category) {
+        whereCondition = arrayContains(schema.chemicals.chemicalCategories, [filter.category]);
+      }
+
+      const pathArray = propertyPath.split('.');
+      const valueJson = JSON.stringify(newValue);
+
+      const results = await db
+        .update(schema.chemicals)
+        .set({
+          chemicalMeta: sql`jsonb_set(COALESCE(${schema.chemicals.chemicalMeta}, '{}'), ${pathArray}::text[], ${valueJson}::jsonb)`,
+          updatedAt: new Date()
+        })
+        .where(whereCondition)
+        .returning();
+
+      return { updatedCount: results.length, updated: results };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'bulkUpdateProperty', error);
+      throw error;
+    }
+  }
+
+  async getChemicalById(chemicalId) {
+    try {
+      const db = this.getDb();
+      
+      const [result] = await db
+        .select()
+        .from(schema.chemicals)
+        .where(eq(schema.chemicals.chemicalId, chemicalId))
+        .limit(1);
+
+      return result || null;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalById', error);
+      throw error;
+    }
+  }
+
+  async getChemicalBySourceId(sourceId) {
+    try {
+      const db = this.getDb();
+      
+      const [result] = await db
+        .select()
+        .from(schema.chemicals)
+        .where(eq(schema.chemicals.sourceId, sourceId))
+        .limit(1);
+
+      return result || null;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalBySourceId', error);
+      throw error;
+    }
+  }
+
+  async getChemicalsByCAS(casNumber) {
+    try {
+      const db = this.getDb();
+      
+      const results = await db
+        .select()
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}->>'CAS' = ${casNumber} OR ${schema.chemicals.chemicalIdentifiers}->'CAS' ? ${casNumber}`);
+
+      return results;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalsByCAS', error);
+      throw error;
+    }
+  }
+
+  async getChemicalsByIdentifier(identifierType, identifierValue) {
+    try {
+      if (!ALLOWED_IDENTIFIER_TYPES.has(identifierType)) {
+        throw new Error(`Invalid identifier type: ${identifierType}`);
+      }
+
+      const db = this.getDb();
+
+      const results = await db
+        .select()
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}->>${identifierType} = ${identifierValue} OR ${schema.chemicals.chemicalIdentifiers}->${identifierType} ? ${identifierValue}`);
+
+      return results;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getChemicalsByIdentifier', error);
+      throw error;
+    }
+  }
+
+  async countByCollection(collectionName) {
+    try {
+      const db = this.getDb();
+      
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(arrayContains(schema.chemicals.chemicalCategories, [collectionName]));
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countByCollection', error);
+      throw error;
+    }
+  }
+
+  async countByIdentifier(identifierValue) {
+    try {
+      const db = this.getDb();
+
+      const searchPattern = `%${escapeLikePattern(identifierValue)}%`;
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}::text LIKE ${searchPattern}`);
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countByIdentifier', error);
+      throw error;
+    }
+  }
+
+  async countByCAS(casNumber) {
+    try {
+      const db = this.getDb();
+      
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}->>'CAS' = ${casNumber} OR ${schema.chemicals.chemicalIdentifiers}->'CAS' ? ${casNumber}`);
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countByCAS', error);
+      throw error;
+    }
+  }
+
+  async getTotalSynonymCount() {
+    try {
+      const db = this.getDb();
+      
+      const result = await db
+        .select({ count: sql`sum(array_length(${schema.chemicals.chemicalSynonyms}, 1))::int` })
+        .from(schema.chemicals);
+
+      return { count: result[0].count || 0 };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getTotalSynonymCount', error);
+      throw error;
+    }
+  }
+
+  async getSynonymCount(synonymTerm) {
+    try {
+      const db = this.getDb();
+      
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(arrayContains(schema.chemicals.chemicalSynonyms, [synonymTerm]));
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'getSynonymCount', error);
+      throw error;
+    }
+  }
+
+  async convertIdentifier(fromIdentifier, toIdentifierType) {
+    try {
+      const db = this.getDb();
+
+      const searchPattern = `%${escapeLikePattern(fromIdentifier)}%`;
+      const chemicals = await db
+        .select()
+        .from(schema.chemicals)
+        .where(sql`${schema.chemicals.chemicalIdentifiers}::text LIKE ${searchPattern}`);
+
+      if (chemicals.length === 0) {
+        return null;
+      }
+
+      const chemical = chemicals[0];
+      const identifiers = chemical.chemicalIdentifiers || {};
+      const toIdentifier = identifiers[toIdentifierType];
+
+      return {
+        fromIdentifier,
+        toIdentifierType,
+        toIdentifier,
+        chemicalId: chemical.chemicalId,
+        chemicalName: chemical.chemicalName
+      };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'convertIdentifier', error);
+      throw error;
+    }
+  }
+
+  async convertIdentifiersBatch(fromIdentifiers, toIdentifierType) {
+    try {
+      const conversions = await Promise.all(
+        fromIdentifiers.map(fromIdentifier => 
+          this.convertIdentifier(fromIdentifier, toIdentifierType)
+        )
+      );
+
+      return conversions.filter(conversion => conversion !== null);
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'convertIdentifiersBatch', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Search for chemicals by name using OpenSearch
+   * @param {string} searchTerm - Name to search for
+   * @param {number} limit - Maximum number of results (default: 10)
+   * @returns {Promise<Object>} Search results
+   */
+  async searchByName(searchTerm, limit = 10) {
+    if (!searchTerm) {
+      return { results: [] };
+    }
+
+    try {
+      const opensearchClient = this.connection.getOpenSearchClient();
+      const indexName = this.connection.getOpenSearchIndex();
+      
+      const response = await opensearchClient.search({
+        index: indexName,
+        body: {
+          size: limit,
+          query: {
+            bool: {
+              should: [
+                { term: { 'chemical_name.keyword': { value: searchTerm, boost: SEARCH_BOOST_EXACT_PRIMARY, case_insensitive: true } } },
+                { prefix: { 'chemical_name.keyword': { value: searchTerm, boost: SEARCH_BOOST_PREFIX_PRIMARY, case_insensitive: true } } },
+                { term: { 'synonyms.keyword': { value: searchTerm, boost: SEARCH_BOOST_EXACT_SECONDARY, case_insensitive: true } } },
+                { prefix: { 'synonyms.keyword': { value: searchTerm, boost: SEARCH_BOOST_PREFIX_SECONDARY, case_insensitive: true } } }
+              ],
+              minimum_should_match: 1
+            }
+          },
+          _source: ['postgres_id', 'chemical_name', 'cas_numbers', 'identifier_values', 'synonyms']
+        }
+      });
+
+      const hits = response.body?.hits?.hits || [];
+      const results = hits.map((hit) => ({
+        id: hit._source.postgres_id,
+        name: hit._source.chemical_name,
+        cas: hit._source.cas_numbers || [],
+        identifiers: hit._source.identifier_values || [],
+        synonyms: hit._source.synonyms || [],
+        score: hit._score
+      }));
+
+      return { results };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'searchByName', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Search for chemicals by synonym using OpenSearch
+   * @param {string} synonymTerm - Synonym to search for
+   * @param {number} limit - Maximum number of results (default: 10)
+   * @returns {Promise<Object>} Search results
+   */
+  async searchBySynonym(synonymTerm, limit = 10) {
+    if (!synonymTerm) {
+      return { results: [] };
+    }
+
+    try {
+      const opensearchClient = this.connection.getOpenSearchClient();
+      const indexName = this.connection.getOpenSearchIndex();
+      
+      const response = await opensearchClient.search({
+        index: indexName,
+        body: {
+          size: limit,
+          query: {
+            bool: {
+              should: [
+                { term: { 'synonyms.keyword': { value: synonymTerm, boost: SEARCH_BOOST_EXACT_PRIMARY, case_insensitive: true } } },
+                { prefix: { 'synonyms.keyword': { value: synonymTerm, boost: SEARCH_BOOST_PREFIX_PRIMARY, case_insensitive: true } } },
+                { term: { 'chemical_name.keyword': { value: synonymTerm, boost: SEARCH_BOOST_EXACT_SECONDARY, case_insensitive: true } } },
+                { prefix: { 'chemical_name.keyword': { value: synonymTerm, boost: SEARCH_BOOST_PREFIX_SECONDARY, case_insensitive: true } } }
+              ],
+              minimum_should_match: 1
+            }
+          },
+          _source: ['postgres_id', 'chemical_name', 'cas_numbers', 'identifier_values', 'synonyms']
+        }
+      });
+
+      const hits = response.body?.hits?.hits || [];
+      const results = hits.map((hit) => ({
+        id: hit._source.postgres_id,
+        name: hit._source.chemical_name,
+        cas: hit._source.cas_numbers || [],
+        identifiers: hit._source.identifier_values || [],
+        synonyms: hit._source.synonyms || [],
+        score: hit._score
+      }));
+
+      return { results };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'searchBySynonym', error);
+      throw error;
+    }
+  }
+
+  async countAll() {
+    try {
+      const db = this.getDb();
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals);
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countAll', error);
+      throw error;
+    }
+  }
+
+  async findChemicalsWithoutDocuments(collectionName, searchTerm, pageSize = 100) {
+    try {
+      const db = this.getDb();
+      
+      let whereConditions = [];
+      
+      if (collectionName) {
+        whereConditions.push(arrayContains(schema.chemicals.chemicalCategories, [collectionName]));
+      }
+      
+      if (searchTerm) {
+        const searchPattern = `%${escapeLikePattern(searchTerm)}%`;
+        whereConditions.push(sql`${schema.chemicals.chemicalName} ILIKE ${searchPattern}`);
+      }
+      
+      const whereClause = whereConditions.length > 0 ? and(...whereConditions) : undefined;
+      
+      const results = await db
+        .select()
+        .from(schema.chemicals)
+        .where(whereClause)
+        .limit(pageSize);
+
+      return results;
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'findChemicalsWithoutDocuments', error);
+      throw error;
+    }
+  }
+
+  async countChemicalsWithoutDocuments(collectionName) {
+    try {
+      const db = this.getDb();
+      
+      const whereClause = collectionName 
+        ? arrayContains(schema.chemicals.chemicalCategories, [collectionName])
+        : undefined;
+      
+      const result = await db
+        .select({ count: sql`count(*)::int` })
+        .from(schema.chemicals)
+        .where(whereClause);
+
+      return { count: result[0].count };
+    } catch (error) {
+      logError('pegasus-sdk', 'ChemicalsService', 'countChemicalsWithoutDocuments', error);
+      throw error;
+    }
+  }
+
+  _buildEsHandlers() {
+    return {
+      index: async (params) => {
+        const chemical = params.body;
+        const result = await this.createChemical(chemical);
+        
+        return {
+          _index: params.index,
+          _id: result.chemicalId,
+          _version: 1,
+          result: 'created',
+          _source: result
+        };
+      },
+      
+      bulk: async (params) => {
+        const operations = params.body || params.operations;
+        
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Starting bulk operation with ${operations?.length || 0} total operations`);
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Params index: ${params.index}`);
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Operations array type: ${Array.isArray(operations) ? 'array' : typeof operations}`);
+        
+        const cdiDocuments = [];
+        let cdiOpCount = 0;
+        let otherOpCount = 0;
+        
+        for (let i = 0; i < operations.length; i++) {
+          const op = operations[i];
+          const isIndexOp = !!(op.index || op.create);
+          const indexName = op.index?._index || op.create?._index || op.delete?._index || op.update?._index;
+          
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] Op[${i}]: action=${Object.keys(op)[0] || 'unknown'}, index=${indexName}`);
+          
+          if ((op.index || op.create) &&
+              (op.index?._index === 'chemical_data_index' || op.create?._index === 'chemical_data_index')) {
+            const doc = operations[i + 1];
+            const sourceId = op.index?._id || op.create?._id;
+            
+            logInfo('pegasus-sdk', `[ChemicalsService.bulk] Found CDI entry: sourceId=${sourceId}, hasDoc=${!!doc}`);
+            
+            if (doc && sourceId) {
+              const cdiDoc = {
+                source_id: sourceId,
+                chemical_name: doc.chemical_primary_name || (doc.chemical_names && doc.chemical_names[0]) || null,
+                chemical_meta: doc.chemical_meta || {},
+                chemical_identifiers: doc.chemical_identifiers || {},
+                chemical_synonyms: doc.chemical_synonyms || [],
+                chemical_categories: doc.chemical_categories || [],
+                created_at: doc.chemical_created_at ? (typeof doc.chemical_created_at === 'string' ? new Date(doc.chemical_created_at) : doc.chemical_created_at) : new Date(),
+                updated_at: doc.chemical_updated_at ? (typeof doc.chemical_updated_at === 'string' ? new Date(doc.chemical_updated_at) : doc.chemical_updated_at) : new Date()
+              };
+              cdiDocuments.push(cdiDoc);
+              logInfo('pegasus-sdk', `[ChemicalsService.bulk] Extracted CDI doc: ${JSON.stringify({ source_id: cdiDoc.source_id, chemical_name: cdiDoc.chemical_name })}`);
+              i++;
+              cdiOpCount++;
+            } else {
+              logInfo('pegasus-sdk', `[ChemicalsService.bulk] CDI entry incomplete: sourceId=${sourceId}, doc=${!!doc}`);
+            }
+          } else {
+            otherOpCount++;
+          }
+        }
+        
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Scan complete: ${cdiOpCount} CDI docs found, ${otherOpCount} other operations skipped`);
+        
+        if (cdiDocuments.length === 0) {
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] No CDI documents to index, returning empty no-op response`);
+          return { took: 0, errors: false, items: [] };
+        }
+        
+        logInfo('pegasus-sdk', `[ChemicalsService.bulk] Calling bulkIndexFielded with ${cdiDocuments.length} CDI documents`);
+        
+        try {
+          const result = await this.bulkIndexFielded(cdiDocuments);
+          logInfo('pegasus-sdk', `[ChemicalsService.bulk] bulkIndexFielded returned: indexed=${result.indexed}, errors=${result.errors.length}`);
+          
+          if (result.errors.length > 0) {
+            logError('pegasus-sdk', 'ChemicalsService.bulk', 'Errors during bulk indexing', result.errors);
+          }
+          
+          return {
+            took: 1,
+            errors: result.errors.length > 0,
+            items: result.results.map((res, idx) => ({
+              index: {
+                _index: 'chemical_data_index',
+                _id: cdiDocuments[idx].source_id,
+                status: res.success ? 200 : 400,
+                result: res.success ? 'created' : 'error',
+                ...(res.success ? {} : { error: { type: 'mapper_parsing_exception', reason: res.error } })
+              }
+            }))
+          };
+        } catch (error) {
+          logError('pegasus-sdk', 'ChemicalsService.bulk', 'Fatal error during bulk indexing', error);
+          throw error;
+        }
+      },
+    
+      get: async (params) => {
+        const result = await this.getChemicalBySourceId(params.id);
+        
+        if (!result) {
+          return {
+            _index: params.index,
+            _id: params.id,
+            found: false
+          };
+        }
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          _version: 1,
+          found: true,
+          _source: result
+        };
+      },
+      
+      update: async (params) => {
+        const result = await this.updateChemical(params.id, params.body);
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          _version: 2,
+          result: result ? 'updated' : 'noop',
+          _source: result
+        };
+      },
+      
+      delete: async (params) => {
+        if (params.index === 'synonym_lookup_index') {
+          return { _index: params.index, _id: params.id, result: 'not_found' };
+        }
+        const result = await this.deleteBySourceId(params.id);
+        
+        return {
+          _index: params.index,
+          _id: params.id,
+          result: result ? 'deleted' : 'not_found'
+        };
+      },
+
+      deleteByQuery: async (params) => {
+        const sourceId = params.body?.query?.term?.chemical_set_identifier
+                      || params.body?.query?.term?.source_id;
+        if (!sourceId) {
+          return { deleted: 0, failures: [] };
+        }
+        const result = await this.deleteBySourceId(sourceId);
+        return {
+          deleted: result ? 1 : 0,
+          failures: []
+        };
+      },
+      
+      search: async (params) => {
+        let searchTerm = '';
+        let limit = params.body?.size || 10;
+        
+        if (params.index === 'synonym_lookup_index') {
+          const query = params.body?.query;
+          searchTerm = query?.match?.chemical_name || 
+                       query?.term?.chemical_name || 
+                       query?.query_string?.query || '';
+          const searchResults = await this.searchBySynonym(searchTerm, limit);
+          
+          return {
+            took: 1,
+            timed_out: false,
+            _shards: {
+              total: 1,
+              successful: 1,
+              skipped: 0,
+              failed: 0
+            },
+            hits: {
+              total: {
+                value: searchResults.results.length,
+                relation: 'eq'
+              },
+              max_score: searchResults.results[0]?.score || 0,
+              hits: searchResults.results.map(result => ({
+                _index: params.index,
+                _id: result.id,
+                _score: result.score,
+                _source: {
+                  postgres_id: result.id,
+                  chemical_name: result.name,
+                  cas_numbers: result.cas,
+                  identifier_values: result.identifiers,
+                  synonyms: result.synonyms
+                }
+              }))
+            }
+          };
+        } else {
+          const query = params.body?.query;
+          searchTerm = query?.match?.chemical_name || 
+                       query?.term?.chemical_name || 
+                       query?.query_string?.query || '';
+          const searchResults = await this.searchByName(searchTerm, limit);
+          
+          return {
+            took: 1,
+            timed_out: false,
+            _shards: {
+              total: 1,
+              successful: 1,
+              skipped: 0,
+              failed: 0
+            },
+            hits: {
+              total: {
+                value: searchResults.results.length,
+                relation: 'eq'
+              },
+              max_score: searchResults.results[0]?.score || 0,
+              hits: searchResults.results.map(result => ({
+                _index: params.index,
+                _id: result.id,
+                _score: result.score,
+                _source: {
+                  postgres_id: result.id,
+                  chemical_name: result.name,
+                  cas_numbers: result.cas,
+                  identifier_values: result.identifiers,
+                  synonyms: result.synonyms
+                }
+              }))
+            }
+          };
+        }
+      },
+      
+      count: async (params) => {
+        if (params.index === 'synonym_lookup_index') {
+          return await this.getTotalSynonymCount();
+        }
+        return await this.countAll();
+      }
+    };
+  }
+
+  registerElasticsearchHandlers(elasticsearchService) {
+    const configurablePatterns = this.connection.config.indexRoutes?.chemicals || ['chemicals*'];
+    const legacyPatterns = ['synonym_lookup_index', 'chemical_data_index', 'chemical_converter_index'];
+    const allPatterns = [...new Set([...configurablePatterns, ...legacyPatterns])];
+    const handlers = this._buildEsHandlers();
+    allPatterns.forEach(pattern => {
+      elasticsearchService.registerIndexRoute(pattern, handlers);
+    });
+  }
+}
+
 module.exports = ChemicalsService;