@toxplanet/pegasus-sdk 1.2.1 → 1.2.3

package/lib/connection.js

@@ -1,208 +1,217 @@
-const { Client: OpenSearchClient } = require('@opensearch-project/opensearch');
-const { RDSDataClient, ExecuteStatementCommand } = require('@aws-sdk/client-rds-data');
-const { AwsSigv4Signer } = require('@opensearch-project/opensearch/aws');
-const { fromNodeProviderChain } = require('@aws-sdk/credential-providers');
-const { mapRecords } = require('./db');
-const { loadConfig } = require('../config');
-const { logInfo, logError } = require('@toxplanet/tphelper/logging');
-
-class PegasusConnection {
-  constructor(config = {}) {
-    const envConfig = loadConfig(config.environment);
-    
-    this.config = { ...envConfig, ...config };
-    this.environment = this.config.environment;
-    this.region = this.config.region;
-    this.secretArn = this.config.secretArn;
-    this.clusterArn = this.config.clusterArn;
-    this.openSearchEndpoint = this.config.openSearchEndpoint;
-    this.openSearchIndex = this.config.openSearchIndex;
-    this.databaseName = this.config.database?.name;
-    
-    this.rdsDataClient = null;
-    this.osClient = null;
-    this.isConnected = false;
-  }
-
-  async connect() {
-    if (this.isConnected) {
-      return;
-    }
-
-    this.rdsDataClient = new RDSDataClient({ region: this.region });
-
-    logInfo('pegasus-sdk', 'RDS Data API client initialized');
-
-    try {
-      const command = new ExecuteStatementCommand({
-        resourceArn: this.clusterArn,
-        secretArn: this.secretArn,
-        database: this.databaseName,
-        sql: 'SELECT 1'
-      });
-      await this.rdsDataClient.send(command);
-      logInfo('pegasus-sdk', 'RDS Data API connection verified and ready');
-    } catch (err) {
-      logError('pegasus-sdk', 'PegasusConnection', 'connect.verification', err);
-      throw err;
-    }
-
-    if (this.openSearchEndpoint) {
-      this.osClient = new OpenSearchClient({
-        ...AwsSigv4Signer({
-          region: this.region,
-          service: 'aoss',
-          getCredentials: () => {
-            const credentialsProvider = fromNodeProviderChain();
-            return credentialsProvider();
-          }
-        }),
-        node: this.openSearchEndpoint
-      });
-    }
-
-    this.isConnected = true;
-  }
-
-  /**
-   * With the RDS Data API, there are no stale connections or pool idleness.
-   * Each call is stateless. Just ensure the client is initialized.
-   *
-   * @returns {Promise<boolean>} true if a connect happened, false if already connected.
-   */
-  async ensureConnected() {
-    if (!this.isConnected) {
-      await this.connect();
-      return true;
-    }
-    return false;
-  }
-
-  /**
-   * Data API is stateless, so reconnect simply means re-initializing.
-   * Called in fallback scenarios but not necessary for stale connections.
-   */
-  async reconnect() {
-    logInfo('pegasus-sdk', 'Reconnecting RDS Data API client');
-    this.isConnected = false;
-    await this.connect();
-  }
-
-  /**
-   * With stateless Data API, there is no meaningful "activity" tracking needed.
-   * This is a no-op for backward compatibility.
-   */
-  recordActivity() {
-  }
-
-  async disconnect() {
-    if (!this.isConnected) {
-      return;
-    }
-
-    this.rdsDataClient = null;
-    this.osClient = null;
-    this.isConnected = false;
-    logInfo('pegasus-sdk', 'RDS Data API client disconnected');
-  }
-
-  getOpenSearchClient() {
-    if (!this.osClient) {
-      throw new Error('OpenSearch connection not established. Call connect() first or provide openSearchEndpoint.');
-    }
-    return this.osClient;
-  }
-
-  getOpenSearchIndex() {
-    return this.openSearchIndex || 'chemicals';
-  }
-
-  async testConnection() {
-    try {
-      if (!this.rdsDataClient) {
-        throw new Error('RDS Data API not initialized');
-      }
-
-      const command = new ExecuteStatementCommand({
-        resourceArn: this.clusterArn,
-        secretArn: this.secretArn,
-        database: this.databaseName,
-        sql: 'SELECT NOW() as current_time, version() as pg_version',
-        includeResultMetadata: true
-      });
-      
-      const result = await this.rdsDataClient.send(command);
-      const rows = mapRecords(result.records, result.columnMetadata);
-      const row = rows?.[0];
-      
-      const pgStatus = {
-        connected: true,
-        timestamp: row?.current_time,
-        version: row?.pg_version
-      };
-
-      let osStatus = null;
-      if (this.osClient) {
-        try {
-          const indexName = this.getOpenSearchIndex();
-          const testSearch = await this.osClient.search({
-            index: indexName,
-            body: {
-              size: 1,
-              query: {
-                match: { chemical_name: 'benzene' }
-              }
-            }
-          });
-          osStatus = {
-            connected: true,
-            resultsFound: testSearch.body.hits.total.value || 0
-          };
-        } catch (osError) {
-          osStatus = {
-            connected: false,
-            error: osError.message
-          };
-        }
-      }
-
-      return {
-        postgres: pgStatus,
-        opensearch: osStatus,
-        environment: this.environment,
-        region: this.region
-      };
-    } catch (error) {
-      return {
-        postgres: { connected: false, error: error.message },
-        opensearch: null,
-        environment: this.environment,
-        region: this.region
-      };
-    }
-  }
-
-  async query(sql, params) {
-    const start = Date.now();
-    logInfo('pegasus-sdk', `[SQL] ${sql}${params ? ` -- params: ${JSON.stringify(params)}` : ''}`);
-    
-    const command = new ExecuteStatementCommand({
-      resourceArn: this.clusterArn,
-      secretArn: this.secretArn,
-      database: this.databaseName,
-      sql,
-      parameters: params || [],
-      includeResultMetadata: true
-    });
-
-    const result = await this.rdsDataClient.send(command);
-    logInfo('pegasus-sdk', `[SQL] rowCount: ${result.numberOfRecordsUpdated || 0} duration: ${Date.now() - start}ms`);
-    
-    return {
-      rowCount: result.numberOfRecordsUpdated || result.records?.length || 0,
-      rows: mapRecords(result.records, result.columnMetadata)
-    };
-  }
-}
-
-module.exports = PegasusConnection;
+const { RDSDataClient, ExecuteStatementCommand } = require('@aws-sdk/client-rds-data');
+const { LambdaClient, InvokeCommand } = require('@aws-sdk/client-lambda');
+const { mapRecords } = require('./db');
+const { loadConfig } = require('../config');
+const { logInfo, logError } = require('@toxplanet/tphelper/logging');
+
+class PegasusConnection {
+  constructor(config = {}) {
+    const envConfig = loadConfig(config.environment);
+    
+    this.config = { ...envConfig, ...config };
+    this.environment = this.config.environment;
+    this.region = this.config.region;
+    this.secretArn = this.config.secretArn;
+    this.clusterArn = this.config.clusterArn;
+    this.databaseName = this.config.database?.name;
+    
+    this.rdsDataClient = null;
+    this.lambdaClient = null;
+    this.isConnected = false;
+  }
+
+  async connect() {
+    if (this.isConnected) {
+      return;
+    }
+
+    this.rdsDataClient = new RDSDataClient({ region: this.region });
+    this.lambdaClient = new LambdaClient({ region: this.region });
+
+    logInfo('pegasus-sdk', 'RDS Data API client initialized');
+
+    try {
+      const command = new ExecuteStatementCommand({
+        resourceArn: this.clusterArn,
+        secretArn: this.secretArn,
+        database: this.databaseName,
+        sql: 'SELECT 1'
+      });
+      await this.rdsDataClient.send(command);
+      logInfo('pegasus-sdk', 'RDS Data API connection verified and ready');
+    } catch (err) {
+      logError('pegasus-sdk', 'PegasusConnection', 'connect.verification', err);
+      throw err;
+    }
+
+    this.isConnected = true;
+  }
+
+  /**
+   * With the RDS Data API, there are no stale connections or pool idleness.
+   * Each call is stateless. Just ensure the client is initialized.
+   *
+   * @returns {Promise<boolean>} true if a connect happened, false if already connected.
+   */
+  async ensureConnected() {
+    if (!this.isConnected) {
+      await this.connect();
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Data API is stateless, so reconnect simply means re-initializing.
+   * Called in fallback scenarios but not necessary for stale connections.
+   */
+  async reconnect() {
+    logInfo('pegasus-sdk', 'Reconnecting RDS Data API client');
+    this.isConnected = false;
+    await this.connect();
+  }
+
+  /**
+   * With stateless Data API, there is no meaningful "activity" tracking needed.
+   * This is a no-op for backward compatibility.
+   */
+  recordActivity() {
+  }
+
+  async disconnect() {
+    if (!this.isConnected) {
+      return;
+    }
+
+    this.rdsDataClient = null;
+    this.lambdaClient = null;
+    this.isConnected = false;
+    logInfo('pegasus-sdk', 'RDS Data API client disconnected');
+  }
+
+  async invokeOpenSearch(event) {
+    if (!this.lambdaClient) {
+      throw new Error('Lambda client not initialized. Call connect() first.');
+    }
+
+    const lambdaArn = process.env.PEGASUS_SDK_OPENSEARCH_LAMBDA_ARN
+      || this.config.openSearchLambdaArn
+      || null;
+
+    if (!lambdaArn) {
+      throw new Error('No OpenSearch Lambda ARN configured. Set PEGASUS_SDK_OPENSEARCH_LAMBDA_ARN or provide awsAccountId in config.');
+    }
+
+    try {
+      const response = await this.lambdaClient.send(new InvokeCommand({
+        FunctionName: lambdaArn,
+        Payload: JSON.stringify(event)
+      }));
+
+      const rawPayload = response.Payload;
+      const payload = (rawPayload instanceof Uint8Array || Buffer.isBuffer(rawPayload))
+        ? Buffer.from(rawPayload).toString('utf-8')
+        : rawPayload;
+      const parsed = JSON.parse(payload);
+      const body = JSON.parse(parsed.body);
+
+      if (!body.success) {
+        const error = new Error(body.error);
+        error.statusCode = body.statusCode;
+        error.responseBody = body.body;
+        throw error;
+      }
+
+      return body.result;
+    } catch (error) {
+      logError('pegasus-sdk', 'PegasusConnection', 'invokeOpenSearch', error);
+      throw error;
+    }
+  }
+
+  async testConnection() {
+    try {
+      if (!this.rdsDataClient) {
+        throw new Error('RDS Data API not initialized');
+      }
+
+      const command = new ExecuteStatementCommand({
+        resourceArn: this.clusterArn,
+        secretArn: this.secretArn,
+        database: this.databaseName,
+        sql: 'SELECT NOW() as current_time, version() as pg_version',
+        includeResultMetadata: true
+      });
+      
+      const result = await this.rdsDataClient.send(command);
+      const rows = mapRecords(result.records, result.columnMetadata);
+      const row = rows?.[0];
+      
+      const pgStatus = {
+        connected: true,
+        timestamp: row?.current_time,
+        version: row?.pg_version
+      };
+
+      let osStatus = null;
+      try {
+        const testSearch = await this.invokeOpenSearch({
+          operation: 'search',
+          body: {
+            size: 1,
+            query: {
+              match: { chemical_name: 'benzene' }
+            }
+          }
+        });
+        osStatus = {
+          connected: true,
+          resultsFound: testSearch.hits.total.value || 0
+        };
+      } catch (osError) {
+        osStatus = {
+          connected: false,
+          error: osError.message
+        };
+      }
+
+      return {
+        postgres: pgStatus,
+        opensearch: osStatus,
+        environment: this.environment,
+        region: this.region
+      };
+    } catch (error) {
+      return {
+        postgres: { connected: false, error: error.message },
+        opensearch: null,
+        environment: this.environment,
+        region: this.region
+      };
+    }
+  }
+
+  async query(sql, params) {
+    const start = Date.now();
+    logInfo('pegasus-sdk', `[SQL] ${sql}${params ? ` -- params: ${JSON.stringify(params)}` : ''}`);
+    
+    const command = new ExecuteStatementCommand({
+      resourceArn: this.clusterArn,
+      secretArn: this.secretArn,
+      database: this.databaseName,
+      sql,
+      parameters: params || [],
+      includeResultMetadata: true
+    });
+
+    const result = await this.rdsDataClient.send(command);
+    logInfo('pegasus-sdk', `[SQL] rowCount: ${result.numberOfRecordsUpdated || 0} duration: ${Date.now() - start}ms`);
+    
+    return {
+      rowCount: result.numberOfRecordsUpdated || result.records?.length || 0,
+      rows: mapRecords(result.records, result.columnMetadata)
+    };
+  }
+}
+
+module.exports = PegasusConnection;

package/lib/db/index.js

@@ -1,26 +1,26 @@
-function getFieldValue(field) {
-  if (!field || field.isNull) return null;
-  if ('stringValue' in field) return field.stringValue;
-  if ('longValue' in field) return field.longValue;
-  if ('doubleValue' in field) return field.doubleValue;
-  if ('booleanValue' in field) return field.booleanValue;
-  return null;
-}
-
-function mapRecord(record, columnMetadata) {
-  const obj = {};
-  columnMetadata.forEach((col, i) => {
-    obj[col.name] = getFieldValue(record[i]);
-  });
-  return obj;
-}
-
-function mapRecords(records = [], columnMetadata = []) {
-  return records.map(r => mapRecord(r, columnMetadata));
-}
-
-module.exports = {
-  getFieldValue,
-  mapRecord,
-  mapRecords
-};
+function getFieldValue(field) {
+  if (!field || field.isNull) return null;
+  if ('stringValue' in field) return field.stringValue;
+  if ('longValue' in field) return field.longValue;
+  if ('doubleValue' in field) return field.doubleValue;
+  if ('booleanValue' in field) return field.booleanValue;
+  return null;
+}
+
+function mapRecord(record, columnMetadata) {
+  const obj = {};
+  columnMetadata.forEach((col, i) => {
+    obj[col.name] = getFieldValue(record[i]);
+  });
+  return obj;
+}
+
+function mapRecords(records = [], columnMetadata = []) {
+  return records.map(r => mapRecord(r, columnMetadata));
+}
+
+module.exports = {
+  getFieldValue,
+  mapRecord,
+  mapRecords
+};

package/lib/search.js

@@ -109,8 +109,6 @@ class SearchService {
     const synonymPrefix = options.synonymPrefix !== undefined ? options.synonymPrefix : 3;
 
     try {
-      const opensearchClient = this.connection.getOpenSearchClient();
-      
       // Get CAS number variations (if applicable)
       const queryVariations = getCasNumberVariations(query);
       
@@ -123,24 +121,28 @@ class SearchService {
       const shouldClauses = [];
       
       for (const queryVariation of queryVariations) {
-        // Exact matches (configurable priority)
         shouldClauses.push(
+          // keyword / array-of-keyword fields — exact term match
           { term: { 'cas_numbers': { value: queryVariation, boost: casExact } } },
-          { term: { 'chemical_name.keyword': { value: queryVariation, boost: nameExact, case_insensitive: true } } },
           { term: { 'identifier_values': { value: queryVariation, boost: identifierExact } } },
+          // keyword sub-fields — exact + prefix (present when index has .keyword multi-field)
+          { term: { 'chemical_name.keyword': { value: queryVariation, boost: nameExact, case_insensitive: true } } },
           { term: { 'synonyms.keyword': { value: queryVariation, boost: synonymExact, case_insensitive: true } } },
-          // Prefix matches (configurable priority)
-          { prefix: { 'cas_numbers': { value: queryVariation, boost: casPrefix } } },
           { prefix: { 'chemical_name.keyword': { value: queryVariation, boost: namePrefix, case_insensitive: true } } },
+          { prefix: { 'synonyms.keyword': { value: queryVariation, boost: synonymPrefix, case_insensitive: true } } },
+          // keyword field prefix matching for cas / identifiers
+          { prefix: { 'cas_numbers': { value: queryVariation, boost: casPrefix } } },
           { prefix: { 'identifier_values': { value: queryVariation, boost: identifierPrefix } } },
-          { prefix: { 'synonyms.keyword': { value: queryVariation, boost: synonymPrefix, case_insensitive: true } } }
+          // match queries work on plain text fields even without .keyword sub-fields
+          { match: { 'chemical_name': { query: queryVariation, boost: nameExact } } },
+          { match: { 'synonyms': { query: queryVariation, boost: synonymExact } } },
+          { match_phrase_prefix: { 'chemical_name': { query: queryVariation, boost: namePrefix } } },
+          { match_phrase_prefix: { 'synonyms': { query: queryVariation, boost: synonymPrefix } } }
         );
       }
       
-      const indexName = this.connection.getOpenSearchIndex();
-      
-      const response = await opensearchClient.search({
-        index: indexName,
+      const result = await this.connection.invokeOpenSearch({
+        operation: 'search',
         body: {
           size: limit,
           query: {
@@ -153,7 +155,7 @@ class SearchService {
         }
       });
 
-      const hits = response.body?.hits?.hits || [];
+      const hits = result?.hits?.hits || [];
       const results = hits.map((hit) => ({
         id: hit._source.postgres_id,
         name: hit._source.chemical_name,

package/package.json

@@ -1,47 +1,47 @@
-{
-  "name": "@toxplanet/pegasus-sdk",
-  "version": "1.2.1",
-  "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
-  "main": "index.js",
-  "type": "commonjs",
-  "scripts": {
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "test:ui": "vitest --ui"
-  },
-  "keywords": [
-    "elasticsearch",
-    "opensearch",
-    "postgresql",
-    "aws",
-    "chemicals",
-    "database",
-    "search",
-    "sdk",
-    "pegasus",
-    "migration"
-  ],
-  "author": "Chemical Research Development Team",
-  "license": "MIT",
-  "dependencies": {
-    "@toxplanet/tphelper": "1.2.8",
-    "@opensearch-project/opensearch": "^2.5.0",
-    "@aws-sdk/client-rds-data": "^3.490.0",
-    "@aws-sdk/client-sqs": "^3.490.0",
-    "@aws-sdk/credential-providers": "^3.490.0"
-  },
-  "engines": {
-    "node": ">=18.0.0"
-  },
-  "files": [
-    "index.js",
-    "lib/",
-    "config/",
-    "README.md",
-    "ELASTICSEARCH_CLIENT.md",
-    "LICENSE"
-  ],
-  "devDependencies": {
-    "vitest": "^1.2.0"
-  }
-}
+{
+  "name": "@toxplanet/pegasus-sdk",
+  "version": "1.2.3",
+  "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
+  "main": "index.js",
+  "type": "commonjs",
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:ui": "vitest --ui",
+    "test:e2e": "vitest run --config vitest.e2e.config.js"
+  },
+  "keywords": [
+    "elasticsearch",
+    "opensearch",
+    "postgresql",
+    "aws",
+    "chemicals",
+    "database",
+    "search",
+    "sdk",
+    "pegasus",
+    "migration"
+  ],
+  "author": "Chemical Research Development Team",
+  "license": "MIT",
+  "dependencies": {
+    "@toxplanet/tphelper": "1.2.8",
+    "@aws-sdk/client-lambda": "^3.490.0",
+    "@aws-sdk/client-rds-data": "^3.490.0",
+    "@aws-sdk/client-sqs": "^3.490.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "index.js",
+    "lib/",
+    "config/",
+    "README.md",
+    "ELASTICSEARCH_CLIENT.md",
+    "LICENSE"
+  ],
+  "devDependencies": {
+    "vitest": "^1.2.0"
+  }
+}