@toxplanet/pegasus-sdk 1.0.0

package/env.example

@@ -0,0 +1,3 @@
+NODE_ENV=development
+AWS_REGION=us-east-1
+OPENSEARCH_ENDPOINT=https://your-collection.us-east-1.aoss.amazonaws.com

package/index.d.ts

@@ -0,0 +1,215 @@
+export interface PegasusConfig {
+  environment?: string;
+  region?: string;
+  secretName?: string;
+  openSearchEndpoint?: string;
+  maxConnections?: number;
+  minConnections?: number;
+  idleTimeoutMillis?: number;
+  connectionTimeoutMillis?: number;
+  statementTimeout?: number;
+  queryTimeout?: number;
+}
+
+export interface Chemical {
+  chemical_id?: string;
+  source_id: string;
+  chemical_name: string;
+  chemical_identifiers: Identifier[];
+  chemical_synonyms: string[];
+  chemical_categories: string[];
+  chemical_meta: Record<string, any>;
+  created_at?: Date;
+  updated_at?: Date;
+}
+
+export interface Identifier {
+  type: string;
+  value: string;
+}
+
+export interface Document {
+  document_id?: string;
+  document_path: string;
+  processed_at?: Date;
+  cas_count?: number;
+}
+
+export interface OutboxEntry {
+  outbox_id: number;
+  chemical_id: string;
+  operation: 'INSERT' | 'UPDATE' | 'DELETE';
+  created_at: Date;
+  processed_at?: Date;
+  retry_count: number;
+  last_error?: string;
+}
+
+export interface ConnectionStatus {
+  postgres: {
+    connected: boolean;
+    timestamp?: Date;
+    version?: string;
+    poolSize?: number;
+    idleConnections?: number;
+    waitingRequests?: number;
+    error?: string;
+  };
+  opensearch: {
+    connected: boolean;
+    version?: string;
+    cluster?: string;
+    error?: string;
+  } | null;
+  environment: string;
+  region: string;
+}
+
+export declare class PegasusConnection {
+  constructor(config?: PegasusConfig);
+  getSecret(): Promise<any>;
+  connect(): Promise<void>;
+  disconnect(): Promise<void>;
+  getPostgresClient(): any;
+  getOpenSearchClient(): any;
+  testConnection(): Promise<ConnectionStatus>;
+  query(sql: string, params?: any[]): Promise<any>;
+  getClient(): Promise<any>;
+  transaction(callback: (client: any) => Promise<any>): Promise<any>;
+}
+
+export declare class ChemicalsService {
+  constructor(connection: PegasusConnection);
+  bulkIndexFielded(documents: any[]): Promise<void>;
+  bulkIndexFulltext(documents: any[]): Promise<void>;
+  bulkIndexSubstances(substances: any[]): Promise<void>;
+  createChemical(chemical: Chemical): Promise<Chemical>;
+  updateChemical(chemicalId: string, updates: Partial<Chemical>): Promise<Chemical>;
+  deleteChemical(chemicalId: string): Promise<void>;
+  deleteBySourceId(sourceId: string): Promise<void>;
+  deleteCollection(collectionName: string): Promise<number>;
+  updateCollectionProperty(collectionName: string, propertyPath: string, newValue: any): Promise<number>;
+  bulkUpdateProperty(filter: any, propertyPath: string, newValue: any): Promise<number>;
+  getChemicalById(chemicalId: string): Promise<Chemical>;
+  getChemicalBySourceId(sourceId: string): Promise<Chemical>;
+  getChemicalsByCAS(casNumber: string): Promise<Chemical[]>;
+  getChemicalsByIdentifier(identifierType: string, identifierValue: string): Promise<Chemical[]>;
+  countAll(): Promise<number>;
+  countByCollection(collectionName: string): Promise<number>;
+  countByIdentifier(identifierValue: string): Promise<number>;
+  countByCAS(casNumber: string): Promise<number>;
+  getTotalSynonymCount(): Promise<number>;
+  getSynonymCount(synonymTerm: string): Promise<number>;
+  convertIdentifier(fromIdentifier: string, toIdentifierType: string): Promise<any>;
+  convertIdentifiersBatch(fromIdentifiers: string[], toIdentifierType: string): Promise<any[]>;
+  searchByName(searchTerm: string, limit: number): Promise<Chemical[]>;
+  searchBySynonym(synonymTerm: string, limit: number): Promise<Chemical[]>;
+  findChemicalsWithoutDocuments(collectionName: string, searchTerm: string, pageSize: number): Promise<Chemical[]>;
+  countChemicalsWithoutDocuments(collectionName: string): Promise<number>;
+}
+
+export declare class DocumentsService {
+  constructor(connection: PegasusConnection);
+  createDocument(documentPath: string, casNumbers: string[]): Promise<Document>;
+  bulkCreateDocuments(documents: any[]): Promise<void>;
+  updateDocument(documentId: string, updates: Partial<Document>): Promise<Document>;
+  deleteDocument(documentId: string): Promise<void>;
+  deleteDocumentByPath(documentPath: string): Promise<void>;
+  getDocumentById(documentId: string): Promise<Document>;
+  getDocumentByPath(documentPath: string): Promise<Document>;
+  getDocumentsByCAS(casNumber: string): Promise<Document[]>;
+  getCASByDocument(documentId: string): Promise<string[]>;
+  getCASByDocumentPath(documentPath: string): Promise<string[]>;
+  addCASToDocument(documentId: string, casNumber: string): Promise<void>;
+  addCASToDocumentBatch(documentId: string, casNumbers: string[]): Promise<void>;
+  removeCASFromDocument(documentId: string, casNumber: string): Promise<void>;
+  findDocumentsWithMultipleCAS(casNumbers: string[], requireAll: boolean): Promise<Document[]>;
+  countDocuments(): Promise<number>;
+  countDocumentsByCAS(casNumber: string): Promise<number>;
+  countUniqueCAS(): Promise<number>;
+  getTopCASByDocumentCount(limit: number): Promise<any[]>;
+  extractTextFromPDF(pdfBuffer: Buffer): Promise<string>;
+  extractCASFromText(text: string): Promise<string[]>;
+  processDocument(documentPath: string, documentData: any): Promise<Document>;
+}
+
+export declare class SearchService {
+  constructor(connection: PegasusConnection);
+  searchChemicals(searchTerm: string, searchType: string, limit: number, offset: number): Promise<any[]>;
+  searchStartsWith(searchTerm: string, limit: number): Promise<any[]>;
+  searchContains(searchTerm: string, limit: number): Promise<any[]>;
+  searchExact(searchTerm: string, limit: number): Promise<any[]>;
+  searchByCAS(casNumber: string, searchType: string): Promise<any[]>;
+  searchByIdentifier(identifierValue: string, searchType: string): Promise<any[]>;
+  searchBySynonym(synonymTerm: string, searchType: string): Promise<any[]>;
+  advancedSearch(queryBuilder: any): Promise<any[]>;
+  searchWithFilters(searchTerm: string, filters: any, limit: number): Promise<any[]>;
+  searchByCollection(collectionName: string, searchTerm: string, limit: number): Promise<any[]>;
+  aggregateByCategory(): Promise<any>;
+  aggregateByIdentifierType(): Promise<any>;
+  getSearchSuggestions(partialTerm: string, limit: number): Promise<string[]>;
+  findSimilarChemicals(chemicalId: string, limit: number): Promise<any[]>;
+}
+
+export declare class SyncService {
+  constructor(connection: PegasusConnection);
+  syncBatch(batchSize: number): Promise<number>;
+  syncAll(): Promise<number>;
+  syncContinuous(intervalMs: number): Promise<void>;
+  stopContinuousSync(): Promise<void>;
+  getPendingCount(): Promise<number>;
+  getOldestPending(): Promise<Date>;
+  getSyncLag(): Promise<number>;
+  getFailedEntries(minRetryCount: number): Promise<OutboxEntry[]>;
+  retryFailed(outboxId: number): Promise<void>;
+  retryAllFailed(): Promise<number>;
+  markAsProcessed(outboxId: number): Promise<void>;
+  deleteProcessedOlderThan(days: number): Promise<number>;
+  cleanupOutbox(daysToKeep: number): Promise<number>;
+  getSyncStats(timeWindowMinutes: number): Promise<any>;
+  getSyncThroughput(): Promise<number>;
+  verifySync(chemicalId: string): Promise<boolean>;
+  forceResync(chemicalId: string): Promise<void>;
+  getOutboxHealth(): Promise<any>;
+}
+
+
+export declare class UtilsService {
+  constructor(connection: PegasusConnection);
+  executeBatch(operations: any[], batchSize: number, concurrency: number): Promise<void>;
+  withTransaction(callback: () => Promise<any>): Promise<any>;
+  withRetry(operation: () => Promise<any>, maxRetries: number, backoffMs: number): Promise<any>;
+  validateChemical(chemical: Chemical): boolean;
+  validateDocument(document: Document): boolean;
+  validateIdentifier(identifier: Identifier): boolean;
+  validateCAS(casNumber: string): boolean;
+  transformForOpenSearch(chemical: Chemical): any;
+  transformFromElasticsearch(esDocument: any): Chemical;
+  transformFromDynamoDB(dynamoItem: any): Chemical;
+  buildOpenSearchQuery(searchTerm: string, searchType: string): any;
+  buildPostgresFilter(filters: any): any;
+  parseChemicalIdentifiers(identifiers: any): Identifier[];
+  parseSynonyms(synonyms: any): string[];
+  extractCASFromText(text: string): string[];
+  sanitizeSearchTerm(term: string): string;
+  generateSourceId(chemical: Chemical): string;
+  calculateChecksum(data: any): string;
+  formatError(error: Error): any;
+  logOperation(operation: string, duration: number, metadata: any): void;
+  getTimestamp(): Date;
+}
+
+
+export default class PegasusSDK {
+  connection: PegasusConnection;
+  chemicals: ChemicalsService;
+  documents: DocumentsService;
+  search: SearchService;
+  sync: SyncService;
+  utils: UtilsService;
+
+  constructor(config: PegasusConfig);
+  connect(): Promise<void>;
+  disconnect(): Promise<void>;
+  healthCheck(): Promise<any>;
+}

package/index.js

@@ -0,0 +1,37 @@
+const PegasusConnection = require('./lib/connection');
+const ChemicalsService = require('./lib/chemicals');
+const DocumentsService = require('./lib/documents');
+const SearchService = require('./lib/search');
+const SyncService = require('./lib/sync');
+const UtilsService = require('./lib/utils');
+
+class PegasusSDK {
+  constructor(config) {
+    this.connection = new PegasusConnection(config);
+    this.chemicals = new ChemicalsService(this.connection);
+    this.documents = new DocumentsService(this.connection);
+    this.search = new SearchService(this.connection);
+    this.sync = new SyncService(this.connection);
+    this.utils = new UtilsService(this.connection);
+  }
+
+  async connect() {
+    return this.connection.connect();
+  }
+
+  async disconnect() {
+    return this.connection.disconnect();
+  }
+
+  async healthCheck() {
+    return this.connection.testConnection();
+  }
+}
+
+module.exports = PegasusSDK;
+module.exports.PegasusConnection = PegasusConnection;
+module.exports.ChemicalsService = ChemicalsService;
+module.exports.DocumentsService = DocumentsService;
+module.exports.SearchService = SearchService;
+module.exports.SyncService = SyncService;
+module.exports.UtilsService = UtilsService;

package/lib/chemicals.js

@@ -0,0 +1,57 @@
+class ChemicalsService {
+  constructor(connection) {}
+
+  async bulkIndexFielded(documents) {}
+
+  async bulkIndexFulltext(documents) {}
+
+  async bulkIndexSubstances(substances) {}
+
+  async createChemical(chemical) {}
+
+  async updateChemical(chemicalId, updates) {}
+
+  async deleteChemical(chemicalId) {}
+
+  async deleteBySourceId(sourceId) {}
+
+  async deleteCollection(collectionName) {}
+
+  async updateCollectionProperty(collectionName, propertyPath, newValue) {}
+
+  async bulkUpdateProperty(filter, propertyPath, newValue) {}
+
+  async getChemicalById(chemicalId) {}
+
+  async getChemicalBySourceId(sourceId) {}
+
+  async getChemicalsByCAS(casNumber) {}
+
+  async getChemicalsByIdentifier(identifierType, identifierValue) {}
+
+  async countAll() {}
+
+  async countByCollection(collectionName) {}
+
+  async countByIdentifier(identifierValue) {}
+
+  async countByCAS(casNumber) {}
+
+  async getTotalSynonymCount() {}
+
+  async getSynonymCount(synonymTerm) {}
+
+  async convertIdentifier(fromIdentifier, toIdentifierType) {}
+
+  async convertIdentifiersBatch(fromIdentifiers, toIdentifierType) {}
+
+  async searchByName(searchTerm, limit) {}
+
+  async searchBySynonym(synonymTerm, limit) {}
+
+  async findChemicalsWithoutDocuments(collectionName, searchTerm, pageSize) {}
+
+  async countChemicalsWithoutDocuments(collectionName) {}
+}
+
+module.exports = ChemicalsService;

package/lib/connection.js

@@ -0,0 +1,206 @@
+const { Pool } = require('pg');
+const { Client } = require('@opensearch-project/opensearch');
+const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager');
+const { AwsSigv4Signer } = require('@opensearch-project/opensearch/aws');
+const { defaultProvider } = require('@aws-sdk/credential-providers');
+
+class PegasusConnection {
+  constructor(config = {}) {
+    this.config = config;
+    this.environment = config.environment || process.env.NODE_ENV || 'development';
+    this.region = config.region || process.env.AWS_REGION || 'us-east-1';
+    this.secretName = config.secretName || `pegasus/${this.environment}/database`;
+    this.openSearchEndpoint = config.openSearchEndpoint || process.env.OPENSEARCH_ENDPOINT;
+    
+    this.pgPool = null;
+    this.osClient = null;
+    this.secretsClient = null;
+    this.cachedSecret = null;
+    this.isConnected = false;
+  }
+
+  async getSecret() {
+    if (this.cachedSecret) {
+      return this.cachedSecret;
+    }
+
+    if (!this.secretsClient) {
+      this.secretsClient = new SecretsManagerClient({ region: this.region });
+    }
+
+    const command = new GetSecretValueCommand({ SecretId: this.secretName });
+    const response = await this.secretsClient.send(command);
+    
+    if (response.SecretString) {
+      this.cachedSecret = JSON.parse(response.SecretString);
+      return this.cachedSecret;
+    }
+    
+    throw new Error(`Secret ${this.secretName} does not contain SecretString`);
+  }
+
+  async connect() {
+    if (this.isConnected) {
+      return;
+    }
+
+    const secret = await this.getSecret();
+    
+    const poolConfig = {
+      host: secret.host || secret.endpoint,
+      port: secret.port || 5432,
+      database: secret.database || secret.dbname || 'chemicals',
+      user: secret.username || secret.user,
+      password: secret.password,
+      max: this.config.maxConnections || 2,
+      min: this.config.minConnections || 0,
+      idleTimeoutMillis: this.config.idleTimeoutMillis || 30000,
+      connectionTimeoutMillis: this.config.connectionTimeoutMillis || 5000,
+      allowExitOnIdle: true,
+      ssl: {
+        rejectUnauthorized: true
+      },
+      statement_timeout: this.config.statementTimeout || 30000,
+      query_timeout: this.config.queryTimeout || 30000
+    };
+
+    this.pgPool = new Pool(poolConfig);
+
+    this.pgPool.on('error', (err) => {
+      console.error('Unexpected PostgreSQL pool error:', err);
+    });
+
+    this.pgPool.on('connect', () => {
+      console.log('PostgreSQL client connected');
+    });
+
+    this.pgPool.on('remove', () => {
+      console.log('PostgreSQL client removed from pool');
+    });
+
+    if (this.openSearchEndpoint) {
+      this.osClient = new Client({
+        ...AwsSigv4Signer({
+          region: this.region,
+          service: 'aoss',
+          getCredentials: () => {
+            const credentialsProvider = defaultProvider();
+            return credentialsProvider();
+          }
+        }),
+        node: this.openSearchEndpoint
+      });
+    }
+
+    this.isConnected = true;
+  }
+
+  async disconnect() {
+    if (!this.isConnected) {
+      return;
+    }
+
+    if (this.pgPool) {
+      await this.pgPool.end();
+      this.pgPool = null;
+    }
+
+    this.osClient = null;
+    this.isConnected = false;
+    this.cachedSecret = null;
+  }
+
+  getPostgresClient() {
+    if (!this.pgPool) {
+      throw new Error('PostgreSQL connection not established. Call connect() first.');
+    }
+    return this.pgPool;
+  }
+
+  getOpenSearchClient() {
+    if (!this.osClient) {
+      throw new Error('OpenSearch connection not established. Call connect() first or provide openSearchEndpoint.');
+    }
+    return this.osClient;
+  }
+
+  async testConnection() {
+    try {
+      if (this.pgPool) {
+        const client = await this.pgPool.connect();
+        const result = await client.query('SELECT NOW() as current_time, version() as pg_version');
+        client.release();
+        
+        const pgStatus = {
+          connected: true,
+          timestamp: result.rows[0].current_time,
+          version: result.rows[0].pg_version,
+          poolSize: this.pgPool.totalCount,
+          idleConnections: this.pgPool.idleCount,
+          waitingRequests: this.pgPool.waitingCount
+        };
+
+        let osStatus = null;
+        if (this.osClient) {
+          try {
+            const osInfo = await this.osClient.info();
+            osStatus = {
+              connected: true,
+              version: osInfo.body.version.number,
+              cluster: osInfo.body.cluster_name
+            };
+          } catch (osError) {
+            osStatus = {
+              connected: false,
+              error: osError.message
+            };
+          }
+        }
+
+        return {
+          postgres: pgStatus,
+          opensearch: osStatus,
+          environment: this.environment,
+          region: this.region
+        };
+      }
+      
+      throw new Error('No active connections');
+    } catch (error) {
+      return {
+        postgres: { connected: false, error: error.message },
+        opensearch: null,
+        environment: this.environment,
+        region: this.region
+      };
+    }
+  }
+
+  async query(sql, params) {
+    const pool = this.getPostgresClient();
+    return pool.query(sql, params);
+  }
+
+  async getClient() {
+    const pool = this.getPostgresClient();
+    return pool.connect();
+  }
+
+  async transaction(callback) {
+    const client = await this.getClient();
+    
+    try {
+      await client.query('BEGIN');
+      const result = await callback(client);
+      await client.query('COMMIT');
+      return result;
+    } catch (error) {
+      await client.query('ROLLBACK');
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+}
+
+module.exports = PegasusConnection;

package/lib/documents.js

@@ -0,0 +1,47 @@
+class DocumentsService {
+  constructor(connection) {}
+
+  async createDocument(documentPath, casNumbers) {}
+
+  async bulkCreateDocuments(documents) {}
+
+  async updateDocument(documentId, updates) {}
+
+  async deleteDocument(documentId) {}
+
+  async deleteDocumentByPath(documentPath) {}
+
+  async getDocumentById(documentId) {}
+
+  async getDocumentByPath(documentPath) {}
+
+  async getDocumentsByCAS(casNumber) {}
+
+  async getCASByDocument(documentId) {}
+
+  async getCASByDocumentPath(documentPath) {}
+
+  async addCASToDocument(documentId, casNumber) {}
+
+  async addCASToDocumentBatch(documentId, casNumbers) {}
+
+  async removeCASFromDocument(documentId, casNumber) {}
+
+  async findDocumentsWithMultipleCAS(casNumbers, requireAll) {}
+
+  async countDocuments() {}
+
+  async countDocumentsByCAS(casNumber) {}
+
+  async countUniqueCAS() {}
+
+  async getTopCASByDocumentCount(limit) {}
+
+  async extractTextFromPDF(pdfBuffer) {}
+
+  async extractCASFromText(text) {}
+
+  async processDocument(documentPath, documentData) {}
+}
+
+module.exports = DocumentsService;

package/lib/search.js

@@ -0,0 +1,33 @@
+class SearchService {
+  constructor(connection) {}
+
+  async searchChemicals(searchTerm, searchType, limit, offset) {}
+
+  async searchStartsWith(searchTerm, limit) {}
+
+  async searchContains(searchTerm, limit) {}
+
+  async searchExact(searchTerm, limit) {}
+
+  async searchByCAS(casNumber, searchType) {}
+
+  async searchByIdentifier(identifierValue, searchType) {}
+
+  async searchBySynonym(synonymTerm, searchType) {}
+
+  async advancedSearch(queryBuilder) {}
+
+  async searchWithFilters(searchTerm, filters, limit) {}
+
+  async searchByCollection(collectionName, searchTerm, limit) {}
+
+  async aggregateByCategory() {}
+
+  async aggregateByIdentifierType() {}
+
+  async getSearchSuggestions(partialTerm, limit) {}
+
+  async findSimilarChemicals(chemicalId, limit) {}
+}
+
+module.exports = SearchService;

package/lib/sync.js

@@ -0,0 +1,41 @@
+class SyncService {
+  constructor(connection) {}
+
+  async syncBatch(batchSize) {}
+
+  async syncAll() {}
+
+  async syncContinuous(intervalMs) {}
+
+  async stopContinuousSync() {}
+
+  async getPendingCount() {}
+
+  async getOldestPending() {}
+
+  async getSyncLag() {}
+
+  async getFailedEntries(minRetryCount) {}
+
+  async retryFailed(outboxId) {}
+
+  async retryAllFailed() {}
+
+  async markAsProcessed(outboxId) {}
+
+  async deleteProcessedOlderThan(days) {}
+
+  async cleanupOutbox(daysToKeep) {}
+
+  async getSyncStats(timeWindowMinutes) {}
+
+  async getSyncThroughput() {}
+
+  async verifySync(chemicalId) {}
+
+  async forceResync(chemicalId) {}
+
+  async getOutboxHealth() {}
+}
+
+module.exports = SyncService;

package/lib/utils.js

@@ -0,0 +1,47 @@
+class UtilsService {
+  constructor(connection) {}
+
+  async executeBatch(operations, batchSize, concurrency) {}
+
+  async withTransaction(callback) {}
+
+  async withRetry(operation, maxRetries, backoffMs) {}
+
+  validateChemical(chemical) {}
+
+  validateDocument(document) {}
+
+  validateIdentifier(identifier) {}
+
+  validateCAS(casNumber) {}
+
+  transformForOpenSearch(chemical) {}
+
+  transformFromElasticsearch(esDocument) {}
+
+  transformFromDynamoDB(dynamoItem) {}
+
+  buildOpenSearchQuery(searchTerm, searchType) {}
+
+  buildPostgresFilter(filters) {}
+
+  parseChemicalIdentifiers(identifiers) {}
+
+  parseSynonyms(synonyms) {}
+
+  extractCASFromText(text) {}
+
+  sanitizeSearchTerm(term) {}
+
+  generateSourceId(chemical) {}
+
+  calculateChecksum(data) {}
+
+  formatError(error) {}
+
+  logOperation(operation, duration, metadata) {}
+
+  getTimestamp() {}
+}
+
+module.exports = UtilsService;

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@toxplanet/pegasus-sdk",
+  "version": "1.0.0",
+  "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture",
+  "main": "index.js",
+  "scripts": {
+  },
+  "keywords": [
+  ],
+  "author": "Chemical Research Development Team",
+  "license": "MIT",
+  "dependencies": {
+    "pg": "^8.11.3",
+    "@opensearch-project/opensearch": "^2.5.0",
+    "@aws-sdk/client-opensearch-serverless": "^3.490.0",
+    "@aws-sdk/client-secrets-manager": "^3.490.0",
+    "@aws-sdk/credential-providers": "^3.490.0",
+    "aws4fetch": "^1.0.18"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}