@toxplanet/pegasus-sdk 1.0.1 → 1.1.0

package/lib/sync.js

@@ -1,41 +1,43 @@
-class SyncService {
-  constructor(connection) {}
-
-  async syncBatch(batchSize) {}
-
-  async syncAll() {}
-
-  async syncContinuous(intervalMs) {}
-
-  async stopContinuousSync() {}
-
-  async getPendingCount() {}
-
-  async getOldestPending() {}
-
-  async getSyncLag() {}
-
-  async getFailedEntries(minRetryCount) {}
-
-  async retryFailed(outboxId) {}
-
-  async retryAllFailed() {}
-
-  async markAsProcessed(outboxId) {}
-
-  async deleteProcessedOlderThan(days) {}
-
-  async cleanupOutbox(daysToKeep) {}
-
-  async getSyncStats(timeWindowMinutes) {}
-
-  async getSyncThroughput() {}
-
-  async verifySync(chemicalId) {}
-
-  async forceResync(chemicalId) {}
-
-  async getOutboxHealth() {}
-}
-
-module.exports = SyncService;
+class SyncService {
+  constructor(connection) {
+    this.connection = connection;
+  }
+
+  async syncBatch(batchSize) {}
+
+  async syncAll() {}
+
+  async syncContinuous(intervalMs) {}
+
+  async stopContinuousSync() {}
+
+  async getPendingCount() {}
+
+  async getOldestPending() {}
+
+  async getSyncLag() {}
+
+  async getFailedEntries(minRetryCount) {}
+
+  async retryFailed(outboxId) {}
+
+  async retryAllFailed() {}
+
+  async markAsProcessed(outboxId) {}
+
+  async deleteProcessedOlderThan(days) {}
+
+  async cleanupOutbox(daysToKeep) {}
+
+  async getSyncStats(timeWindowMinutes) {}
+
+  async getSyncThroughput() {}
+
+  async verifySync(chemicalId) {}
+
+  async forceResync(chemicalId) {}
+
+  async getOutboxHealth() {}
+}
+
+module.exports = SyncService;

package/lib/utils.js

@@ -1,47 +1,49 @@
-class UtilsService {
-  constructor(connection) {}
-
-  async executeBatch(operations, batchSize, concurrency) {}
-
-  async withTransaction(callback) {}
-
-  async withRetry(operation, maxRetries, backoffMs) {}
-
-  validateChemical(chemical) {}
-
-  validateDocument(document) {}
-
-  validateIdentifier(identifier) {}
-
-  validateCAS(casNumber) {}
-
-  transformForOpenSearch(chemical) {}
-
-  transformFromElasticsearch(esDocument) {}
-
-  transformFromDynamoDB(dynamoItem) {}
-
-  buildOpenSearchQuery(searchTerm, searchType) {}
-
-  buildPostgresFilter(filters) {}
-
-  parseChemicalIdentifiers(identifiers) {}
-
-  parseSynonyms(synonyms) {}
-
-  extractCASFromText(text) {}
-
-  sanitizeSearchTerm(term) {}
-
-  generateSourceId(chemical) {}
-
-  calculateChecksum(data) {}
-
-  formatError(error) {}
-
-  logOperation(operation, duration, metadata) {}
-
-  getTimestamp() {}
-}
-
-module.exports = UtilsService;
+class UtilsService {
+  constructor(connection) {
+    this.connection = connection;
+  }
+
+  async executeBatch(operations, batchSize, concurrency) {}
+
+  async withTransaction(callback) {}
+
+  async withRetry(operation, maxRetries, backoffMs) {}
+
+  validateChemical(chemical) {}
+
+  validateDocument(document) {}
+
+  validateIdentifier(identifier) {}
+
+  validateCAS(casNumber) {}
+
+  transformForOpenSearch(chemical) {}
+
+  transformFromElasticsearch(esDocument) {}
+
+  transformFromDynamoDB(dynamoItem) {}
+
+  buildOpenSearchQuery(searchTerm, searchType) {}
+
+  buildPostgresFilter(filters) {}
+
+  parseChemicalIdentifiers(identifiers) {}
+
+  parseSynonyms(synonyms) {}
+
+  extractCASFromText(text) {}
+
+  sanitizeSearchTerm(term) {}
+
+  generateSourceId(chemical) {}
+
+  calculateChecksum(data) {}
+
+  formatError(error) {}
+
+  logOperation(operation, duration, metadata) {}
+
+  getTimestamp() {}
+}
+
+module.exports = UtilsService;

package/package.json

@@ -1,25 +1,48 @@
-{
-  "name": "@toxplanet/pegasus-sdk",
-  "version": "1.0.1",
-  "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture",
-  "main": "index.js",
-  "scripts": {
-    "test:search": "node tests/search.js",
-    "test:chemicals": "node tests/chemicals.js"
-  },
-  "keywords": [
-  ],
-  "author": "Chemical Research Development Team",
-  "license": "MIT",
-  "dependencies": {
-    "@toxplanet/tphelper": "1.2.8",
-    "pg": "^8.11.3",
-    "drizzle-orm": "^0.30.0",
-    "@opensearch-project/opensearch": "^2.5.0",
-    "@aws-sdk/client-secrets-manager": "^3.490.0",
-    "@aws-sdk/credential-providers": "^3.490.0"
-  },
-  "engines": {
-    "node": ">=18.0.0"
-  }
-}
+{
+  "name": "@toxplanet/pegasus-sdk",
+  "version": "1.1.0",
+  "description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
+  "main": "index.js",
+  "type": "commonjs",
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:ui": "vitest --ui"
+  },
+  "keywords": [
+    "elasticsearch",
+    "opensearch",
+    "postgresql",
+    "aws",
+    "chemicals",
+    "database",
+    "search",
+    "sdk",
+    "pegasus",
+    "migration"
+  ],
+  "author": "Chemical Research Development Team",
+  "license": "MIT",
+  "dependencies": {
+    "@toxplanet/tphelper": "1.2.8",
+    "pg": "^8.11.3",
+    "drizzle-orm": "^0.30.0",
+    "@opensearch-project/opensearch": "^2.5.0",
+    "@aws-sdk/client-secrets-manager": "^3.490.0",
+    "@aws-sdk/credential-providers": "^3.490.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "index.js",
+    "lib/",
+    "config/",
+    "README.md",
+    "ELASTICSEARCH_CLIENT.md",
+    "LICENSE"
+  ],
+  "devDependencies": {
+    "vitest": "^1.2.0"
+  }
+}

package/env.example

@@ -1,21 +0,0 @@
-NODE_ENV=development
-
-# Pegasus SDK Configuration Overrides
-PEGASUS_SDK_DB_SECRET_ARN=arn:aws:secretsmanager:us-east-1:292931567094:secret:rds!cluster-b851c3ce-58cc-41cd-aeae-05cc7f5e031a-ZYSjiI
-PEGASUS_SDK_OPENSEARCH_ENDPOINT=https://war8lk73nzswquk8dcz1.us-east-1.aoss.amazonaws.com
-PEGASUS_SDK_AWS_REGION=us-east-1
-
-# Database Configuration
-PEGASUS_SDK_DATABASE_HOST=cr-chemicals.cluster-cz0iqdg8irhb.us-east-1.rds.amazonaws.com
-PEGASUS_SDK_DATABASE_NAME=chemicals
-
-# OpenSearch Configuration
-PEGASUS_SDK_OPENSEARCH_INDEX=chemicals
-
-# PostgreSQL Connection Pool Settings (optional)
-PEGASUS_SDK_MAX_CONNECTIONS=2
-PEGASUS_SDK_MIN_CONNECTIONS=0
-PEGASUS_SDK_IDLE_TIMEOUT=30000
-PEGASUS_SDK_CONNECTION_TIMEOUT=5000
-PEGASUS_SDK_STATEMENT_TIMEOUT=30000
-PEGASUS_SDK_QUERY_TIMEOUT=30000

package/index.d.ts

@@ -1,252 +0,0 @@
-export interface PostgresConfig {
-  maxConnections?: number;
-  minConnections?: number;
-  idleTimeoutMillis?: number;
-  connectionTimeoutMillis?: number;
-  statementTimeout?: number;
-  queryTimeout?: number;
-}
-
-export interface DatabaseConfig {
-  host?: string;
-  name?: string;
-}
-
-export interface PegasusConfig {
-  environment?: string;
-  region?: string;
-  secretName?: string;
-  openSearchEndpoint?: string;
-  openSearchIndex?: string;
-  database?: DatabaseConfig;
-  postgres?: PostgresConfig;
-}
-
-export interface Chemical {
-  chemical_id?: string;
-  source_id: string;
-  chemical_name: string;
-  chemical_identifiers: Identifier[];
-  chemical_synonyms: string[];
-  chemical_categories: string[];
-  chemical_meta: Record<string, any>;
-  created_at?: Date;
-  updated_at?: Date;
-}
-
-export interface Identifier {
-  type: string;
-  value: string;
-}
-
-export interface Document {
-  document_id?: string;
-  document_path: string;
-  processed_at?: Date;
-  cas_count?: number;
-}
-
-export interface OutboxEntry {
-  outbox_id: number;
-  chemical_id: string;
-  operation: 'INSERT' | 'UPDATE' | 'DELETE';
-  created_at: Date;
-  processed_at?: Date;
-  retry_count: number;
-  last_error?: string;
-}
-
-export interface ConnectionStatus {
-  postgres: {
-    connected: boolean;
-    timestamp?: Date;
-    version?: string;
-    poolSize?: number;
-    idleConnections?: number;
-    waitingRequests?: number;
-    error?: string;
-  };
-  opensearch: {
-    connected: boolean;
-    version?: string;
-    cluster?: string;
-    error?: string;
-  } | null;
-  environment: string;
-  region: string;
-}
-
-export declare class PegasusConnection {
-  constructor(config?: PegasusConfig);
-  getSecret(): Promise<any>;
-  connect(): Promise<void>;
-  disconnect(): Promise<void>;
-  getPostgresClient(): any;
-  getOpenSearchClient(): any;
-  getOpenSearchIndex(): string;
-  testConnection(): Promise<ConnectionStatus>;
-  query(sql: string, params?: any[]): Promise<any>;
-  getClient(): Promise<any>;
-  transaction(callback: (client: any) => Promise<any>): Promise<any>;
-}
-
-export declare class ChemicalsService {
-  constructor(connection: PegasusConnection);
-  bulkIndexFielded(documents: any[]): Promise<void>;
-  bulkIndexFulltext(documents: any[]): Promise<void>;
-  bulkIndexSubstances(substances: any[]): Promise<void>;
-  createChemical(chemical: Chemical): Promise<Chemical>;
-  updateChemical(chemicalId: string, updates: Partial<Chemical>): Promise<Chemical>;
-  deleteChemical(chemicalId: string): Promise<void>;
-  deleteBySourceId(sourceId: string): Promise<void>;
-  deleteCollection(collectionName: string): Promise<number>;
-  updateCollectionProperty(collectionName: string, propertyPath: string, newValue: any): Promise<number>;
-  bulkUpdateProperty(filter: any, propertyPath: string, newValue: any): Promise<number>;
-  getChemicalById(chemicalId: string): Promise<Chemical>;
-  getChemicalBySourceId(sourceId: string): Promise<Chemical>;
-  getChemicalsByCAS(casNumber: string): Promise<Chemical[]>;
-  getChemicalsByIdentifier(identifierType: string, identifierValue: string): Promise<Chemical[]>;
-  countAll(): Promise<number>;
-  countByCollection(collectionName: string): Promise<number>;
-  countByIdentifier(identifierValue: string): Promise<number>;
-  countByCAS(casNumber: string): Promise<number>;
-  getTotalSynonymCount(): Promise<number>;
-  getSynonymCount(synonymTerm: string): Promise<number>;
-  convertIdentifier(fromIdentifier: string, toIdentifierType: string): Promise<any>;
-  convertIdentifiersBatch(fromIdentifiers: string[], toIdentifierType: string): Promise<any[]>;
-  searchByName(searchTerm: string, limit?: number): Promise<SearchResults>;
-  searchBySynonym(synonymTerm: string, limit?: number): Promise<SearchResults>;
-  findChemicalsWithoutDocuments(collectionName: string, searchTerm: string, pageSize: number): Promise<Chemical[]>;
-  countChemicalsWithoutDocuments(collectionName: string): Promise<number>;
-}
-
-export declare class DocumentsService {
-  constructor(connection: PegasusConnection);
-  createDocument(documentPath: string, casNumbers: string[]): Promise<Document>;
-  bulkCreateDocuments(documents: any[]): Promise<void>;
-  updateDocument(documentId: string, updates: Partial<Document>): Promise<Document>;
-  deleteDocument(documentId: string): Promise<void>;
-  deleteDocumentByPath(documentPath: string): Promise<void>;
-  getDocumentById(documentId: string): Promise<Document>;
-  getDocumentByPath(documentPath: string): Promise<Document>;
-  getDocumentsByCAS(casNumber: string): Promise<Document[]>;
-  getCASByDocument(documentId: string): Promise<string[]>;
-  getCASByDocumentPath(documentPath: string): Promise<string[]>;
-  addCASToDocument(documentId: string, casNumber: string): Promise<void>;
-  addCASToDocumentBatch(documentId: string, casNumbers: string[]): Promise<void>;
-  removeCASFromDocument(documentId: string, casNumber: string): Promise<void>;
-  findDocumentsWithMultipleCAS(casNumbers: string[], requireAll: boolean): Promise<Document[]>;
-  countDocuments(): Promise<number>;
-  countDocumentsByCAS(casNumber: string): Promise<number>;
-  countUniqueCAS(): Promise<number>;
-  getTopCASByDocumentCount(limit: number): Promise<any[]>;
-  extractTextFromPDF(pdfBuffer: Buffer): Promise<string>;
-  extractCASFromText(text: string): Promise<string[]>;
-  processDocument(documentPath: string, documentData: any): Promise<Document>;
-}
-
-export interface SearchOptions {
-  limit?: number;
-  casExact?: number;
-  casPrefix?: number;
-  nameExact?: number;
-  namePrefix?: number;
-  identifierExact?: number;
-  identifierPrefix?: number;
-  synonymExact?: number;
-  synonymPrefix?: number;
-}
-
-export interface ChemicalSearchResult {
-  id: number;
-  name: string;
-  cas: string[];
-  identifiers: string[];
-  synonyms: string[];
-  score: number;
-}
-
-export interface SearchResults {
-  results: ChemicalSearchResult[];
-}
-
-export declare class SearchService {
-  constructor(connection: PegasusConnection);
-  searchChemicals(query: string, options?: SearchOptions): Promise<SearchResults>;
-  searchStartsWith(searchTerm: string, limit?: number): Promise<SearchResults>;
-  searchContains(searchTerm: string, limit?: number): Promise<SearchResults>;
-  searchExact(searchTerm: string, limit?: number): Promise<SearchResults>;
-  searchByCAS(casNumber: string, searchType?: string): Promise<SearchResults>;
-  searchByIdentifier(identifierValue: string, searchType?: string): Promise<SearchResults>;
-  searchBySynonym(synonymTerm: string, searchType?: string): Promise<SearchResults>;
-  advancedSearch(queryBuilder: any): Promise<any[]>;
-  searchWithFilters(searchTerm: string, filters: any, limit: number): Promise<any[]>;
-  searchByCollection(collectionName: string, searchTerm: string, limit: number): Promise<any[]>;
-  aggregateByCategory(): Promise<any>;
-  aggregateByIdentifierType(): Promise<any>;
-  getSearchSuggestions(partialTerm: string, limit: number): Promise<string[]>;
-  findSimilarChemicals(chemicalId: string, limit: number): Promise<any[]>;
-}
-
-export declare class SyncService {
-  constructor(connection: PegasusConnection);
-  syncBatch(batchSize: number): Promise<number>;
-  syncAll(): Promise<number>;
-  syncContinuous(intervalMs: number): Promise<void>;
-  stopContinuousSync(): Promise<void>;
-  getPendingCount(): Promise<number>;
-  getOldestPending(): Promise<Date>;
-  getSyncLag(): Promise<number>;
-  getFailedEntries(minRetryCount: number): Promise<OutboxEntry[]>;
-  retryFailed(outboxId: number): Promise<void>;
-  retryAllFailed(): Promise<number>;
-  markAsProcessed(outboxId: number): Promise<void>;
-  deleteProcessedOlderThan(days: number): Promise<number>;
-  cleanupOutbox(daysToKeep: number): Promise<number>;
-  getSyncStats(timeWindowMinutes: number): Promise<any>;
-  getSyncThroughput(): Promise<number>;
-  verifySync(chemicalId: string): Promise<boolean>;
-  forceResync(chemicalId: string): Promise<void>;
-  getOutboxHealth(): Promise<any>;
-}
-
-
-export declare class UtilsService {
-  constructor(connection: PegasusConnection);
-  executeBatch(operations: any[], batchSize: number, concurrency: number): Promise<void>;
-  withTransaction(callback: () => Promise<any>): Promise<any>;
-  withRetry(operation: () => Promise<any>, maxRetries: number, backoffMs: number): Promise<any>;
-  validateChemical(chemical: Chemical): boolean;
-  validateDocument(document: Document): boolean;
-  validateIdentifier(identifier: Identifier): boolean;
-  validateCAS(casNumber: string): boolean;
-  transformForOpenSearch(chemical: Chemical): any;
-  transformFromElasticsearch(esDocument: any): Chemical;
-  transformFromDynamoDB(dynamoItem: any): Chemical;
-  buildOpenSearchQuery(searchTerm: string, searchType: string): any;
-  buildPostgresFilter(filters: any): any;
-  parseChemicalIdentifiers(identifiers: any): Identifier[];
-  parseSynonyms(synonyms: any): string[];
-  extractCASFromText(text: string): string[];
-  sanitizeSearchTerm(term: string): string;
-  generateSourceId(chemical: Chemical): string;
-  calculateChecksum(data: any): string;
-  formatError(error: Error): any;
-  logOperation(operation: string, duration: number, metadata: any): void;
-  getTimestamp(): Date;
-}
-
-
-export default class PegasusSDK {
-  connection: PegasusConnection;
-  chemicals: ChemicalsService;
-  documents: DocumentsService;
-  search: SearchService;
-  sync: SyncService;
-  utils: UtilsService;
-
-  constructor(config: PegasusConfig);
-  connect(): Promise<void>;
-  disconnect(): Promise<void>;
-  healthCheck(): Promise<any>;
-}