@toxplanet/pegasus-sdk 1.1.19 → 1.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/environment.acc.js +4 -15
- package/config/environment.dev.js +4 -15
- package/config/environment.prod.js +4 -15
- package/config/environment.qa.js +4 -15
- package/lib/chemicals.js +29 -58
- package/lib/connection.js +122 -204
- package/lib/db/index.js +3 -3
- package/package.json +2 -3
|
@@ -3,24 +3,13 @@ module.exports = {
|
|
|
3
3
|
region: 'us-east-1',
|
|
4
4
|
awsAccountId: '605134466764',
|
|
5
5
|
sourceService: 'pegasus-sdk',
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
openSearchIndex: 'chemicals',
|
|
6
|
+
secretArn: 'arn:aws:secretsmanager:us-east-1:605134466764:secret:rds!cluster-9b502dde-5e2a-49db-b2c5-9801141ee40b-gkHbLm',
|
|
7
|
+
clusterArn: 'arn:aws:rds:us-east-1:605134466764:cluster:cr-chemicals-acc',
|
|
9
8
|
database: {
|
|
10
|
-
host: 'cr-chemicals-acc.cluster-czgc4c20yerz.us-east-1.rds.amazonaws.com',
|
|
11
9
|
name: 'chemicals'
|
|
12
10
|
},
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
minConnections: 0,
|
|
16
|
-
idleTimeoutMillis: 30000,
|
|
17
|
-
connectionTimeoutMillis: 5000,
|
|
18
|
-
statementTimeout: 30000,
|
|
19
|
-
queryTimeout: 30000,
|
|
20
|
-
ssl: {
|
|
21
|
-
rejectUnauthorized: false
|
|
22
|
-
}
|
|
23
|
-
},
|
|
11
|
+
openSearchEndpoint: 'https://1pbu0yqr197lq07hfcjh.us-east-1.aoss.amazonaws.com',
|
|
12
|
+
openSearchIndex: 'chemicals',
|
|
24
13
|
indexRoutes: {
|
|
25
14
|
chemicals: ['chemicals*'],
|
|
26
15
|
documents: ['documents*'],
|
|
@@ -3,24 +3,13 @@ module.exports = {
|
|
|
3
3
|
region: 'us-east-1',
|
|
4
4
|
awsAccountId: '292931567094',
|
|
5
5
|
sourceService: 'pegasus-sdk',
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
openSearchIndex: 'chemicals',
|
|
6
|
+
secretArn: 'arn:aws:secretsmanager:us-east-1:292931567094:secret:rds!cluster-b851c3ce-58cc-41cd-aeae-05cc7f5e031a-ZYSjiI',
|
|
7
|
+
clusterArn: 'arn:aws:rds:us-east-1:292931567094:cluster:cr-chemicals',
|
|
9
8
|
database: {
|
|
10
|
-
host: 'cr-chemicals.cluster-cz0iqdg8irhb.us-east-1.rds.amazonaws.com',
|
|
11
9
|
name: 'chemicals'
|
|
12
10
|
},
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
minConnections: 1,
|
|
16
|
-
idleTimeoutMillis: 0,
|
|
17
|
-
connectionTimeoutMillis: 15000,
|
|
18
|
-
statementTimeout: 30000,
|
|
19
|
-
queryTimeout: 30000,
|
|
20
|
-
ssl: {
|
|
21
|
-
rejectUnauthorized: false
|
|
22
|
-
}
|
|
23
|
-
},
|
|
11
|
+
openSearchEndpoint: 'https://war8lk73nzswquk8dcz1.us-east-1.aoss.amazonaws.com',
|
|
12
|
+
openSearchIndex: 'chemicals',
|
|
24
13
|
indexRoutes: {
|
|
25
14
|
chemicals: ['chemicals*'],
|
|
26
15
|
documents: ['documents*'],
|
|
@@ -3,24 +3,13 @@ module.exports = {
|
|
|
3
3
|
region: 'us-east-1',
|
|
4
4
|
awsAccountId: '147997144422',
|
|
5
5
|
sourceService: 'pegasus-sdk',
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
openSearchIndex: 'chemicals',
|
|
6
|
+
secretArn: 'arn:aws:secretsmanager:us-east-1:147997144422:secret:rds!cluster-25483b3f-3758-43ed-9548-26c91de16c2d',
|
|
7
|
+
clusterArn: 'arn:aws:rds:us-east-1:147997144422:cluster:cr-chemicals-prod',
|
|
9
8
|
database: {
|
|
10
|
-
host: 'cr-chemicals-qa.cluster-c7gakqksq9m4.us-east-1.rds.amazonaws.com',
|
|
11
9
|
name: 'chemicals'
|
|
12
10
|
},
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
minConnections: 2,
|
|
16
|
-
idleTimeoutMillis: 120000,
|
|
17
|
-
connectionTimeoutMillis: 15000,
|
|
18
|
-
statementTimeout: 120000,
|
|
19
|
-
queryTimeout: 120000,
|
|
20
|
-
ssl: {
|
|
21
|
-
rejectUnauthorized: true
|
|
22
|
-
}
|
|
23
|
-
},
|
|
11
|
+
openSearchEndpoint: 'https://odusb11s00j5hyy5r6.us-east-1.aoss.amazonaws.com',
|
|
12
|
+
openSearchIndex: 'chemicals',
|
|
24
13
|
indexRoutes: {
|
|
25
14
|
chemicals: ['chemicals*'],
|
|
26
15
|
documents: ['documents*'],
|
package/config/environment.qa.js
CHANGED
|
@@ -3,24 +3,13 @@ module.exports = {
|
|
|
3
3
|
region: 'us-east-1',
|
|
4
4
|
awsAccountId: '147997144422',
|
|
5
5
|
sourceService: 'pegasus-sdk',
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
openSearchIndex: 'chemicals',
|
|
6
|
+
secretArn: 'arn:aws:secretsmanager:us-east-1:147997144422:secret:rds!cluster-25483b3f-3758-43ed-9548-26c91de16c2d-oYjysU',
|
|
7
|
+
clusterArn: 'arn:aws:rds:us-east-1:147997144422:cluster:cr-chemicals-qa',
|
|
9
8
|
database: {
|
|
10
|
-
host: 'cr-chemicals-qa.cluster-c7gakqksq9m4.us-east-1.rds.amazonaws.com',
|
|
11
9
|
name: 'chemicals'
|
|
12
10
|
},
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
minConnections: 1,
|
|
16
|
-
idleTimeoutMillis: 60000,
|
|
17
|
-
connectionTimeoutMillis: 10000,
|
|
18
|
-
statementTimeout: 60000,
|
|
19
|
-
queryTimeout: 60000,
|
|
20
|
-
ssl: {
|
|
21
|
-
rejectUnauthorized: true
|
|
22
|
-
}
|
|
23
|
-
},
|
|
11
|
+
openSearchEndpoint: 'https://odusb11s00j5hyy5r6.us-east-1.aoss.amazonaws.com',
|
|
12
|
+
openSearchIndex: 'chemicals',
|
|
24
13
|
indexRoutes: {
|
|
25
14
|
chemicals: ['chemicals*'],
|
|
26
15
|
documents: ['documents*'],
|
package/lib/chemicals.js
CHANGED
|
@@ -22,11 +22,8 @@ class ChemicalsService {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
async getDb() {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
this.db = getDrizzle(this.connection.pgPool);
|
|
28
|
-
}
|
|
29
|
-
return this.db;
|
|
25
|
+
await this.connection.ensureConnected();
|
|
26
|
+
return this.connection.db;
|
|
30
27
|
}
|
|
31
28
|
|
|
32
29
|
async sendSqlWriteFailure({ sql, parameters, error, retryCount, failedAt }) {
|
|
@@ -76,28 +73,28 @@ class ChemicalsService {
|
|
|
76
73
|
_buildChemicalUpsertSql(chemical) {
|
|
77
74
|
const sql = [
|
|
78
75
|
'INSERT INTO chemicals (source_id, chemical_name, chemical_meta, chemical_identifiers, chemical_synonyms, chemical_categories, created_at, updated_at)',
|
|
79
|
-
'VALUES (
|
|
76
|
+
'VALUES (:source_id, :chemical_name, :chemical_meta::jsonb, :chemical_identifiers::jsonb, :chemical_synonyms, :chemical_categories, :created_at, :updated_at)',
|
|
80
77
|
'ON CONFLICT (source_id) DO UPDATE SET',
|
|
81
|
-
' chemical_name =
|
|
82
|
-
' chemical_meta =
|
|
83
|
-
' chemical_identifiers =
|
|
84
|
-
' chemical_synonyms =
|
|
85
|
-
' chemical_categories =
|
|
86
|
-
' updated_at =
|
|
78
|
+
' chemical_name = :chemical_name,',
|
|
79
|
+
' chemical_meta = :chemical_meta::jsonb,',
|
|
80
|
+
' chemical_identifiers = :chemical_identifiers::jsonb,',
|
|
81
|
+
' chemical_synonyms = :chemical_synonyms,',
|
|
82
|
+
' chemical_categories = :chemical_categories,',
|
|
83
|
+
' updated_at = :updated_at'
|
|
87
84
|
].join('\n');
|
|
88
85
|
|
|
89
86
|
const serializeDate = (d) => d instanceof Date ? d.toISOString() : d;
|
|
90
87
|
|
|
91
|
-
const parameters =
|
|
92
|
-
'
|
|
93
|
-
'
|
|
94
|
-
'
|
|
95
|
-
'
|
|
96
|
-
'
|
|
97
|
-
'
|
|
98
|
-
'
|
|
99
|
-
'
|
|
100
|
-
|
|
88
|
+
const parameters = [
|
|
89
|
+
{ name: 'source_id', value: { stringValue: chemical.sourceId } },
|
|
90
|
+
{ name: 'chemical_name', value: { stringValue: chemical.chemicalName } },
|
|
91
|
+
{ name: 'chemical_meta', value: { stringValue: JSON.stringify(chemical.chemicalMeta ?? {}) } },
|
|
92
|
+
{ name: 'chemical_identifiers', value: { stringValue: JSON.stringify(chemical.chemicalIdentifiers ?? {}) } },
|
|
93
|
+
{ name: 'chemical_synonyms', value: { stringValue: JSON.stringify(chemical.chemicalSynonyms ?? []) } },
|
|
94
|
+
{ name: 'chemical_categories', value: { stringValue: JSON.stringify(chemical.chemicalCategories ?? []) } },
|
|
95
|
+
{ name: 'created_at', value: { stringValue: serializeDate(chemical.createdAt) } },
|
|
96
|
+
{ name: 'updated_at', value: { stringValue: serializeDate(chemical.updatedAt) } }
|
|
97
|
+
];
|
|
101
98
|
|
|
102
99
|
return { sql, parameters };
|
|
103
100
|
}
|
|
@@ -175,14 +172,6 @@ class ChemicalsService {
|
|
|
175
172
|
logInfo('pegasus-sdk', `[bulkIndexFielded] Prepared chemical object: sourceId=${chemical.sourceId}, chemicalName=${chemical.chemicalName}`);
|
|
176
173
|
logInfo('pegasus-sdk', `[bulkIndexFielded] DEBUG SQL for document ${i}:\n${this._buildDebugSql(chemical)}`);
|
|
177
174
|
|
|
178
|
-
const isConnectionError = (err) =>
|
|
179
|
-
err.message?.toLowerCase().includes('timeout') ||
|
|
180
|
-
err.message?.toLowerCase().includes('connection') ||
|
|
181
|
-
err.code === 'ECONNREFUSED' ||
|
|
182
|
-
err.code === 'ETIMEDOUT';
|
|
183
|
-
|
|
184
|
-
// Use this.getDb() on each attempt so a reconnect mid-loop automatically
|
|
185
|
-
// gets a fresh Drizzle instance bound to the new pool.
|
|
186
175
|
const attemptUpsert = async () => {
|
|
187
176
|
const freshDb = await this.getDb();
|
|
188
177
|
return freshDb.insert(schema.chemicals)
|
|
@@ -211,38 +200,20 @@ class ChemicalsService {
|
|
|
211
200
|
try {
|
|
212
201
|
const [result] = await attemptUpsert();
|
|
213
202
|
logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully: ${result?.chemicalId || 'no ID returned'}`);
|
|
214
|
-
this.connection.recordActivity();
|
|
215
203
|
results.push({ index: i, success: true, result });
|
|
216
204
|
continue;
|
|
217
205
|
} catch (firstErr) {
|
|
218
206
|
lastError = firstErr;
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
continue;
|
|
230
|
-
} catch (reconnectErr) {
|
|
231
|
-
lastError = reconnectErr;
|
|
232
|
-
retryCount = 1;
|
|
233
|
-
}
|
|
234
|
-
} else {
|
|
235
|
-
logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} first attempt failed (${firstErr.message}), retrying once`);
|
|
236
|
-
try {
|
|
237
|
-
const [result] = await attemptUpsert();
|
|
238
|
-
logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully on retry: ${result?.chemicalId || 'no ID returned'}`);
|
|
239
|
-
this.connection.recordActivity();
|
|
240
|
-
results.push({ index: i, success: true, result });
|
|
241
|
-
continue;
|
|
242
|
-
} catch (retryErr) {
|
|
243
|
-
lastError = retryErr;
|
|
244
|
-
retryCount = 1;
|
|
245
|
-
}
|
|
207
|
+
logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} first attempt failed (${firstErr.message}), retrying once`);
|
|
208
|
+
|
|
209
|
+
try {
|
|
210
|
+
const [result] = await attemptUpsert();
|
|
211
|
+
logInfo('pegasus-sdk', `[bulkIndexFielded] Document ${i} indexed successfully on retry: ${result?.chemicalId || 'no ID returned'}`);
|
|
212
|
+
results.push({ index: i, success: true, result });
|
|
213
|
+
continue;
|
|
214
|
+
} catch (retryErr) {
|
|
215
|
+
lastError = retryErr;
|
|
216
|
+
retryCount = 1;
|
|
246
217
|
}
|
|
247
218
|
}
|
|
248
219
|
|
package/lib/connection.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
const {
|
|
2
|
-
const {
|
|
3
|
-
const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager');
|
|
1
|
+
const { Client: OpenSearchClient } = require('@opensearch-project/opensearch');
|
|
2
|
+
const { RDSDataClient, ExecuteStatementCommand } = require('@aws-sdk/client-rds-data');
|
|
4
3
|
const { AwsSigv4Signer } = require('@opensearch-project/opensearch/aws');
|
|
5
4
|
const { fromNodeProviderChain } = require('@aws-sdk/credential-providers');
|
|
5
|
+
const { getDrizzle, schema } = require('./db');
|
|
6
6
|
const { loadConfig } = require('../config');
|
|
7
7
|
const { logInfo, logError } = require('@toxplanet/tphelper/logging');
|
|
8
8
|
|
|
@@ -13,38 +13,16 @@ class PegasusConnection {
|
|
|
13
13
|
this.config = { ...envConfig, ...config };
|
|
14
14
|
this.environment = this.config.environment;
|
|
15
15
|
this.region = this.config.region;
|
|
16
|
-
this.
|
|
16
|
+
this.secretArn = this.config.secretArn;
|
|
17
|
+
this.clusterArn = this.config.clusterArn;
|
|
17
18
|
this.openSearchEndpoint = this.config.openSearchEndpoint;
|
|
18
19
|
this.openSearchIndex = this.config.openSearchIndex;
|
|
19
|
-
this.databaseHost = this.config.database?.host;
|
|
20
20
|
this.databaseName = this.config.database?.name;
|
|
21
21
|
|
|
22
|
-
this.
|
|
22
|
+
this.rdsDataClient = null;
|
|
23
|
+
this.db = null;
|
|
23
24
|
this.osClient = null;
|
|
24
|
-
this.secretsClient = null;
|
|
25
|
-
this.cachedSecret = null;
|
|
26
25
|
this.isConnected = false;
|
|
27
|
-
this.lastActivityAt = null;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
async getSecret() {
|
|
31
|
-
if (this.cachedSecret) {
|
|
32
|
-
return this.cachedSecret;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
if (!this.secretsClient) {
|
|
36
|
-
this.secretsClient = new SecretsManagerClient({ region: this.region });
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
const command = new GetSecretValueCommand({ SecretId: this.secretName });
|
|
40
|
-
const response = await this.secretsClient.send(command);
|
|
41
|
-
|
|
42
|
-
if (response.SecretString) {
|
|
43
|
-
this.cachedSecret = JSON.parse(response.SecretString);
|
|
44
|
-
return this.cachedSecret;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
throw new Error(`Secret ${this.secretName} does not contain SecretString`);
|
|
48
26
|
}
|
|
49
27
|
|
|
50
28
|
async connect() {
|
|
@@ -52,52 +30,32 @@ class PegasusConnection {
|
|
|
52
30
|
return;
|
|
53
31
|
}
|
|
54
32
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
const poolConfig = {
|
|
58
|
-
host: this.databaseHost,
|
|
59
|
-
port: 5432,
|
|
60
|
-
database: this.databaseName,
|
|
61
|
-
user: secret.username,
|
|
62
|
-
password: secret.password,
|
|
63
|
-
max: this.config.postgres.maxConnections,
|
|
64
|
-
min: this.config.postgres.minConnections,
|
|
65
|
-
idleTimeoutMillis: this.config.postgres.idleTimeoutMillis,
|
|
66
|
-
connectionTimeoutMillis: this.config.postgres.connectionTimeoutMillis,
|
|
67
|
-
keepAlive: true,
|
|
68
|
-
keepAliveInitialDelayMillis: 10000,
|
|
69
|
-
ssl: this.config.postgres.ssl,
|
|
70
|
-
statement_timeout: this.config.postgres.statementTimeout,
|
|
71
|
-
query_timeout: this.config.postgres.queryTimeout
|
|
72
|
-
};
|
|
73
|
-
|
|
74
|
-
this.pgPool = new Pool(poolConfig);
|
|
33
|
+
this.rdsDataClient = new RDSDataClient({ region: this.region });
|
|
75
34
|
|
|
76
|
-
this.
|
|
77
|
-
|
|
35
|
+
this.db = getDrizzle(this.rdsDataClient, {
|
|
36
|
+
resourceArn: this.clusterArn,
|
|
37
|
+
secretArn: this.secretArn,
|
|
38
|
+
database: this.databaseName
|
|
78
39
|
});
|
|
79
40
|
|
|
80
|
-
|
|
81
|
-
this.lastActivityAt = Date.now();
|
|
82
|
-
logInfo('pegasus-sdk', 'PostgreSQL client connected');
|
|
83
|
-
});
|
|
41
|
+
logInfo('pegasus-sdk', 'RDS Data API client initialized');
|
|
84
42
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
43
|
+
try {
|
|
44
|
+
const command = new ExecuteStatementCommand({
|
|
45
|
+
resourceArn: this.clusterArn,
|
|
46
|
+
secretArn: this.secretArn,
|
|
47
|
+
database: this.databaseName,
|
|
48
|
+
sql: 'SELECT 1'
|
|
49
|
+
});
|
|
50
|
+
await this.rdsDataClient.send(command);
|
|
51
|
+
logInfo('pegasus-sdk', 'RDS Data API connection verified and ready');
|
|
52
|
+
} catch (err) {
|
|
53
|
+
logError('pegasus-sdk', 'PegasusConnection', 'connect.verification', err);
|
|
54
|
+
throw err;
|
|
55
|
+
}
|
|
98
56
|
|
|
99
57
|
if (this.openSearchEndpoint) {
|
|
100
|
-
this.osClient = new
|
|
58
|
+
this.osClient = new OpenSearchClient({
|
|
101
59
|
...AwsSigv4Signer({
|
|
102
60
|
region: this.region,
|
|
103
61
|
service: 'aoss',
|
|
@@ -113,77 +71,36 @@ class PegasusConnection {
|
|
|
113
71
|
this.isConnected = true;
|
|
114
72
|
}
|
|
115
73
|
|
|
116
|
-
async reconnect() {
|
|
117
|
-
logInfo('pegasus-sdk', 'Reconnecting PostgreSQL pool (stale connection detected)');
|
|
118
|
-
if (this.pgPool) {
|
|
119
|
-
try {
|
|
120
|
-
await this.pgPool.end();
|
|
121
|
-
} catch (err) {
|
|
122
|
-
logError('pegasus-sdk', 'PegasusConnection', 'reconnect.end', err);
|
|
123
|
-
}
|
|
124
|
-
this.pgPool = null;
|
|
125
|
-
}
|
|
126
|
-
this.isConnected = false;
|
|
127
|
-
this.lastActivityAt = null;
|
|
128
|
-
await this.connect();
|
|
129
|
-
}
|
|
130
|
-
|
|
131
74
|
/**
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*/
|
|
135
|
-
recordActivity() {
|
|
136
|
-
this.lastActivityAt = Date.now();
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
/**
|
|
140
|
-
* Proactively verify the connection is alive before executing a batch of
|
|
141
|
-
* queries. If the pool has been idle long enough that connections may have
|
|
142
|
-
* gone stale, a lightweight SELECT 1 is issued first. If that fails, the
|
|
143
|
-
* pool is rebuilt before the caller's real query fires — avoiding the full
|
|
144
|
-
* connectionTimeoutMillis wait on the real query.
|
|
75
|
+
* With the RDS Data API, there are no stale connections or pool idleness.
|
|
76
|
+
* Each call is stateless. Just ensure the client is initialized.
|
|
145
77
|
*
|
|
146
|
-
* @returns {Promise<boolean>} true if a
|
|
147
|
-
* reset any cached Drizzle db instance), false if connection is healthy.
|
|
78
|
+
* @returns {Promise<boolean>} true if a connect happened, false if already connected.
|
|
148
79
|
*/
|
|
149
80
|
async ensureConnected() {
|
|
150
|
-
if (!this.
|
|
81
|
+
if (!this.isConnected || !this.db) {
|
|
151
82
|
await this.connect();
|
|
152
83
|
return true;
|
|
153
84
|
}
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
154
87
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
88
|
+
/**
|
|
89
|
+
* Data API is stateless, so reconnect simply means re-initializing.
|
|
90
|
+
* Called in fallback scenarios but not necessary for stale connections.
|
|
91
|
+
*/
|
|
92
|
+
async reconnect() {
|
|
93
|
+
logInfo('pegasus-sdk', 'Reconnecting RDS Data API client');
|
|
94
|
+
this.db = null;
|
|
95
|
+
this.isConnected = false;
|
|
96
|
+
await this.connect();
|
|
97
|
+
}
|
|
163
98
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
this.pgPool.connect(),
|
|
170
|
-
new Promise((_, reject) =>
|
|
171
|
-
setTimeout(() => reject(new Error('liveness check timeout')), 3000)
|
|
172
|
-
)
|
|
173
|
-
]);
|
|
174
|
-
await client.query('SELECT 1');
|
|
175
|
-
client.release();
|
|
176
|
-
this.lastActivityAt = Date.now();
|
|
177
|
-
logInfo('pegasus-sdk', '[ensureConnected] Liveness check passed');
|
|
178
|
-
return false;
|
|
179
|
-
} catch (err) {
|
|
180
|
-
if (client) {
|
|
181
|
-
try { client.release(true); } catch (_) {}
|
|
182
|
-
}
|
|
183
|
-
logInfo('pegasus-sdk', `[ensureConnected] Liveness check failed (${err.message}) — reconnecting pool proactively`);
|
|
184
|
-
await this.reconnect();
|
|
185
|
-
return true;
|
|
186
|
-
}
|
|
99
|
+
/**
|
|
100
|
+
* With stateless Data API, there is no meaningful "activity" tracking needed.
|
|
101
|
+
* This is a no-op for backward compatibility.
|
|
102
|
+
*/
|
|
103
|
+
recordActivity() {
|
|
187
104
|
}
|
|
188
105
|
|
|
189
106
|
async disconnect() {
|
|
@@ -191,21 +108,18 @@ class PegasusConnection {
|
|
|
191
108
|
return;
|
|
192
109
|
}
|
|
193
110
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
this.pgPool = null;
|
|
197
|
-
}
|
|
198
|
-
|
|
111
|
+
this.db = null;
|
|
112
|
+
this.rdsDataClient = null;
|
|
199
113
|
this.osClient = null;
|
|
200
114
|
this.isConnected = false;
|
|
201
|
-
|
|
115
|
+
logInfo('pegasus-sdk', 'RDS Data API client disconnected');
|
|
202
116
|
}
|
|
203
117
|
|
|
204
118
|
getPostgresClient() {
|
|
205
|
-
if (!this.
|
|
206
|
-
throw new Error('
|
|
119
|
+
if (!this.db) {
|
|
120
|
+
throw new Error('RDS Data API not initialized. Call connect() first.');
|
|
207
121
|
}
|
|
208
|
-
return this.
|
|
122
|
+
return this.db;
|
|
209
123
|
}
|
|
210
124
|
|
|
211
125
|
getOpenSearchClient() {
|
|
@@ -221,54 +135,57 @@ class PegasusConnection {
|
|
|
221
135
|
|
|
222
136
|
async testConnection() {
|
|
223
137
|
try {
|
|
224
|
-
if (this.
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
138
|
+
if (!this.db) {
|
|
139
|
+
throw new Error('RDS Data API not initialized');
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const command = new ExecuteStatementCommand({
|
|
143
|
+
resourceArn: this.clusterArn,
|
|
144
|
+
secretArn: this.secretArn,
|
|
145
|
+
database: this.databaseName,
|
|
146
|
+
sql: 'SELECT NOW() as current_time, version() as pg_version'
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
const result = await this.rdsDataClient.send(command);
|
|
150
|
+
const row = result.records?.[0];
|
|
151
|
+
|
|
152
|
+
const pgStatus = {
|
|
153
|
+
connected: true,
|
|
154
|
+
timestamp: row?.[0]?.stringValue,
|
|
155
|
+
version: row?.[1]?.stringValue
|
|
156
|
+
};
|
|
237
157
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
}
|
|
158
|
+
let osStatus = null;
|
|
159
|
+
if (this.osClient) {
|
|
160
|
+
try {
|
|
161
|
+
const indexName = this.getOpenSearchIndex();
|
|
162
|
+
const testSearch = await this.osClient.search({
|
|
163
|
+
index: indexName,
|
|
164
|
+
body: {
|
|
165
|
+
size: 1,
|
|
166
|
+
query: {
|
|
167
|
+
match: { chemical_name: 'benzene' }
|
|
249
168
|
}
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
}
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
osStatus = {
|
|
172
|
+
connected: true,
|
|
173
|
+
resultsFound: testSearch.body.hits.total.value || 0
|
|
174
|
+
};
|
|
175
|
+
} catch (osError) {
|
|
176
|
+
osStatus = {
|
|
177
|
+
connected: false,
|
|
178
|
+
error: osError.message
|
|
179
|
+
};
|
|
261
180
|
}
|
|
262
|
-
|
|
263
|
-
return {
|
|
264
|
-
postgres: pgStatus,
|
|
265
|
-
opensearch: osStatus,
|
|
266
|
-
environment: this.environment,
|
|
267
|
-
region: this.region
|
|
268
|
-
};
|
|
269
181
|
}
|
|
270
|
-
|
|
271
|
-
|
|
182
|
+
|
|
183
|
+
return {
|
|
184
|
+
postgres: pgStatus,
|
|
185
|
+
opensearch: osStatus,
|
|
186
|
+
environment: this.environment,
|
|
187
|
+
region: this.region
|
|
188
|
+
};
|
|
272
189
|
} catch (error) {
|
|
273
190
|
return {
|
|
274
191
|
postgres: { connected: false, error: error.message },
|
|
@@ -280,33 +197,34 @@ class PegasusConnection {
|
|
|
280
197
|
}
|
|
281
198
|
|
|
282
199
|
async query(sql, params) {
|
|
283
|
-
const pool = this.getPostgresClient();
|
|
284
200
|
const start = Date.now();
|
|
285
201
|
logInfo('pegasus-sdk', `[SQL] ${sql}${params ? ` -- params: ${JSON.stringify(params)}` : ''}`);
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
202
|
+
|
|
203
|
+
const command = new ExecuteStatementCommand({
|
|
204
|
+
resourceArn: this.clusterArn,
|
|
205
|
+
secretArn: this.secretArn,
|
|
206
|
+
database: this.databaseName,
|
|
207
|
+
sql,
|
|
208
|
+
parameters: params || []
|
|
209
|
+
});
|
|
290
210
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
211
|
+
const result = await this.rdsDataClient.send(command);
|
|
212
|
+
logInfo('pegasus-sdk', `[SQL] rowCount: ${result.numberOfRecordsUpdated || 0} duration: ${Date.now() - start}ms`);
|
|
213
|
+
|
|
214
|
+
return {
|
|
215
|
+
rowCount: result.numberOfRecordsUpdated || result.records?.length || 0,
|
|
216
|
+
rows: result.records || [],
|
|
217
|
+
command: result
|
|
218
|
+
};
|
|
294
219
|
}
|
|
295
220
|
|
|
296
221
|
async transaction(callback) {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
try {
|
|
300
|
-
await client.query('BEGIN');
|
|
301
|
-
const result = await callback(client);
|
|
302
|
-
await client.query('COMMIT');
|
|
303
|
-
return result;
|
|
304
|
-
} catch (error) {
|
|
305
|
-
await client.query('ROLLBACK');
|
|
306
|
-
throw error;
|
|
307
|
-
} finally {
|
|
308
|
-
client.release();
|
|
222
|
+
if (!this.db) {
|
|
223
|
+
throw new Error('RDS Data API not initialized. Call connect() first.');
|
|
309
224
|
}
|
|
225
|
+
return await this.db.transaction(async (trx) => {
|
|
226
|
+
return await callback(trx);
|
|
227
|
+
});
|
|
310
228
|
}
|
|
311
229
|
}
|
|
312
230
|
|
package/lib/db/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const { drizzle } = require('drizzle-orm/
|
|
1
|
+
const { drizzle } = require('drizzle-orm/aws-data-api/pg');
|
|
2
2
|
const { logInfo } = require('@toxplanet/tphelper/logging');
|
|
3
3
|
const schema = require('./schema');
|
|
4
4
|
|
|
@@ -8,8 +8,8 @@ const logger = {
|
|
|
8
8
|
}
|
|
9
9
|
};
|
|
10
10
|
|
|
11
|
-
function getDrizzle(
|
|
12
|
-
return drizzle(
|
|
11
|
+
function getDrizzle(rdsDataClient, { resourceArn, secretArn, database }) {
|
|
12
|
+
return drizzle(rdsDataClient, { schema, logger, resourceArn, secretArn, database });
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
module.exports = {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toxplanet/pegasus-sdk",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.20",
|
|
4
4
|
"description": "SDK for migrating chemical data to Pegasus PostgreSQL + OpenSearch architecture with Elasticsearch client compatibility",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -25,10 +25,9 @@
|
|
|
25
25
|
"license": "MIT",
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@toxplanet/tphelper": "1.2.8",
|
|
28
|
-
"pg": "^8.11.3",
|
|
29
28
|
"drizzle-orm": "^0.30.0",
|
|
30
29
|
"@opensearch-project/opensearch": "^2.5.0",
|
|
31
|
-
"@aws-sdk/client-
|
|
30
|
+
"@aws-sdk/client-rds-data": "^3.490.0",
|
|
32
31
|
"@aws-sdk/client-sqs": "^3.490.0",
|
|
33
32
|
"@aws-sdk/credential-providers": "^3.490.0"
|
|
34
33
|
},
|