@cumulus/es-client 18.2.2 → 18.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.js +10 -9
- package/collections.js +5 -4
- package/esAmazonConnection.js +72 -0
- package/esScrollSearch.js +2 -2
- package/esSearchAfter.js +3 -2
- package/indexer.js +55 -35
- package/package.json +14 -12
- package/search.js +202 -50
- package/stats.js +4 -4
- package/testUtils.js +11 -4
package/bootstrap.js
CHANGED
|
@@ -15,7 +15,7 @@ const Logger = require('@cumulus/logger');
|
|
|
15
15
|
const { inTestMode } = require('@cumulus/common/test-utils');
|
|
16
16
|
const { IndexExistsError } = require('@cumulus/errors');
|
|
17
17
|
|
|
18
|
-
const {
|
|
18
|
+
const { EsClient, defaultIndexAlias } = require('./search');
|
|
19
19
|
const { createIndex } = require('./indexer');
|
|
20
20
|
const mappings = require('./config/mappings.json');
|
|
21
21
|
|
|
@@ -31,7 +31,7 @@ const logger = new Logger({ sender: '@cumulus/es-client/bootstrap' });
|
|
|
31
31
|
* @returns {Array<string>} - list of missing indices
|
|
32
32
|
*/
|
|
33
33
|
async function findMissingMappings(esClient, index, newMappings) {
|
|
34
|
-
const typesResponse = await esClient.indices.getMapping({
|
|
34
|
+
const typesResponse = await esClient.client.indices.getMapping({
|
|
35
35
|
index,
|
|
36
36
|
}).then((response) => response.body);
|
|
37
37
|
|
|
@@ -61,7 +61,7 @@ async function removeIndexAsAlias(esClient, alias, removeAliasConflict) {
|
|
|
61
61
|
// We can't do a simple exists check here, because it'll return true if the alias
|
|
62
62
|
// is actually an alias assigned to an index. We do a get and check that the alias
|
|
63
63
|
// name is not the key, which would indicate it's an index
|
|
64
|
-
const { body: existingIndex } = await esClient.indices.get(
|
|
64
|
+
const { body: existingIndex } = await esClient.client.indices.get(
|
|
65
65
|
{ index: alias },
|
|
66
66
|
{ ignore: [404] }
|
|
67
67
|
);
|
|
@@ -72,7 +72,7 @@ async function removeIndexAsAlias(esClient, alias, removeAliasConflict) {
|
|
|
72
72
|
throw new Error('Aborting ES recreation as configuration does not allow removal of index');
|
|
73
73
|
}
|
|
74
74
|
logger.warn(`Deleting alias as index: ${alias}`);
|
|
75
|
-
await esClient.indices.delete({ index: alias });
|
|
75
|
+
await esClient.client.indices.delete({ index: alias });
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
|
|
@@ -97,10 +97,11 @@ async function bootstrapElasticSearch({
|
|
|
97
97
|
}) {
|
|
98
98
|
if (!host) return;
|
|
99
99
|
|
|
100
|
-
const esClient =
|
|
100
|
+
const esClient = new EsClient(host);
|
|
101
|
+
await esClient.initializeEsClient();
|
|
101
102
|
|
|
102
103
|
// Make sure that indexes are not automatically created
|
|
103
|
-
await esClient.cluster.putSettings({
|
|
104
|
+
await esClient.client.cluster.putSettings({
|
|
104
105
|
body: {
|
|
105
106
|
persistent: { 'action.auto_create_index': false },
|
|
106
107
|
},
|
|
@@ -110,7 +111,7 @@ async function bootstrapElasticSearch({
|
|
|
110
111
|
|
|
111
112
|
let aliasedIndex = index;
|
|
112
113
|
|
|
113
|
-
const indices = await esClient.indices.getAlias({ name: alias }, { ignore: [404] })
|
|
114
|
+
const indices = await esClient.client.indices.getAlias({ name: alias }, { ignore: [404] })
|
|
114
115
|
.then((response) => response.body);
|
|
115
116
|
|
|
116
117
|
const aliasExists = !isNil(indices) && !indices.error;
|
|
@@ -130,7 +131,7 @@ async function bootstrapElasticSearch({
|
|
|
130
131
|
}
|
|
131
132
|
}
|
|
132
133
|
|
|
133
|
-
await esClient.indices.putAlias({
|
|
134
|
+
await esClient.client.indices.putAlias({
|
|
134
135
|
index: index,
|
|
135
136
|
name: alias,
|
|
136
137
|
});
|
|
@@ -145,7 +146,7 @@ async function bootstrapElasticSearch({
|
|
|
145
146
|
const concurrencyLimit = inTestMode() ? 1 : 3;
|
|
146
147
|
const limit = pLimit(concurrencyLimit);
|
|
147
148
|
const addMissingTypesPromises = missingTypes.map((type) =>
|
|
148
|
-
limit(() => esClient.indices.putMapping({
|
|
149
|
+
limit(() => esClient.client.indices.putMapping({
|
|
149
150
|
index: aliasedIndex,
|
|
150
151
|
type,
|
|
151
152
|
body: get(mappings, type),
|
package/collections.js
CHANGED
|
@@ -18,8 +18,8 @@ class Collection extends BaseSearch {
|
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
async getStats(records, ids) {
|
|
21
|
-
if (!this.
|
|
22
|
-
|
|
21
|
+
if (!this._esClient) {
|
|
22
|
+
await this.initializeEsClient();
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
const aggs = await this.client.search({
|
|
@@ -126,9 +126,10 @@ class Collection extends BaseSearch {
|
|
|
126
126
|
* @returns {Promise<Array<string>>} - list of collection ids with active granules
|
|
127
127
|
*/
|
|
128
128
|
async aggregateGranuleCollections() {
|
|
129
|
-
if (!this.
|
|
130
|
-
|
|
129
|
+
if (!this._esClient) {
|
|
130
|
+
await this.initializeEsClient();
|
|
131
131
|
}
|
|
132
|
+
|
|
132
133
|
// granules
|
|
133
134
|
const searchParams = this._buildSearch();
|
|
134
135
|
delete searchParams.from;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { Connection } = require('@elastic/elasticsearch');
|
|
4
|
+
const aws4 = require('aws4');
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Builds and returns a custom subclass of Connection that is configured to sign requests
|
|
8
|
+
* for AWS Elasticsearch service. Request signing is provided by the aws4 library and requires
|
|
9
|
+
* valid AWS credentials.
|
|
10
|
+
*
|
|
11
|
+
* @param {object} awsConfig - AWS configuration values to be used to build the Connection.
|
|
12
|
+
* @param {string} [awsConfig.region] - Optionally specify the AWS region in the request.
|
|
13
|
+
* @param {object} awsConfig.credentials - Valid AWS credentials object.
|
|
14
|
+
* @returns {AmazonConnection} - Connection configured and signed to work with AWS Elasticsearch
|
|
15
|
+
* service.
|
|
16
|
+
*/
|
|
17
|
+
const createAmazonConnection = (awsConfig) => {
|
|
18
|
+
class AmazonConnection extends Connection {
|
|
19
|
+
constructor(opts = {}) {
|
|
20
|
+
super(opts);
|
|
21
|
+
if (awsConfig.credentials) {
|
|
22
|
+
this.accessKeyId = awsConfig.credentials.accessKeyId;
|
|
23
|
+
this.secretAccessKey = awsConfig.credentials.secretAccessKey;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
buildRequestObject(params) {
|
|
28
|
+
const req = super.buildRequestObject(params);
|
|
29
|
+
|
|
30
|
+
req.service = 'es';
|
|
31
|
+
|
|
32
|
+
if (awsConfig.region) {
|
|
33
|
+
req.region = awsConfig.region;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (!req.headers) {
|
|
37
|
+
req.headers = {};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Fix the Host header, since HttpConnector.makeReqParams() appends
|
|
41
|
+
// the port number which will cause signature verification to fail
|
|
42
|
+
req.headers.host = req.hostname;
|
|
43
|
+
|
|
44
|
+
// This fix allows the connector to work with the older 6.x elastic branch.
|
|
45
|
+
// The problem with that version, is that the Transport object would add a
|
|
46
|
+
// `Content-Length` header (yes with Pascal Case), thus duplicating headers
|
|
47
|
+
// (`Content-Length` and `content-length`), which makes the signature fail.
|
|
48
|
+
let contentLength = 0;
|
|
49
|
+
if (params.body) {
|
|
50
|
+
contentLength = Buffer.byteLength(params.body, 'utf8');
|
|
51
|
+
req.body = params.body;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const lengthHeader = 'content-length';
|
|
55
|
+
const headerFound = Object.keys(req.headers).find(
|
|
56
|
+
(header) => header.toLowerCase() === lengthHeader
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
if (headerFound === undefined) {
|
|
60
|
+
req.headers[lengthHeader] = contentLength;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return aws4.sign(req, awsConfig.credentials);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return AmazonConnection;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
module.exports = (awsConfig) => ({
|
|
71
|
+
Connection: createAmazonConnection(awsConfig),
|
|
72
|
+
});
|
package/esScrollSearch.js
CHANGED
|
@@ -36,8 +36,8 @@ const {
|
|
|
36
36
|
*/
|
|
37
37
|
class ESScrollSearch extends Search {
|
|
38
38
|
async query() {
|
|
39
|
-
if (!this.
|
|
40
|
-
this.
|
|
39
|
+
if (!this._esClient) {
|
|
40
|
+
await this.initializeEsClient(undefined, this.metrics);
|
|
41
41
|
}
|
|
42
42
|
let response;
|
|
43
43
|
if (!this.scrollId) {
|
package/esSearchAfter.js
CHANGED
|
@@ -33,9 +33,10 @@ class ESSearchAfter extends Search {
|
|
|
33
33
|
* @returns {Promise<Object>} Object containing query meta and results
|
|
34
34
|
*/
|
|
35
35
|
async query() {
|
|
36
|
-
if (!this.
|
|
37
|
-
|
|
36
|
+
if (!this._esClient) {
|
|
37
|
+
await this.initializeEsClient();
|
|
38
38
|
}
|
|
39
|
+
|
|
39
40
|
const searchParams = this._buildSearch();
|
|
40
41
|
const response = await this.client.search(searchParams);
|
|
41
42
|
|
package/indexer.js
CHANGED
|
@@ -21,20 +21,20 @@ const { IndexExistsError, ValidationError } = require('@cumulus/errors');
|
|
|
21
21
|
const { constructCollectionId } = require('@cumulus/message/Collections');
|
|
22
22
|
const { removeNilProperties } = require('@cumulus/common/util');
|
|
23
23
|
|
|
24
|
-
const { Search, defaultIndexAlias } = require('./search');
|
|
24
|
+
const { EsClient, Search, defaultIndexAlias } = require('./search');
|
|
25
25
|
const mappings = require('./config/mappings.json');
|
|
26
26
|
|
|
27
27
|
const logger = new Logger({ sender: '@cumulus/es-client/indexer' });
|
|
28
28
|
|
|
29
29
|
async function createIndex(esClient, indexName) {
|
|
30
|
-
const indexExists = await esClient.indices.exists({ index: indexName })
|
|
30
|
+
const indexExists = await esClient.client.indices.exists({ index: indexName })
|
|
31
31
|
.then((response) => response.body);
|
|
32
32
|
|
|
33
33
|
if (indexExists) {
|
|
34
34
|
throw new IndexExistsError(`Index ${indexName} exists and cannot be created.`);
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
await esClient.indices.create({
|
|
37
|
+
await esClient.client.indices.create({
|
|
38
38
|
index: indexName,
|
|
39
39
|
body: {
|
|
40
40
|
mappings,
|
|
@@ -57,7 +57,7 @@ async function createIndex(esClient, indexName) {
|
|
|
57
57
|
* @param {Object} doc - the record
|
|
58
58
|
* @param {string} index - Elasticsearch index alias
|
|
59
59
|
* @param {string} type - Elasticsearch type
|
|
60
|
-
* @param {string} parent - the optional parent id
|
|
60
|
+
* @param {string} [parent] - the optional parent id
|
|
61
61
|
* @returns {Promise} Elasticsearch response
|
|
62
62
|
*/
|
|
63
63
|
async function genericRecordUpdate(esClient, id, doc, index, type, parent) {
|
|
@@ -76,11 +76,28 @@ async function genericRecordUpdate(esClient, id, doc, index, type, parent) {
|
|
|
76
76
|
|
|
77
77
|
if (parent) params.parent = parent;
|
|
78
78
|
|
|
79
|
-
|
|
80
|
-
|
|
79
|
+
let actualEsClient;
|
|
80
|
+
if (esClient) {
|
|
81
|
+
actualEsClient = esClient;
|
|
82
|
+
} else {
|
|
83
|
+
const defaultEsClient = new EsClient();
|
|
84
|
+
await defaultEsClient.initializeEsClient();
|
|
85
|
+
actualEsClient = defaultEsClient;
|
|
86
|
+
}
|
|
87
|
+
|
|
81
88
|
let indexResponse;
|
|
82
89
|
try {
|
|
83
|
-
|
|
90
|
+
try {
|
|
91
|
+
indexResponse = await actualEsClient.client.index(params);
|
|
92
|
+
} catch (error) {
|
|
93
|
+
if (error.name === 'ResponseError' && error.meta.body.message.includes('The security token included in the request is expired')) {
|
|
94
|
+
logger.warn(`genericRecordUpdate encountered a ResponseError ${JSON.stringify(error)}, updating credentials and retrying`);
|
|
95
|
+
await actualEsClient.refreshClient();
|
|
96
|
+
indexResponse = await actualEsClient.client.index(params);
|
|
97
|
+
} else {
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
84
101
|
} catch (error) {
|
|
85
102
|
logger.error(`Error thrown on index ${JSON.stringify(error)}`);
|
|
86
103
|
throw error;
|
|
@@ -99,7 +116,7 @@ async function genericRecordUpdate(esClient, id, doc, index, type, parent) {
|
|
|
99
116
|
* @returns {Promise} Elasticsearch response
|
|
100
117
|
*/
|
|
101
118
|
async function updateExistingRecord(esClient, id, doc, index, type) {
|
|
102
|
-
return await esClient.update({
|
|
119
|
+
return await esClient.client.update({
|
|
103
120
|
index,
|
|
104
121
|
type,
|
|
105
122
|
id,
|
|
@@ -155,6 +172,7 @@ async function upsertExecution({
|
|
|
155
172
|
type = 'execution',
|
|
156
173
|
refresh,
|
|
157
174
|
}, writeConstraints = true) {
|
|
175
|
+
const cumulusEsClient = await esClient.client;
|
|
158
176
|
Object.keys(updates).forEach((key) => {
|
|
159
177
|
if (updates[key] === null && executionInvalidNullFields.includes(key)) {
|
|
160
178
|
throw new ValidationError(`Attempted Elasticsearch write with invalid key ${key} set to null. Please remove or change this field and retry`);
|
|
@@ -190,7 +208,7 @@ async function upsertExecution({
|
|
|
190
208
|
}
|
|
191
209
|
`;
|
|
192
210
|
}
|
|
193
|
-
return await
|
|
211
|
+
return await cumulusEsClient.update({
|
|
194
212
|
index,
|
|
195
213
|
type,
|
|
196
214
|
id: upsertDoc.arn,
|
|
@@ -309,8 +327,7 @@ async function indexGranule(esClient, payload, index = defaultIndexAlias, type =
|
|
|
309
327
|
parent: payload.collectionId,
|
|
310
328
|
refresh: inTestMode(),
|
|
311
329
|
};
|
|
312
|
-
await esClient.delete(delGranParams, { ignore: [404] });
|
|
313
|
-
|
|
330
|
+
await esClient.client.delete(delGranParams, { ignore: [404] });
|
|
314
331
|
return genericRecordUpdate(
|
|
315
332
|
esClient,
|
|
316
333
|
payload.granuleId,
|
|
@@ -356,6 +373,8 @@ async function upsertGranule({
|
|
|
356
373
|
throw new ValidationError(`Attempted Elasticsearch write with invalid key ${key} set to null. Please remove or change this field and retry`);
|
|
357
374
|
}
|
|
358
375
|
});
|
|
376
|
+
|
|
377
|
+
const cumulusEsClient = await esClient.client;
|
|
359
378
|
// If the granule exists in 'deletedgranule', delete it first before inserting the granule
|
|
360
379
|
// into ES. Ignore 404 error, so the deletion still succeeds if the record doesn't exist.
|
|
361
380
|
const delGranParams = {
|
|
@@ -365,7 +384,7 @@ async function upsertGranule({
|
|
|
365
384
|
parent: updates.collectionId,
|
|
366
385
|
refresh: inTestMode(),
|
|
367
386
|
};
|
|
368
|
-
await
|
|
387
|
+
await cumulusEsClient.delete(delGranParams, { ignore: [404] });
|
|
369
388
|
|
|
370
389
|
// Remove nils in case there isn't a collision
|
|
371
390
|
const upsertDoc = removeNilProperties(updates);
|
|
@@ -412,7 +431,7 @@ async function upsertGranule({
|
|
|
412
431
|
}
|
|
413
432
|
}
|
|
414
433
|
|
|
415
|
-
return await
|
|
434
|
+
return await cumulusEsClient.update({
|
|
416
435
|
index,
|
|
417
436
|
type,
|
|
418
437
|
id: updates.granuleId,
|
|
@@ -473,7 +492,7 @@ async function upsertPdr({
|
|
|
473
492
|
...updates,
|
|
474
493
|
timestamp: Date.now(),
|
|
475
494
|
};
|
|
476
|
-
return await esClient.update({
|
|
495
|
+
return await esClient.client.update({
|
|
477
496
|
index,
|
|
478
497
|
type,
|
|
479
498
|
id: upsertDoc.pdrName,
|
|
@@ -506,13 +525,13 @@ async function upsertPdr({
|
|
|
506
525
|
* @param {Object} params.esClient - Elasticsearch Connection object
|
|
507
526
|
* @param {string} params.id - id of the Elasticsearch record
|
|
508
527
|
* @param {string} params.type - Elasticsearch type (default: execution)
|
|
509
|
-
* @param {
|
|
528
|
+
* @param {string} params.parent - id of the parent (optional)
|
|
510
529
|
* @param {string} params.index - Elasticsearch index (default: cumulus)
|
|
511
530
|
* @param {Array} params.ignore - Response codes to ignore (optional)
|
|
512
531
|
* @returns {Promise} elasticsearch delete response
|
|
513
532
|
*/
|
|
514
533
|
async function deleteRecord({
|
|
515
|
-
esClient,
|
|
534
|
+
esClient = new EsClient(),
|
|
516
535
|
id,
|
|
517
536
|
type,
|
|
518
537
|
parent,
|
|
@@ -531,8 +550,8 @@ async function deleteRecord({
|
|
|
531
550
|
if (parent) params.parent = parent;
|
|
532
551
|
if (ignore) options = { ignore };
|
|
533
552
|
|
|
534
|
-
|
|
535
|
-
const deleteResponse = await
|
|
553
|
+
await esClient.initializeEsClient();
|
|
554
|
+
const deleteResponse = await esClient.client.delete(params, options);
|
|
536
555
|
return deleteResponse.body;
|
|
537
556
|
}
|
|
538
557
|
|
|
@@ -778,27 +797,28 @@ async function deleteGranule({
|
|
|
778
797
|
|
|
779
798
|
module.exports = {
|
|
780
799
|
createIndex,
|
|
781
|
-
indexCollection,
|
|
782
|
-
indexProvider,
|
|
783
|
-
indexReconciliationReport,
|
|
784
|
-
indexRule,
|
|
785
|
-
indexGranule,
|
|
786
|
-
upsertGranule,
|
|
787
|
-
indexPdr,
|
|
788
|
-
upsertPdr,
|
|
789
|
-
indexExecution,
|
|
790
|
-
indexAsyncOperation,
|
|
791
|
-
deleteRecord,
|
|
792
800
|
deleteAsyncOperation,
|
|
793
|
-
updateAsyncOperation,
|
|
794
|
-
upsertExecution,
|
|
795
801
|
deleteCollection,
|
|
796
|
-
deleteProvider,
|
|
797
|
-
deleteRule,
|
|
798
|
-
deletePdr,
|
|
799
|
-
deleteGranule,
|
|
800
802
|
deleteExecution,
|
|
803
|
+
deleteGranule,
|
|
804
|
+
deletePdr,
|
|
805
|
+
deleteProvider,
|
|
801
806
|
deleteReconciliationReport,
|
|
807
|
+
deleteRecord,
|
|
808
|
+
deleteRule,
|
|
802
809
|
executionInvalidNullFields,
|
|
803
810
|
granuleInvalidNullFields,
|
|
811
|
+
genericRecordUpdate,
|
|
812
|
+
indexAsyncOperation,
|
|
813
|
+
indexCollection,
|
|
814
|
+
indexExecution,
|
|
815
|
+
indexGranule,
|
|
816
|
+
indexPdr,
|
|
817
|
+
indexProvider,
|
|
818
|
+
indexReconciliationReport,
|
|
819
|
+
indexRule,
|
|
820
|
+
updateAsyncOperation,
|
|
821
|
+
upsertExecution,
|
|
822
|
+
upsertGranule,
|
|
823
|
+
upsertPdr,
|
|
804
824
|
};
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/es-client",
|
|
3
|
-
"version": "18.
|
|
3
|
+
"version": "18.3.1",
|
|
4
4
|
"description": "Utilities for working with Elasticsearch",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"CUMULUS",
|
|
7
7
|
"NASA"
|
|
8
8
|
],
|
|
9
9
|
"engines": {
|
|
10
|
-
"node": ">=
|
|
10
|
+
"node": ">=20.12.2"
|
|
11
11
|
},
|
|
12
12
|
"publishConfig": {
|
|
13
13
|
"access": "public"
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
21
|
"test": "../../node_modules/.bin/ava",
|
|
22
|
+
"test:ci": "../../scripts/run_package_ci_unit.sh",
|
|
22
23
|
"test:coverage": "../../node_modules/.bin/nyc npm test",
|
|
23
24
|
"coverage": "python ../../scripts/coverage_handler/coverage.py"
|
|
24
25
|
},
|
|
@@ -26,26 +27,27 @@
|
|
|
26
27
|
"files": [
|
|
27
28
|
"tests/*.js"
|
|
28
29
|
],
|
|
29
|
-
"verbose": true
|
|
30
|
+
"verbose": true,
|
|
31
|
+
"failFast": true
|
|
30
32
|
},
|
|
31
33
|
"author": "Cumulus Authors",
|
|
32
34
|
"license": "Apache-2.0",
|
|
33
35
|
"dependencies": {
|
|
34
|
-
"@
|
|
35
|
-
"@cumulus/
|
|
36
|
-
"@cumulus/
|
|
37
|
-
"@cumulus/
|
|
36
|
+
"@aws-sdk/credential-providers": "^3.535.0",
|
|
37
|
+
"@cumulus/common": "18.3.1",
|
|
38
|
+
"@cumulus/errors": "18.3.1",
|
|
39
|
+
"@cumulus/logger": "18.3.1",
|
|
40
|
+
"@cumulus/message": "18.3.1",
|
|
38
41
|
"@elastic/elasticsearch": "^5.6.20",
|
|
39
|
-
"
|
|
40
|
-
"aws-sdk": "^2.1492.0",
|
|
42
|
+
"aws4": "^1.12.0",
|
|
41
43
|
"lodash": "~4.17.21",
|
|
42
44
|
"moment": "2.29.4",
|
|
43
45
|
"p-limit": "^1.2.0"
|
|
44
46
|
},
|
|
45
47
|
"devDependencies": {
|
|
46
|
-
"@cumulus/aws-client": "18.
|
|
47
|
-
"@cumulus/test-data": "18.
|
|
48
|
+
"@cumulus/aws-client": "18.3.1",
|
|
49
|
+
"@cumulus/test-data": "18.3.1",
|
|
48
50
|
"p-each-series": "^2.1.0"
|
|
49
51
|
},
|
|
50
|
-
"gitHead": "
|
|
52
|
+
"gitHead": "0393f90c6401ef0c524068e4636c1dcc389020b8"
|
|
51
53
|
}
|
package/search.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
//@ts-check
|
|
1
2
|
/* This code is copied from sat-api-lib library
|
|
2
3
|
* with some alterations.
|
|
3
4
|
* source: https://raw.githubusercontent.com/sat-utils/sat-api-lib/master/libs/search.js
|
|
@@ -8,12 +9,13 @@
|
|
|
8
9
|
|
|
9
10
|
const has = require('lodash/has');
|
|
10
11
|
const omit = require('lodash/omit');
|
|
11
|
-
const
|
|
12
|
-
const { AmazonConnection } = require('aws-elasticsearch-connector');
|
|
12
|
+
const { fromNodeProviderChain } = require('@aws-sdk/credential-providers');
|
|
13
13
|
const elasticsearch = require('@elastic/elasticsearch');
|
|
14
14
|
|
|
15
15
|
const { inTestMode } = require('@cumulus/common/test-utils');
|
|
16
|
+
const Logger = require('@cumulus/logger');
|
|
16
17
|
|
|
18
|
+
const createEsAmazonConnection = require('./esAmazonConnection');
|
|
17
19
|
const queries = require('./queries');
|
|
18
20
|
const aggs = require('./aggregations');
|
|
19
21
|
|
|
@@ -25,16 +27,11 @@ const logDetails = {
|
|
|
25
27
|
const defaultIndexAlias = 'cumulus-alias';
|
|
26
28
|
const multipleRecordFoundString = 'More than one record was found!';
|
|
27
29
|
const recordNotFoundString = 'Record not found';
|
|
28
|
-
|
|
29
|
-
const getCredentials = () =>
|
|
30
|
-
new Promise((resolve, reject) => aws.config.getCredentials((err) => {
|
|
31
|
-
if (err) return reject(err);
|
|
32
|
-
return resolve();
|
|
33
|
-
}));
|
|
30
|
+
const logger = new Logger({ sender: '@cumulus/es-client/search' });
|
|
34
31
|
|
|
35
32
|
/**
|
|
36
|
-
*
|
|
37
|
-
*
|
|
33
|
+
* Returns the local address of elasticsearch based on
|
|
34
|
+
* environment variables
|
|
38
35
|
*
|
|
39
36
|
* @returns {string} elasticsearch local address
|
|
40
37
|
*/
|
|
@@ -46,6 +43,19 @@ const getLocalEsHost = () => {
|
|
|
46
43
|
return `${protocol}://localhost:9200`;
|
|
47
44
|
};
|
|
48
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Retrieves AWS credentials using the `fromNodeProviderChain` function.
|
|
48
|
+
*/
|
|
49
|
+
const getAwsCredentials = async () => {
|
|
50
|
+
const credentialsProvider = fromNodeProviderChain({
|
|
51
|
+
clientConfig: {
|
|
52
|
+
region: process.env.AWS_REGION,
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
const creds = await credentialsProvider();
|
|
56
|
+
return creds;
|
|
57
|
+
};
|
|
58
|
+
|
|
49
59
|
const esTestConfig = () => ({
|
|
50
60
|
node: getLocalEsHost(),
|
|
51
61
|
requestTimeout: 5000,
|
|
@@ -54,9 +64,18 @@ const esTestConfig = () => ({
|
|
|
54
64
|
},
|
|
55
65
|
});
|
|
56
66
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
67
|
+
/**
|
|
68
|
+
* Generates a configuration for Elasticsearch in a production environment.
|
|
69
|
+
*
|
|
70
|
+
* @param {string | undefined} host - The host URL for the Elasticsearch instance.
|
|
71
|
+
* If not provided, the function will use the `ES_HOST` environment variable.
|
|
72
|
+
* @param {import('@aws-sdk/types').AwsCredentialIdentity | undefined} credentials - The
|
|
73
|
+
* AWS credentials for accessing the Elasticsearch instance.
|
|
74
|
+
* @returns
|
|
75
|
+
* - The configuration object for Elasticsearch, including the node address,
|
|
76
|
+
* AWS connection details, and request timeout.
|
|
77
|
+
*/
|
|
78
|
+
const esProdConfig = (host, credentials) => {
|
|
60
79
|
let node = 'http://localhost:9200';
|
|
61
80
|
|
|
62
81
|
if (process.env.ES_HOST) {
|
|
@@ -64,13 +83,12 @@ const esProdConfig = async (host) => {
|
|
|
64
83
|
} else if (host) {
|
|
65
84
|
node = `https://${host}`;
|
|
66
85
|
}
|
|
67
|
-
|
|
68
86
|
return {
|
|
69
87
|
node,
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
},
|
|
88
|
+
...createEsAmazonConnection({
|
|
89
|
+
credentials,
|
|
90
|
+
region: process.env.AWS_REGION,
|
|
91
|
+
}),
|
|
74
92
|
|
|
75
93
|
// Note that this doesn't abort the query.
|
|
76
94
|
requestTimeout: 50000, // milliseconds
|
|
@@ -93,29 +111,147 @@ const esMetricsConfig = () => {
|
|
|
93
111
|
};
|
|
94
112
|
};
|
|
95
113
|
|
|
114
|
+
/**
|
|
115
|
+
* Generates a configuration for Elasticsearch based on the environment
|
|
116
|
+
* and provided parameters.
|
|
117
|
+
*
|
|
118
|
+
* @param {string} [host] - The host URL for the Elasticsearch instance.
|
|
119
|
+
* @param {boolean} [metrics=false] - A flag indicating whether metrics are enabled.
|
|
120
|
+
* @returns {Promise<[Object, import('@aws-sdk/types').Credentials | undefined]>} A
|
|
121
|
+
* promise that resolves to a tuple containing the configuration object and
|
|
122
|
+
* AWS credentials (if applicable).
|
|
123
|
+
*/
|
|
96
124
|
const esConfig = async (host, metrics = false) => {
|
|
97
125
|
let config;
|
|
126
|
+
let credentials;
|
|
98
127
|
if (inTestMode() || 'LOCAL_ES_HOST' in process.env) {
|
|
99
128
|
config = esTestConfig();
|
|
100
129
|
} else if (metrics) {
|
|
101
130
|
config = esMetricsConfig();
|
|
102
131
|
} else {
|
|
103
|
-
|
|
132
|
+
credentials = await getAwsCredentials();
|
|
133
|
+
config = esProdConfig(host, credentials);
|
|
104
134
|
}
|
|
105
|
-
return config;
|
|
135
|
+
return [config, credentials];
|
|
106
136
|
};
|
|
107
137
|
|
|
138
|
+
/**
|
|
139
|
+
* `EsClient` is a class for managing an Elasticsearch client.
|
|
140
|
+
*
|
|
141
|
+
* @property {string} host - The host URL for the Elasticsearch instance.
|
|
142
|
+
* @property {boolean} metrics - A flag indicating whether metrics are enabled.
|
|
143
|
+
* @property {elasticsearch.Client} _client - The Elasticsearch client instance.
|
|
144
|
+
*
|
|
145
|
+
* @method constructor - Initializes a new instance of the `EsClient` class.
|
|
146
|
+
* @method initializeEsClient - Initializes the Elasticsearch client (this._client/client)
|
|
147
|
+
* if it hasn't been initialized yet.
|
|
148
|
+
* @method refreshClient - Refreshes the Elasticsearch client if the AWS credentials have changed,
|
|
149
|
+
* by creating a new Elasticsearch `Client` instance.
|
|
150
|
+
* @method client - Getter that returns the current Elasticsearch Client
|
|
151
|
+
*/
|
|
152
|
+
class EsClient {
|
|
153
|
+
/**
|
|
154
|
+
* Initializes the Elasticsearch client if it hasn't been initialized yet,
|
|
155
|
+
* fetching AWS credentials if necessary.
|
|
156
|
+
*
|
|
157
|
+
* @returns {Promise<elasticsearch.Client>} A promise that resolves to an instance of
|
|
158
|
+
* `elasticsearch.Client`.
|
|
159
|
+
*/
|
|
160
|
+
async initializeEsClient() {
|
|
161
|
+
/** @type {elasticsearch.Client | undefined} */
|
|
162
|
+
let client = this._client;
|
|
163
|
+
if (!client) {
|
|
164
|
+
const [config, credentials] = await esConfig(this.host, this.metrics);
|
|
165
|
+
if (credentials) {
|
|
166
|
+
this._awsKeyId = credentials.accessKeyId;
|
|
167
|
+
}
|
|
168
|
+
client = new elasticsearch.Client(config);
|
|
169
|
+
this._client = client;
|
|
170
|
+
}
|
|
171
|
+
return client;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Asynchronously refreshes the Elasticsearch client if the AWS credentials have changed,
|
|
176
|
+
* by creating a new Elasticsearch `Client` instance.
|
|
177
|
+
*
|
|
178
|
+
* @returns {Promise<void>} A promise that resolves when the credentials have been refreshed.
|
|
179
|
+
*/
|
|
180
|
+
async refreshClient() {
|
|
181
|
+
const { host, metrics } = this;
|
|
182
|
+
if (this.metrics || inTestMode() || process.env.LOCAL_ES_HOST) {
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
const oldKey = this._awsKeyId;
|
|
186
|
+
const newCreds = await getAwsCredentials();
|
|
187
|
+
if (oldKey !== newCreds.accessKeyId) {
|
|
188
|
+
logger.info('AWS Credentials updated, updating to new ESClient');
|
|
189
|
+
const [config] = await esConfig(host, metrics); // Removed unused variable _creds
|
|
190
|
+
this._client = new elasticsearch.Client(config);
|
|
191
|
+
this._awsKeyId = newCreds.accessKeyId;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Getter that returns the Elasticsearch client instance if it's been initialized
|
|
197
|
+
*
|
|
198
|
+
* @returns {elasticsearch.Client | undefined} The Elasticsearch client instance.
|
|
199
|
+
*/
|
|
200
|
+
get client() {
|
|
201
|
+
return this._client;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Initializes a new instance of the `EsClient` class.
|
|
206
|
+
*
|
|
207
|
+
* @param {string} host - The host URL for the Elasticsearch instance.
|
|
208
|
+
* @param {boolean} [metrics=false] - A flag indicating whether metrics are enabled.
|
|
209
|
+
*/
|
|
210
|
+
constructor(host, metrics = false) {
|
|
211
|
+
this.host = host;
|
|
212
|
+
this.metrics = metrics;
|
|
213
|
+
if (metrics) {
|
|
214
|
+
this.host = process.env.METRICS_ES_HOST;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* `BaseSearch` is a class for managing certain Cumulus Elasticsearch queries.
|
|
221
|
+
*
|
|
222
|
+
* @property {string | undefined} host - The host URL for the Elasticsearch instance.
|
|
223
|
+
* @property {boolean} metrics - A flag indicating whether metrics are enabled.
|
|
224
|
+
* @property {EsClient} _esClient - The Elasticsearch client instance.
|
|
225
|
+
* @property {string | null} type - The type of the Elasticsearch index.
|
|
226
|
+
* @property {Object} params - The query parameters.
|
|
227
|
+
* @property {number} size - The number of results to return per page.
|
|
228
|
+
* @property {number} frm - The starting index for the results.
|
|
229
|
+
* @property {number} page - The current page number.
|
|
230
|
+
* @property {string} index - The Elasticsearch index to query.
|
|
231
|
+
*
|
|
232
|
+
* @method initializeEsClient - Initializes the EsClient associated with the instance of this class
|
|
233
|
+
* @method client - Returns the Elasticsearch client instance.
|
|
234
|
+
* @method constructor - Initializes the `BaseSearch` instance, including the EsClient instance.
|
|
235
|
+
* @method get - Retrieves a single document by id and/or parentId.
|
|
236
|
+
* @method exists - Checks if a document exists by id and/or parentId.
|
|
237
|
+
* @method query - Performs a search query.
|
|
238
|
+
* @method count - Counts the number of documents in the index
|
|
239
|
+
*/
|
|
108
240
|
class BaseSearch {
|
|
109
|
-
|
|
110
|
-
|
|
241
|
+
async initializeEsClient(host, metrics) {
|
|
242
|
+
this._esClient = new EsClient(host, metrics);
|
|
243
|
+
await this._esClient.initializeEsClient();
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
get client() {
|
|
247
|
+
return this._esClient ? this._esClient.client : undefined;
|
|
111
248
|
}
|
|
112
249
|
|
|
113
|
-
constructor(event, type = null, index, metrics = false) {
|
|
250
|
+
constructor(event = {}, type = null, index, metrics = false) {
|
|
114
251
|
let params = {};
|
|
115
252
|
const logLimit = 10;
|
|
116
253
|
|
|
117
254
|
this.type = type;
|
|
118
|
-
this.client = null;
|
|
119
255
|
this.metrics = metrics;
|
|
120
256
|
|
|
121
257
|
// this will allow us to receive payload
|
|
@@ -125,17 +261,16 @@ class BaseSearch {
|
|
|
125
261
|
}
|
|
126
262
|
|
|
127
263
|
// get page number
|
|
128
|
-
const page = Number.parseInt(
|
|
264
|
+
const page = Number.parseInt(params.page ? params.page : 1, 10);
|
|
129
265
|
this.params = params;
|
|
130
|
-
//log.debug('Generated params:', params, logDetails);
|
|
131
266
|
|
|
132
|
-
this.size = Number.parseInt(
|
|
267
|
+
this.size = Number.parseInt(params.limit ? params.limit : logLimit, 10);
|
|
133
268
|
|
|
134
269
|
// max size is 100 for performance reasons
|
|
135
270
|
this.size = this.size > 100 ? 100 : this.size;
|
|
136
271
|
|
|
137
272
|
this.frm = (page - 1) * this.size;
|
|
138
|
-
this.page = Number.parseInt(
|
|
273
|
+
this.page = Number.parseInt(params.skip ? params.skip : page, 10);
|
|
139
274
|
this.index = index || defaultIndexAlias;
|
|
140
275
|
}
|
|
141
276
|
|
|
@@ -197,37 +332,38 @@ class BaseSearch {
|
|
|
197
332
|
const body = {
|
|
198
333
|
query: {
|
|
199
334
|
bool: {
|
|
200
|
-
must: [
|
|
201
|
-
|
|
202
|
-
|
|
335
|
+
must: [
|
|
336
|
+
{
|
|
337
|
+
term: {
|
|
338
|
+
_id: id,
|
|
339
|
+
},
|
|
203
340
|
},
|
|
204
|
-
|
|
341
|
+
],
|
|
205
342
|
},
|
|
206
343
|
},
|
|
207
344
|
};
|
|
208
345
|
|
|
209
346
|
if (parentId) {
|
|
210
|
-
body.query.bool.must.push(
|
|
211
|
-
{
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
}
|
|
217
|
-
);
|
|
347
|
+
body.query.bool.must.push({
|
|
348
|
+
parent_id: {
|
|
349
|
+
id: parentId,
|
|
350
|
+
type: this.type,
|
|
351
|
+
},
|
|
352
|
+
});
|
|
218
353
|
}
|
|
219
354
|
|
|
220
355
|
logDetails.granuleId = id;
|
|
221
356
|
|
|
222
|
-
if (!this.
|
|
223
|
-
|
|
357
|
+
if (!this._esClient) {
|
|
358
|
+
await this.initializeEsClient();
|
|
224
359
|
}
|
|
225
360
|
|
|
226
361
|
const result = await this.client.search({
|
|
227
362
|
index: this.index,
|
|
228
363
|
type: this.type,
|
|
229
364
|
body,
|
|
230
|
-
})
|
|
365
|
+
})
|
|
366
|
+
.then((response) => response.body);
|
|
231
367
|
|
|
232
368
|
if (result.hits.total > 1) {
|
|
233
369
|
return { detail: multipleRecordFoundString };
|
|
@@ -251,8 +387,8 @@ class BaseSearch {
|
|
|
251
387
|
|
|
252
388
|
try {
|
|
253
389
|
// search ES with the generated parameters
|
|
254
|
-
if (!this.
|
|
255
|
-
|
|
390
|
+
if (!this._esClient) {
|
|
391
|
+
await this.initializeEsClient(null, this.metrics);
|
|
256
392
|
}
|
|
257
393
|
const response = await this.client.search(searchParams);
|
|
258
394
|
const hits = response.body.hits.hits;
|
|
@@ -262,7 +398,9 @@ class BaseSearch {
|
|
|
262
398
|
meta.page = this.page;
|
|
263
399
|
meta.count = response.body.hits.total;
|
|
264
400
|
if (hits.length > 0) {
|
|
265
|
-
meta.searchContext = encodeURIComponent(
|
|
401
|
+
meta.searchContext = encodeURIComponent(
|
|
402
|
+
JSON.stringify(hits[hits.length - 1].sort)
|
|
403
|
+
);
|
|
266
404
|
}
|
|
267
405
|
|
|
268
406
|
return {
|
|
@@ -278,11 +416,11 @@ class BaseSearch {
|
|
|
278
416
|
const searchParams = this._buildAggregation();
|
|
279
417
|
|
|
280
418
|
try {
|
|
281
|
-
if (!this.
|
|
282
|
-
this.
|
|
419
|
+
if (!this.esClient) {
|
|
420
|
+
this.esClient = await this.initializeEsClient();
|
|
283
421
|
}
|
|
284
422
|
|
|
285
|
-
const result = await this.
|
|
423
|
+
const result = await this.esClient.search(searchParams);
|
|
286
424
|
const count = result.body.hits.total;
|
|
287
425
|
|
|
288
426
|
return {
|
|
@@ -293,7 +431,6 @@ class BaseSearch {
|
|
|
293
431
|
counts: result.body.aggregations,
|
|
294
432
|
};
|
|
295
433
|
} catch (error) {
|
|
296
|
-
//log.error(e, logDetails);
|
|
297
434
|
return error;
|
|
298
435
|
}
|
|
299
436
|
}
|
|
@@ -301,9 +438,24 @@ class BaseSearch {
|
|
|
301
438
|
|
|
302
439
|
class Search extends BaseSearch {}
|
|
303
440
|
|
|
441
|
+
/**
|
|
442
|
+
* Initializes and returns an instance of an `EsClient` Class
|
|
443
|
+
*
|
|
444
|
+
* @param {string} host - The host URL for the Elasticsearch instance.
|
|
445
|
+
* @param {boolean} metrics - A flag indicating whether metrics are enabled.
|
|
446
|
+
* @returns {Promise<EsClient>} A promise that resolves to an instance of `EsClient`.
|
|
447
|
+
*/
|
|
448
|
+
const getEsClient = async (host, metrics) => {
|
|
449
|
+
const esClient = new EsClient(host, metrics);
|
|
450
|
+
await esClient.initializeEsClient();
|
|
451
|
+
return esClient;
|
|
452
|
+
};
|
|
453
|
+
|
|
304
454
|
module.exports = {
|
|
305
455
|
BaseSearch,
|
|
306
456
|
Search,
|
|
457
|
+
EsClient,
|
|
458
|
+
getEsClient,
|
|
307
459
|
defaultIndexAlias,
|
|
308
460
|
multipleRecordFoundString,
|
|
309
461
|
recordNotFoundString,
|
package/stats.js
CHANGED
|
@@ -28,8 +28,8 @@ class Stats extends BaseSearch {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
async query() {
|
|
31
|
-
if (!this.
|
|
32
|
-
this.
|
|
31
|
+
if (!this._esClient) {
|
|
32
|
+
await this.initializeEsClient(undefined, this.metrics);
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
// granules
|
|
@@ -105,8 +105,8 @@ class Stats extends BaseSearch {
|
|
|
105
105
|
}
|
|
106
106
|
|
|
107
107
|
async count() {
|
|
108
|
-
if (!this.
|
|
109
|
-
this.
|
|
108
|
+
if (!this._esClient) {
|
|
109
|
+
await this.initializeEsClient(undefined, this.metrics);
|
|
110
110
|
}
|
|
111
111
|
|
|
112
112
|
const originalField = this.params.field || 'status';
|
package/testUtils.js
CHANGED
|
@@ -2,7 +2,7 @@ const { randomString } = require('@cumulus/common/test-utils');
|
|
|
2
2
|
|
|
3
3
|
const bootstrap = require('./bootstrap');
|
|
4
4
|
|
|
5
|
-
const { Search } = require('./search');
|
|
5
|
+
const { EsClient, Search } = require('./search');
|
|
6
6
|
|
|
7
7
|
const createTestIndex = async () => {
|
|
8
8
|
const esIndex = randomString();
|
|
@@ -13,12 +13,19 @@ const createTestIndex = async () => {
|
|
|
13
13
|
index: esIndex,
|
|
14
14
|
alias: esAlias,
|
|
15
15
|
});
|
|
16
|
-
const esClient = await
|
|
17
|
-
|
|
16
|
+
const esClient = await new EsClient('fakehost');
|
|
17
|
+
await esClient.initializeEsClient();
|
|
18
|
+
const searchClient = await new Search();
|
|
19
|
+
await searchClient.initializeEsClient('fakehost');
|
|
20
|
+
return {
|
|
21
|
+
esClient,
|
|
22
|
+
esIndex,
|
|
23
|
+
searchClient,
|
|
24
|
+
};
|
|
18
25
|
};
|
|
19
26
|
|
|
20
27
|
const cleanupTestIndex = async ({ esClient, esIndex }) => {
|
|
21
|
-
await esClient.indices.delete({ index: esIndex });
|
|
28
|
+
await esClient.client.indices.delete({ index: esIndex });
|
|
22
29
|
};
|
|
23
30
|
|
|
24
31
|
module.exports = {
|