@cumulus/db 19.2.0-alpha.0 → 19.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/execution.d.ts +14 -12
- package/dist/lib/execution.js +23 -27
- package/dist/models/file.d.ts +4 -0
- package/dist/models/file.js +8 -0
- package/dist/search/CollectionSearch.d.ts +14 -11
- package/dist/search/CollectionSearch.js +47 -35
- package/dist/search/ExecutionSearch.d.ts +4 -4
- package/dist/search/ExecutionSearch.js +4 -4
- package/dist/search/GranuleSearch.d.ts +14 -1
- package/dist/search/GranuleSearch.js +55 -3
- package/dist/search/StatsSearch.d.ts +0 -1
- package/dist/search/StatsSearch.js +1 -2
- package/dist/translate/async_operations.js +2 -1
- package/package.json +8 -8
package/dist/lib/execution.d.ts
CHANGED
|
@@ -6,7 +6,20 @@ export interface ArnRecord {
|
|
|
6
6
|
arn: string;
|
|
7
7
|
}
|
|
8
8
|
/**
|
|
9
|
-
* Returns execution
|
|
9
|
+
* Returns execution records sorted by most recent first for an input
|
|
10
|
+
* set of Granule Cumulus IDs.
|
|
11
|
+
* @returns Array of arn objects with the most recent first.
|
|
12
|
+
*/
|
|
13
|
+
export declare const getExecutionInfoByGranuleCumulusIds: ({ knexOrTransaction, granuleCumulusIds, limit, }: {
|
|
14
|
+
knexOrTransaction: Knex | Knex.Transaction;
|
|
15
|
+
granuleCumulusIds: number[];
|
|
16
|
+
limit?: number | undefined;
|
|
17
|
+
}) => Promise<{
|
|
18
|
+
granule_cumulus_id: number;
|
|
19
|
+
url: string;
|
|
20
|
+
}[]>;
|
|
21
|
+
/**
|
|
22
|
+
* Returns execution records sorted by most recent first for an input
|
|
10
23
|
* Granule Cumulus ID.
|
|
11
24
|
*
|
|
12
25
|
* @param {Object} params
|
|
@@ -24,17 +37,6 @@ export declare const getExecutionInfoByGranuleCumulusId: ({ knexOrTransaction, g
|
|
|
24
37
|
executionColumns: string[];
|
|
25
38
|
limit?: number | undefined;
|
|
26
39
|
}) => Promise<Partial<PostgresExecutionRecord>[]>;
|
|
27
|
-
/**
|
|
28
|
-
* Returns a list of executionArns sorted by most recent first, for an input
|
|
29
|
-
* Granule Cumulus ID.
|
|
30
|
-
*
|
|
31
|
-
* @param {Knex | Knex.Transaction} knexOrTransaction
|
|
32
|
-
* Knex client for reading from RDS database
|
|
33
|
-
* @param {number} granuleCumulusId - The primary ID for a Granule
|
|
34
|
-
* @param {number} limit - limit to number of executions to query
|
|
35
|
-
* @returns {Promise<ArnRecord[]>} - Array of arn objects with the most recent first.
|
|
36
|
-
*/
|
|
37
|
-
export declare const getExecutionArnsByGranuleCumulusId: (knexOrTransaction: Knex | Knex.Transaction, granuleCumulusId: Number, limit?: number) => Promise<ArnRecord[]>;
|
|
38
40
|
/**
|
|
39
41
|
* Returns a list of executionArns sorted by most recent first, for an input
|
|
40
42
|
* list of granuleIds and workflowNames.
|
package/dist/lib/execution.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.
|
|
3
|
+
exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.getExecutionInfoByGranuleCumulusId = exports.getExecutionInfoByGranuleCumulusIds = void 0;
|
|
4
4
|
const errors_1 = require("@cumulus/errors");
|
|
5
5
|
const execution_1 = require("../models/execution");
|
|
6
6
|
const granules_executions_1 = require("../models/granules-executions");
|
|
@@ -9,54 +9,50 @@ const Logger = require('@cumulus/logger');
|
|
|
9
9
|
const { getKnexClient } = require('../connection');
|
|
10
10
|
const log = new Logger({ sender: '@cumulus/db/lib/execution' });
|
|
11
11
|
/**
|
|
12
|
-
* Returns execution
|
|
13
|
-
* Granule Cumulus
|
|
14
|
-
*
|
|
15
|
-
* @param {Object} params
|
|
16
|
-
* @param {Knex | Knex.Transaction} params.knexOrTransaction
|
|
17
|
-
* Knex client for reading from RDS database
|
|
18
|
-
* @param {Array<string>} params.executionColumns - Columns to return from executions table
|
|
19
|
-
* @param {number} params.granuleCumulusId - The primary ID for a Granule
|
|
20
|
-
* @param {number} [params.limit] - limit to number of executions to query
|
|
21
|
-
* @returns {Promise<Partial<PostgresExecutionRecord>[]>}
|
|
22
|
-
* Array of arn objects with the most recent first.
|
|
12
|
+
* Returns execution records sorted by most recent first for an input
|
|
13
|
+
* set of Granule Cumulus IDs.
|
|
14
|
+
* @returns Array of arn objects with the most recent first.
|
|
23
15
|
*/
|
|
24
|
-
const
|
|
16
|
+
const getExecutionInfoByGranuleCumulusIds = async ({ knexOrTransaction, granuleCumulusIds, limit, }) => {
|
|
25
17
|
const knexQuery = knexOrTransaction(tables_1.TableNames.executions)
|
|
26
|
-
.column(
|
|
27
|
-
|
|
18
|
+
.column([
|
|
19
|
+
`${tables_1.TableNames.executions}.url`,
|
|
20
|
+
`${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`,
|
|
21
|
+
])
|
|
22
|
+
.whereIn(`${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusIds)
|
|
28
23
|
.join(tables_1.TableNames.granulesExecutions, `${tables_1.TableNames.executions}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.execution_cumulus_id`)
|
|
29
|
-
.join(tables_1.TableNames.granules, `${tables_1.TableNames.granules}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`)
|
|
30
24
|
.orderBy(`${tables_1.TableNames.executions}.timestamp`, 'desc');
|
|
31
25
|
if (limit) {
|
|
32
26
|
knexQuery.limit(limit);
|
|
33
27
|
}
|
|
34
28
|
return await knexQuery;
|
|
35
29
|
};
|
|
36
|
-
exports.
|
|
30
|
+
exports.getExecutionInfoByGranuleCumulusIds = getExecutionInfoByGranuleCumulusIds;
|
|
37
31
|
/**
|
|
38
|
-
* Returns
|
|
32
|
+
* Returns execution records sorted by most recent first for an input
|
|
39
33
|
* Granule Cumulus ID.
|
|
40
34
|
*
|
|
41
|
-
* @param {
|
|
35
|
+
* @param {Object} params
|
|
36
|
+
* @param {Knex | Knex.Transaction} params.knexOrTransaction
|
|
42
37
|
* Knex client for reading from RDS database
|
|
43
|
-
* @param {
|
|
44
|
-
* @param {number}
|
|
45
|
-
* @
|
|
38
|
+
* @param {Array<string>} params.executionColumns - Columns to return from executions table
|
|
39
|
+
* @param {number} params.granuleCumulusId - The primary ID for a Granule
|
|
40
|
+
* @param {number} [params.limit] - limit to number of executions to query
|
|
41
|
+
* @returns {Promise<Partial<PostgresExecutionRecord>[]>}
|
|
42
|
+
* Array of arn objects with the most recent first.
|
|
46
43
|
*/
|
|
47
|
-
const
|
|
44
|
+
const getExecutionInfoByGranuleCumulusId = async ({ knexOrTransaction, granuleCumulusId, executionColumns = ['arn'], limit, }) => {
|
|
48
45
|
const knexQuery = knexOrTransaction(tables_1.TableNames.executions)
|
|
49
|
-
.
|
|
50
|
-
.where(`${tables_1.TableNames.
|
|
46
|
+
.column(executionColumns.map((column) => `${tables_1.TableNames.executions}.${column}`))
|
|
47
|
+
.where(`${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusId)
|
|
51
48
|
.join(tables_1.TableNames.granulesExecutions, `${tables_1.TableNames.executions}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.execution_cumulus_id`)
|
|
52
|
-
.join(tables_1.TableNames.granules, `${tables_1.TableNames.granules}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`)
|
|
53
49
|
.orderBy(`${tables_1.TableNames.executions}.timestamp`, 'desc');
|
|
54
50
|
if (limit) {
|
|
55
51
|
knexQuery.limit(limit);
|
|
56
52
|
}
|
|
57
53
|
return await knexQuery;
|
|
58
54
|
};
|
|
59
|
-
exports.
|
|
55
|
+
exports.getExecutionInfoByGranuleCumulusId = getExecutionInfoByGranuleCumulusId;
|
|
60
56
|
/**
|
|
61
57
|
* Returns a list of executionArns sorted by most recent first, for an input
|
|
62
58
|
* list of granuleIds and workflowNames.
|
package/dist/models/file.d.ts
CHANGED
|
@@ -12,6 +12,10 @@ declare class FilePgModel extends BasePgModel<PostgresFile, PostgresFileRecord>
|
|
|
12
12
|
_intersectProps: {};
|
|
13
13
|
_unionProps: never;
|
|
14
14
|
}[]>;
|
|
15
|
+
/**
|
|
16
|
+
* Retrieves all files for all granules given
|
|
17
|
+
*/
|
|
18
|
+
searchByGranuleCumulusIds(knexOrTrx: Knex | Knex.Transaction, granule_cumulus_ids: number[], columns?: string | string[]): Promise<PostgresFileRecord[]>;
|
|
15
19
|
}
|
|
16
20
|
export { FilePgModel };
|
|
17
21
|
//# sourceMappingURL=file.d.ts.map
|
package/dist/models/file.js
CHANGED
|
@@ -16,6 +16,14 @@ class FilePgModel extends base_1.BasePgModel {
|
|
|
16
16
|
.merge()
|
|
17
17
|
.returning('*');
|
|
18
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Retrieves all files for all granules given
|
|
21
|
+
*/
|
|
22
|
+
searchByGranuleCumulusIds(knexOrTrx, granule_cumulus_ids, columns = '*') {
|
|
23
|
+
return knexOrTrx(this.tableName)
|
|
24
|
+
.select(columns)
|
|
25
|
+
.whereIn('granule_cumulus_id', granule_cumulus_ids);
|
|
26
|
+
}
|
|
19
27
|
}
|
|
20
28
|
exports.FilePgModel = FilePgModel;
|
|
21
29
|
//# sourceMappingURL=file.js.map
|
|
@@ -34,20 +34,23 @@ export declare class CollectionSearch extends BaseSearch {
|
|
|
34
34
|
dbQueryParameters?: DbQueryParameters;
|
|
35
35
|
}): void;
|
|
36
36
|
/**
|
|
37
|
-
* Build
|
|
37
|
+
* Build subquery for active collections
|
|
38
|
+
* The subquery will search granules
|
|
38
39
|
*
|
|
39
|
-
* @param
|
|
40
|
-
* @
|
|
41
|
-
* @param [params.countQuery] - query builder for getting count
|
|
42
|
-
* @param params.searchQuery - query builder for search
|
|
43
|
-
* @param [params.dbQueryParameters] - db query parameters
|
|
40
|
+
* @param knex - db client
|
|
41
|
+
* @returns granule query
|
|
44
42
|
*/
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
43
|
+
private buildSubQueryForActiveCollections;
|
|
44
|
+
/**
|
|
45
|
+
* Build the search query
|
|
46
|
+
*
|
|
47
|
+
* @param knex - DB client
|
|
48
|
+
* @returns queries for getting count and search result
|
|
49
|
+
*/
|
|
50
|
+
protected buildSearch(knex: Knex): {
|
|
51
|
+
countQuery?: Knex.QueryBuilder;
|
|
48
52
|
searchQuery: Knex.QueryBuilder;
|
|
49
|
-
|
|
50
|
-
}): void;
|
|
53
|
+
};
|
|
51
54
|
/**
|
|
52
55
|
* Executes stats query to get granules' status aggregation
|
|
53
56
|
*
|
|
@@ -4,12 +4,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.CollectionSearch = void 0;
|
|
7
|
+
const omitBy_1 = __importDefault(require("lodash/omitBy"));
|
|
7
8
|
const pick_1 = __importDefault(require("lodash/pick"));
|
|
8
9
|
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
9
10
|
const BaseSearch_1 = require("./BaseSearch");
|
|
11
|
+
const queries_1 = require("./queries");
|
|
12
|
+
const GranuleSearch_1 = require("./GranuleSearch");
|
|
10
13
|
const collections_1 = require("../translate/collections");
|
|
11
14
|
const tables_1 = require("../tables");
|
|
12
15
|
const log = new logger_1.default({ sender: '@cumulus/db/CollectionSearch' });
|
|
16
|
+
const granuleFields = ['createdAt', 'granuleId', 'timestamp', 'updatedAt'];
|
|
17
|
+
const isGranuleField = (_value, key) => granuleFields.includes(key.split('__')[0]);
|
|
13
18
|
/**
|
|
14
19
|
* Class to build and execute db search query for collections
|
|
15
20
|
*/
|
|
@@ -19,6 +24,10 @@ class CollectionSearch extends BaseSearch_1.BaseSearch {
|
|
|
19
24
|
super({ queryStringParameters }, 'collection');
|
|
20
25
|
this.active = (active === 'true');
|
|
21
26
|
this.includeStats = (includeStats === 'true');
|
|
27
|
+
// for active collection search, omit the fields which are for searching granules
|
|
28
|
+
if (this.active) {
|
|
29
|
+
this.dbQueryParameters = (0, queries_1.convertQueryStringToDbQueryParameters)(this.type, (0, omitBy_1.default)(this.queryStringParameters, isGranuleField));
|
|
30
|
+
}
|
|
22
31
|
}
|
|
23
32
|
/**
|
|
24
33
|
* Build queries for infix and prefix
|
|
@@ -39,34 +48,39 @@ class CollectionSearch extends BaseSearch_1.BaseSearch {
|
|
|
39
48
|
}
|
|
40
49
|
}
|
|
41
50
|
/**
|
|
42
|
-
* Build
|
|
51
|
+
* Build subquery for active collections
|
|
52
|
+
* The subquery will search granules
|
|
43
53
|
*
|
|
44
|
-
* @param
|
|
45
|
-
* @
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
54
|
+
* @param knex - db client
|
|
55
|
+
* @returns granule query
|
|
56
|
+
*/
|
|
57
|
+
buildSubQueryForActiveCollections(knex) {
|
|
58
|
+
const granulesTable = tables_1.TableNames.granules;
|
|
59
|
+
const granuleSearch = new GranuleSearch_1.GranuleSearch({ queryStringParameters: this.queryStringParameters });
|
|
60
|
+
const { countQuery: subQuery } = granuleSearch.buildSearchForActiveCollections(knex);
|
|
61
|
+
subQuery
|
|
62
|
+
.clear('select')
|
|
63
|
+
.select(1)
|
|
64
|
+
.where(`${granulesTable}.collection_cumulus_id`, knex.raw(`${this.tableName}.cumulus_id`))
|
|
65
|
+
.limit(1);
|
|
66
|
+
return subQuery;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Build the search query
|
|
70
|
+
*
|
|
71
|
+
* @param knex - DB client
|
|
72
|
+
* @returns queries for getting count and search result
|
|
49
73
|
*/
|
|
50
|
-
|
|
74
|
+
buildSearch(knex) {
|
|
75
|
+
const queries = super.buildSearch(knex);
|
|
51
76
|
if (!this.active) {
|
|
52
|
-
|
|
53
|
-
return;
|
|
77
|
+
return queries;
|
|
54
78
|
}
|
|
55
|
-
const
|
|
56
|
-
const {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
Object.entries(range).forEach(([name, rangeValues]) => {
|
|
61
|
-
if (rangeValues.gte) {
|
|
62
|
-
subQuery.where(`${granulesTable}.${name}`, '>=', rangeValues.gte);
|
|
63
|
-
}
|
|
64
|
-
if (rangeValues.lte) {
|
|
65
|
-
subQuery.where(`${granulesTable}.${name}`, '<=', rangeValues.lte);
|
|
66
|
-
}
|
|
67
|
-
});
|
|
68
|
-
subQuery.limit(1);
|
|
69
|
-
[countQuery, searchQuery].forEach((query) => query.whereExists(subQuery));
|
|
79
|
+
const subQuery = this.buildSubQueryForActiveCollections(knex);
|
|
80
|
+
const { countQuery, searchQuery } = queries;
|
|
81
|
+
[countQuery, searchQuery].forEach((query) => query?.whereExists(subQuery));
|
|
82
|
+
log.debug(`buildSearch returns countQuery: ${countQuery?.toSQL().sql}, searchQuery: ${searchQuery.toSQL().sql}`);
|
|
83
|
+
return { countQuery, searchQuery };
|
|
70
84
|
}
|
|
71
85
|
/**
|
|
72
86
|
* Executes stats query to get granules' status aggregation
|
|
@@ -77,21 +91,19 @@ class CollectionSearch extends BaseSearch_1.BaseSearch {
|
|
|
77
91
|
*/
|
|
78
92
|
async retrieveGranuleStats(collectionCumulusIds, knex) {
|
|
79
93
|
const granulesTable = tables_1.TableNames.granules;
|
|
80
|
-
|
|
94
|
+
let statsQuery = knex(granulesTable);
|
|
95
|
+
if (this.active) {
|
|
96
|
+
const granuleSearch = new GranuleSearch_1.GranuleSearch({
|
|
97
|
+
queryStringParameters: this.queryStringParameters,
|
|
98
|
+
});
|
|
99
|
+
const { countQuery } = granuleSearch.buildSearchForActiveCollections(knex);
|
|
100
|
+
statsQuery = countQuery.clear('select');
|
|
101
|
+
}
|
|
102
|
+
statsQuery
|
|
81
103
|
.select(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`)
|
|
82
104
|
.count('*')
|
|
83
105
|
.groupBy(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`)
|
|
84
106
|
.whereIn(`${granulesTable}.collection_cumulus_id`, collectionCumulusIds);
|
|
85
|
-
if (this.active) {
|
|
86
|
-
Object.entries(this.dbQueryParameters?.range ?? {}).forEach(([name, rangeValues]) => {
|
|
87
|
-
if (rangeValues.gte) {
|
|
88
|
-
statsQuery.where(`${granulesTable}.${name}`, '>=', rangeValues.gte);
|
|
89
|
-
}
|
|
90
|
-
if (rangeValues.lte) {
|
|
91
|
-
statsQuery.where(`${granulesTable}.${name}`, '<=', rangeValues.lte);
|
|
92
|
-
}
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
107
|
log.debug(`retrieveGranuleStats statsQuery: ${statsQuery?.toSQL().sql}`);
|
|
96
108
|
const results = await statsQuery;
|
|
97
109
|
const reduced = results.reduce((acc, record) => {
|
|
@@ -16,15 +16,15 @@ interface ExecutionRecord extends BaseRecord, PostgresExecutionRecord {
|
|
|
16
16
|
export declare class ExecutionSearch extends BaseSearch {
|
|
17
17
|
constructor(event: QueryEvent);
|
|
18
18
|
/**
|
|
19
|
-
* check if joined
|
|
19
|
+
* check if joined async_operations table search is needed
|
|
20
20
|
*
|
|
21
|
-
* @returns whether
|
|
21
|
+
* @returns whether async_operations search is needed
|
|
22
22
|
*/
|
|
23
23
|
protected searchAsync(): boolean;
|
|
24
24
|
/**
|
|
25
|
-
* check if joined
|
|
25
|
+
* check if joined parent execution table search is needed
|
|
26
26
|
*
|
|
27
|
-
* @returns whether
|
|
27
|
+
* @returns whether parent execution search is needed
|
|
28
28
|
*/
|
|
29
29
|
protected searchParent(): boolean;
|
|
30
30
|
/**
|
|
@@ -24,18 +24,18 @@ class ExecutionSearch extends BaseSearch_1.BaseSearch {
|
|
|
24
24
|
super(event, 'execution');
|
|
25
25
|
}
|
|
26
26
|
/**
|
|
27
|
-
* check if joined
|
|
27
|
+
* check if joined async_operations table search is needed
|
|
28
28
|
*
|
|
29
|
-
* @returns whether
|
|
29
|
+
* @returns whether async_operations search is needed
|
|
30
30
|
*/
|
|
31
31
|
searchAsync() {
|
|
32
32
|
const { not, term, terms } = this.dbQueryParameters;
|
|
33
33
|
return (!!(not?.asyncOperationId || term?.asyncOperationId || terms?.asyncOperationId));
|
|
34
34
|
}
|
|
35
35
|
/**
|
|
36
|
-
* check if joined
|
|
36
|
+
* check if joined parent execution table search is needed
|
|
37
37
|
*
|
|
38
|
-
* @returns whether
|
|
38
|
+
* @returns whether parent execution search is needed
|
|
39
39
|
*/
|
|
40
40
|
searchParent() {
|
|
41
41
|
const { not, term, terms } = this.dbQueryParameters;
|
|
@@ -38,13 +38,26 @@ export declare class GranuleSearch extends BaseSearch {
|
|
|
38
38
|
searchQuery: Knex.QueryBuilder;
|
|
39
39
|
dbQueryParameters?: DbQueryParameters;
|
|
40
40
|
}): void;
|
|
41
|
+
/**
|
|
42
|
+
* Build the search query for active collections.
|
|
43
|
+
* If time params are specified the query will search granules that have been updated
|
|
44
|
+
* in that time frame. If granuleId or providerId are provided, it will filter those as well.
|
|
45
|
+
*
|
|
46
|
+
* @param knex - DB client
|
|
47
|
+
* @returns queries for getting count and search result
|
|
48
|
+
*/
|
|
49
|
+
buildSearchForActiveCollections(knex: Knex): {
|
|
50
|
+
countQuery: Knex.QueryBuilder;
|
|
51
|
+
searchQuery: Knex.QueryBuilder;
|
|
52
|
+
};
|
|
41
53
|
/**
|
|
42
54
|
* Translate postgres records to api records
|
|
43
55
|
*
|
|
44
56
|
* @param pgRecords - postgres records returned from query
|
|
57
|
+
* @param knex - DB client
|
|
45
58
|
* @returns translated api records
|
|
46
59
|
*/
|
|
47
|
-
protected translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[]): Partial<ApiGranuleRecord>[]
|
|
60
|
+
protected translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[], knex: Knex): Promise<Partial<ApiGranuleRecord>[]>;
|
|
48
61
|
}
|
|
49
62
|
export {};
|
|
50
63
|
//# sourceMappingURL=GranuleSearch.d.ts.map
|
|
@@ -10,6 +10,8 @@ const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
|
10
10
|
const BaseSearch_1 = require("./BaseSearch");
|
|
11
11
|
const granules_1 = require("../translate/granules");
|
|
12
12
|
const tables_1 = require("../tables");
|
|
13
|
+
const file_1 = require("../models/file");
|
|
14
|
+
const execution_1 = require("../lib/execution");
|
|
13
15
|
const log = new logger_1.default({ sender: '@cumulus/db/GranuleSearch' });
|
|
14
16
|
/**
|
|
15
17
|
* Class to build and execute db search query for granules
|
|
@@ -78,15 +80,56 @@ class GranuleSearch extends BaseSearch_1.BaseSearch {
|
|
|
78
80
|
[countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.granule_id`, `${prefix}%`));
|
|
79
81
|
}
|
|
80
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* Build the search query for active collections.
|
|
85
|
+
* If time params are specified the query will search granules that have been updated
|
|
86
|
+
* in that time frame. If granuleId or providerId are provided, it will filter those as well.
|
|
87
|
+
*
|
|
88
|
+
* @param knex - DB client
|
|
89
|
+
* @returns queries for getting count and search result
|
|
90
|
+
*/
|
|
91
|
+
buildSearchForActiveCollections(knex) {
|
|
92
|
+
const { countQuery, searchQuery } = this.buildBasicQuery(knex);
|
|
93
|
+
this.buildTermQuery({ countQuery, searchQuery });
|
|
94
|
+
this.buildTermsQuery({ countQuery, searchQuery });
|
|
95
|
+
this.buildRangeQuery({ knex, countQuery, searchQuery });
|
|
96
|
+
log.debug(`buildSearchForActiveCollections returns countQuery: ${countQuery?.toSQL().sql}, searchQuery: ${searchQuery.toSQL().sql}`);
|
|
97
|
+
return { countQuery, searchQuery };
|
|
98
|
+
}
|
|
81
99
|
/**
|
|
82
100
|
* Translate postgres records to api records
|
|
83
101
|
*
|
|
84
102
|
* @param pgRecords - postgres records returned from query
|
|
103
|
+
* @param knex - DB client
|
|
85
104
|
* @returns translated api records
|
|
86
105
|
*/
|
|
87
|
-
translatePostgresRecordsToApiRecords(pgRecords) {
|
|
106
|
+
async translatePostgresRecordsToApiRecords(pgRecords, knex) {
|
|
88
107
|
log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `);
|
|
89
|
-
const { fields } = this.dbQueryParameters;
|
|
108
|
+
const { fields, includeFullRecord } = this.dbQueryParameters;
|
|
109
|
+
const fileMapping = {};
|
|
110
|
+
const executionMapping = {};
|
|
111
|
+
const cumulusIds = pgRecords.map((record) => record.cumulus_id);
|
|
112
|
+
if (includeFullRecord) {
|
|
113
|
+
//get Files
|
|
114
|
+
const fileModel = new file_1.FilePgModel();
|
|
115
|
+
const files = await fileModel.searchByGranuleCumulusIds(knex, cumulusIds);
|
|
116
|
+
files.forEach((file) => {
|
|
117
|
+
if (!(file.granule_cumulus_id in fileMapping)) {
|
|
118
|
+
fileMapping[file.granule_cumulus_id] = [];
|
|
119
|
+
}
|
|
120
|
+
fileMapping[file.granule_cumulus_id].push(file);
|
|
121
|
+
});
|
|
122
|
+
//get Executions
|
|
123
|
+
const executions = await (0, execution_1.getExecutionInfoByGranuleCumulusIds)({
|
|
124
|
+
knexOrTransaction: knex,
|
|
125
|
+
granuleCumulusIds: cumulusIds,
|
|
126
|
+
});
|
|
127
|
+
executions.forEach((execution) => {
|
|
128
|
+
if (!(execution.granule_cumulus_id in executionMapping)) {
|
|
129
|
+
executionMapping[execution.granule_cumulus_id] = execution;
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
}
|
|
90
133
|
const apiRecords = pgRecords.map((item) => {
|
|
91
134
|
const granulePgRecord = item;
|
|
92
135
|
const collectionPgRecord = {
|
|
@@ -94,10 +137,19 @@ class GranuleSearch extends BaseSearch_1.BaseSearch {
|
|
|
94
137
|
name: item.collectionName,
|
|
95
138
|
version: item.collectionVersion,
|
|
96
139
|
};
|
|
140
|
+
const executionUrls = executionMapping[item.cumulus_id]?.url
|
|
141
|
+
? [{ url: executionMapping[item.cumulus_id].url }]
|
|
142
|
+
: [];
|
|
97
143
|
const pdr = item.pdrName ? { name: item.pdrName } : undefined;
|
|
98
144
|
const providerPgRecord = item.providerName ? { name: item.providerName } : undefined;
|
|
145
|
+
const fileRecords = fileMapping[granulePgRecord.cumulus_id] || [];
|
|
99
146
|
const apiRecord = (0, granules_1.translatePostgresGranuleToApiGranuleWithoutDbQuery)({
|
|
100
|
-
granulePgRecord,
|
|
147
|
+
granulePgRecord,
|
|
148
|
+
collectionPgRecord,
|
|
149
|
+
pdr,
|
|
150
|
+
providerPgRecord,
|
|
151
|
+
files: fileRecords,
|
|
152
|
+
executionUrls,
|
|
101
153
|
});
|
|
102
154
|
return fields ? (0, pick_1.default)(apiRecord, fields) : apiRecord;
|
|
103
155
|
});
|
|
@@ -93,7 +93,6 @@ declare class StatsSearch extends BaseSearch {
|
|
|
93
93
|
* @param params
|
|
94
94
|
* @param params.searchQuery - the search query
|
|
95
95
|
* @param [params.dbQueryParameters] - the db query parameters
|
|
96
|
-
* @returns the updated search query based on queryStringParams
|
|
97
96
|
*/
|
|
98
97
|
protected buildTermQuery(params: {
|
|
99
98
|
searchQuery: Knex.QueryBuilder;
|
|
@@ -180,7 +180,6 @@ class StatsSearch extends BaseSearch_1.BaseSearch {
|
|
|
180
180
|
* @param params
|
|
181
181
|
* @param params.searchQuery - the search query
|
|
182
182
|
* @param [params.dbQueryParameters] - the db query parameters
|
|
183
|
-
* @returns the updated search query based on queryStringParams
|
|
184
183
|
*/
|
|
185
184
|
buildTermQuery(params) {
|
|
186
185
|
const { dbQueryParameters, searchQuery } = params;
|
|
@@ -188,7 +187,7 @@ class StatsSearch extends BaseSearch_1.BaseSearch {
|
|
|
188
187
|
if (this.field?.includes('error.Error')) {
|
|
189
188
|
searchQuery.whereRaw(`${this.tableName}.error ->> 'Error' is not null`);
|
|
190
189
|
}
|
|
191
|
-
|
|
190
|
+
super.buildTermQuery({
|
|
192
191
|
...params,
|
|
193
192
|
dbQueryParameters: { term: (0, omit_1.default)(term, 'error.Error') },
|
|
194
193
|
});
|
|
@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.translatePostgresAsyncOperationToApiAsyncOperation = exports.translateApiAsyncOperationToPostgresAsyncOperation = void 0;
|
|
7
|
+
const omit_1 = __importDefault(require("lodash/omit"));
|
|
7
8
|
const snake_camel_1 = require("snake-camel");
|
|
8
9
|
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
9
10
|
const log = new logger_1.default({ sender: '@cumulus/db/translate/async-operations' });
|
|
@@ -38,7 +39,7 @@ const convertOutputToObject = (output) => {
|
|
|
38
39
|
*/
|
|
39
40
|
const translateApiAsyncOperationToPostgresAsyncOperation = (record) => {
|
|
40
41
|
// fix for old implementation of async-operation output assignment
|
|
41
|
-
const translatedRecord = (0, snake_camel_1.toSnake)(record);
|
|
42
|
+
const translatedRecord = (0, snake_camel_1.toSnake)((0, omit_1.default)(record, 'timestamp'));
|
|
42
43
|
if (record.output === 'none') {
|
|
43
44
|
delete translatedRecord.output;
|
|
44
45
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/db",
|
|
3
|
-
"version": "19.2.0-alpha.
|
|
3
|
+
"version": "19.2.0-alpha.1",
|
|
4
4
|
"description": "Utilities for working with the Cumulus DB",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -33,12 +33,12 @@
|
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@aws-sdk/client-secrets-manager": "^3.621.0",
|
|
36
|
-
"@cumulus/aws-client": "19.2.0-alpha.
|
|
37
|
-
"@cumulus/common": "19.2.0-alpha.
|
|
38
|
-
"@cumulus/errors": "19.2.0-alpha.
|
|
39
|
-
"@cumulus/logger": "19.2.0-alpha.
|
|
40
|
-
"@cumulus/message": "19.2.0-alpha.
|
|
41
|
-
"@cumulus/types": "19.2.0-alpha.
|
|
36
|
+
"@cumulus/aws-client": "19.2.0-alpha.1",
|
|
37
|
+
"@cumulus/common": "19.2.0-alpha.1",
|
|
38
|
+
"@cumulus/errors": "19.2.0-alpha.1",
|
|
39
|
+
"@cumulus/logger": "19.2.0-alpha.1",
|
|
40
|
+
"@cumulus/message": "19.2.0-alpha.1",
|
|
41
|
+
"@cumulus/types": "19.2.0-alpha.1",
|
|
42
42
|
"crypto-random-string": "^3.2.0",
|
|
43
43
|
"is-valid-hostname": "1.0.2",
|
|
44
44
|
"knex": "2.4.1",
|
|
@@ -50,5 +50,5 @@
|
|
|
50
50
|
"devDependencies": {
|
|
51
51
|
"@types/uuid": "^8.0.0"
|
|
52
52
|
},
|
|
53
|
-
"gitHead": "
|
|
53
|
+
"gitHead": "3b8d4b03abefe6a623245903b48685ebb7ae4ca7"
|
|
54
54
|
}
|