@cumulus/db 18.3.4 → 19.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/config.d.ts +19 -0
- package/dist/config.js +40 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +15 -4
- package/dist/lib/execution.d.ts +5 -0
- package/dist/lib/execution.js +16 -1
- package/dist/migrations/20240125171703_update_granule_execution_cumulus_id_type.d.ts +4 -0
- package/dist/migrations/20240125171703_update_granule_execution_cumulus_id_type.js +22 -0
- package/dist/migrations/20240126135619_granules_add_indexes.d.ts +4 -0
- package/dist/migrations/20240126135619_granules_add_indexes.js +19 -0
- package/dist/migrations/20240126165019_granules_update_constraints.d.ts +4 -0
- package/dist/migrations/20240126165019_granules_update_constraints.js +44 -0
- package/dist/migrations/20240606060726_alter_async_operations_add_operation_type_bulk_execution_delete.d.ts +4 -0
- package/dist/migrations/20240606060726_alter_async_operations_add_operation_type_bulk_execution_delete.js +43 -0
- package/dist/migrations/20240613174614_add_execution_parent_and_collection_indexes.d.ts +4 -0
- package/dist/migrations/20240613174614_add_execution_parent_and_collection_indexes.js +17 -0
- package/dist/migrations/20240617204826_update_executions_deletion_constraint.d.ts +4 -0
- package/dist/migrations/20240617204826_update_executions_deletion_constraint.js +14 -0
- package/dist/migrations/20240728101230_add_table_indexes.d.ts +4 -0
- package/dist/migrations/20240728101230_add_table_indexes.js +53 -0
- package/dist/models/execution.d.ts +2 -2
- package/dist/models/execution.js +1 -1
- package/dist/search/BaseSearch.d.ts +187 -0
- package/dist/search/BaseSearch.js +416 -0
- package/dist/search/CollectionSearch.d.ts +79 -0
- package/dist/search/CollectionSearch.js +162 -0
- package/dist/search/ExecutionSearch.d.ts +62 -0
- package/dist/search/ExecutionSearch.js +133 -0
- package/dist/search/GranuleSearch.d.ts +55 -0
- package/dist/search/GranuleSearch.js +109 -0
- package/dist/search/StatsSearch.d.ts +111 -0
- package/dist/search/StatsSearch.js +214 -0
- package/dist/search/field-mapping.d.ts +16 -0
- package/dist/search/field-mapping.js +304 -0
- package/dist/search/queries.d.ts +10 -0
- package/dist/search/queries.js +235 -0
- package/dist/translate/executions.d.ts +6 -0
- package/dist/translate/executions.js +32 -23
- package/dist/translate/granules.d.ts +25 -1
- package/dist/translate/granules.js +48 -27
- package/dist/types/search.d.ts +52 -0
- package/dist/types/search.js +3 -0
- package/package.json +9 -9
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.CollectionSearch = void 0;
|
|
7
|
+
const pick_1 = __importDefault(require("lodash/pick"));
|
|
8
|
+
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
9
|
+
const BaseSearch_1 = require("./BaseSearch");
|
|
10
|
+
const collections_1 = require("../translate/collections");
|
|
11
|
+
const tables_1 = require("../tables");
|
|
12
|
+
const log = new logger_1.default({ sender: '@cumulus/db/CollectionSearch' });
|
|
13
|
+
/**
|
|
14
|
+
* Class to build and execute db search query for collections
|
|
15
|
+
*/
|
|
16
|
+
class CollectionSearch extends BaseSearch_1.BaseSearch {
|
|
17
|
+
constructor(event) {
|
|
18
|
+
const { active, includeStats, ...queryStringParameters } = event.queryStringParameters || {};
|
|
19
|
+
super({ queryStringParameters }, 'collection');
|
|
20
|
+
this.active = (active === 'true');
|
|
21
|
+
this.includeStats = (includeStats === 'true');
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Build basic query
|
|
25
|
+
*
|
|
26
|
+
* @param knex - DB client
|
|
27
|
+
* @returns queries for getting count and search result
|
|
28
|
+
*/
|
|
29
|
+
buildBasicQuery(knex) {
|
|
30
|
+
const countQuery = knex(this.tableName)
|
|
31
|
+
.count('*');
|
|
32
|
+
const searchQuery = knex(this.tableName)
|
|
33
|
+
.select(`${this.tableName}.*`);
|
|
34
|
+
return { countQuery, searchQuery };
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Build queries for infix and prefix
|
|
38
|
+
*
|
|
39
|
+
* @param params
|
|
40
|
+
* @param params.countQuery - query builder for getting count
|
|
41
|
+
* @param params.searchQuery - query builder for search
|
|
42
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
43
|
+
*/
|
|
44
|
+
buildInfixPrefixQuery(params) {
|
|
45
|
+
const { countQuery, searchQuery, dbQueryParameters } = params;
|
|
46
|
+
const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters;
|
|
47
|
+
if (infix) {
|
|
48
|
+
[countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${infix}%`));
|
|
49
|
+
}
|
|
50
|
+
if (prefix) {
|
|
51
|
+
[countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${prefix}%`));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Build queries for range fields
|
|
56
|
+
*
|
|
57
|
+
* @param params
|
|
58
|
+
* @param params.knex - db client
|
|
59
|
+
* @param [params.countQuery] - query builder for getting count
|
|
60
|
+
* @param params.searchQuery - query builder for search
|
|
61
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
62
|
+
*/
|
|
63
|
+
buildRangeQuery(params) {
|
|
64
|
+
if (!this.active) {
|
|
65
|
+
super.buildRangeQuery(params);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
const granulesTable = tables_1.TableNames.granules;
|
|
69
|
+
const { knex, countQuery, searchQuery, dbQueryParameters } = params;
|
|
70
|
+
const { range = {} } = dbQueryParameters ?? this.dbQueryParameters;
|
|
71
|
+
const subQuery = knex.select(1).from(granulesTable)
|
|
72
|
+
.where(`${granulesTable}.collection_cumulus_id`, knex.raw(`${this.tableName}.cumulus_id`));
|
|
73
|
+
Object.entries(range).forEach(([name, rangeValues]) => {
|
|
74
|
+
if (rangeValues.gte) {
|
|
75
|
+
subQuery.where(`${granulesTable}.${name}`, '>=', rangeValues.gte);
|
|
76
|
+
}
|
|
77
|
+
if (rangeValues.lte) {
|
|
78
|
+
subQuery.where(`${granulesTable}.${name}`, '<=', rangeValues.lte);
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
subQuery.limit(1);
|
|
82
|
+
[countQuery, searchQuery].forEach((query) => query.whereExists(subQuery));
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Executes stats query to get granules' status aggregation
|
|
86
|
+
*
|
|
87
|
+
* @param collectionCumulusIds - array of cumulusIds of the collections
|
|
88
|
+
* @param knex - knex for the stats query
|
|
89
|
+
* @returns the collection's granules status' aggregation
|
|
90
|
+
*/
|
|
91
|
+
async retrieveGranuleStats(collectionCumulusIds, knex) {
|
|
92
|
+
const granulesTable = tables_1.TableNames.granules;
|
|
93
|
+
const statsQuery = knex(granulesTable)
|
|
94
|
+
.select(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`)
|
|
95
|
+
.count('*')
|
|
96
|
+
.groupBy(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`)
|
|
97
|
+
.whereIn(`${granulesTable}.collection_cumulus_id`, collectionCumulusIds);
|
|
98
|
+
if (this.active) {
|
|
99
|
+
Object.entries(this.dbQueryParameters?.range ?? {}).forEach(([name, rangeValues]) => {
|
|
100
|
+
if (rangeValues.gte) {
|
|
101
|
+
statsQuery.where(`${granulesTable}.${name}`, '>=', rangeValues.gte);
|
|
102
|
+
}
|
|
103
|
+
if (rangeValues.lte) {
|
|
104
|
+
statsQuery.where(`${granulesTable}.${name}`, '<=', rangeValues.lte);
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
log.debug(`retrieveGranuleStats statsQuery: ${statsQuery?.toSQL().sql}`);
|
|
109
|
+
const results = await statsQuery;
|
|
110
|
+
const reduced = results.reduce((acc, record) => {
|
|
111
|
+
const cumulusId = Number(record.collection_cumulus_id);
|
|
112
|
+
if (!acc[cumulusId]) {
|
|
113
|
+
acc[cumulusId] = {
|
|
114
|
+
queued: 0,
|
|
115
|
+
completed: 0,
|
|
116
|
+
failed: 0,
|
|
117
|
+
running: 0,
|
|
118
|
+
total: 0,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
acc[cumulusId][record.status] += Number(record.count);
|
|
122
|
+
acc[cumulusId]['total'] += Number(record.count);
|
|
123
|
+
return acc;
|
|
124
|
+
}, {});
|
|
125
|
+
return reduced;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Translate postgres records to api records
|
|
129
|
+
*
|
|
130
|
+
* @param pgRecords - postgres Collection records returned from query
|
|
131
|
+
* @param knex - knex for the stats query if incldueStats is true
|
|
132
|
+
* @returns translated api records
|
|
133
|
+
*/
|
|
134
|
+
async translatePostgresRecordsToApiRecords(pgRecords, knex) {
|
|
135
|
+
log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `);
|
|
136
|
+
let statsRecords;
|
|
137
|
+
const cumulusIds = pgRecords.map((record) => record.cumulus_id);
|
|
138
|
+
if (this.includeStats) {
|
|
139
|
+
statsRecords = await this.retrieveGranuleStats(cumulusIds, knex);
|
|
140
|
+
}
|
|
141
|
+
const apiRecords = pgRecords.map((record) => {
|
|
142
|
+
const apiRecord = (0, collections_1.translatePostgresCollectionToApiCollection)(record);
|
|
143
|
+
const apiRecordFinal = this.dbQueryParameters.fields
|
|
144
|
+
? (0, pick_1.default)(apiRecord, this.dbQueryParameters.fields)
|
|
145
|
+
: apiRecord;
|
|
146
|
+
if (statsRecords) {
|
|
147
|
+
apiRecordFinal.stats = statsRecords[record.cumulus_id] ? statsRecords[record.cumulus_id]
|
|
148
|
+
: {
|
|
149
|
+
queued: 0,
|
|
150
|
+
completed: 0,
|
|
151
|
+
failed: 0,
|
|
152
|
+
running: 0,
|
|
153
|
+
total: 0,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
return apiRecordFinal;
|
|
157
|
+
});
|
|
158
|
+
return apiRecords;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
exports.CollectionSearch = CollectionSearch;
|
|
162
|
+
//# sourceMappingURL=CollectionSearch.js.map
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Knex } from 'knex';
|
|
2
|
+
import { ApiExecutionRecord } from '@cumulus/types/api/executions';
|
|
3
|
+
import { BaseSearch } from './BaseSearch';
|
|
4
|
+
import { DbQueryParameters, QueryEvent } from '../types/search';
|
|
5
|
+
import { PostgresExecutionRecord } from '../types/execution';
|
|
6
|
+
import { BaseRecord } from '../types/base';
|
|
7
|
+
interface ExecutionRecord extends BaseRecord, PostgresExecutionRecord {
|
|
8
|
+
collectionName?: string;
|
|
9
|
+
collectionVersion?: string;
|
|
10
|
+
asyncOperationId?: string;
|
|
11
|
+
parentArn?: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Class to build and execute db search query for executions
|
|
15
|
+
*/
|
|
16
|
+
export declare class ExecutionSearch extends BaseSearch {
|
|
17
|
+
constructor(event: QueryEvent);
|
|
18
|
+
/**
|
|
19
|
+
* check if joined async_ops table search is needed
|
|
20
|
+
*
|
|
21
|
+
* @returns whether collection search is needed
|
|
22
|
+
*/
|
|
23
|
+
protected searchAsync(): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* check if joined async_ops table search is needed
|
|
26
|
+
*
|
|
27
|
+
* @returns whether collection search is needed
|
|
28
|
+
*/
|
|
29
|
+
protected searchParent(): boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Build basic query
|
|
32
|
+
*
|
|
33
|
+
* @param knex - DB client
|
|
34
|
+
* @returns queries for getting count and search result
|
|
35
|
+
*/
|
|
36
|
+
protected buildBasicQuery(knex: Knex): {
|
|
37
|
+
countQuery: Knex.QueryBuilder;
|
|
38
|
+
searchQuery: Knex.QueryBuilder;
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Build queries for infix and prefix
|
|
42
|
+
*
|
|
43
|
+
* @param params
|
|
44
|
+
* @param params.countQuery - query builder for getting count
|
|
45
|
+
* @param params.searchQuery - query builder for search
|
|
46
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
47
|
+
*/
|
|
48
|
+
protected buildInfixPrefixQuery(params: {
|
|
49
|
+
countQuery: Knex.QueryBuilder;
|
|
50
|
+
searchQuery: Knex.QueryBuilder;
|
|
51
|
+
dbQueryParameters?: DbQueryParameters;
|
|
52
|
+
}): void;
|
|
53
|
+
/**
|
|
54
|
+
* Translate postgres records to api records
|
|
55
|
+
*
|
|
56
|
+
* @param pgRecords - postgres records returned from query
|
|
57
|
+
* @returns translated api records
|
|
58
|
+
*/
|
|
59
|
+
protected translatePostgresRecordsToApiRecords(pgRecords: ExecutionRecord[]): Partial<ApiExecutionRecord>[];
|
|
60
|
+
}
|
|
61
|
+
export {};
|
|
62
|
+
//# sourceMappingURL=ExecutionSearch.d.ts.map
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.ExecutionSearch = void 0;
|
|
7
|
+
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
8
|
+
const pick_1 = __importDefault(require("lodash/pick"));
|
|
9
|
+
const set_1 = __importDefault(require("lodash/set"));
|
|
10
|
+
const Collections_1 = require("@cumulus/message/Collections");
|
|
11
|
+
const BaseSearch_1 = require("./BaseSearch");
|
|
12
|
+
const executions_1 = require("../translate/executions");
|
|
13
|
+
const tables_1 = require("../tables");
|
|
14
|
+
const log = new logger_1.default({ sender: '@cumulus/db/ExecutionSearch' });
|
|
15
|
+
/**
|
|
16
|
+
* Class to build and execute db search query for executions
|
|
17
|
+
*/
|
|
18
|
+
class ExecutionSearch extends BaseSearch_1.BaseSearch {
|
|
19
|
+
constructor(event) {
|
|
20
|
+
// estimate the table rowcount by default
|
|
21
|
+
if (event?.queryStringParameters?.estimateTableRowCount !== 'false') {
|
|
22
|
+
(0, set_1.default)(event, 'queryStringParameters.estimateTableRowCount', 'true');
|
|
23
|
+
}
|
|
24
|
+
super(event, 'execution');
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* check if joined async_ops table search is needed
|
|
28
|
+
*
|
|
29
|
+
* @returns whether collection search is needed
|
|
30
|
+
*/
|
|
31
|
+
searchAsync() {
|
|
32
|
+
const { not, term, terms } = this.dbQueryParameters;
|
|
33
|
+
return (!!(not?.asyncOperationId || term?.asyncOperationId || terms?.asyncOperationId));
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* check if joined async_ops table search is needed
|
|
37
|
+
*
|
|
38
|
+
* @returns whether collection search is needed
|
|
39
|
+
*/
|
|
40
|
+
searchParent() {
|
|
41
|
+
const { not, term, terms } = this.dbQueryParameters;
|
|
42
|
+
return (!!(not?.parentArn || term?.parentArn || terms?.parentArn));
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Build basic query
|
|
46
|
+
*
|
|
47
|
+
* @param knex - DB client
|
|
48
|
+
* @returns queries for getting count and search result
|
|
49
|
+
*/
|
|
50
|
+
buildBasicQuery(knex) {
|
|
51
|
+
const { collections: collectionsTable, asyncOperations: asyncOperationsTable, executions: executionsTable, } = tables_1.TableNames;
|
|
52
|
+
const searchQuery = knex(`${this.tableName}`)
|
|
53
|
+
.select(`${this.tableName}.*`)
|
|
54
|
+
.select({
|
|
55
|
+
collectionName: `${collectionsTable}.name`,
|
|
56
|
+
collectionVersion: `${collectionsTable}.version`,
|
|
57
|
+
});
|
|
58
|
+
if (this.searchAsync() || this.dbQueryParameters.includeFullRecord) {
|
|
59
|
+
searchQuery.select({ asyncOperationId: `${asyncOperationsTable}.id` });
|
|
60
|
+
}
|
|
61
|
+
if (this.searchParent() || this.dbQueryParameters.includeFullRecord) {
|
|
62
|
+
searchQuery.select({ parentArn: `${executionsTable}_parent.arn` });
|
|
63
|
+
}
|
|
64
|
+
const countQuery = knex(this.tableName)
|
|
65
|
+
.count('*');
|
|
66
|
+
if (this.searchCollection()) {
|
|
67
|
+
countQuery.innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`);
|
|
68
|
+
searchQuery.innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`);
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
searchQuery.leftJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`);
|
|
72
|
+
}
|
|
73
|
+
if (this.searchAsync()) {
|
|
74
|
+
countQuery.innerJoin(asyncOperationsTable, `${this.tableName}.async_operation_cumulus_id`, `${asyncOperationsTable}.cumulus_id`);
|
|
75
|
+
searchQuery.innerJoin(asyncOperationsTable, `${this.tableName}.async_operation_cumulus_id`, `${asyncOperationsTable}.cumulus_id`);
|
|
76
|
+
}
|
|
77
|
+
else if (this.dbQueryParameters.includeFullRecord) {
|
|
78
|
+
searchQuery.leftJoin(asyncOperationsTable, `${this.tableName}.async_operation_cumulus_id`, `${asyncOperationsTable}.cumulus_id`);
|
|
79
|
+
}
|
|
80
|
+
if (this.searchParent()) {
|
|
81
|
+
countQuery.innerJoin(`${this.tableName} as ${this.tableName}_parent`, `${this.tableName}.parent_cumulus_id`, `${this.tableName}_parent.cumulus_id`);
|
|
82
|
+
searchQuery.innerJoin(`${this.tableName} as ${this.tableName}_parent`, `${this.tableName}.parent_cumulus_id`, `${this.tableName}_parent.cumulus_id`);
|
|
83
|
+
}
|
|
84
|
+
else if (this.dbQueryParameters.includeFullRecord) {
|
|
85
|
+
searchQuery.leftJoin(`${this.tableName} as ${this.tableName}_parent`, `${this.tableName}.parent_cumulus_id`, `${this.tableName}_parent.cumulus_id`);
|
|
86
|
+
}
|
|
87
|
+
return { countQuery, searchQuery };
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Build queries for infix and prefix
|
|
91
|
+
*
|
|
92
|
+
* @param params
|
|
93
|
+
* @param params.countQuery - query builder for getting count
|
|
94
|
+
* @param params.searchQuery - query builder for search
|
|
95
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
96
|
+
*/
|
|
97
|
+
buildInfixPrefixQuery(params) {
|
|
98
|
+
const { countQuery, searchQuery, dbQueryParameters } = params;
|
|
99
|
+
const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters;
|
|
100
|
+
if (infix) {
|
|
101
|
+
[countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.arn`, `%${infix}%`));
|
|
102
|
+
}
|
|
103
|
+
if (prefix) {
|
|
104
|
+
[countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.arn`, `%${prefix}%`));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Translate postgres records to api records
|
|
109
|
+
*
|
|
110
|
+
* @param pgRecords - postgres records returned from query
|
|
111
|
+
* @returns translated api records
|
|
112
|
+
*/
|
|
113
|
+
translatePostgresRecordsToApiRecords(pgRecords) {
|
|
114
|
+
log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `);
|
|
115
|
+
const apiRecords = pgRecords.map((executionRecord) => {
|
|
116
|
+
const { collectionName, collectionVersion, asyncOperationId, parentArn } = executionRecord;
|
|
117
|
+
const collectionId = collectionName && collectionVersion
|
|
118
|
+
? (0, Collections_1.constructCollectionId)(collectionName, collectionVersion) : undefined;
|
|
119
|
+
const apiRecord = (0, executions_1.translatePostgresExecutionToApiExecutionWithoutDbQuery)({
|
|
120
|
+
executionRecord,
|
|
121
|
+
collectionId,
|
|
122
|
+
asyncOperationId,
|
|
123
|
+
parentArn,
|
|
124
|
+
});
|
|
125
|
+
return this.dbQueryParameters.fields
|
|
126
|
+
? (0, pick_1.default)(apiRecord, this.dbQueryParameters.fields)
|
|
127
|
+
: apiRecord;
|
|
128
|
+
});
|
|
129
|
+
return apiRecords;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
exports.ExecutionSearch = ExecutionSearch;
|
|
133
|
+
//# sourceMappingURL=ExecutionSearch.js.map
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { Knex } from 'knex';
|
|
2
|
+
import { ApiGranuleRecord } from '@cumulus/types/api/granules';
|
|
3
|
+
import { BaseRecord } from '../types/base';
|
|
4
|
+
import { BaseSearch } from './BaseSearch';
|
|
5
|
+
import { DbQueryParameters, QueryEvent } from '../types/search';
|
|
6
|
+
import { PostgresGranuleRecord } from '../types/granule';
|
|
7
|
+
interface GranuleRecord extends BaseRecord, PostgresGranuleRecord {
|
|
8
|
+
cumulus_id: number;
|
|
9
|
+
updated_at: Date;
|
|
10
|
+
collection_cumulus_id: number;
|
|
11
|
+
collectionName: string;
|
|
12
|
+
collectionVersion: string;
|
|
13
|
+
pdr_cumulus_id: number;
|
|
14
|
+
pdrName?: string;
|
|
15
|
+
provider_cumulus_id?: number;
|
|
16
|
+
providerName?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Class to build and execute db search query for granules
|
|
20
|
+
*/
|
|
21
|
+
export declare class GranuleSearch extends BaseSearch {
|
|
22
|
+
constructor(event: QueryEvent);
|
|
23
|
+
/**
|
|
24
|
+
* Build basic query
|
|
25
|
+
*
|
|
26
|
+
* @param knex - DB client
|
|
27
|
+
* @returns queries for getting count and search result
|
|
28
|
+
*/
|
|
29
|
+
protected buildBasicQuery(knex: Knex): {
|
|
30
|
+
countQuery: Knex.QueryBuilder;
|
|
31
|
+
searchQuery: Knex.QueryBuilder;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Build queries for infix and prefix
|
|
35
|
+
*
|
|
36
|
+
* @param params
|
|
37
|
+
* @param params.countQuery - query builder for getting count
|
|
38
|
+
* @param params.searchQuery - query builder for search
|
|
39
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
40
|
+
*/
|
|
41
|
+
protected buildInfixPrefixQuery(params: {
|
|
42
|
+
countQuery: Knex.QueryBuilder;
|
|
43
|
+
searchQuery: Knex.QueryBuilder;
|
|
44
|
+
dbQueryParameters?: DbQueryParameters;
|
|
45
|
+
}): void;
|
|
46
|
+
/**
|
|
47
|
+
* Translate postgres records to api records
|
|
48
|
+
*
|
|
49
|
+
* @param pgRecords - postgres records returned from query
|
|
50
|
+
* @returns translated api records
|
|
51
|
+
*/
|
|
52
|
+
protected translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[]): Partial<ApiGranuleRecord>[];
|
|
53
|
+
}
|
|
54
|
+
export {};
|
|
55
|
+
//# sourceMappingURL=GranuleSearch.d.ts.map
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.GranuleSearch = void 0;
|
|
7
|
+
const pick_1 = __importDefault(require("lodash/pick"));
|
|
8
|
+
const set_1 = __importDefault(require("lodash/set"));
|
|
9
|
+
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
10
|
+
const BaseSearch_1 = require("./BaseSearch");
|
|
11
|
+
const granules_1 = require("../translate/granules");
|
|
12
|
+
const tables_1 = require("../tables");
|
|
13
|
+
const log = new logger_1.default({ sender: '@cumulus/db/GranuleSearch' });
|
|
14
|
+
/**
|
|
15
|
+
* Class to build and execute db search query for granules
|
|
16
|
+
*/
|
|
17
|
+
class GranuleSearch extends BaseSearch_1.BaseSearch {
|
|
18
|
+
constructor(event) {
|
|
19
|
+
// estimate the table rowcount by default
|
|
20
|
+
if (event?.queryStringParameters?.estimateTableRowCount !== 'false') {
|
|
21
|
+
(0, set_1.default)(event, 'queryStringParameters.estimateTableRowCount', 'true');
|
|
22
|
+
}
|
|
23
|
+
super(event, 'granule');
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Build basic query
|
|
27
|
+
*
|
|
28
|
+
* @param knex - DB client
|
|
29
|
+
* @returns queries for getting count and search result
|
|
30
|
+
*/
|
|
31
|
+
buildBasicQuery(knex) {
|
|
32
|
+
const { collections: collectionsTable, providers: providersTable, pdrs: pdrsTable, } = tables_1.TableNames;
|
|
33
|
+
const countQuery = knex(this.tableName)
|
|
34
|
+
.count('*');
|
|
35
|
+
const searchQuery = knex(this.tableName)
|
|
36
|
+
.select(`${this.tableName}.*`)
|
|
37
|
+
.select({
|
|
38
|
+
providerName: `${providersTable}.name`,
|
|
39
|
+
collectionName: `${collectionsTable}.name`,
|
|
40
|
+
collectionVersion: `${collectionsTable}.version`,
|
|
41
|
+
pdrName: `${pdrsTable}.name`,
|
|
42
|
+
})
|
|
43
|
+
.innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`);
|
|
44
|
+
if (this.searchCollection()) {
|
|
45
|
+
countQuery.innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`);
|
|
46
|
+
}
|
|
47
|
+
if (this.searchProvider()) {
|
|
48
|
+
countQuery.innerJoin(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`);
|
|
49
|
+
searchQuery.innerJoin(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`);
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
searchQuery.leftJoin(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`);
|
|
53
|
+
}
|
|
54
|
+
if (this.searchPdr()) {
|
|
55
|
+
countQuery.innerJoin(pdrsTable, `${this.tableName}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`);
|
|
56
|
+
searchQuery.innerJoin(pdrsTable, `${this.tableName}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`);
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
searchQuery.leftJoin(pdrsTable, `${this.tableName}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`);
|
|
60
|
+
}
|
|
61
|
+
return { countQuery, searchQuery };
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Build queries for infix and prefix
|
|
65
|
+
*
|
|
66
|
+
* @param params
|
|
67
|
+
* @param params.countQuery - query builder for getting count
|
|
68
|
+
* @param params.searchQuery - query builder for search
|
|
69
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
70
|
+
*/
|
|
71
|
+
buildInfixPrefixQuery(params) {
|
|
72
|
+
const { countQuery, searchQuery, dbQueryParameters } = params;
|
|
73
|
+
const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters;
|
|
74
|
+
if (infix) {
|
|
75
|
+
[countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.granule_id`, `%${infix}%`));
|
|
76
|
+
}
|
|
77
|
+
if (prefix) {
|
|
78
|
+
[countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.granule_id`, `${prefix}%`));
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Translate postgres records to api records
|
|
83
|
+
*
|
|
84
|
+
* @param pgRecords - postgres records returned from query
|
|
85
|
+
* @returns translated api records
|
|
86
|
+
*/
|
|
87
|
+
translatePostgresRecordsToApiRecords(pgRecords) {
|
|
88
|
+
log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `);
|
|
89
|
+
const apiRecords = pgRecords.map((item) => {
|
|
90
|
+
const granulePgRecord = item;
|
|
91
|
+
const collectionPgRecord = {
|
|
92
|
+
cumulus_id: item.collection_cumulus_id,
|
|
93
|
+
name: item.collectionName,
|
|
94
|
+
version: item.collectionVersion,
|
|
95
|
+
};
|
|
96
|
+
const pdr = item.pdrName ? { name: item.pdrName } : undefined;
|
|
97
|
+
const providerPgRecord = item.providerName ? { name: item.providerName } : undefined;
|
|
98
|
+
const apiRecord = (0, granules_1.translatePostgresGranuleToApiGranuleWithoutDbQuery)({
|
|
99
|
+
granulePgRecord, collectionPgRecord, pdr, providerPgRecord,
|
|
100
|
+
});
|
|
101
|
+
return this.dbQueryParameters.fields
|
|
102
|
+
? (0, pick_1.default)(apiRecord, this.dbQueryParameters.fields)
|
|
103
|
+
: apiRecord;
|
|
104
|
+
});
|
|
105
|
+
return apiRecords;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
exports.GranuleSearch = GranuleSearch;
|
|
109
|
+
//# sourceMappingURL=GranuleSearch.js.map
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { Knex } from 'knex';
|
|
2
|
+
import { DbQueryParameters, QueryEvent } from '../types/search';
|
|
3
|
+
import { BaseSearch } from './BaseSearch';
|
|
4
|
+
declare type Summary = {
|
|
5
|
+
dateFrom: string;
|
|
6
|
+
dateTo: string;
|
|
7
|
+
value: number;
|
|
8
|
+
aggregation: string;
|
|
9
|
+
unit: string;
|
|
10
|
+
};
|
|
11
|
+
declare type SummaryResult = {
|
|
12
|
+
errors: Summary;
|
|
13
|
+
granules: Summary;
|
|
14
|
+
collections: Summary;
|
|
15
|
+
processingTime: Summary;
|
|
16
|
+
};
|
|
17
|
+
declare type Meta = {
|
|
18
|
+
name: string;
|
|
19
|
+
count: number;
|
|
20
|
+
field: string;
|
|
21
|
+
};
|
|
22
|
+
declare type AggregateRes = {
|
|
23
|
+
key: string;
|
|
24
|
+
count: number;
|
|
25
|
+
};
|
|
26
|
+
declare type ApiAggregateResult = {
|
|
27
|
+
meta: Meta;
|
|
28
|
+
count: AggregateRes[];
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* A class to query postgres for the STATS and STATS/AGGREGATE endpoints
|
|
32
|
+
*/
|
|
33
|
+
declare class StatsSearch extends BaseSearch {
|
|
34
|
+
readonly field: string;
|
|
35
|
+
constructor(event: QueryEvent, type: string);
|
|
36
|
+
/**
|
|
37
|
+
* Formats the postgres records into an API stats/aggregate response
|
|
38
|
+
*
|
|
39
|
+
* @param result - the postgres query results
|
|
40
|
+
* @returns the api object with the aggregate statistics
|
|
41
|
+
*/
|
|
42
|
+
private formatAggregateResult;
|
|
43
|
+
/**
|
|
44
|
+
* Formats the postgres results into an API stats/summary response
|
|
45
|
+
*
|
|
46
|
+
* @param result - the knex summary query results
|
|
47
|
+
* @returns the api object with the summary statistics
|
|
48
|
+
*/
|
|
49
|
+
private formatSummaryResult;
|
|
50
|
+
/**
|
|
51
|
+
* Queries postgres for a summary of statistics around the granules in the system
|
|
52
|
+
*
|
|
53
|
+
* @param testKnex - the knex client to be used
|
|
54
|
+
* @returns the postgres aggregations based on query
|
|
55
|
+
*/
|
|
56
|
+
summary(testKnex?: Knex): Promise<SummaryResult>;
|
|
57
|
+
/**
|
|
58
|
+
* Performs joins on the collections/pdrs/providers table if neccessary
|
|
59
|
+
*
|
|
60
|
+
* @param query - the knex query to be joined or not
|
|
61
|
+
*/
|
|
62
|
+
private joinTables;
|
|
63
|
+
/**
|
|
64
|
+
* Aggregates the search query based on queryStringParameters
|
|
65
|
+
*
|
|
66
|
+
* @param query - the knex query to be aggregated
|
|
67
|
+
* @param knex - the knex client to be used
|
|
68
|
+
*/
|
|
69
|
+
private aggregateQueryField;
|
|
70
|
+
/**
|
|
71
|
+
* Builds basic query
|
|
72
|
+
*
|
|
73
|
+
* @param knex - the knex client
|
|
74
|
+
* @returns the search query
|
|
75
|
+
*/
|
|
76
|
+
protected buildBasicQuery(knex: Knex): {
|
|
77
|
+
searchQuery: Knex.QueryBuilder;
|
|
78
|
+
};
|
|
79
|
+
/**
|
|
80
|
+
* Builds queries for infix and prefix
|
|
81
|
+
*
|
|
82
|
+
* @param params
|
|
83
|
+
* @param params.searchQuery - the search query
|
|
84
|
+
* @param [params.dbQueryParameters] - the db query parameters
|
|
85
|
+
*/
|
|
86
|
+
protected buildInfixPrefixQuery(params: {
|
|
87
|
+
searchQuery: Knex.QueryBuilder;
|
|
88
|
+
dbQueryParameters?: DbQueryParameters;
|
|
89
|
+
}): void;
|
|
90
|
+
/**
|
|
91
|
+
* Builds queries for term fields
|
|
92
|
+
*
|
|
93
|
+
* @param params
|
|
94
|
+
* @param params.searchQuery - the search query
|
|
95
|
+
* @param [params.dbQueryParameters] - the db query parameters
|
|
96
|
+
* @returns the updated search query based on queryStringParams
|
|
97
|
+
*/
|
|
98
|
+
protected buildTermQuery(params: {
|
|
99
|
+
searchQuery: Knex.QueryBuilder;
|
|
100
|
+
dbQueryParameters?: DbQueryParameters;
|
|
101
|
+
}): void;
|
|
102
|
+
/**
|
|
103
|
+
* Executes the aggregate search query
|
|
104
|
+
*
|
|
105
|
+
* @param testKnex - the knex client to be used
|
|
106
|
+
* @returns the aggregate query results in api format
|
|
107
|
+
*/
|
|
108
|
+
aggregate(testKnex?: Knex): Promise<ApiAggregateResult>;
|
|
109
|
+
}
|
|
110
|
+
export { StatsSearch };
|
|
111
|
+
//# sourceMappingURL=StatsSearch.d.ts.map
|