@cumulus/db 18.3.4 → 19.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/config.d.ts +19 -0
- package/dist/config.js +40 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +15 -4
- package/dist/lib/execution.d.ts +5 -0
- package/dist/lib/execution.js +16 -1
- package/dist/migrations/20240125171703_update_granule_execution_cumulus_id_type.d.ts +4 -0
- package/dist/migrations/20240125171703_update_granule_execution_cumulus_id_type.js +22 -0
- package/dist/migrations/20240126135619_granules_add_indexes.d.ts +4 -0
- package/dist/migrations/20240126135619_granules_add_indexes.js +19 -0
- package/dist/migrations/20240126165019_granules_update_constraints.d.ts +4 -0
- package/dist/migrations/20240126165019_granules_update_constraints.js +44 -0
- package/dist/migrations/20240606060726_alter_async_operations_add_operation_type_bulk_execution_delete.d.ts +4 -0
- package/dist/migrations/20240606060726_alter_async_operations_add_operation_type_bulk_execution_delete.js +43 -0
- package/dist/migrations/20240613174614_add_execution_parent_and_collection_indexes.d.ts +4 -0
- package/dist/migrations/20240613174614_add_execution_parent_and_collection_indexes.js +17 -0
- package/dist/migrations/20240617204826_update_executions_deletion_constraint.d.ts +4 -0
- package/dist/migrations/20240617204826_update_executions_deletion_constraint.js +14 -0
- package/dist/migrations/20240728101230_add_table_indexes.d.ts +4 -0
- package/dist/migrations/20240728101230_add_table_indexes.js +53 -0
- package/dist/models/execution.d.ts +2 -2
- package/dist/models/execution.js +1 -1
- package/dist/search/BaseSearch.d.ts +187 -0
- package/dist/search/BaseSearch.js +416 -0
- package/dist/search/CollectionSearch.d.ts +79 -0
- package/dist/search/CollectionSearch.js +162 -0
- package/dist/search/ExecutionSearch.d.ts +62 -0
- package/dist/search/ExecutionSearch.js +133 -0
- package/dist/search/GranuleSearch.d.ts +55 -0
- package/dist/search/GranuleSearch.js +109 -0
- package/dist/search/StatsSearch.d.ts +111 -0
- package/dist/search/StatsSearch.js +214 -0
- package/dist/search/field-mapping.d.ts +16 -0
- package/dist/search/field-mapping.js +304 -0
- package/dist/search/queries.d.ts +10 -0
- package/dist/search/queries.js +235 -0
- package/dist/translate/executions.d.ts +6 -0
- package/dist/translate/executions.js +32 -23
- package/dist/translate/granules.d.ts +25 -1
- package/dist/translate/granules.js +48 -27
- package/dist/types/search.d.ts +52 -0
- package/dist/types/search.js +3 -0
- package/package.json +9 -9
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { Knex } from 'knex';
|
|
2
|
+
import { BaseRecord } from '../types/base';
|
|
3
|
+
import { DbQueryParameters, QueryEvent, QueryStringParameters } from '../types/search';
|
|
4
|
+
export declare const typeToTable: {
|
|
5
|
+
[key: string]: string;
|
|
6
|
+
};
|
|
7
|
+
/**
|
|
8
|
+
* Class to build and execute db search query
|
|
9
|
+
*/
|
|
10
|
+
declare class BaseSearch {
|
|
11
|
+
readonly type: string;
|
|
12
|
+
readonly tableName: string;
|
|
13
|
+
readonly queryStringParameters: QueryStringParameters;
|
|
14
|
+
dbQueryParameters: DbQueryParameters;
|
|
15
|
+
constructor(event: QueryEvent, type: string);
|
|
16
|
+
/**
|
|
17
|
+
* check if joined collections table search is needed
|
|
18
|
+
*
|
|
19
|
+
* @returns whether collection search is needed
|
|
20
|
+
*/
|
|
21
|
+
protected searchCollection(): boolean;
|
|
22
|
+
/**
|
|
23
|
+
* check if joined pdrs table search is needed
|
|
24
|
+
*
|
|
25
|
+
* @returns whether pdr search is needed
|
|
26
|
+
*/
|
|
27
|
+
protected searchPdr(): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* check if joined providers table search is needed
|
|
30
|
+
*
|
|
31
|
+
* @returns whether provider search is needed
|
|
32
|
+
*/
|
|
33
|
+
protected searchProvider(): boolean;
|
|
34
|
+
/**
|
|
35
|
+
* Determine if an estimated row count should be returned
|
|
36
|
+
*
|
|
37
|
+
* @param countSql - sql statement for count
|
|
38
|
+
* @returns whether an estimated row count should be returned
|
|
39
|
+
*/
|
|
40
|
+
protected shouldEstimateRowcount(countSql: string): boolean;
|
|
41
|
+
/**
|
|
42
|
+
* Build the search query
|
|
43
|
+
*
|
|
44
|
+
* @param knex - DB client
|
|
45
|
+
* @returns queries for getting count and search result
|
|
46
|
+
*/
|
|
47
|
+
protected buildSearch(knex: Knex): {
|
|
48
|
+
countQuery?: Knex.QueryBuilder;
|
|
49
|
+
searchQuery: Knex.QueryBuilder;
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* Get metadata template for query result
|
|
53
|
+
*
|
|
54
|
+
* @returns metadata template
|
|
55
|
+
*/
|
|
56
|
+
private _metaTemplate;
|
|
57
|
+
/**
|
|
58
|
+
* Build basic query
|
|
59
|
+
*
|
|
60
|
+
* @param knex - DB client
|
|
61
|
+
* @throws - function is not implemented
|
|
62
|
+
*/
|
|
63
|
+
protected buildBasicQuery(knex: Knex): {
|
|
64
|
+
countQuery?: Knex.QueryBuilder;
|
|
65
|
+
searchQuery: Knex.QueryBuilder;
|
|
66
|
+
};
|
|
67
|
+
/**
|
|
68
|
+
* Build queries for infix and prefix
|
|
69
|
+
*
|
|
70
|
+
* @param params
|
|
71
|
+
* @param [params.countQuery] - query builder for getting count
|
|
72
|
+
* @param params.searchQuery - query builder for search
|
|
73
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
74
|
+
*/
|
|
75
|
+
protected buildInfixPrefixQuery(params: {
|
|
76
|
+
countQuery?: Knex.QueryBuilder;
|
|
77
|
+
searchQuery: Knex.QueryBuilder;
|
|
78
|
+
dbQueryParameters?: DbQueryParameters;
|
|
79
|
+
}): void;
|
|
80
|
+
/**
|
|
81
|
+
* Build queries for checking if field 'exists'
|
|
82
|
+
*
|
|
83
|
+
* @param params
|
|
84
|
+
* @param [params.countQuery] - query builder for getting count
|
|
85
|
+
* @param params.searchQuery - query builder for search
|
|
86
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
87
|
+
*/
|
|
88
|
+
protected buildExistsQuery(params: {
|
|
89
|
+
countQuery?: Knex.QueryBuilder;
|
|
90
|
+
searchQuery: Knex.QueryBuilder;
|
|
91
|
+
dbQueryParameters?: DbQueryParameters;
|
|
92
|
+
}): void;
|
|
93
|
+
/**
|
|
94
|
+
* Build queries for range fields
|
|
95
|
+
*
|
|
96
|
+
* @param params
|
|
97
|
+
* @param params.knex - db client
|
|
98
|
+
* @param [params.countQuery] - query builder for getting count
|
|
99
|
+
* @param params.searchQuery - query builder for search
|
|
100
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
101
|
+
*/
|
|
102
|
+
protected buildRangeQuery(params: {
|
|
103
|
+
knex?: Knex;
|
|
104
|
+
countQuery?: Knex.QueryBuilder;
|
|
105
|
+
searchQuery: Knex.QueryBuilder;
|
|
106
|
+
dbQueryParameters?: DbQueryParameters;
|
|
107
|
+
}): void;
|
|
108
|
+
/**
|
|
109
|
+
* Build queries for term fields
|
|
110
|
+
*
|
|
111
|
+
* @param params
|
|
112
|
+
* @param [params.countQuery] - query builder for getting count
|
|
113
|
+
* @param params.searchQuery - query builder for search
|
|
114
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
115
|
+
*/
|
|
116
|
+
protected buildTermQuery(params: {
|
|
117
|
+
countQuery?: Knex.QueryBuilder;
|
|
118
|
+
searchQuery: Knex.QueryBuilder;
|
|
119
|
+
dbQueryParameters?: DbQueryParameters;
|
|
120
|
+
}): void;
|
|
121
|
+
/**
|
|
122
|
+
* Build queries for terms fields
|
|
123
|
+
*
|
|
124
|
+
* @param params
|
|
125
|
+
* @param [params.countQuery] - query builder for getting count
|
|
126
|
+
* @param params.searchQuery - query builder for search
|
|
127
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
128
|
+
*/
|
|
129
|
+
protected buildTermsQuery(params: {
|
|
130
|
+
countQuery?: Knex.QueryBuilder;
|
|
131
|
+
searchQuery: Knex.QueryBuilder;
|
|
132
|
+
dbQueryParameters?: DbQueryParameters;
|
|
133
|
+
}): void;
|
|
134
|
+
/**
|
|
135
|
+
* Build queries for checking if field doesn't match the given value
|
|
136
|
+
*
|
|
137
|
+
* @param params
|
|
138
|
+
* @param [params.countQuery] - query builder for getting count
|
|
139
|
+
* @param params.searchQuery - query builder for search
|
|
140
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
141
|
+
*/
|
|
142
|
+
protected buildNotMatchQuery(params: {
|
|
143
|
+
countQuery?: Knex.QueryBuilder;
|
|
144
|
+
searchQuery: Knex.QueryBuilder;
|
|
145
|
+
dbQueryParameters?: DbQueryParameters;
|
|
146
|
+
}): void;
|
|
147
|
+
/**
|
|
148
|
+
* Build queries for sort keys and fields
|
|
149
|
+
*
|
|
150
|
+
* @param params
|
|
151
|
+
* @param params.searchQuery - query builder for search
|
|
152
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
153
|
+
*/
|
|
154
|
+
protected buildSortQuery(params: {
|
|
155
|
+
searchQuery: Knex.QueryBuilder;
|
|
156
|
+
dbQueryParameters?: DbQueryParameters;
|
|
157
|
+
}): void;
|
|
158
|
+
/**
|
|
159
|
+
* Translate postgres records to api records
|
|
160
|
+
*
|
|
161
|
+
* @param pgRecords - postgres records returned from query
|
|
162
|
+
* @param [knex] - knex client for additional queries if neccessary
|
|
163
|
+
* @throws - function is not implemented
|
|
164
|
+
*/
|
|
165
|
+
protected translatePostgresRecordsToApiRecords(pgRecords: BaseRecord[], knex?: Knex): void;
|
|
166
|
+
/**
|
|
167
|
+
* Get estimated table rowcount
|
|
168
|
+
*
|
|
169
|
+
* @param params
|
|
170
|
+
* @param params.knex - DB client
|
|
171
|
+
* @param [params.tableName] - table name
|
|
172
|
+
* @returns rowcount
|
|
173
|
+
*/
|
|
174
|
+
protected getEstimatedRowcount(params: {
|
|
175
|
+
knex: Knex;
|
|
176
|
+
tableName?: string;
|
|
177
|
+
}): Promise<number>;
|
|
178
|
+
/**
|
|
179
|
+
* Build and execute search query
|
|
180
|
+
*
|
|
181
|
+
* @param testKnex - knex for testing
|
|
182
|
+
* @returns search result
|
|
183
|
+
*/
|
|
184
|
+
query(testKnex?: Knex): Promise<any>;
|
|
185
|
+
}
|
|
186
|
+
export { BaseSearch };
|
|
187
|
+
//# sourceMappingURL=BaseSearch.d.ts.map
|
|
@@ -0,0 +1,416 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.BaseSearch = exports.typeToTable = void 0;
|
|
7
|
+
const get_1 = __importDefault(require("lodash/get"));
|
|
8
|
+
const omit_1 = __importDefault(require("lodash/omit"));
|
|
9
|
+
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
10
|
+
const connection_1 = require("../connection");
|
|
11
|
+
const tables_1 = require("../tables");
|
|
12
|
+
const queries_1 = require("./queries");
|
|
13
|
+
const log = new logger_1.default({ sender: '@cumulus/db/BaseSearch' });
|
|
14
|
+
exports.typeToTable = {
|
|
15
|
+
asyncOperation: tables_1.TableNames.asyncOperations,
|
|
16
|
+
collection: tables_1.TableNames.collections,
|
|
17
|
+
execution: tables_1.TableNames.executions,
|
|
18
|
+
granule: tables_1.TableNames.granules,
|
|
19
|
+
pdr: tables_1.TableNames.pdrs,
|
|
20
|
+
provider: tables_1.TableNames.providers,
|
|
21
|
+
rule: tables_1.TableNames.rules,
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Class to build and execute db search query
|
|
25
|
+
*/
|
|
26
|
+
class BaseSearch {
|
|
27
|
+
constructor(event, type) {
|
|
28
|
+
// parsed from queryStringParameters for query build
|
|
29
|
+
this.dbQueryParameters = {};
|
|
30
|
+
this.type = type;
|
|
31
|
+
this.tableName = exports.typeToTable[this.type];
|
|
32
|
+
this.queryStringParameters = event?.queryStringParameters ?? {};
|
|
33
|
+
this.dbQueryParameters = (0, queries_1.convertQueryStringToDbQueryParameters)(this.type, this.queryStringParameters);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* check if joined collections table search is needed
|
|
37
|
+
*
|
|
38
|
+
* @returns whether collection search is needed
|
|
39
|
+
*/
|
|
40
|
+
searchCollection() {
|
|
41
|
+
const { not, term, terms } = this.dbQueryParameters;
|
|
42
|
+
return !!(not?.collectionName
|
|
43
|
+
|| not?.collectionVersion
|
|
44
|
+
|| term?.collectionName
|
|
45
|
+
|| term?.collectionVersion
|
|
46
|
+
|| terms?.collectionName
|
|
47
|
+
|| terms?.collectionVersion);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* check if joined pdrs table search is needed
|
|
51
|
+
*
|
|
52
|
+
* @returns whether pdr search is needed
|
|
53
|
+
*/
|
|
54
|
+
searchPdr() {
|
|
55
|
+
const { not, term, terms } = this.dbQueryParameters;
|
|
56
|
+
return !!(not?.pdrName || term?.pdrName || terms?.pdrName);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* check if joined providers table search is needed
|
|
60
|
+
*
|
|
61
|
+
* @returns whether provider search is needed
|
|
62
|
+
*/
|
|
63
|
+
searchProvider() {
|
|
64
|
+
const { not, term, terms } = this.dbQueryParameters;
|
|
65
|
+
return !!(not?.providerName || term?.providerName || terms?.providerName);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Determine if an estimated row count should be returned
|
|
69
|
+
*
|
|
70
|
+
* @param countSql - sql statement for count
|
|
71
|
+
* @returns whether an estimated row count should be returned
|
|
72
|
+
*/
|
|
73
|
+
shouldEstimateRowcount(countSql) {
|
|
74
|
+
const isBasicQuery = (countSql === `select count(*) from "${this.tableName}"`);
|
|
75
|
+
return this.dbQueryParameters.estimateTableRowCount === true && isBasicQuery;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Build the search query
|
|
79
|
+
*
|
|
80
|
+
* @param knex - DB client
|
|
81
|
+
* @returns queries for getting count and search result
|
|
82
|
+
*/
|
|
83
|
+
buildSearch(knex) {
|
|
84
|
+
const { countQuery, searchQuery } = this.buildBasicQuery(knex);
|
|
85
|
+
this.buildTermQuery({ countQuery, searchQuery });
|
|
86
|
+
this.buildTermsQuery({ countQuery, searchQuery });
|
|
87
|
+
this.buildNotMatchQuery({ countQuery, searchQuery });
|
|
88
|
+
this.buildRangeQuery({ knex, countQuery, searchQuery });
|
|
89
|
+
this.buildExistsQuery({ countQuery, searchQuery });
|
|
90
|
+
this.buildInfixPrefixQuery({ countQuery, searchQuery });
|
|
91
|
+
this.buildSortQuery({ searchQuery });
|
|
92
|
+
const { limit, offset } = this.dbQueryParameters;
|
|
93
|
+
if (limit)
|
|
94
|
+
searchQuery.limit(limit);
|
|
95
|
+
if (offset)
|
|
96
|
+
searchQuery.offset(offset);
|
|
97
|
+
log.debug(`buildSearch returns countQuery: ${countQuery?.toSQL().sql}, searchQuery: ${searchQuery.toSQL().sql}`);
|
|
98
|
+
return { countQuery, searchQuery };
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Get metadata template for query result
|
|
102
|
+
*
|
|
103
|
+
* @returns metadata template
|
|
104
|
+
*/
|
|
105
|
+
_metaTemplate() {
|
|
106
|
+
return {
|
|
107
|
+
name: 'cumulus-api',
|
|
108
|
+
stack: process.env.stackName,
|
|
109
|
+
table: this.tableName,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Build basic query
|
|
114
|
+
*
|
|
115
|
+
* @param knex - DB client
|
|
116
|
+
* @throws - function is not implemented
|
|
117
|
+
*/
|
|
118
|
+
buildBasicQuery(knex) {
|
|
119
|
+
log.debug(`buildBasicQuery is not implemented ${knex.constructor.name}`);
|
|
120
|
+
throw new Error('buildBasicQuery is not implemented');
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Build queries for infix and prefix
|
|
124
|
+
*
|
|
125
|
+
* @param params
|
|
126
|
+
* @param [params.countQuery] - query builder for getting count
|
|
127
|
+
* @param params.searchQuery - query builder for search
|
|
128
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
129
|
+
*/
|
|
130
|
+
buildInfixPrefixQuery(params) {
|
|
131
|
+
log.debug(`buildInfixPrefixQuery is not implemented ${Object.keys(params)}`);
|
|
132
|
+
throw new Error('buildInfixPrefixQuery is not implemented');
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Build queries for checking if field 'exists'
|
|
136
|
+
*
|
|
137
|
+
* @param params
|
|
138
|
+
* @param [params.countQuery] - query builder for getting count
|
|
139
|
+
* @param params.searchQuery - query builder for search
|
|
140
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
141
|
+
*/
|
|
142
|
+
buildExistsQuery(params) {
|
|
143
|
+
const { countQuery, searchQuery, dbQueryParameters } = params;
|
|
144
|
+
const { exists = {} } = dbQueryParameters ?? this.dbQueryParameters;
|
|
145
|
+
Object.entries(exists).forEach(([name, value]) => {
|
|
146
|
+
const queryMethod = value ? 'whereNotNull' : 'whereNull';
|
|
147
|
+
const checkNull = value ? 'not null' : 'null';
|
|
148
|
+
switch (name) {
|
|
149
|
+
case 'collectionName':
|
|
150
|
+
case 'collectionVersion':
|
|
151
|
+
[countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.collection_cumulus_id`));
|
|
152
|
+
break;
|
|
153
|
+
case 'providerName':
|
|
154
|
+
[countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.provider_cumulus_id`));
|
|
155
|
+
break;
|
|
156
|
+
case 'pdrName':
|
|
157
|
+
[countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.pdr_cumulus_id`));
|
|
158
|
+
break;
|
|
159
|
+
case 'asyncOperationId':
|
|
160
|
+
[countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.async_operation_cumulus_id`));
|
|
161
|
+
break;
|
|
162
|
+
case 'error':
|
|
163
|
+
case 'error.Error':
|
|
164
|
+
[countQuery, searchQuery].forEach((query) => query?.whereRaw(`${this.tableName}.error ->> 'Error' is ${checkNull}`));
|
|
165
|
+
break;
|
|
166
|
+
case 'parentArn':
|
|
167
|
+
[countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.parent_cumulus_id`));
|
|
168
|
+
break;
|
|
169
|
+
default:
|
|
170
|
+
[countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.${name}`));
|
|
171
|
+
break;
|
|
172
|
+
}
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Build queries for range fields
|
|
177
|
+
*
|
|
178
|
+
* @param params
|
|
179
|
+
* @param params.knex - db client
|
|
180
|
+
* @param [params.countQuery] - query builder for getting count
|
|
181
|
+
* @param params.searchQuery - query builder for search
|
|
182
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
183
|
+
*/
|
|
184
|
+
buildRangeQuery(params) {
|
|
185
|
+
const { countQuery, searchQuery, dbQueryParameters } = params;
|
|
186
|
+
const { range = {} } = dbQueryParameters ?? this.dbQueryParameters;
|
|
187
|
+
Object.entries(range).forEach(([name, rangeValues]) => {
|
|
188
|
+
if (rangeValues.gte) {
|
|
189
|
+
countQuery?.where(`${this.tableName}.${name}`, '>=', rangeValues.gte);
|
|
190
|
+
searchQuery.where(`${this.tableName}.${name}`, '>=', rangeValues.gte);
|
|
191
|
+
}
|
|
192
|
+
if (rangeValues.lte) {
|
|
193
|
+
countQuery?.where(`${this.tableName}.${name}`, '<=', rangeValues.lte);
|
|
194
|
+
searchQuery.where(`${this.tableName}.${name}`, '<=', rangeValues.lte);
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Build queries for term fields
|
|
200
|
+
*
|
|
201
|
+
* @param params
|
|
202
|
+
* @param [params.countQuery] - query builder for getting count
|
|
203
|
+
* @param params.searchQuery - query builder for search
|
|
204
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
205
|
+
*/
|
|
206
|
+
buildTermQuery(params) {
|
|
207
|
+
const { collections: collectionsTable, providers: providersTable, pdrs: pdrsTable, asyncOperations: asyncOperationsTable, executions: executionsTable, } = tables_1.TableNames;
|
|
208
|
+
const { countQuery, searchQuery, dbQueryParameters } = params;
|
|
209
|
+
const { term = {} } = dbQueryParameters ?? this.dbQueryParameters;
|
|
210
|
+
Object.entries(term).forEach(([name, value]) => {
|
|
211
|
+
switch (name) {
|
|
212
|
+
case 'collectionName':
|
|
213
|
+
[countQuery, searchQuery].forEach((query) => query?.where(`${collectionsTable}.name`, value));
|
|
214
|
+
break;
|
|
215
|
+
case 'collectionVersion':
|
|
216
|
+
[countQuery, searchQuery].forEach((query) => query?.where(`${collectionsTable}.version`, value));
|
|
217
|
+
break;
|
|
218
|
+
case 'providerName':
|
|
219
|
+
[countQuery, searchQuery].forEach((query) => query?.where(`${providersTable}.name`, value));
|
|
220
|
+
break;
|
|
221
|
+
case 'pdrName':
|
|
222
|
+
[countQuery, searchQuery].forEach((query) => query?.where(`${pdrsTable}.name`, value));
|
|
223
|
+
break;
|
|
224
|
+
case 'error.Error':
|
|
225
|
+
[countQuery, searchQuery]
|
|
226
|
+
.forEach((query) => query?.whereRaw(`${this.tableName}.error->>'Error' = '${value}'`));
|
|
227
|
+
break;
|
|
228
|
+
case 'asyncOperationId':
|
|
229
|
+
[countQuery, searchQuery].forEach((query) => query?.where(`${asyncOperationsTable}.id`, value));
|
|
230
|
+
break;
|
|
231
|
+
case 'parentArn':
|
|
232
|
+
[countQuery, searchQuery].forEach((query) => query?.where(`${executionsTable}_parent.arn`, value));
|
|
233
|
+
break;
|
|
234
|
+
default:
|
|
235
|
+
[countQuery, searchQuery].forEach((query) => query?.where(`${this.tableName}.${name}`, value));
|
|
236
|
+
break;
|
|
237
|
+
}
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Build queries for terms fields
|
|
242
|
+
*
|
|
243
|
+
* @param params
|
|
244
|
+
* @param [params.countQuery] - query builder for getting count
|
|
245
|
+
* @param params.searchQuery - query builder for search
|
|
246
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
247
|
+
*/
|
|
248
|
+
buildTermsQuery(params) {
|
|
249
|
+
const { collections: collectionsTable, providers: providersTable, pdrs: pdrsTable, asyncOperations: asyncOperationsTable, executions: executionsTable, } = tables_1.TableNames;
|
|
250
|
+
const { countQuery, searchQuery, dbQueryParameters } = params;
|
|
251
|
+
const { terms = {} } = dbQueryParameters ?? this.dbQueryParameters;
|
|
252
|
+
// collection name and version are searched in pair
|
|
253
|
+
if (terms.collectionName && terms.collectionVersion
|
|
254
|
+
&& terms.collectionName.length > 0
|
|
255
|
+
&& terms.collectionVersion.length > 0) {
|
|
256
|
+
const collectionPair = [];
|
|
257
|
+
for (let i = 0; i < terms.collectionName.length; i += 1) {
|
|
258
|
+
const name = terms.collectionName[i];
|
|
259
|
+
const version = terms.collectionVersion[i];
|
|
260
|
+
if (name && version)
|
|
261
|
+
collectionPair.push([name, version]);
|
|
262
|
+
}
|
|
263
|
+
[countQuery, searchQuery]
|
|
264
|
+
.forEach((query) => query?.whereIn([`${collectionsTable}.name`, `${collectionsTable}.version`], collectionPair));
|
|
265
|
+
}
|
|
266
|
+
Object.entries((0, omit_1.default)(terms, ['collectionName', 'collectionVersion'])).forEach(([name, value]) => {
|
|
267
|
+
switch (name) {
|
|
268
|
+
case 'providerName':
|
|
269
|
+
[countQuery, searchQuery].forEach((query) => query?.whereIn(`${providersTable}.name`, value));
|
|
270
|
+
break;
|
|
271
|
+
case 'pdrName':
|
|
272
|
+
[countQuery, searchQuery].forEach((query) => query?.whereIn(`${pdrsTable}.name`, value));
|
|
273
|
+
break;
|
|
274
|
+
case 'error.Error':
|
|
275
|
+
[countQuery, searchQuery]
|
|
276
|
+
.forEach((query) => query?.whereRaw(`${this.tableName}.error->>'Error' in ('${value.join('\',\'')}')`));
|
|
277
|
+
break;
|
|
278
|
+
case 'asyncOperationId':
|
|
279
|
+
[countQuery, searchQuery].forEach((query) => query?.whereIn(`${asyncOperationsTable}.id`, value));
|
|
280
|
+
break;
|
|
281
|
+
case 'parentArn':
|
|
282
|
+
[countQuery, searchQuery].forEach((query) => query?.whereIn(`${executionsTable}_parent.arn`, value));
|
|
283
|
+
break;
|
|
284
|
+
default:
|
|
285
|
+
[countQuery, searchQuery].forEach((query) => query?.whereIn(`${this.tableName}.${name}`, value));
|
|
286
|
+
break;
|
|
287
|
+
}
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Build queries for checking if field doesn't match the given value
|
|
292
|
+
*
|
|
293
|
+
* @param params
|
|
294
|
+
* @param [params.countQuery] - query builder for getting count
|
|
295
|
+
* @param params.searchQuery - query builder for search
|
|
296
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
297
|
+
*/
|
|
298
|
+
buildNotMatchQuery(params) {
|
|
299
|
+
const { collections: collectionsTable, providers: providersTable, pdrs: pdrsTable, asyncOperations: asyncOperationsTable, executions: executionsTable, } = tables_1.TableNames;
|
|
300
|
+
const { countQuery, searchQuery, dbQueryParameters } = params;
|
|
301
|
+
const { not: term = {} } = dbQueryParameters ?? this.dbQueryParameters;
|
|
302
|
+
// collection name and version are searched in pair
|
|
303
|
+
if (term.collectionName && term.collectionVersion) {
|
|
304
|
+
[countQuery, searchQuery].forEach((query) => query?.whereNot({
|
|
305
|
+
[`${collectionsTable}.name`]: term.collectionName,
|
|
306
|
+
[`${collectionsTable}.version`]: term.collectionVersion,
|
|
307
|
+
}));
|
|
308
|
+
}
|
|
309
|
+
Object.entries((0, omit_1.default)(term, ['collectionName', 'collectionVersion'])).forEach(([name, value]) => {
|
|
310
|
+
switch (name) {
|
|
311
|
+
case 'providerName':
|
|
312
|
+
[countQuery, searchQuery].forEach((query) => query?.whereNot(`${providersTable}.name`, value));
|
|
313
|
+
break;
|
|
314
|
+
case 'pdrName':
|
|
315
|
+
[countQuery, searchQuery].forEach((query) => query?.whereNot(`${pdrsTable}.name`, value));
|
|
316
|
+
break;
|
|
317
|
+
case 'asyncOperationId':
|
|
318
|
+
[countQuery, searchQuery].forEach((query) => query?.whereNot(`${asyncOperationsTable}.id`, value));
|
|
319
|
+
break;
|
|
320
|
+
case 'parentArn':
|
|
321
|
+
[countQuery, searchQuery].forEach((query) => query?.whereNot(`${executionsTable}_parent.arn`, value));
|
|
322
|
+
break;
|
|
323
|
+
case 'error.Error':
|
|
324
|
+
[countQuery, searchQuery].forEach((query) => query?.whereRaw(`${this.tableName}.error->>'Error' != '${value}'`));
|
|
325
|
+
break;
|
|
326
|
+
default:
|
|
327
|
+
[countQuery, searchQuery].forEach((query) => query?.whereNot(`${this.tableName}.${name}`, value));
|
|
328
|
+
break;
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Build queries for sort keys and fields
|
|
334
|
+
*
|
|
335
|
+
* @param params
|
|
336
|
+
* @param params.searchQuery - query builder for search
|
|
337
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
338
|
+
*/
|
|
339
|
+
buildSortQuery(params) {
|
|
340
|
+
const { searchQuery, dbQueryParameters } = params;
|
|
341
|
+
const { sort } = dbQueryParameters || this.dbQueryParameters;
|
|
342
|
+
sort?.forEach((key) => {
|
|
343
|
+
if (key.column.startsWith('error')) {
|
|
344
|
+
searchQuery.orderByRaw(`${this.tableName}.error ->> 'Error' ${key.order}`);
|
|
345
|
+
}
|
|
346
|
+
else {
|
|
347
|
+
searchQuery.orderBy([key]);
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Translate postgres records to api records
|
|
353
|
+
*
|
|
354
|
+
* @param pgRecords - postgres records returned from query
|
|
355
|
+
* @param [knex] - knex client for additional queries if neccessary
|
|
356
|
+
* @throws - function is not implemented
|
|
357
|
+
*/
|
|
358
|
+
translatePostgresRecordsToApiRecords(pgRecords, knex) {
|
|
359
|
+
log.error(`translatePostgresRecordsToApiRecords is not implemented ${pgRecords[0]} with client ${knex}`);
|
|
360
|
+
throw new Error('translatePostgresRecordsToApiRecords is not implemented');
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Get estimated table rowcount
|
|
364
|
+
*
|
|
365
|
+
* @param params
|
|
366
|
+
* @param params.knex - DB client
|
|
367
|
+
* @param [params.tableName] - table name
|
|
368
|
+
* @returns rowcount
|
|
369
|
+
*/
|
|
370
|
+
async getEstimatedRowcount(params) {
|
|
371
|
+
const { knex, tableName = this.tableName } = params;
|
|
372
|
+
const query = knex.raw(`EXPLAIN (FORMAT JSON) select * from "${tableName}"`);
|
|
373
|
+
log.debug(`Estimating the row count ${query.toSQL().sql}`);
|
|
374
|
+
const countResult = await query;
|
|
375
|
+
const countPath = 'rows[0]["QUERY PLAN"][0].Plan["Plan Rows"]';
|
|
376
|
+
const estimatedCount = (0, get_1.default)(countResult, countPath);
|
|
377
|
+
const count = Number(estimatedCount ?? 0);
|
|
378
|
+
return count;
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Build and execute search query
|
|
382
|
+
*
|
|
383
|
+
* @param testKnex - knex for testing
|
|
384
|
+
* @returns search result
|
|
385
|
+
*/
|
|
386
|
+
async query(testKnex) {
|
|
387
|
+
const knex = testKnex ?? await (0, connection_1.getKnexClient)();
|
|
388
|
+
const { countQuery, searchQuery } = this.buildSearch(knex);
|
|
389
|
+
const shouldEstimateRowcount = countQuery
|
|
390
|
+
? this.shouldEstimateRowcount(countQuery?.toSQL().sql)
|
|
391
|
+
: false;
|
|
392
|
+
const getEstimate = shouldEstimateRowcount
|
|
393
|
+
? this.getEstimatedRowcount({ knex })
|
|
394
|
+
: undefined;
|
|
395
|
+
try {
|
|
396
|
+
const [countResult, pgRecords] = await Promise.all([
|
|
397
|
+
getEstimate || countQuery, searchQuery,
|
|
398
|
+
]);
|
|
399
|
+
const meta = this._metaTemplate();
|
|
400
|
+
meta.limit = this.dbQueryParameters.limit;
|
|
401
|
+
meta.page = this.dbQueryParameters.page;
|
|
402
|
+
meta.count = shouldEstimateRowcount ? countResult : Number(countResult[0]?.count ?? 0);
|
|
403
|
+
const apiRecords = await this.translatePostgresRecordsToApiRecords(pgRecords, knex);
|
|
404
|
+
return {
|
|
405
|
+
meta,
|
|
406
|
+
results: apiRecords,
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
catch (error) {
|
|
410
|
+
log.error(`Error caught in search query for ${JSON.stringify(this.queryStringParameters)}`, error);
|
|
411
|
+
return error;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
exports.BaseSearch = BaseSearch;
|
|
416
|
+
//# sourceMappingURL=BaseSearch.js.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { Knex } from 'knex';
|
|
2
|
+
import { CollectionRecord } from '@cumulus/types/api/collections';
|
|
3
|
+
import { BaseSearch } from './BaseSearch';
|
|
4
|
+
import { DbQueryParameters, QueryEvent } from '../types/search';
|
|
5
|
+
import { PostgresCollectionRecord } from '../types/collection';
|
|
6
|
+
declare type Statuses = {
|
|
7
|
+
queued: number;
|
|
8
|
+
completed: number;
|
|
9
|
+
failed: number;
|
|
10
|
+
running: number;
|
|
11
|
+
total: number;
|
|
12
|
+
};
|
|
13
|
+
interface CollectionRecordApi extends CollectionRecord {
|
|
14
|
+
stats?: Statuses;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Class to build and execute db search query for collections
|
|
18
|
+
*/
|
|
19
|
+
export declare class CollectionSearch extends BaseSearch {
|
|
20
|
+
readonly active: boolean;
|
|
21
|
+
readonly includeStats: boolean;
|
|
22
|
+
constructor(event: QueryEvent);
|
|
23
|
+
/**
|
|
24
|
+
* Build basic query
|
|
25
|
+
*
|
|
26
|
+
* @param knex - DB client
|
|
27
|
+
* @returns queries for getting count and search result
|
|
28
|
+
*/
|
|
29
|
+
protected buildBasicQuery(knex: Knex): {
|
|
30
|
+
countQuery: Knex.QueryBuilder;
|
|
31
|
+
searchQuery: Knex.QueryBuilder;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Build queries for infix and prefix
|
|
35
|
+
*
|
|
36
|
+
* @param params
|
|
37
|
+
* @param params.countQuery - query builder for getting count
|
|
38
|
+
* @param params.searchQuery - query builder for search
|
|
39
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
40
|
+
*/
|
|
41
|
+
protected buildInfixPrefixQuery(params: {
|
|
42
|
+
countQuery: Knex.QueryBuilder;
|
|
43
|
+
searchQuery: Knex.QueryBuilder;
|
|
44
|
+
dbQueryParameters?: DbQueryParameters;
|
|
45
|
+
}): void;
|
|
46
|
+
/**
|
|
47
|
+
* Build queries for range fields
|
|
48
|
+
*
|
|
49
|
+
* @param params
|
|
50
|
+
* @param params.knex - db client
|
|
51
|
+
* @param [params.countQuery] - query builder for getting count
|
|
52
|
+
* @param params.searchQuery - query builder for search
|
|
53
|
+
* @param [params.dbQueryParameters] - db query parameters
|
|
54
|
+
*/
|
|
55
|
+
protected buildRangeQuery(params: {
|
|
56
|
+
knex: Knex;
|
|
57
|
+
countQuery: Knex.QueryBuilder;
|
|
58
|
+
searchQuery: Knex.QueryBuilder;
|
|
59
|
+
dbQueryParameters?: DbQueryParameters;
|
|
60
|
+
}): void;
|
|
61
|
+
/**
|
|
62
|
+
* Executes stats query to get granules' status aggregation
|
|
63
|
+
*
|
|
64
|
+
* @param collectionCumulusIds - array of cumulusIds of the collections
|
|
65
|
+
* @param knex - knex for the stats query
|
|
66
|
+
* @returns the collection's granules status' aggregation
|
|
67
|
+
*/
|
|
68
|
+
private retrieveGranuleStats;
|
|
69
|
+
/**
|
|
70
|
+
* Translate postgres records to api records
|
|
71
|
+
*
|
|
72
|
+
* @param pgRecords - postgres Collection records returned from query
|
|
73
|
+
* @param knex - knex for the stats query if incldueStats is true
|
|
74
|
+
* @returns translated api records
|
|
75
|
+
*/
|
|
76
|
+
protected translatePostgresRecordsToApiRecords(pgRecords: PostgresCollectionRecord[], knex: Knex): Promise<Partial<CollectionRecordApi>[]>;
|
|
77
|
+
}
|
|
78
|
+
export {};
|
|
79
|
+
//# sourceMappingURL=CollectionSearch.d.ts.map
|