@cumulus/db 10.1.2 → 11.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +13 -8
- package/dist/index.js +28 -11
- package/dist/lib/QuerySearchClient.d.ts +36 -0
- package/dist/lib/QuerySearchClient.js +51 -0
- package/dist/lib/collection.d.ts +10 -0
- package/dist/lib/collection.js +21 -0
- package/dist/lib/errors.d.ts +6 -0
- package/dist/lib/errors.js +8 -0
- package/dist/lib/execution.d.ts +21 -0
- package/dist/lib/execution.js +28 -1
- package/dist/lib/file.d.ts +33 -0
- package/dist/lib/file.js +61 -0
- package/dist/lib/granule.d.ts +35 -3
- package/dist/lib/granule.js +104 -8
- package/dist/migrations/20210914051414_add_type_to_files.d.ts +4 -0
- package/dist/migrations/20210914051414_add_type_to_files.js +18 -0
- package/dist/models/base.d.ts +63 -9
- package/dist/models/base.js +89 -12
- package/dist/models/collection.d.ts +4 -3
- package/dist/models/collection.js +4 -1
- package/dist/models/execution.d.ts +2 -0
- package/dist/models/execution.js +6 -2
- package/dist/models/file.d.ts +2 -2
- package/dist/models/file.js +2 -1
- package/dist/models/granule.d.ts +13 -2
- package/dist/models/granule.js +27 -13
- package/dist/models/pdr.js +2 -2
- package/dist/test-utils.js +3 -0
- package/dist/translate/async_operations.d.ts +13 -0
- package/dist/translate/async_operations.js +56 -2
- package/dist/translate/collections.d.ts +13 -2
- package/dist/translate/collections.js +31 -2
- package/dist/translate/file.js +3 -1
- package/dist/translate/granules.d.ts +15 -4
- package/dist/translate/granules.js +38 -14
- package/dist/translate/pdr.d.ts +29 -0
- package/dist/translate/pdr.js +82 -0
- package/dist/translate/providers.d.ts +2 -1
- package/dist/translate/providers.js +24 -1
- package/dist/translate/rules.d.ts +2 -1
- package/dist/translate/rules.js +34 -1
- package/dist/types/base.d.ts +5 -0
- package/dist/types/base.js +3 -0
- package/dist/types/collection.d.ts +6 -4
- package/dist/types/file.d.ts +3 -1
- package/dist/types/granule.d.ts +2 -1
- package/dist/types/pdr.d.ts +2 -1
- package/dist/types/provider.d.ts +1 -1
- package/dist/types/provider.js +6 -0
- package/dist/types/query.d.ts +10 -0
- package/dist/types/query.js +3 -0
- package/dist/types/record.d.ts +5 -0
- package/dist/types/record.js +3 -0
- package/dist/types/rule.d.ts +3 -2
- package/package.json +8 -8
- package/dist/translate/pdrs.d.ts +0 -19
- package/dist/translate/pdrs.js +0 -44
package/dist/lib/granule.js
CHANGED
|
@@ -1,10 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.upsertGranuleWithExecutionJoinRecord = void 0;
|
|
6
|
+
exports.getGranulesByApiPropertiesQuery = exports.getApiGranuleExecutionCumulusIds = exports.getUniqueGranuleByGranuleId = exports.getApiGranuleCumulusIds = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleCollectionId = void 0;
|
|
7
|
+
const Collections_1 = require("@cumulus/message/Collections");
|
|
8
|
+
const errors_1 = require("@cumulus/errors");
|
|
9
|
+
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
4
10
|
const collection_1 = require("../models/collection");
|
|
5
11
|
const granule_1 = require("../models/granule");
|
|
6
12
|
const granules_executions_1 = require("../models/granules-executions");
|
|
7
|
-
const {
|
|
13
|
+
const { TableNames } = require('../tables');
|
|
14
|
+
const getGranuleCollectionId = async (knexOrTransaction, granule) => {
|
|
15
|
+
const collectionPgModel = new collection_1.CollectionPgModel();
|
|
16
|
+
const collection = await collectionPgModel.get(knexOrTransaction, { cumulus_id: granule.collection_cumulus_id });
|
|
17
|
+
return (0, Collections_1.constructCollectionId)(collection.name, collection.version);
|
|
18
|
+
};
|
|
19
|
+
exports.getGranuleCollectionId = getGranuleCollectionId;
|
|
8
20
|
/**
|
|
9
21
|
* Upsert a granule and a record in the granules/executions join table.
|
|
10
22
|
*
|
|
@@ -14,24 +26,24 @@ const { deconstructCollectionId } = require('@cumulus/message/Collections');
|
|
|
14
26
|
* @param {Object} [granulePgModel] - Granule PG model class instance
|
|
15
27
|
* @param {Object} [granulesExecutionsPgModel]
|
|
16
28
|
* Granules/executions PG model class instance
|
|
17
|
-
* @returns {Promise<
|
|
29
|
+
* @returns {Promise<PostgresGranuleRecord[]>}
|
|
18
30
|
*/
|
|
19
31
|
const upsertGranuleWithExecutionJoinRecord = async (knexTransaction, granule, executionCumulusId, granulePgModel = new granule_1.GranulePgModel(), granulesExecutionsPgModel = new granules_executions_1.GranulesExecutionsPgModel()) => {
|
|
20
|
-
const [
|
|
32
|
+
const [pgGranule] = await granulePgModel.upsert(knexTransaction, granule, executionCumulusId);
|
|
21
33
|
// granuleCumulusId could be undefined if the upsert affected no rows due to its
|
|
22
34
|
// conditional logic. In that case, we assume that the execution history for the
|
|
23
35
|
// granule was already written and return early. Execution history cannot be written
|
|
24
36
|
// without granuleCumulusId regardless.
|
|
25
|
-
if (!
|
|
37
|
+
if (!pgGranule) {
|
|
26
38
|
return [];
|
|
27
39
|
}
|
|
28
40
|
if (executionCumulusId) {
|
|
29
41
|
await granulesExecutionsPgModel.upsert(knexTransaction, {
|
|
30
|
-
granule_cumulus_id:
|
|
42
|
+
granule_cumulus_id: pgGranule.cumulus_id,
|
|
31
43
|
execution_cumulus_id: executionCumulusId,
|
|
32
44
|
});
|
|
33
45
|
}
|
|
34
|
-
return [
|
|
46
|
+
return [pgGranule];
|
|
35
47
|
};
|
|
36
48
|
exports.upsertGranuleWithExecutionJoinRecord = upsertGranuleWithExecutionJoinRecord;
|
|
37
49
|
/**
|
|
@@ -50,7 +62,7 @@ const getApiGranuleCumulusIds = async (knexOrTransaction, granules, collectionPg
|
|
|
50
62
|
const { collectionId } = granule;
|
|
51
63
|
let collectionCumulusId = collectionMap[collectionId];
|
|
52
64
|
if (!collectionCumulusId) {
|
|
53
|
-
const { name, version } = deconstructCollectionId(collectionId);
|
|
65
|
+
const { name, version } = (0, Collections_1.deconstructCollectionId)(collectionId);
|
|
54
66
|
collectionCumulusId = await collectionPgModel.getRecordCumulusId(knexOrTransaction, { name, version });
|
|
55
67
|
collectionMap[collectionId] = collectionCumulusId;
|
|
56
68
|
}
|
|
@@ -62,6 +74,30 @@ const getApiGranuleCumulusIds = async (knexOrTransaction, granules, collectionPg
|
|
|
62
74
|
return [...new Set(granuleCumulusIds)];
|
|
63
75
|
};
|
|
64
76
|
exports.getApiGranuleCumulusIds = getApiGranuleCumulusIds;
|
|
77
|
+
/**
|
|
78
|
+
* Get one Granule for a granule_id. If more than one or none are found, throw an error
|
|
79
|
+
*
|
|
80
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction -
|
|
81
|
+
* DB client or transaction
|
|
82
|
+
* @param {string} granuleId - a Granule ID
|
|
83
|
+
* @param {GranulePgModel} granulePgModel - Granule PG model class instance
|
|
84
|
+
* @returns {Promise<PostgresGranuleRecord>}
|
|
85
|
+
*/
|
|
86
|
+
const getUniqueGranuleByGranuleId = async (knexOrTransaction, granuleId, granulePgModel = new granule_1.GranulePgModel()) => {
|
|
87
|
+
const logger = new logger_1.default({ sender: '@cumulus/api/granules' });
|
|
88
|
+
const PgGranuleRecords = await granulePgModel.search(knexOrTransaction, {
|
|
89
|
+
granule_id: granuleId,
|
|
90
|
+
});
|
|
91
|
+
if (PgGranuleRecords.length > 1) {
|
|
92
|
+
logger.warn(`Granule ID ${granuleId} is not unique across collections, cannot make an update action based on granule Id alone`);
|
|
93
|
+
throw new Error(`Failed to write ${granuleId} due to granuleId duplication on postgres granule record`);
|
|
94
|
+
}
|
|
95
|
+
if (PgGranuleRecords.length === 0) {
|
|
96
|
+
throw new errors_1.RecordDoesNotExist(`Granule ${granuleId} does not exist or was already deleted`);
|
|
97
|
+
}
|
|
98
|
+
return PgGranuleRecords[0];
|
|
99
|
+
};
|
|
100
|
+
exports.getUniqueGranuleByGranuleId = getUniqueGranuleByGranuleId;
|
|
65
101
|
/**
|
|
66
102
|
* Get cumulus IDs for all executions associated to a set of granules
|
|
67
103
|
*
|
|
@@ -81,4 +117,64 @@ const getApiGranuleExecutionCumulusIds = async (knexOrTransaction, granules, col
|
|
|
81
117
|
return executionCumulusIds;
|
|
82
118
|
};
|
|
83
119
|
exports.getApiGranuleExecutionCumulusIds = getApiGranuleExecutionCumulusIds;
|
|
120
|
+
/**
|
|
121
|
+
* Helper to build a query to search granules by various API granule record properties.
|
|
122
|
+
*
|
|
123
|
+
* @param {Knex} knex - DB client
|
|
124
|
+
* @param {Object} searchParams
|
|
125
|
+
* @param {string | Array<string>} [searchParams.collectionIds] - Collection ID
|
|
126
|
+
* @param {string | Array<string>} [searchParams.granuleIds] - array of granule IDs
|
|
127
|
+
* @param {string} [searchParams.providerName] - Provider name
|
|
128
|
+
* @param {UpdatedAtRange} [searchParams.updatedAtRange] - Date range for updated_at column
|
|
129
|
+
* @param {string} [searchParams.status] - Granule status to search by
|
|
130
|
+
* @param {string | Array<string>} [sortByFields] - Field(s) to sort by
|
|
131
|
+
* @returns {Knex.QueryBuilder}
|
|
132
|
+
*/
|
|
133
|
+
const getGranulesByApiPropertiesQuery = (knex, { collectionIds, granuleIds, providerNames, updatedAtRange = {}, status, }, sortByFields) => {
|
|
134
|
+
const { granules: granulesTable, collections: collectionsTable, providers: providersTable, } = TableNames;
|
|
135
|
+
return knex(granulesTable)
|
|
136
|
+
.select(`${granulesTable}.*`)
|
|
137
|
+
.select({
|
|
138
|
+
providerName: `${providersTable}.name`,
|
|
139
|
+
collectionName: `${collectionsTable}.name`,
|
|
140
|
+
collectionVersion: `${collectionsTable}.version`,
|
|
141
|
+
})
|
|
142
|
+
.innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`)
|
|
143
|
+
.leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`)
|
|
144
|
+
.modify((queryBuilder) => {
|
|
145
|
+
if (collectionIds) {
|
|
146
|
+
const collectionIdFilters = [collectionIds].flat();
|
|
147
|
+
const collectionIdConcatField = `(${collectionsTable}.name || '${Collections_1.collectionIdSeparator}' || ${collectionsTable}.version)`;
|
|
148
|
+
const collectionIdInClause = collectionIdFilters.map(() => '?').join(',');
|
|
149
|
+
queryBuilder.whereRaw(`${collectionIdConcatField} IN (${collectionIdInClause})`, collectionIdFilters);
|
|
150
|
+
}
|
|
151
|
+
if (granuleIds) {
|
|
152
|
+
const granuleIdFilters = [granuleIds].flat();
|
|
153
|
+
queryBuilder.where((nestedQueryBuilder) => {
|
|
154
|
+
granuleIdFilters.forEach((granuleId) => {
|
|
155
|
+
nestedQueryBuilder.orWhere(`${granulesTable}.granule_id`, 'LIKE', `%${granuleId}%`);
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
if (providerNames) {
|
|
160
|
+
queryBuilder.whereIn(`${providersTable}.name`, providerNames);
|
|
161
|
+
}
|
|
162
|
+
if (updatedAtRange.updatedAtFrom) {
|
|
163
|
+
queryBuilder.where(`${granulesTable}.updated_at`, '>=', updatedAtRange.updatedAtFrom);
|
|
164
|
+
}
|
|
165
|
+
if (updatedAtRange.updatedAtTo) {
|
|
166
|
+
queryBuilder.where(`${granulesTable}.updated_at`, '<=', updatedAtRange.updatedAtTo);
|
|
167
|
+
}
|
|
168
|
+
if (status) {
|
|
169
|
+
queryBuilder.where(`${granulesTable}.status`, status);
|
|
170
|
+
}
|
|
171
|
+
if (sortByFields) {
|
|
172
|
+
queryBuilder.orderBy([sortByFields].flat());
|
|
173
|
+
}
|
|
174
|
+
})
|
|
175
|
+
.groupBy(`${granulesTable}.cumulus_id`)
|
|
176
|
+
.groupBy(`${collectionsTable}.cumulus_id`)
|
|
177
|
+
.groupBy(`${providersTable}.cumulus_id`);
|
|
178
|
+
};
|
|
179
|
+
exports.getGranulesByApiPropertiesQuery = getGranulesByApiPropertiesQuery;
|
|
84
180
|
//# sourceMappingURL=granule.js.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.down = exports.up = void 0;
|
|
4
|
+
const up = async (knex) => {
|
|
5
|
+
await knex.schema.table('files', (table) => {
|
|
6
|
+
table
|
|
7
|
+
.text('type')
|
|
8
|
+
.comment('file "type"');
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
exports.up = up;
|
|
12
|
+
const down = async (knex) => {
|
|
13
|
+
await knex.schema.table('files', (table) => {
|
|
14
|
+
table.dropColumn('type');
|
|
15
|
+
});
|
|
16
|
+
};
|
|
17
|
+
exports.down = down;
|
|
18
|
+
//# sourceMappingURL=20210914051414_add_type_to_files.js.map
|
package/dist/models/base.d.ts
CHANGED
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
import { Knex } from 'knex';
|
|
2
|
+
import { UpdatedAtRange } from '../types/record';
|
|
3
|
+
import { BaseRecord } from '../types/base';
|
|
2
4
|
import { TableNames } from '../tables';
|
|
3
|
-
declare class BasePgModel<ItemType, RecordType extends {
|
|
4
|
-
cumulus_id: number;
|
|
5
|
-
}> {
|
|
5
|
+
declare class BasePgModel<ItemType, RecordType extends BaseRecord> {
|
|
6
6
|
readonly tableName: TableNames;
|
|
7
|
-
constructor({ tableName
|
|
7
|
+
constructor({ tableName }: {
|
|
8
8
|
tableName: TableNames;
|
|
9
9
|
});
|
|
10
|
+
/**
|
|
11
|
+
* Fetches multiple items from Postgres
|
|
12
|
+
*
|
|
13
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
|
|
14
|
+
* @param {Partial<RecordType>} params - An object or any portion of an object of type RecordType
|
|
15
|
+
* @param {UpdatedAtRange} updatedAtParams - An object with Date search bounds for updatedAt
|
|
16
|
+
* @returns {Promise<PostgresCollectionRecord[]>} List of returned records
|
|
17
|
+
*/
|
|
18
|
+
searchWithUpdatedAtRange(knexOrTransaction: Knex | Knex.Transaction, params: Partial<RecordType>, updatedAtParams: UpdatedAtRange): Promise<RecordType[]>;
|
|
10
19
|
count(knexOrTransaction: Knex | Knex.Transaction, params: ([string, string, string] | [Partial<RecordType>])[]): Promise<{
|
|
11
20
|
[k: string]: string | number;
|
|
12
21
|
}[]>;
|
|
@@ -18,6 +27,24 @@ declare class BasePgModel<ItemType, RecordType extends {
|
|
|
18
27
|
* @returns {Promise<RecordType>} The returned record
|
|
19
28
|
*/
|
|
20
29
|
get(knexOrTransaction: Knex | Knex.Transaction, params: Partial<RecordType>): Promise<RecordType>;
|
|
30
|
+
getMaxCumulusId(knexOrTransaction: Knex | Knex.Transaction): Promise<number>;
|
|
31
|
+
paginateByCumulusId(knexOrTransaction: Knex | Knex.Transaction, startId?: number, pageSize?: number): Promise<RecordType[]>;
|
|
32
|
+
/**
|
|
33
|
+
* Builds query to fetch mulitple items from postgres
|
|
34
|
+
*
|
|
35
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
|
|
36
|
+
* @param {Object} params - An object or any portion of an object of type RecordType
|
|
37
|
+
* @returns {Object} - Knex querybuilder object
|
|
38
|
+
*/
|
|
39
|
+
queryBuilderSearch(knexOrTransaction: Knex | Knex.Transaction, params: Partial<RecordType>): Knex.QueryBuilder<any, {
|
|
40
|
+
_base: any;
|
|
41
|
+
_hasSelection: false;
|
|
42
|
+
_keys: never;
|
|
43
|
+
_aliases: {};
|
|
44
|
+
_single: false;
|
|
45
|
+
_intersectProps: {};
|
|
46
|
+
_unionProps: never;
|
|
47
|
+
}[]>;
|
|
21
48
|
/**
|
|
22
49
|
* Fetches multiple items from Postgres
|
|
23
50
|
*
|
|
@@ -39,11 +66,11 @@ declare class BasePgModel<ItemType, RecordType extends {
|
|
|
39
66
|
/**
|
|
40
67
|
* Get cumulus_id column value for multiple records in Postgres
|
|
41
68
|
*
|
|
42
|
-
* @param {Knex | Knex.Transaction} knexOrTransaction
|
|
69
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction
|
|
43
70
|
* DB client or transaction
|
|
44
71
|
* @param {Array<string>} columnNames - column names for whereIn query
|
|
45
|
-
* @param {Array<
|
|
46
|
-
* @returns {Promise<Array
|
|
72
|
+
* @param {Array<any>} values - record values for whereIn query
|
|
73
|
+
* @returns {Promise<Array>} An array of cumulus_ids for the returned records
|
|
47
74
|
*/
|
|
48
75
|
getRecordsCumulusIds(knexOrTransaction: Knex | Knex.Transaction, columnNames: Array<keyof RecordType>, values: Array<any>): Promise<Array<number>>;
|
|
49
76
|
/**
|
|
@@ -61,9 +88,21 @@ declare class BasePgModel<ItemType, RecordType extends {
|
|
|
61
88
|
* @param {ItemType} item - A record to insert into the DB
|
|
62
89
|
* @param {string | Array<string>} returningFields - A string or array of strings
|
|
63
90
|
* of columns to return. Defaults to 'cumulus_id'.
|
|
64
|
-
* @returns {Promise<
|
|
91
|
+
* @returns {Promise<unknown[] | Object[]>} Returns an array of objects or an
|
|
92
|
+
* array of values from the specified column(s) from returningFields.
|
|
65
93
|
*/
|
|
66
|
-
create(knexOrTransaction: Knex | Knex.Transaction, item: ItemType, returningFields?: string | string[]): Promise<
|
|
94
|
+
create(knexOrTransaction: Knex | Knex.Transaction, item: ItemType, returningFields?: string | string[]): Promise<unknown[] | Object[]>;
|
|
95
|
+
/**
|
|
96
|
+
* Creates multiple items in Postgres
|
|
97
|
+
*
|
|
98
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
|
|
99
|
+
* @param {ItemType[]} items - Records to insert into the DB
|
|
100
|
+
* @param {string | Array<string>} returningFields - A string or array of strings
|
|
101
|
+
* of columns to return. Defaults to 'cumulus_id'.
|
|
102
|
+
* @returns {Promise<unknown[] | Object[]>} Returns an array of objects or an
|
|
103
|
+
* array of values from the specified column(s) from returningFields.
|
|
104
|
+
*/
|
|
105
|
+
insert(knexOrTransaction: Knex | Knex.Transaction, items: ItemType[], returningFields?: string | string[]): Promise<unknown[] | Object[]>;
|
|
67
106
|
/**
|
|
68
107
|
* Deletes the item from Postgres
|
|
69
108
|
*
|
|
@@ -82,6 +121,21 @@ declare class BasePgModel<ItemType, RecordType extends {
|
|
|
82
121
|
* @returns {Promise<RecordType[]>} List of returned records
|
|
83
122
|
*/
|
|
84
123
|
update(knexOrTransaction: Knex | Knex.Transaction, whereClause: Partial<RecordType>, updateParams: Partial<RecordType>, returning?: Array<string>): Promise<any[]>;
|
|
124
|
+
/**
|
|
125
|
+
* Deletes items from postgres based on params, excluding any cumulus_ids in the excludeCumulusIds
|
|
126
|
+
* @param {Object} params
|
|
127
|
+
* @param {Knex | Knex.Transaction} params.knexOrTransaction - DB client or transaction
|
|
128
|
+
* @param {Partial<RecordType>} params.queryParams - An object or any portion
|
|
129
|
+
* of an object of type RecordType
|
|
130
|
+
* @param {[number]} params.excludeCumulusIds - A list of cumulus_ids to exclude from the deletion
|
|
131
|
+
* request
|
|
132
|
+
* @returns {Promise<number>} The number of rows deleted
|
|
133
|
+
*/
|
|
134
|
+
deleteExcluding({ knexOrTransaction, excludeCumulusIds, queryParams, }: {
|
|
135
|
+
knexOrTransaction: Knex | Knex.Transaction;
|
|
136
|
+
excludeCumulusIds: Number[];
|
|
137
|
+
queryParams: Partial<RecordType>;
|
|
138
|
+
}): Promise<number>;
|
|
85
139
|
}
|
|
86
140
|
export { BasePgModel };
|
|
87
141
|
//# sourceMappingURL=base.d.ts.map
|
package/dist/models/base.js
CHANGED
|
@@ -4,11 +4,33 @@ exports.BasePgModel = void 0;
|
|
|
4
4
|
const errors_1 = require("@cumulus/errors");
|
|
5
5
|
const database_1 = require("../database");
|
|
6
6
|
class BasePgModel {
|
|
7
|
-
constructor({ tableName
|
|
7
|
+
constructor({ tableName }) {
|
|
8
8
|
this.tableName = tableName;
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* Fetches multiple items from Postgres
|
|
12
|
+
*
|
|
13
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
|
|
14
|
+
* @param {Partial<RecordType>} params - An object or any portion of an object of type RecordType
|
|
15
|
+
* @param {UpdatedAtRange} updatedAtParams - An object with Date search bounds for updatedAt
|
|
16
|
+
* @returns {Promise<PostgresCollectionRecord[]>} List of returned records
|
|
17
|
+
*/
|
|
18
|
+
async searchWithUpdatedAtRange(knexOrTransaction, params, updatedAtParams) {
|
|
19
|
+
const records = await knexOrTransaction(this.tableName).where((builder) => {
|
|
20
|
+
var _a, _b;
|
|
21
|
+
builder.where(params);
|
|
22
|
+
if (updatedAtParams.updatedAtFrom || updatedAtParams.updatedAtTo) {
|
|
23
|
+
builder.whereBetween('updated_at', [
|
|
24
|
+
(_a = updatedAtParams === null || updatedAtParams === void 0 ? void 0 : updatedAtParams.updatedAtFrom) !== null && _a !== void 0 ? _a : new Date(0),
|
|
25
|
+
(_b = updatedAtParams === null || updatedAtParams === void 0 ? void 0 : updatedAtParams.updatedAtTo) !== null && _b !== void 0 ? _b : new Date(),
|
|
26
|
+
]);
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
return records;
|
|
30
|
+
}
|
|
10
31
|
async count(knexOrTransaction, params) {
|
|
11
|
-
const query = knexOrTransaction(this.tableName)
|
|
32
|
+
const query = knexOrTransaction(this.tableName)
|
|
33
|
+
.where((builder) => {
|
|
12
34
|
params.forEach((param) => {
|
|
13
35
|
if (param.length === 3) {
|
|
14
36
|
builder.where(...param);
|
|
@@ -17,7 +39,8 @@ class BasePgModel {
|
|
|
17
39
|
builder.where(param[0]);
|
|
18
40
|
}
|
|
19
41
|
});
|
|
20
|
-
})
|
|
42
|
+
})
|
|
43
|
+
.count();
|
|
21
44
|
return await query;
|
|
22
45
|
}
|
|
23
46
|
/**
|
|
@@ -36,6 +59,31 @@ class BasePgModel {
|
|
|
36
59
|
}
|
|
37
60
|
return record;
|
|
38
61
|
}
|
|
62
|
+
async getMaxCumulusId(knexOrTransaction) {
|
|
63
|
+
const result = await knexOrTransaction(this.tableName)
|
|
64
|
+
.max('cumulus_id')
|
|
65
|
+
.first();
|
|
66
|
+
if (!result) {
|
|
67
|
+
throw new Error(`Invalid .max "cumulus_id" query on ${this.tableName}, MAX cumulus_id cannot be returned`);
|
|
68
|
+
}
|
|
69
|
+
return Number(result.max);
|
|
70
|
+
}
|
|
71
|
+
async paginateByCumulusId(knexOrTransaction, startId = 0, pageSize = 100) {
|
|
72
|
+
return await knexOrTransaction
|
|
73
|
+
.select()
|
|
74
|
+
.from(this.tableName)
|
|
75
|
+
.whereBetween('cumulus_id', [startId, startId + pageSize - 1]);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Builds query to fetch mulitple items from postgres
|
|
79
|
+
*
|
|
80
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
|
|
81
|
+
* @param {Object} params - An object or any portion of an object of type RecordType
|
|
82
|
+
* @returns {Object} - Knex querybuilder object
|
|
83
|
+
*/
|
|
84
|
+
queryBuilderSearch(knexOrTransaction, params) {
|
|
85
|
+
return knexOrTransaction(this.tableName).where(params);
|
|
86
|
+
}
|
|
39
87
|
/**
|
|
40
88
|
* Fetches multiple items from Postgres
|
|
41
89
|
*
|
|
@@ -44,8 +92,7 @@ class BasePgModel {
|
|
|
44
92
|
* @returns {Promise<RecordType[]>} List of returned records
|
|
45
93
|
*/
|
|
46
94
|
async search(knexOrTransaction, params) {
|
|
47
|
-
const records = await
|
|
48
|
-
.where(params);
|
|
95
|
+
const records = await this.queryBuilderSearch(knexOrTransaction, params);
|
|
49
96
|
return records;
|
|
50
97
|
}
|
|
51
98
|
/**
|
|
@@ -70,11 +117,11 @@ class BasePgModel {
|
|
|
70
117
|
/**
|
|
71
118
|
* Get cumulus_id column value for multiple records in Postgres
|
|
72
119
|
*
|
|
73
|
-
* @param {Knex | Knex.Transaction} knexOrTransaction
|
|
120
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction
|
|
74
121
|
* DB client or transaction
|
|
75
122
|
* @param {Array<string>} columnNames - column names for whereIn query
|
|
76
|
-
* @param {Array<
|
|
77
|
-
* @returns {Promise<Array
|
|
123
|
+
* @param {Array<any>} values - record values for whereIn query
|
|
124
|
+
* @returns {Promise<Array>} An array of cumulus_ids for the returned records
|
|
78
125
|
*/
|
|
79
126
|
async getRecordsCumulusIds(knexOrTransaction, columnNames, values) {
|
|
80
127
|
const records = await knexOrTransaction(this.tableName)
|
|
@@ -108,13 +155,29 @@ class BasePgModel {
|
|
|
108
155
|
* @param {ItemType} item - A record to insert into the DB
|
|
109
156
|
* @param {string | Array<string>} returningFields - A string or array of strings
|
|
110
157
|
* of columns to return. Defaults to 'cumulus_id'.
|
|
111
|
-
* @returns {Promise<
|
|
158
|
+
* @returns {Promise<unknown[] | Object[]>} Returns an array of objects or an
|
|
159
|
+
* array of values from the specified column(s) from returningFields.
|
|
112
160
|
*/
|
|
113
161
|
async create(knexOrTransaction, item, returningFields = 'cumulus_id') {
|
|
114
162
|
return await knexOrTransaction(this.tableName)
|
|
115
163
|
.insert(item)
|
|
116
164
|
.returning(returningFields);
|
|
117
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* Creates multiple items in Postgres
|
|
168
|
+
*
|
|
169
|
+
* @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
|
|
170
|
+
* @param {ItemType[]} items - Records to insert into the DB
|
|
171
|
+
* @param {string | Array<string>} returningFields - A string or array of strings
|
|
172
|
+
* of columns to return. Defaults to 'cumulus_id'.
|
|
173
|
+
* @returns {Promise<unknown[] | Object[]>} Returns an array of objects or an
|
|
174
|
+
* array of values from the specified column(s) from returningFields.
|
|
175
|
+
*/
|
|
176
|
+
async insert(knexOrTransaction, items, returningFields = 'cumulus_id') {
|
|
177
|
+
return await knexOrTransaction(this.tableName)
|
|
178
|
+
.insert(items)
|
|
179
|
+
.returning(returningFields);
|
|
180
|
+
}
|
|
118
181
|
/**
|
|
119
182
|
* Deletes the item from Postgres
|
|
120
183
|
*
|
|
@@ -123,9 +186,7 @@ class BasePgModel {
|
|
|
123
186
|
* @returns {Promise<number>} The number of rows deleted
|
|
124
187
|
*/
|
|
125
188
|
async delete(knexOrTransaction, params) {
|
|
126
|
-
return await knexOrTransaction(this.tableName)
|
|
127
|
-
.where(params)
|
|
128
|
-
.del();
|
|
189
|
+
return await knexOrTransaction(this.tableName).where(params).del();
|
|
129
190
|
}
|
|
130
191
|
/**
|
|
131
192
|
* Updates the item or items in Postgres
|
|
@@ -141,6 +202,22 @@ class BasePgModel {
|
|
|
141
202
|
.where(whereClause)
|
|
142
203
|
.update(updateParams, returning);
|
|
143
204
|
}
|
|
205
|
+
/**
|
|
206
|
+
* Deletes items from postgres based on params, excluding any cumulus_ids in the excludeCumulusIds
|
|
207
|
+
* @param {Object} params
|
|
208
|
+
* @param {Knex | Knex.Transaction} params.knexOrTransaction - DB client or transaction
|
|
209
|
+
* @param {Partial<RecordType>} params.queryParams - An object or any portion
|
|
210
|
+
* of an object of type RecordType
|
|
211
|
+
* @param {[number]} params.excludeCumulusIds - A list of cumulus_ids to exclude from the deletion
|
|
212
|
+
* request
|
|
213
|
+
* @returns {Promise<number>} The number of rows deleted
|
|
214
|
+
*/
|
|
215
|
+
async deleteExcluding({ knexOrTransaction, excludeCumulusIds = [], queryParams, }) {
|
|
216
|
+
return await knexOrTransaction(this.tableName)
|
|
217
|
+
.where(queryParams)
|
|
218
|
+
.whereNotIn('cumulus_id', excludeCumulusIds)
|
|
219
|
+
.del();
|
|
220
|
+
}
|
|
144
221
|
}
|
|
145
222
|
exports.BasePgModel = BasePgModel;
|
|
146
223
|
//# sourceMappingURL=base.js.map
|
|
@@ -3,12 +3,13 @@ import { BasePgModel } from './base';
|
|
|
3
3
|
import { PostgresCollection, PostgresCollectionRecord } from '../types/collection';
|
|
4
4
|
declare class CollectionPgModel extends BasePgModel<PostgresCollection, PostgresCollectionRecord> {
|
|
5
5
|
constructor();
|
|
6
|
+
create(knexOrTransaction: Knex | Knex.Transaction, item: PostgresCollection): Promise<Object[] | unknown[]>;
|
|
6
7
|
upsert(knexOrTransaction: Knex | Knex.Transaction, collection: PostgresCollection): Knex.QueryBuilder<any, {
|
|
7
8
|
_base: any;
|
|
8
|
-
_hasSelection:
|
|
9
|
-
_keys:
|
|
9
|
+
_hasSelection: false;
|
|
10
|
+
_keys: never;
|
|
10
11
|
_aliases: {};
|
|
11
|
-
_single:
|
|
12
|
+
_single: false;
|
|
12
13
|
_intersectProps: {};
|
|
13
14
|
_unionProps: never;
|
|
14
15
|
}[]>;
|
|
@@ -9,12 +9,15 @@ class CollectionPgModel extends base_1.BasePgModel {
|
|
|
9
9
|
tableName: tables_1.TableNames.collections,
|
|
10
10
|
});
|
|
11
11
|
}
|
|
12
|
+
create(knexOrTransaction, item) {
|
|
13
|
+
return super.create(knexOrTransaction, item, '*');
|
|
14
|
+
}
|
|
12
15
|
upsert(knexOrTransaction, collection) {
|
|
13
16
|
return knexOrTransaction(this.tableName)
|
|
14
17
|
.insert(collection)
|
|
15
18
|
.onConflict(['name', 'version'])
|
|
16
19
|
.merge()
|
|
17
|
-
.returning('
|
|
20
|
+
.returning('*');
|
|
18
21
|
}
|
|
19
22
|
}
|
|
20
23
|
exports.CollectionPgModel = CollectionPgModel;
|
|
@@ -3,6 +3,8 @@ import { BasePgModel } from './base';
|
|
|
3
3
|
import { PostgresExecution, PostgresExecutionRecord } from '../types/execution';
|
|
4
4
|
declare class ExecutionPgModel extends BasePgModel<PostgresExecution, PostgresExecutionRecord> {
|
|
5
5
|
constructor();
|
|
6
|
+
static nonActiveStatuses: string[];
|
|
7
|
+
create(knexOrTransaction: Knex | Knex.Transaction, item: PostgresExecution): Promise<Object[] | unknown[]>;
|
|
6
8
|
upsert(knexOrTrx: Knex | Knex.Transaction, execution: PostgresExecution): Promise<any[]>;
|
|
7
9
|
/**
|
|
8
10
|
* Get executions from the execution cumulus_id
|
package/dist/models/execution.js
CHANGED
|
@@ -10,6 +10,9 @@ class ExecutionPgModel extends base_1.BasePgModel {
|
|
|
10
10
|
tableName: tables_1.TableNames.executions,
|
|
11
11
|
});
|
|
12
12
|
}
|
|
13
|
+
create(knexOrTransaction, item) {
|
|
14
|
+
return super.create(knexOrTransaction, item, '*');
|
|
15
|
+
}
|
|
13
16
|
async upsert(knexOrTrx, execution) {
|
|
14
17
|
if (execution.status === 'running') {
|
|
15
18
|
return await knexOrTrx(this.tableName)
|
|
@@ -21,13 +24,13 @@ class ExecutionPgModel extends base_1.BasePgModel {
|
|
|
21
24
|
timestamp: execution.timestamp,
|
|
22
25
|
original_payload: execution.original_payload,
|
|
23
26
|
})
|
|
24
|
-
.returning('
|
|
27
|
+
.returning('*');
|
|
25
28
|
}
|
|
26
29
|
return await knexOrTrx(this.tableName)
|
|
27
30
|
.insert(execution)
|
|
28
31
|
.onConflict('arn')
|
|
29
32
|
.merge()
|
|
30
|
-
.returning('
|
|
33
|
+
.returning('*');
|
|
31
34
|
}
|
|
32
35
|
/**
|
|
33
36
|
* Get executions from the execution cumulus_id
|
|
@@ -64,4 +67,5 @@ class ExecutionPgModel extends base_1.BasePgModel {
|
|
|
64
67
|
}
|
|
65
68
|
}
|
|
66
69
|
exports.ExecutionPgModel = ExecutionPgModel;
|
|
70
|
+
ExecutionPgModel.nonActiveStatuses = ['completed', 'failed', 'unknown'];
|
|
67
71
|
//# sourceMappingURL=execution.js.map
|
package/dist/models/file.d.ts
CHANGED
|
@@ -5,8 +5,8 @@ declare class FilePgModel extends BasePgModel<PostgresFile, PostgresFileRecord>
|
|
|
5
5
|
constructor();
|
|
6
6
|
upsert(knexOrTrx: Knex | Knex.Transaction, file: PostgresFile): Knex.QueryBuilder<any, {
|
|
7
7
|
_base: any;
|
|
8
|
-
_hasSelection:
|
|
9
|
-
_keys:
|
|
8
|
+
_hasSelection: false;
|
|
9
|
+
_keys: never;
|
|
10
10
|
_aliases: {};
|
|
11
11
|
_single: false;
|
|
12
12
|
_intersectProps: {};
|
package/dist/models/file.js
CHANGED
package/dist/models/granule.d.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { Knex } from 'knex';
|
|
2
2
|
import { PostgresGranule, PostgresGranuleRecord, PostgresGranuleUniqueColumns } from '../types/granule';
|
|
3
3
|
import { BasePgModel } from './base';
|
|
4
|
-
import {
|
|
4
|
+
import { ExecutionPgModel } from './execution';
|
|
5
5
|
interface RecordSelect {
|
|
6
6
|
cumulus_id: number;
|
|
7
7
|
}
|
|
8
8
|
export default class GranulePgModel extends BasePgModel<PostgresGranule, PostgresGranuleRecord> {
|
|
9
9
|
constructor();
|
|
10
|
+
create(knexOrTransaction: Knex | Knex.Transaction, item: PostgresGranule): Promise<Object[] | unknown[]>;
|
|
10
11
|
/**
|
|
11
12
|
* Deletes the item from Postgres
|
|
12
13
|
*
|
|
@@ -17,6 +18,7 @@ export default class GranulePgModel extends BasePgModel<PostgresGranule, Postgre
|
|
|
17
18
|
delete(knexOrTransaction: Knex | Knex.Transaction, params: PostgresGranuleUniqueColumns | {
|
|
18
19
|
cumulus_id: number;
|
|
19
20
|
}): Promise<number>;
|
|
21
|
+
deleteExcluding(): Promise<never>;
|
|
20
22
|
/**
|
|
21
23
|
* Checks if a granule is present in PostgreSQL
|
|
22
24
|
*
|
|
@@ -35,7 +37,16 @@ export default class GranulePgModel extends BasePgModel<PostgresGranule, Postgre
|
|
|
35
37
|
* @returns {Promise<PostgresGranuleRecord>} The returned record
|
|
36
38
|
*/
|
|
37
39
|
get(knexOrTransaction: Knex | Knex.Transaction, params: PostgresGranuleUniqueColumns | RecordSelect): Promise<PostgresGranuleRecord>;
|
|
38
|
-
|
|
40
|
+
_buildExclusionClause(executionPgModel: ExecutionPgModel, executionCumulusId: number, knexOrTrx: Knex | Knex.Transaction, status: 'queued' | 'running'): Knex.QueryBuilder<any, {
|
|
41
|
+
_base: any;
|
|
42
|
+
_hasSelection: false;
|
|
43
|
+
_keys: never;
|
|
44
|
+
_aliases: {};
|
|
45
|
+
_single: false;
|
|
46
|
+
_intersectProps: {};
|
|
47
|
+
_unionProps: never;
|
|
48
|
+
}[]>;
|
|
49
|
+
upsert(knexOrTrx: Knex | Knex.Transaction, granule: PostgresGranule, executionCumulusId?: number, executionPgModel?: ExecutionPgModel): Promise<any[]>;
|
|
39
50
|
/**
|
|
40
51
|
* Get granules from the granule cumulus_id
|
|
41
52
|
*
|