@cumulus/db 19.2.0-alpha.1 → 20.0.1

package/dist/index.d.ts

@@ -30,7 +30,7 @@ export { translateApiReconReportToPostgresReconReport, translatePostgresReconRep
 export { getCollectionsByGranuleIds, getUniqueCollectionsByGranuleFilter, } from './lib/collection';
 export { batchDeleteExecutionFromDatabaseByCumulusCollectionId, executionArnsFromGranuleIdsAndWorkflowNames, getApiExecutionCumulusIds, getApiGranuleExecutionCumulusIdsByExecution, getExecutionInfoByGranuleCumulusId, getWorkflowNameIntersectFromGranuleIds, newestExecutionArnFromGranuleIdWorkflowName, } from './lib/execution';
 export { getFilesAndGranuleInfoQuery, } from './lib/file';
-export { getApiGranuleCumulusIds, getApiGranuleExecutionCumulusIds, getGranuleCollectionId, getUniqueGranuleByGranuleId, getGranuleByUniqueColumns, upsertGranuleWithExecutionJoinRecord, getGranulesByApiPropertiesQuery, getGranulesByGranuleId, getGranuleAndCollection, } from './lib/granule';
+export { getApiGranuleCumulusIds, getApiGranuleExecutionCumulusIds, getGranuleCollectionId, getUniqueGranuleByGranuleId, getGranuleByUniqueColumns, upsertGranuleWithExecutionJoinRecord, getGranulesByApiPropertiesQuery, getGranulesByGranuleId, getGranuleAndCollection, updateBatchGranulesCollection, } from './lib/granule';
 export { QuerySearchClient, } from './lib/QuerySearchClient';
 export { AsyncOperationSearch, } from './search/AsyncOperationSearch';
 export { CollectionSearch, } from './search/CollectionSearch';

package/dist/index.js

@@ -24,7 +24,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getApiExecutionCumulusIds = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getUniqueCollectionsByGranuleFilter = exports.getCollectionsByGranuleIds = exports.translatePostgresReconReportToApiReconReport = exports.translateApiReconReportToPostgresReconReport = exports.translatePostgresPdrToApiPdr = exports.translateApiPdrToPostgresPdr = exports.translatePostgresGranuleResultToApiGranule = exports.translatePostgresGranuleToApiGranule = exports.translateApiGranuleToPostgresGranuleWithoutNilsRemoved = exports.translateApiGranuleToPostgresGranule = exports.translatePostgresExecutionToApiExecution = exports.translateApiExecutionToPostgresExecutionWithoutNilsRemoved = exports.translateApiExecutionToPostgresExecution = exports.translateApiRuleToPostgresRuleRaw = exports.translateApiRuleToPostgresRule = exports.translatePostgresRuleToApiRule = exports.translatePostgresProviderToApiProvider = exports.translateApiProviderToPostgresProvider = exports.translatePostgresCollectionToApiCollection = exports.translateApiCollectionToPostgresCollection = exports.translatePostgresFileToApiFile = exports.translateApiFiletoPostgresFile = exports.translatePostgresAsyncOperationToApiAsyncOperation = exports.translateApiAsyncOperationToPostgresAsyncOperation = exports.nullifyUndefinedProviderValues = exports.validateProviderHost = exports.migrationDir = exports.TableNames = exports.createRejectableTransaction = exports.localStackConnectionEnv = exports.getKnexConfig = exports.getKnexClient = exports.isCollisionError = exports.generateLocalTestDb = exports.fakeRuleRecordFactory = exports.fakeReconciliationReportRecordFactory = exports.fakeProviderRecordFactory = exports.fakePdrRecordFactory = exports.fakeGranuleRecordFactory = exports.fakeFileRecordFactory = exports.fakeExecutionRecordFactory = exports.fakeCollectionRecordFactory = exports.fakeAsyncOperationRecordFactory = exports.destroyLocalTestDb = exports.deleteTestDatabase = exports.createTestDatabase = exports.Knex = void 0;
-exports.RulePgModel = exports.ReconciliationReportPgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.ReconciliationReportSearch = exports.StatsSearch = exports.RuleSearch = exports.ProviderSearch = exports.PdrSearch = exports.GranuleSearch = exports.ExecutionSearch = exports.CollectionSearch = exports.AsyncOperationSearch = exports.QuerySearchClient = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getFilesAndGranuleInfoQuery = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.getWorkflowNameIntersectFromGranuleIds = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = void 0;
+exports.RulePgModel = exports.ReconciliationReportPgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.ReconciliationReportSearch = exports.StatsSearch = exports.RuleSearch = exports.ProviderSearch = exports.PdrSearch = exports.GranuleSearch = exports.ExecutionSearch = exports.CollectionSearch = exports.AsyncOperationSearch = exports.QuerySearchClient = exports.updateBatchGranulesCollection = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getFilesAndGranuleInfoQuery = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.getWorkflowNameIntersectFromGranuleIds = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = void 0;
 const path = __importStar(require("path"));
 var knex_1 = require("knex");
 Object.defineProperty(exports, "Knex", { enumerable: true, get: function () { return knex_1.Knex; } });
@@ -111,6 +111,7 @@ Object.defineProperty(exports, "upsertGranuleWithExecutionJoinRecord", { enumera
 Object.defineProperty(exports, "getGranulesByApiPropertiesQuery", { enumerable: true, get: function () { return granule_1.getGranulesByApiPropertiesQuery; } });
 Object.defineProperty(exports, "getGranulesByGranuleId", { enumerable: true, get: function () { return granule_1.getGranulesByGranuleId; } });
 Object.defineProperty(exports, "getGranuleAndCollection", { enumerable: true, get: function () { return granule_1.getGranuleAndCollection; } });
+Object.defineProperty(exports, "updateBatchGranulesCollection", { enumerable: true, get: function () { return granule_1.updateBatchGranulesCollection; } });
 var QuerySearchClient_1 = require("./lib/QuerySearchClient");
 Object.defineProperty(exports, "QuerySearchClient", { enumerable: true, get: function () { return QuerySearchClient_1.QuerySearchClient; } });
 var AsyncOperationSearch_1 = require("./search/AsyncOperationSearch");

package/dist/lib/granule.d.ts

@@ -114,4 +114,13 @@ export declare const getGranuleAndCollection: (knexOrTransaction: Knex | Knex.Tr
  * @returns {Promise<PostgresGranuleRecord[]>} The returned list of records
  */
 export declare const getGranulesByGranuleId: (knexOrTransaction: Knex | Knex.Transaction, granuleId: string) => Promise<PostgresGranuleRecord[]>;
+/**
+ * Update a list of granuleIds to a new collection_cumulus_id in postgres
+ *
+ * @param {Knex} knex - DB client or transaction
+ * @param {Array<String>} granuleIds - list of Granule IDs
+ * @param {number} collectionCumulusId - collection_cumulus_id to update to
+ * @returns {Promise<void>}
+ */
+export declare const updateBatchGranulesCollection: (knex: Knex, granuleIds: Array<String>, collectionCumulusId: number) => Promise<void>;
 //# sourceMappingURL=granule.d.ts.map

package/dist/lib/granule.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getGranulesByGranuleId = exports.getGranuleAndCollection = exports.getGranulesByApiPropertiesQuery = exports.getApiGranuleExecutionCumulusIds = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getApiGranuleCumulusIds = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleCollectionId = void 0;
+exports.updateBatchGranulesCollection = exports.getGranulesByGranuleId = exports.getGranuleAndCollection = exports.getGranulesByApiPropertiesQuery = exports.getApiGranuleExecutionCumulusIds = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getApiGranuleCumulusIds = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleCollectionId = void 0;
 const Collections_1 = require("@cumulus/message/Collections");
 const errors_1 = require("@cumulus/errors");
 const logger_1 = __importDefault(require("@cumulus/logger"));
@@ -12,6 +12,7 @@ const granule_1 = require("../models/granule");
 const granules_executions_1 = require("../models/granules-executions");
 const { deprecate } = require('@cumulus/common/util');
 const { TableNames } = require('../tables');
+const log = new logger_1.default({ sender: '@cumulus/db/lib/granules' });
 const getGranuleCollectionId = async (knexOrTransaction, granule) => {
     const collectionPgModel = new collection_1.CollectionPgModel();
     const collection = await collectionPgModel.get(knexOrTransaction, { cumulus_id: granule.collection_cumulus_id });
@@ -248,4 +249,23 @@ const getGranulesByGranuleId = async (knexOrTransaction, granuleId) => {
     return records;
 };
 exports.getGranulesByGranuleId = getGranulesByGranuleId;
+/**
+ * Update a list of granuleIds to a new collection_cumulus_id in postgres
+ *
+ * @param {Knex} knex - DB client or transaction
+ * @param {Array<String>} granuleIds - list of Granule IDs
+ * @param {number} collectionCumulusId - collection_cumulus_id to update to
+ * @returns {Promise<void>}
+ */
+const updateBatchGranulesCollection = async (knex, granuleIds, collectionCumulusId) => {
+    const { granules: granulesTable, } = TableNames;
+    try {
+        await knex(granulesTable).whereIn('granule_id', granuleIds).update({ collection_cumulus_id: collectionCumulusId });
+    }
+    catch (thrownError) {
+        log.error(`Write Granules failed: ${JSON.stringify(thrownError)}`);
+        throw thrownError;
+    }
+};
+exports.updateBatchGranulesCollection = updateBatchGranulesCollection;
 //# sourceMappingURL=granule.js.map

package/dist/migrations/20240125171703_update_granule_execution_cumulus_id_type.js

@@ -6,10 +6,6 @@ const up = async (knex) => {
     await knex.raw('ALTER TABLE executions ALTER COLUMN cumulus_id TYPE BIGINT, ALTER COLUMN parent_cumulus_id TYPE BIGINT');
     await knex.raw('ALTER TABLE granules_executions ALTER COLUMN granule_cumulus_id TYPE BIGINT, ALTER COLUMN execution_cumulus_id TYPE BIGINT');
     await knex.raw('ALTER TABLE pdrs ALTER COLUMN execution_cumulus_id TYPE BIGINT');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) executions');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) files');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) granules_executions');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) pdrs');
 };
 exports.up = up;
 const down = async () => {

package/dist/migrations/20240728101230_add_table_indexes.js

@@ -18,14 +18,6 @@ const up = async (knex) => {
     await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_provider_collection_cumulus_id_name_index ON pdrs(provider_cumulus_id, collection_cumulus_id, name)');
     await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS providers_updated_at_index ON providers(updated_at)');
     await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS rules_updated_at_index ON rules(updated_at)');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) async_operations');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) collections');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) executions');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) files');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) granules');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) pdrs');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) providers');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) rules');
 };
 exports.up = up;
 const down = async (knex) => {

package/dist/search/StatsSearch.js

@@ -24,9 +24,8 @@ const infixMapping = {
 class StatsSearch extends BaseSearch_1.BaseSearch {
     constructor(event, type) {
         const { field, ...queryStringParameters } = event.queryStringParameters || {};
-        super({ queryStringParameters }, type);
+        super({ queryStringParameters: { ...queryStringParameters, limit: null } }, type);
         this.field = field ?? 'status';
-        this.dbQueryParameters = (0, omit_1.default)(this.dbQueryParameters, ['limit', 'offset']);
     }
     /**
      * Formats the postgres records into an API stats/aggregate response

package/dist/search/queries.js

@@ -33,6 +33,13 @@ const regexes = {
     exists: /^(.*)__exists$/,
     range: /^(.*)__(from|to)$/,
 };
+/**
+ * Based on PostgreSQL documentation, when using LIMIT, it is important to use
+ * an ORDER BY clause that constrains the result rows into a unique order.
+ * The following is the default sort column and order for the pagination.
+ */
+const defaultSortColumn = 'cumulus_id';
+const defaultSortOrder = 'asc';
 /**
  * Convert 'exists' query fields to db query parameters from api query string fields
  *
@@ -166,7 +173,7 @@ const convertTerms = (type, queryStringFields) => {
  */
 const convertSort = (type, queryStringParameters) => {
     const sortArray = [];
-    const { sort_by: sortBy, sort_key: sortKey } = queryStringParameters;
+    const { sort_by: sortBy, sort_key: sortKey, limit } = queryStringParameters;
     let { order } = queryStringParameters;
     if (sortBy) {
         order = order ?? 'asc';
@@ -180,6 +187,9 @@ const convertSort = (type, queryStringParameters) => {
             return Object.keys(queryParam ?? {}).map((key) => sortArray.push({ column: key, order }));
         });
     }
+    if (limit !== null) {
+        sortArray.push({ column: defaultSortColumn, order: defaultSortOrder });
+    }
     return sortArray;
 };
 /**

package/dist/test-utils.js

@@ -104,7 +104,7 @@ exports.fakeFileRecordFactory = fakeFileRecordFactory;
 const fakeAsyncOperationRecordFactory = (params) => ({
     id: (0, uuid_1.v4)(),
     description: (0, crypto_random_string_1.default)({ length: 10 }),
-    operation_type: 'ES Index',
+    operation_type: 'Reconciliation Report',
     status: 'RUNNING',
     output: { test: 'output' },
     task_arn: (0, crypto_random_string_1.default)({ length: 3 }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/db",
-  "version": "19.2.0-alpha.1",
+  "version": "20.0.1",
   "description": "Utilities for working with the Cumulus DB",
   "license": "Apache-2.0",
   "main": "./dist/index.js",
@@ -33,12 +33,12 @@
   },
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.621.0",
-    "@cumulus/aws-client": "19.2.0-alpha.1",
-    "@cumulus/common": "19.2.0-alpha.1",
-    "@cumulus/errors": "19.2.0-alpha.1",
-    "@cumulus/logger": "19.2.0-alpha.1",
-    "@cumulus/message": "19.2.0-alpha.1",
-    "@cumulus/types": "19.2.0-alpha.1",
+    "@cumulus/aws-client": "20.0.1",
+    "@cumulus/common": "20.0.1",
+    "@cumulus/errors": "20.0.1",
+    "@cumulus/logger": "20.0.1",
+    "@cumulus/message": "20.0.1",
+    "@cumulus/types": "20.0.1",
     "crypto-random-string": "^3.2.0",
     "is-valid-hostname": "1.0.2",
     "knex": "2.4.1",
@@ -50,5 +50,5 @@
   "devDependencies": {
     "@types/uuid": "^8.0.0"
   },
-  "gitHead": "3b8d4b03abefe6a623245903b48685ebb7ae4ca7"
+  "gitHead": "3bfee176543ede36c4ca902f4af9c6d108454468"
 }