@cumulus/db 19.2.0-alpha.0 → 20.0.0

package/dist/index.d.ts

@@ -30,7 +30,7 @@ export { translateApiReconReportToPostgresReconReport, translatePostgresReconRep
 export { getCollectionsByGranuleIds, getUniqueCollectionsByGranuleFilter, } from './lib/collection';
 export { batchDeleteExecutionFromDatabaseByCumulusCollectionId, executionArnsFromGranuleIdsAndWorkflowNames, getApiExecutionCumulusIds, getApiGranuleExecutionCumulusIdsByExecution, getExecutionInfoByGranuleCumulusId, getWorkflowNameIntersectFromGranuleIds, newestExecutionArnFromGranuleIdWorkflowName, } from './lib/execution';
 export { getFilesAndGranuleInfoQuery, } from './lib/file';
-export { getApiGranuleCumulusIds, getApiGranuleExecutionCumulusIds, getGranuleCollectionId, getUniqueGranuleByGranuleId, getGranuleByUniqueColumns, upsertGranuleWithExecutionJoinRecord, getGranulesByApiPropertiesQuery, getGranulesByGranuleId, getGranuleAndCollection, } from './lib/granule';
+export { getApiGranuleCumulusIds, getApiGranuleExecutionCumulusIds, getGranuleCollectionId, getUniqueGranuleByGranuleId, getGranuleByUniqueColumns, upsertGranuleWithExecutionJoinRecord, getGranulesByApiPropertiesQuery, getGranulesByGranuleId, getGranuleAndCollection, updateBatchGranulesCollection, } from './lib/granule';
 export { QuerySearchClient, } from './lib/QuerySearchClient';
 export { AsyncOperationSearch, } from './search/AsyncOperationSearch';
 export { CollectionSearch, } from './search/CollectionSearch';

package/dist/index.js

@@ -24,7 +24,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getApiExecutionCumulusIds = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getUniqueCollectionsByGranuleFilter = exports.getCollectionsByGranuleIds = exports.translatePostgresReconReportToApiReconReport = exports.translateApiReconReportToPostgresReconReport = exports.translatePostgresPdrToApiPdr = exports.translateApiPdrToPostgresPdr = exports.translatePostgresGranuleResultToApiGranule = exports.translatePostgresGranuleToApiGranule = exports.translateApiGranuleToPostgresGranuleWithoutNilsRemoved = exports.translateApiGranuleToPostgresGranule = exports.translatePostgresExecutionToApiExecution = exports.translateApiExecutionToPostgresExecutionWithoutNilsRemoved = exports.translateApiExecutionToPostgresExecution = exports.translateApiRuleToPostgresRuleRaw = exports.translateApiRuleToPostgresRule = exports.translatePostgresRuleToApiRule = exports.translatePostgresProviderToApiProvider = exports.translateApiProviderToPostgresProvider = exports.translatePostgresCollectionToApiCollection = exports.translateApiCollectionToPostgresCollection = exports.translatePostgresFileToApiFile = exports.translateApiFiletoPostgresFile = exports.translatePostgresAsyncOperationToApiAsyncOperation = exports.translateApiAsyncOperationToPostgresAsyncOperation = exports.nullifyUndefinedProviderValues = exports.validateProviderHost = exports.migrationDir = exports.TableNames = exports.createRejectableTransaction = exports.localStackConnectionEnv = exports.getKnexConfig = exports.getKnexClient = exports.isCollisionError = exports.generateLocalTestDb = exports.fakeRuleRecordFactory = exports.fakeReconciliationReportRecordFactory = exports.fakeProviderRecordFactory = exports.fakePdrRecordFactory = exports.fakeGranuleRecordFactory = exports.fakeFileRecordFactory = exports.fakeExecutionRecordFactory = exports.fakeCollectionRecordFactory = exports.fakeAsyncOperationRecordFactory = exports.destroyLocalTestDb = exports.deleteTestDatabase = exports.createTestDatabase = exports.Knex = void 0;
-exports.RulePgModel = exports.ReconciliationReportPgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.ReconciliationReportSearch = exports.StatsSearch = exports.RuleSearch = exports.ProviderSearch = exports.PdrSearch = exports.GranuleSearch = exports.ExecutionSearch = exports.CollectionSearch = exports.AsyncOperationSearch = exports.QuerySearchClient = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getFilesAndGranuleInfoQuery = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.getWorkflowNameIntersectFromGranuleIds = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = void 0;
+exports.RulePgModel = exports.ReconciliationReportPgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.ReconciliationReportSearch = exports.StatsSearch = exports.RuleSearch = exports.ProviderSearch = exports.PdrSearch = exports.GranuleSearch = exports.ExecutionSearch = exports.CollectionSearch = exports.AsyncOperationSearch = exports.QuerySearchClient = exports.updateBatchGranulesCollection = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getFilesAndGranuleInfoQuery = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.getWorkflowNameIntersectFromGranuleIds = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = void 0;
 const path = __importStar(require("path"));
 var knex_1 = require("knex");
 Object.defineProperty(exports, "Knex", { enumerable: true, get: function () { return knex_1.Knex; } });
@@ -111,6 +111,7 @@ Object.defineProperty(exports, "upsertGranuleWithExecutionJoinRecord", { enumera
 Object.defineProperty(exports, "getGranulesByApiPropertiesQuery", { enumerable: true, get: function () { return granule_1.getGranulesByApiPropertiesQuery; } });
 Object.defineProperty(exports, "getGranulesByGranuleId", { enumerable: true, get: function () { return granule_1.getGranulesByGranuleId; } });
 Object.defineProperty(exports, "getGranuleAndCollection", { enumerable: true, get: function () { return granule_1.getGranuleAndCollection; } });
+Object.defineProperty(exports, "updateBatchGranulesCollection", { enumerable: true, get: function () { return granule_1.updateBatchGranulesCollection; } });
 var QuerySearchClient_1 = require("./lib/QuerySearchClient");
 Object.defineProperty(exports, "QuerySearchClient", { enumerable: true, get: function () { return QuerySearchClient_1.QuerySearchClient; } });
 var AsyncOperationSearch_1 = require("./search/AsyncOperationSearch");

package/dist/lib/execution.d.ts

@@ -6,7 +6,20 @@ export interface ArnRecord {
     arn: string;
 }
 /**
- * Returns execution info sorted by most recent first for an input
+ * Returns execution records sorted by most recent first for an input
+ * set of Granule Cumulus IDs.
+ * @returns Array of arn objects with the most recent first.
+ */
+export declare const getExecutionInfoByGranuleCumulusIds: ({ knexOrTransaction, granuleCumulusIds, limit, }: {
+    knexOrTransaction: Knex | Knex.Transaction;
+    granuleCumulusIds: number[];
+    limit?: number | undefined;
+}) => Promise<{
+    granule_cumulus_id: number;
+    url: string;
+}[]>;
+/**
+ * Returns execution records sorted by most recent first for an input
  * Granule Cumulus ID.
  *
  * @param {Object} params
@@ -24,17 +37,6 @@ export declare const getExecutionInfoByGranuleCumulusId: ({ knexOrTransaction, g
     executionColumns: string[];
     limit?: number | undefined;
 }) => Promise<Partial<PostgresExecutionRecord>[]>;
-/**
- * Returns a list of executionArns sorted by most recent first, for an input
- * Granule Cumulus ID.
- *
- * @param {Knex | Knex.Transaction} knexOrTransaction
- *   Knex client for reading from RDS database
- * @param {number} granuleCumulusId - The primary ID for a Granule
- * @param {number} limit - limit to number of executions to query
- * @returns {Promise<ArnRecord[]>} - Array of arn objects with the most recent first.
- */
-export declare const getExecutionArnsByGranuleCumulusId: (knexOrTransaction: Knex | Knex.Transaction, granuleCumulusId: Number, limit?: number) => Promise<ArnRecord[]>;
 /**
  * Returns a list of executionArns sorted by most recent first, for an input
  * list of granuleIds and workflowNames.

package/dist/lib/execution.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.getExecutionArnsByGranuleCumulusId = exports.getExecutionInfoByGranuleCumulusId = void 0;
+exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.getExecutionInfoByGranuleCumulusId = exports.getExecutionInfoByGranuleCumulusIds = void 0;
 const errors_1 = require("@cumulus/errors");
 const execution_1 = require("../models/execution");
 const granules_executions_1 = require("../models/granules-executions");
@@ -9,54 +9,50 @@ const Logger = require('@cumulus/logger');
 const { getKnexClient } = require('../connection');
 const log = new Logger({ sender: '@cumulus/db/lib/execution' });
 /**
- * Returns execution info sorted by most recent first for an input
- * Granule Cumulus ID.
- *
- * @param {Object} params
- * @param {Knex | Knex.Transaction} params.knexOrTransaction
- *   Knex client for reading from RDS database
- * @param {Array<string>} params.executionColumns - Columns to return from executions table
- * @param {number} params.granuleCumulusId - The primary ID for a Granule
- * @param {number} [params.limit] - limit to number of executions to query
- * @returns {Promise<Partial<PostgresExecutionRecord>[]>}
- *   Array of arn objects with the most recent first.
+ * Returns execution records sorted by most recent first for an input
+ * set of Granule Cumulus IDs.
+ * @returns Array of arn objects with the most recent first.
  */
-const getExecutionInfoByGranuleCumulusId = async ({ knexOrTransaction, granuleCumulusId, executionColumns = ['arn'], limit, }) => {
+const getExecutionInfoByGranuleCumulusIds = async ({ knexOrTransaction, granuleCumulusIds, limit, }) => {
     const knexQuery = knexOrTransaction(tables_1.TableNames.executions)
-        .column(executionColumns.map((column) => `${tables_1.TableNames.executions}.${column}`))
-        .where(`${tables_1.TableNames.granules}.cumulus_id`, granuleCumulusId)
+        .column([
+        `${tables_1.TableNames.executions}.url`,
+        `${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`,
+    ])
+        .whereIn(`${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusIds)
         .join(tables_1.TableNames.granulesExecutions, `${tables_1.TableNames.executions}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.execution_cumulus_id`)
-        .join(tables_1.TableNames.granules, `${tables_1.TableNames.granules}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`)
         .orderBy(`${tables_1.TableNames.executions}.timestamp`, 'desc');
     if (limit) {
         knexQuery.limit(limit);
     }
     return await knexQuery;
 };
-exports.getExecutionInfoByGranuleCumulusId = getExecutionInfoByGranuleCumulusId;
+exports.getExecutionInfoByGranuleCumulusIds = getExecutionInfoByGranuleCumulusIds;
 /**
- * Returns a list of executionArns sorted by most recent first, for an input
+ * Returns execution records sorted by most recent first for an input
  * Granule Cumulus ID.
  *
- * @param {Knex | Knex.Transaction} knexOrTransaction
+ * @param {Object} params
+ * @param {Knex | Knex.Transaction} params.knexOrTransaction
  *   Knex client for reading from RDS database
- * @param {number} granuleCumulusId - The primary ID for a Granule
- * @param {number} limit - limit to number of executions to query
- * @returns {Promise<ArnRecord[]>} - Array of arn objects with the most recent first.
+ * @param {Array<string>} params.executionColumns - Columns to return from executions table
+ * @param {number} params.granuleCumulusId - The primary ID for a Granule
+ * @param {number} [params.limit] - limit to number of executions to query
+ * @returns {Promise<Partial<PostgresExecutionRecord>[]>}
+ *   Array of arn objects with the most recent first.
  */
-const getExecutionArnsByGranuleCumulusId = async (knexOrTransaction, granuleCumulusId, limit) => {
+const getExecutionInfoByGranuleCumulusId = async ({ knexOrTransaction, granuleCumulusId, executionColumns = ['arn'], limit, }) => {
     const knexQuery = knexOrTransaction(tables_1.TableNames.executions)
-        .select(`${tables_1.TableNames.executions}.arn`)
-        .where(`${tables_1.TableNames.granules}.cumulus_id`, granuleCumulusId)
+        .column(executionColumns.map((column) => `${tables_1.TableNames.executions}.${column}`))
+        .where(`${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusId)
         .join(tables_1.TableNames.granulesExecutions, `${tables_1.TableNames.executions}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.execution_cumulus_id`)
-        .join(tables_1.TableNames.granules, `${tables_1.TableNames.granules}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`)
         .orderBy(`${tables_1.TableNames.executions}.timestamp`, 'desc');
     if (limit) {
         knexQuery.limit(limit);
     }
     return await knexQuery;
 };
-exports.getExecutionArnsByGranuleCumulusId = getExecutionArnsByGranuleCumulusId;
+exports.getExecutionInfoByGranuleCumulusId = getExecutionInfoByGranuleCumulusId;
 /**
  * Returns a list of executionArns sorted by most recent first, for an input
  * list of granuleIds and workflowNames.

package/dist/lib/granule.d.ts

@@ -114,4 +114,13 @@ export declare const getGranuleAndCollection: (knexOrTransaction: Knex | Knex.Tr
  * @returns {Promise<PostgresGranuleRecord[]>} The returned list of records
  */
 export declare const getGranulesByGranuleId: (knexOrTransaction: Knex | Knex.Transaction, granuleId: string) => Promise<PostgresGranuleRecord[]>;
+/**
+ * Update a list of granuleIds to a new collection_cumulus_id in postgres
+ *
+ * @param {Knex} knex - DB client or transaction
+ * @param {Array<String>} granuleIds - list of Granule IDs
+ * @param {number} collectionCumulusId - collection_cumulus_id to update to
+ * @returns {Promise<void>}
+ */
+export declare const updateBatchGranulesCollection: (knex: Knex, granuleIds: Array<String>, collectionCumulusId: number) => Promise<void>;
 //# sourceMappingURL=granule.d.ts.map

package/dist/lib/granule.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getGranulesByGranuleId = exports.getGranuleAndCollection = exports.getGranulesByApiPropertiesQuery = exports.getApiGranuleExecutionCumulusIds = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getApiGranuleCumulusIds = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleCollectionId = void 0;
+exports.updateBatchGranulesCollection = exports.getGranulesByGranuleId = exports.getGranuleAndCollection = exports.getGranulesByApiPropertiesQuery = exports.getApiGranuleExecutionCumulusIds = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getApiGranuleCumulusIds = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleCollectionId = void 0;
 const Collections_1 = require("@cumulus/message/Collections");
 const errors_1 = require("@cumulus/errors");
 const logger_1 = __importDefault(require("@cumulus/logger"));
@@ -12,6 +12,7 @@ const granule_1 = require("../models/granule");
 const granules_executions_1 = require("../models/granules-executions");
 const { deprecate } = require('@cumulus/common/util');
 const { TableNames } = require('../tables');
+const log = new logger_1.default({ sender: '@cumulus/db/lib/granules' });
 const getGranuleCollectionId = async (knexOrTransaction, granule) => {
     const collectionPgModel = new collection_1.CollectionPgModel();
     const collection = await collectionPgModel.get(knexOrTransaction, { cumulus_id: granule.collection_cumulus_id });
@@ -248,4 +249,23 @@ const getGranulesByGranuleId = async (knexOrTransaction, granuleId) => {
     return records;
 };
 exports.getGranulesByGranuleId = getGranulesByGranuleId;
+/**
+ * Update a list of granuleIds to a new collection_cumulus_id in postgres
+ *
+ * @param {Knex} knex - DB client or transaction
+ * @param {Array<String>} granuleIds - list of Granule IDs
+ * @param {number} collectionCumulusId - collection_cumulus_id to update to
+ * @returns {Promise<void>}
+ */
+const updateBatchGranulesCollection = async (knex, granuleIds, collectionCumulusId) => {
+    const { granules: granulesTable, } = TableNames;
+    try {
+        await knex(granulesTable).whereIn('granule_id', granuleIds).update({ collection_cumulus_id: collectionCumulusId });
+    }
+    catch (thrownError) {
+        log.error(`Write Granules failed: ${JSON.stringify(thrownError)}`);
+        throw thrownError;
+    }
+};
+exports.updateBatchGranulesCollection = updateBatchGranulesCollection;
 //# sourceMappingURL=granule.js.map

package/dist/migrations/20240125171703_update_granule_execution_cumulus_id_type.js

@@ -6,10 +6,6 @@ const up = async (knex) => {
     await knex.raw('ALTER TABLE executions ALTER COLUMN cumulus_id TYPE BIGINT, ALTER COLUMN parent_cumulus_id TYPE BIGINT');
     await knex.raw('ALTER TABLE granules_executions ALTER COLUMN granule_cumulus_id TYPE BIGINT, ALTER COLUMN execution_cumulus_id TYPE BIGINT');
     await knex.raw('ALTER TABLE pdrs ALTER COLUMN execution_cumulus_id TYPE BIGINT');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) executions');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) files');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) granules_executions');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) pdrs');
 };
 exports.up = up;
 const down = async () => {

package/dist/migrations/20240728101230_add_table_indexes.js

@@ -18,14 +18,6 @@ const up = async (knex) => {
     await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_provider_collection_cumulus_id_name_index ON pdrs(provider_cumulus_id, collection_cumulus_id, name)');
     await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS providers_updated_at_index ON providers(updated_at)');
     await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS rules_updated_at_index ON rules(updated_at)');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) async_operations');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) collections');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) executions');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) files');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) granules');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) pdrs');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) providers');
-    await knex.raw('VACUUM (ANALYZE, VERBOSE) rules');
 };
 exports.up = up;
 const down = async (knex) => {

package/dist/models/file.d.ts

@@ -12,6 +12,10 @@ declare class FilePgModel extends BasePgModel<PostgresFile, PostgresFileRecord>
         _intersectProps: {};
         _unionProps: never;
     }[]>;
+    /**
+     * Retrieves all files for all granules given
+    */
+    searchByGranuleCumulusIds(knexOrTrx: Knex | Knex.Transaction, granule_cumulus_ids: number[], columns?: string | string[]): Promise<PostgresFileRecord[]>;
 }
 export { FilePgModel };
 //# sourceMappingURL=file.d.ts.map

package/dist/models/file.js

@@ -16,6 +16,14 @@ class FilePgModel extends base_1.BasePgModel {
             .merge()
             .returning('*');
     }
+    /**
+     * Retrieves all files for all granules given
+    */
+    searchByGranuleCumulusIds(knexOrTrx, granule_cumulus_ids, columns = '*') {
+        return knexOrTrx(this.tableName)
+            .select(columns)
+            .whereIn('granule_cumulus_id', granule_cumulus_ids);
+    }
 }
 exports.FilePgModel = FilePgModel;
 //# sourceMappingURL=file.js.map

package/dist/search/CollectionSearch.d.ts

@@ -34,20 +34,23 @@ export declare class CollectionSearch extends BaseSearch {
         dbQueryParameters?: DbQueryParameters;
     }): void;
     /**
-     * Build queries for range fields
+     * Build subquery for active collections
+     * The subquery will search granules
      *
-     * @param params
-     * @param params.knex - db client
-     * @param [params.countQuery] - query builder for getting count
-     * @param params.searchQuery - query builder for search
-     * @param [params.dbQueryParameters] - db query parameters
+     * @param knex - db client
+     * @returns granule query
      */
-    protected buildRangeQuery(params: {
-        knex: Knex;
-        countQuery: Knex.QueryBuilder;
+    private buildSubQueryForActiveCollections;
+    /**
+     * Build the search query
+     *
+     * @param knex - DB client
+     * @returns queries for getting count and search result
+     */
+    protected buildSearch(knex: Knex): {
+        countQuery?: Knex.QueryBuilder;
         searchQuery: Knex.QueryBuilder;
-        dbQueryParameters?: DbQueryParameters;
-    }): void;
+    };
     /**
      * Executes stats query to get granules' status aggregation
      *

package/dist/search/CollectionSearch.js

@@ -4,12 +4,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CollectionSearch = void 0;
+const omitBy_1 = __importDefault(require("lodash/omitBy"));
 const pick_1 = __importDefault(require("lodash/pick"));
 const logger_1 = __importDefault(require("@cumulus/logger"));
 const BaseSearch_1 = require("./BaseSearch");
+const queries_1 = require("./queries");
+const GranuleSearch_1 = require("./GranuleSearch");
 const collections_1 = require("../translate/collections");
 const tables_1 = require("../tables");
 const log = new logger_1.default({ sender: '@cumulus/db/CollectionSearch' });
+const granuleFields = ['createdAt', 'granuleId', 'timestamp', 'updatedAt'];
+const isGranuleField = (_value, key) => granuleFields.includes(key.split('__')[0]);
 /**
  * Class to build and execute db search query for collections
  */
@@ -19,6 +24,10 @@ class CollectionSearch extends BaseSearch_1.BaseSearch {
         super({ queryStringParameters }, 'collection');
         this.active = (active === 'true');
         this.includeStats = (includeStats === 'true');
+        // for active collection search, omit the fields which are for searching granules
+        if (this.active) {
+            this.dbQueryParameters = (0, queries_1.convertQueryStringToDbQueryParameters)(this.type, (0, omitBy_1.default)(this.queryStringParameters, isGranuleField));
+        }
     }
     /**
      * Build queries for infix and prefix
@@ -39,34 +48,39 @@ class CollectionSearch extends BaseSearch_1.BaseSearch {
         }
     }
     /**
-     * Build queries for range fields
+     * Build subquery for active collections
+     * The subquery will search granules
      *
-     * @param params
-     * @param params.knex - db client
-     * @param [params.countQuery] - query builder for getting count
-     * @param params.searchQuery - query builder for search
-     * @param [params.dbQueryParameters] - db query parameters
+     * @param knex - db client
+     * @returns granule query
+     */
+    buildSubQueryForActiveCollections(knex) {
+        const granulesTable = tables_1.TableNames.granules;
+        const granuleSearch = new GranuleSearch_1.GranuleSearch({ queryStringParameters: this.queryStringParameters });
+        const { countQuery: subQuery } = granuleSearch.buildSearchForActiveCollections(knex);
+        subQuery
+            .clear('select')
+            .select(1)
+            .where(`${granulesTable}.collection_cumulus_id`, knex.raw(`${this.tableName}.cumulus_id`))
+            .limit(1);
+        return subQuery;
+    }
+    /**
+     * Build the search query
+     *
+     * @param knex - DB client
+     * @returns queries for getting count and search result
      */
-    buildRangeQuery(params) {
+    buildSearch(knex) {
+        const queries = super.buildSearch(knex);
         if (!this.active) {
-            super.buildRangeQuery(params);
-            return;
+            return queries;
         }
-        const granulesTable = tables_1.TableNames.granules;
-        const { knex, countQuery, searchQuery, dbQueryParameters } = params;
-        const { range = {} } = dbQueryParameters ?? this.dbQueryParameters;
-        const subQuery = knex.select(1).from(granulesTable)
-            .where(`${granulesTable}.collection_cumulus_id`, knex.raw(`${this.tableName}.cumulus_id`));
-        Object.entries(range).forEach(([name, rangeValues]) => {
-            if (rangeValues.gte) {
-                subQuery.where(`${granulesTable}.${name}`, '>=', rangeValues.gte);
-            }
-            if (rangeValues.lte) {
-                subQuery.where(`${granulesTable}.${name}`, '<=', rangeValues.lte);
-            }
-        });
-        subQuery.limit(1);
-        [countQuery, searchQuery].forEach((query) => query.whereExists(subQuery));
+        const subQuery = this.buildSubQueryForActiveCollections(knex);
+        const { countQuery, searchQuery } = queries;
+        [countQuery, searchQuery].forEach((query) => query?.whereExists(subQuery));
+        log.debug(`buildSearch returns countQuery: ${countQuery?.toSQL().sql}, searchQuery: ${searchQuery.toSQL().sql}`);
+        return { countQuery, searchQuery };
     }
     /**
      * Executes stats query to get granules' status aggregation
@@ -77,21 +91,19 @@ class CollectionSearch extends BaseSearch_1.BaseSearch {
      */
     async retrieveGranuleStats(collectionCumulusIds, knex) {
         const granulesTable = tables_1.TableNames.granules;
-        const statsQuery = knex(granulesTable)
+        let statsQuery = knex(granulesTable);
+        if (this.active) {
+            const granuleSearch = new GranuleSearch_1.GranuleSearch({
+                queryStringParameters: this.queryStringParameters,
+            });
+            const { countQuery } = granuleSearch.buildSearchForActiveCollections(knex);
+            statsQuery = countQuery.clear('select');
+        }
+        statsQuery
             .select(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`)
             .count('*')
             .groupBy(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`)
             .whereIn(`${granulesTable}.collection_cumulus_id`, collectionCumulusIds);
-        if (this.active) {
-            Object.entries(this.dbQueryParameters?.range ?? {}).forEach(([name, rangeValues]) => {
-                if (rangeValues.gte) {
-                    statsQuery.where(`${granulesTable}.${name}`, '>=', rangeValues.gte);
-                }
-                if (rangeValues.lte) {
-                    statsQuery.where(`${granulesTable}.${name}`, '<=', rangeValues.lte);
-                }
-            });
-        }
         log.debug(`retrieveGranuleStats statsQuery: ${statsQuery?.toSQL().sql}`);
         const results = await statsQuery;
         const reduced = results.reduce((acc, record) => {

package/dist/search/ExecutionSearch.d.ts

@@ -16,15 +16,15 @@ interface ExecutionRecord extends BaseRecord, PostgresExecutionRecord {
 export declare class ExecutionSearch extends BaseSearch {
     constructor(event: QueryEvent);
     /**
-     * check if joined async_ops table search is needed
+     * check if joined async_operations table search is needed
      *
-     * @returns whether collection search is needed
+     * @returns whether async_operations search is needed
      */
     protected searchAsync(): boolean;
     /**
-     * check if joined async_ops table search is needed
+     * check if joined parent execution table search is needed
      *
-     * @returns whether collection search is needed
+     * @returns whether parent execution search is needed
      */
     protected searchParent(): boolean;
     /**

package/dist/search/ExecutionSearch.js

@@ -24,18 +24,18 @@ class ExecutionSearch extends BaseSearch_1.BaseSearch {
         super(event, 'execution');
     }
     /**
-     * check if joined async_ops table search is needed
+     * check if joined async_operations table search is needed
      *
-     * @returns whether collection search is needed
+     * @returns whether async_operations search is needed
      */
     searchAsync() {
         const { not, term, terms } = this.dbQueryParameters;
         return (!!(not?.asyncOperationId || term?.asyncOperationId || terms?.asyncOperationId));
     }
     /**
-     * check if joined async_ops table search is needed
+     * check if joined parent execution table search is needed
      *
-     * @returns whether collection search is needed
+     * @returns whether parent execution search is needed
      */
     searchParent() {
         const { not, term, terms } = this.dbQueryParameters;

package/dist/search/GranuleSearch.d.ts

@@ -38,13 +38,26 @@ export declare class GranuleSearch extends BaseSearch {
         searchQuery: Knex.QueryBuilder;
         dbQueryParameters?: DbQueryParameters;
     }): void;
+    /**
+     * Build the search query for active collections.
+     * If time params are specified the query will search granules that have been updated
+     * in that time frame.  If granuleId or providerId are provided, it will filter those as well.
+     *
+     * @param knex - DB client
+     * @returns queries for getting count and search result
+     */
+    buildSearchForActiveCollections(knex: Knex): {
+        countQuery: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+    };
     /**
      * Translate postgres records to api records
      *
      * @param pgRecords - postgres records returned from query
+     * @param knex - DB client
      * @returns translated api records
      */
-    protected translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[]): Partial<ApiGranuleRecord>[];
+    protected translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[], knex: Knex): Promise<Partial<ApiGranuleRecord>[]>;
 }
 export {};
 //# sourceMappingURL=GranuleSearch.d.ts.map

package/dist/search/GranuleSearch.js

@@ -10,6 +10,8 @@ const logger_1 = __importDefault(require("@cumulus/logger"));
 const BaseSearch_1 = require("./BaseSearch");
 const granules_1 = require("../translate/granules");
 const tables_1 = require("../tables");
+const file_1 = require("../models/file");
+const execution_1 = require("../lib/execution");
 const log = new logger_1.default({ sender: '@cumulus/db/GranuleSearch' });
 /**
  * Class to build and execute db search query for granules
@@ -78,15 +80,56 @@ class GranuleSearch extends BaseSearch_1.BaseSearch {
             [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.granule_id`, `${prefix}%`));
         }
     }
+    /**
+     * Build the search query for active collections.
+     * If time params are specified the query will search granules that have been updated
+     * in that time frame.  If granuleId or providerId are provided, it will filter those as well.
+     *
+     * @param knex - DB client
+     * @returns queries for getting count and search result
+     */
+    buildSearchForActiveCollections(knex) {
+        const { countQuery, searchQuery } = this.buildBasicQuery(knex);
+        this.buildTermQuery({ countQuery, searchQuery });
+        this.buildTermsQuery({ countQuery, searchQuery });
+        this.buildRangeQuery({ knex, countQuery, searchQuery });
+        log.debug(`buildSearchForActiveCollections returns countQuery: ${countQuery?.toSQL().sql}, searchQuery: ${searchQuery.toSQL().sql}`);
+        return { countQuery, searchQuery };
+    }
     /**
      * Translate postgres records to api records
      *
      * @param pgRecords - postgres records returned from query
+     * @param knex - DB client
      * @returns translated api records
      */
-    translatePostgresRecordsToApiRecords(pgRecords) {
+    async translatePostgresRecordsToApiRecords(pgRecords, knex) {
         log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `);
-        const { fields } = this.dbQueryParameters;
+        const { fields, includeFullRecord } = this.dbQueryParameters;
+        const fileMapping = {};
+        const executionMapping = {};
+        const cumulusIds = pgRecords.map((record) => record.cumulus_id);
+        if (includeFullRecord) {
+            //get Files
+            const fileModel = new file_1.FilePgModel();
+            const files = await fileModel.searchByGranuleCumulusIds(knex, cumulusIds);
+            files.forEach((file) => {
+                if (!(file.granule_cumulus_id in fileMapping)) {
+                    fileMapping[file.granule_cumulus_id] = [];
+                }
+                fileMapping[file.granule_cumulus_id].push(file);
+            });
+            //get Executions
+            const executions = await (0, execution_1.getExecutionInfoByGranuleCumulusIds)({
+                knexOrTransaction: knex,
+                granuleCumulusIds: cumulusIds,
+            });
+            executions.forEach((execution) => {
+                if (!(execution.granule_cumulus_id in executionMapping)) {
+                    executionMapping[execution.granule_cumulus_id] = execution;
+                }
+            });
+        }
         const apiRecords = pgRecords.map((item) => {
             const granulePgRecord = item;
             const collectionPgRecord = {
@@ -94,10 +137,19 @@ class GranuleSearch extends BaseSearch_1.BaseSearch {
                 name: item.collectionName,
                 version: item.collectionVersion,
             };
+            const executionUrls = executionMapping[item.cumulus_id]?.url
+                ? [{ url: executionMapping[item.cumulus_id].url }]
+                : [];
             const pdr = item.pdrName ? { name: item.pdrName } : undefined;
             const providerPgRecord = item.providerName ? { name: item.providerName } : undefined;
+            const fileRecords = fileMapping[granulePgRecord.cumulus_id] || [];
             const apiRecord = (0, granules_1.translatePostgresGranuleToApiGranuleWithoutDbQuery)({
-                granulePgRecord, collectionPgRecord, pdr, providerPgRecord,
+                granulePgRecord,
+                collectionPgRecord,
+                pdr,
+                providerPgRecord,
+                files: fileRecords,
+                executionUrls,
             });
             return fields ? (0, pick_1.default)(apiRecord, fields) : apiRecord;
         });

package/dist/search/StatsSearch.d.ts

@@ -93,7 +93,6 @@ declare class StatsSearch extends BaseSearch {
      * @param params
      * @param params.searchQuery - the search query
      * @param [params.dbQueryParameters] - the db query parameters
-     * @returns the updated search query based on queryStringParams
      */
     protected buildTermQuery(params: {
         searchQuery: Knex.QueryBuilder;

package/dist/search/StatsSearch.js

@@ -180,7 +180,6 @@ class StatsSearch extends BaseSearch_1.BaseSearch {
      * @param params
      * @param params.searchQuery - the search query
      * @param [params.dbQueryParameters] - the db query parameters
-     * @returns the updated search query based on queryStringParams
      */
     buildTermQuery(params) {
         const { dbQueryParameters, searchQuery } = params;
@@ -188,7 +187,7 @@ class StatsSearch extends BaseSearch_1.BaseSearch {
         if (this.field?.includes('error.Error')) {
             searchQuery.whereRaw(`${this.tableName}.error ->> 'Error' is not null`);
         }
-        return super.buildTermQuery({
+        super.buildTermQuery({
             ...params,
             dbQueryParameters: { term: (0, omit_1.default)(term, 'error.Error') },
         });

package/dist/test-utils.js

@@ -104,7 +104,7 @@ exports.fakeFileRecordFactory = fakeFileRecordFactory;
 const fakeAsyncOperationRecordFactory = (params) => ({
     id: (0, uuid_1.v4)(),
     description: (0, crypto_random_string_1.default)({ length: 10 }),
-    operation_type: 'ES Index',
+    operation_type: 'Reconciliation Report',
     status: 'RUNNING',
     output: { test: 'output' },
     task_arn: (0, crypto_random_string_1.default)({ length: 3 }),

package/dist/translate/async_operations.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.translatePostgresAsyncOperationToApiAsyncOperation = exports.translateApiAsyncOperationToPostgresAsyncOperation = void 0;
+const omit_1 = __importDefault(require("lodash/omit"));
 const snake_camel_1 = require("snake-camel");
 const logger_1 = __importDefault(require("@cumulus/logger"));
 const log = new logger_1.default({ sender: '@cumulus/db/translate/async-operations' });
@@ -38,7 +39,7 @@ const convertOutputToObject = (output) => {
  */
 const translateApiAsyncOperationToPostgresAsyncOperation = (record) => {
     // fix for old implementation of async-operation output assignment
-    const translatedRecord = (0, snake_camel_1.toSnake)(record);
+    const translatedRecord = (0, snake_camel_1.toSnake)((0, omit_1.default)(record, 'timestamp'));
     if (record.output === 'none') {
         delete translatedRecord.output;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/db",
-  "version": "19.2.0-alpha.0",
+  "version": "20.0.0",
   "description": "Utilities for working with the Cumulus DB",
   "license": "Apache-2.0",
   "main": "./dist/index.js",
@@ -33,12 +33,12 @@
   },
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.621.0",
-    "@cumulus/aws-client": "19.2.0-alpha.0",
-    "@cumulus/common": "19.2.0-alpha.0",
-    "@cumulus/errors": "19.2.0-alpha.0",
-    "@cumulus/logger": "19.2.0-alpha.0",
-    "@cumulus/message": "19.2.0-alpha.0",
-    "@cumulus/types": "19.2.0-alpha.0",
+    "@cumulus/aws-client": "20.0.0",
+    "@cumulus/common": "20.0.0",
+    "@cumulus/errors": "20.0.0",
+    "@cumulus/logger": "20.0.0",
+    "@cumulus/message": "20.0.0",
+    "@cumulus/types": "20.0.0",
     "crypto-random-string": "^3.2.0",
     "is-valid-hostname": "1.0.2",
     "knex": "2.4.1",
@@ -50,5 +50,5 @@
   "devDependencies": {
     "@types/uuid": "^8.0.0"
   },
-  "gitHead": "533280b62564b6ce89fabcde5cbe653afb964205"
+  "gitHead": "d43b620da63c0dc7076b9e619ea6d9b92204fcaa"
 }