@cumulus/db 18.5.1 → 19.1.0

package/dist/index.d.ts

@@ -30,6 +30,11 @@ export { batchDeleteExecutionFromDatabaseByCumulusCollectionId, executionArnsFro
 export { getFilesAndGranuleInfoQuery, } from './lib/file';
 export { getApiGranuleCumulusIds, getApiGranuleExecutionCumulusIds, getGranuleCollectionId, getUniqueGranuleByGranuleId, getGranuleByUniqueColumns, upsertGranuleWithExecutionJoinRecord, getGranulesByApiPropertiesQuery, getGranulesByGranuleId, getGranuleAndCollection, } from './lib/granule';
 export { QuerySearchClient, } from './lib/QuerySearchClient';
+export { BaseSearch, } from './search/BaseSearch';
+export { ExecutionSearch, } from './search/ExecutionSearch';
+export { GranuleSearch, } from './search/GranuleSearch';
+export { StatsSearch, } from './search/StatsSearch';
+export { CollectionSearch, } from './search/CollectionSearch';
 export { AsyncOperationPgModel } from './models/async_operation';
 export { BasePgModel } from './models/base';
 export { CollectionPgModel } from './models/collection';

package/dist/index.js

@@ -24,7 +24,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.newestExecutionArnFromGranuleIdWorkflowName = exports.getWorkflowNameIntersectFromGranuleIds = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getCollectionsByGranuleIds = exports.translatePostgresPdrToApiPdr = exports.translateApiPdrToPostgresPdr = exports.translatePostgresGranuleResultToApiGranule = exports.translatePostgresGranuleToApiGranule = exports.translateApiGranuleToPostgresGranuleWithoutNilsRemoved = exports.translateApiGranuleToPostgresGranule = exports.translatePostgresExecutionToApiExecution = exports.translateApiExecutionToPostgresExecutionWithoutNilsRemoved = exports.translateApiExecutionToPostgresExecution = exports.translateApiRuleToPostgresRuleRaw = exports.translateApiRuleToPostgresRule = exports.translatePostgresRuleToApiRule = exports.translatePostgresProviderToApiProvider = exports.translateApiProviderToPostgresProvider = exports.translatePostgresCollectionToApiCollection = exports.translateApiCollectionToPostgresCollection = exports.translatePostgresFileToApiFile = exports.translateApiFiletoPostgresFile = exports.translatePostgresAsyncOperationToApiAsyncOperation = exports.translateApiAsyncOperationToPostgresAsyncOperation = exports.nullifyUndefinedProviderValues = exports.validateProviderHost = exports.migrationDir = exports.TableNames = exports.createRejectableTransaction = exports.localStackConnectionEnv = exports.getKnexConfig = exports.getKnexClient = exports.isCollisionError = exports.generateLocalTestDb = exports.fakeRuleRecordFactory = exports.fakeProviderRecordFactory = exports.fakePdrRecordFactory = exports.fakeGranuleRecordFactory = exports.fakeFileRecordFactory = exports.fakeExecutionRecordFactory = exports.fakeCollectionRecordFactory = exports.fakeAsyncOperationRecordFactory = exports.destroyLocalTestDb = exports.deleteTestDatabase = exports.createTestDatabase = exports.Knex = void 0;
-exports.RulePgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.QuerySearchClient = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getFilesAndGranuleInfoQuery = void 0;
+exports.RulePgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.CollectionSearch = exports.StatsSearch = exports.GranuleSearch = exports.ExecutionSearch = exports.BaseSearch = exports.QuerySearchClient = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getFilesAndGranuleInfoQuery = void 0;
 const path = __importStar(require("path"));
 var knex_1 = require("knex");
 Object.defineProperty(exports, "Knex", { enumerable: true, get: function () { return knex_1.Knex; } });
@@ -108,6 +108,16 @@ Object.defineProperty(exports, "getGranulesByGranuleId", { enumerable: true, get
 Object.defineProperty(exports, "getGranuleAndCollection", { enumerable: true, get: function () { return granule_1.getGranuleAndCollection; } });
 var QuerySearchClient_1 = require("./lib/QuerySearchClient");
 Object.defineProperty(exports, "QuerySearchClient", { enumerable: true, get: function () { return QuerySearchClient_1.QuerySearchClient; } });
+var BaseSearch_1 = require("./search/BaseSearch");
+Object.defineProperty(exports, "BaseSearch", { enumerable: true, get: function () { return BaseSearch_1.BaseSearch; } });
+var ExecutionSearch_1 = require("./search/ExecutionSearch");
+Object.defineProperty(exports, "ExecutionSearch", { enumerable: true, get: function () { return ExecutionSearch_1.ExecutionSearch; } });
+var GranuleSearch_1 = require("./search/GranuleSearch");
+Object.defineProperty(exports, "GranuleSearch", { enumerable: true, get: function () { return GranuleSearch_1.GranuleSearch; } });
+var StatsSearch_1 = require("./search/StatsSearch");
+Object.defineProperty(exports, "StatsSearch", { enumerable: true, get: function () { return StatsSearch_1.StatsSearch; } });
+var CollectionSearch_1 = require("./search/CollectionSearch");
+Object.defineProperty(exports, "CollectionSearch", { enumerable: true, get: function () { return CollectionSearch_1.CollectionSearch; } });
 var async_operation_1 = require("./models/async_operation");
 Object.defineProperty(exports, "AsyncOperationPgModel", { enumerable: true, get: function () { return async_operation_1.AsyncOperationPgModel; } });
 var base_1 = require("./models/base");

package/dist/lib/execution.js

@@ -115,20 +115,27 @@ const getWorkflowNameIntersectFromGranuleIds = async (knexOrTransaction, granule
     const numberOfGranules = granuleCumulusIdsArray.length;
     const { executions: executionsTable, granulesExecutions: granulesExecutionsTable } = tables_1.TableNames;
     const aggregatedWorkflowCounts = await knexOrTransaction(executionsTable)
-        .select('workflow_name')
-        .countDistinct('granule_cumulus_id')
+        .select(['workflow_name'])
         .innerJoin(granulesExecutionsTable, `${executionsTable}.cumulus_id`, `${granulesExecutionsTable}.execution_cumulus_id`)
         .whereIn('granule_cumulus_id', granuleCumulusIdsArray)
         .groupBy('workflow_name')
+        .countDistinct('granule_cumulus_id')
         .havingRaw('count(distinct granule_cumulus_id) = ?', [numberOfGranules])
         .modify((queryBuilder) => {
         if (numberOfGranules === 1) {
-            queryBuilder.groupBy('timestamp')
-                .orderBy('timestamp', 'desc');
+            queryBuilder.min('timestamp');
         }
     });
-    return aggregatedWorkflowCounts
-        .map((workflowCounts) => workflowCounts.workflow_name);
+    /*
+    sort (and group by) in knex causes an edge case where two distinct workflows
+    of the same name will be returned if they have different timestamps. This means
+    different returns depending on whether you have asked for one or multiple granules
+    hence this sort has been moved to js logic
+    */
+    if (numberOfGranules === 1) {
+        aggregatedWorkflowCounts.sort((a, b) => b.min - a.min);
+    }
+    return aggregatedWorkflowCounts.map((workflowCounts) => workflowCounts.workflow_name);
 };
 exports.getWorkflowNameIntersectFromGranuleIds = getWorkflowNameIntersectFromGranuleIds;
 /**

package/dist/migrations/20240728101230_add_table_indexes.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20240728101230_add_table_indexes.d.ts.map

package/dist/migrations/20240728101230_add_table_indexes.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS async_operations_updated_at_index ON async_operations(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS async_operations_status_operation_type_cumulus_id_index ON async_operations(status, operation_type, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS collections_updated_at_index ON collections(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS executions_updated_at_index ON executions(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS executions_status_collection_cumulus_id_index ON executions(status, collection_cumulus_id, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS files_updated_at_index ON files(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_updated_at_index ON granules(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_coll_status_processendtime_cumulus_id_index ON granules(collection_cumulus_id, status, processing_end_date_time, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_status_provider_collection_cumulus_id_index ON granules(status, provider_cumulus_id, collection_cumulus_id, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_updated_at_index ON pdrs(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_status_provider_collection_cumulus_id_index ON pdrs(status, provider_cumulus_id, collection_cumulus_id, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_execution_cumulus_id_index ON pdrs(execution_cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_coll_status_cumulus_id_index ON pdrs(collection_cumulus_id, status, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_provider_collection_cumulus_id_name_index ON pdrs(provider_cumulus_id, collection_cumulus_id, name)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS providers_updated_at_index ON providers(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS rules_updated_at_index ON rules(updated_at)');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) async_operations');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) collections');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) executions');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) files');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) granules');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) pdrs');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) providers');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) rules');
+};
+exports.up = up;
+const down = async (knex) => {
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS async_operations_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS async_operations_status_operation_type_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS collections_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS executions_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS executions_status_collection_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS files_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS granules_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS granules_coll_status_processendtime_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS granules_status_provider_collection_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_status_provider_collection_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_execution_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_coll_status_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_provider_collection_cumulus_id_name_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS providers_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS rules_updated_at_index');
+};
+exports.down = down;
+exports.config = {
+    transaction: false,
+};
+//# sourceMappingURL=20240728101230_add_table_indexes.js.map

package/dist/models/execution.d.ts

@@ -16,12 +16,12 @@ declare class ExecutionPgModel extends BasePgModel<PostgresExecution, PostgresEx
      * @param {Object} [params] - Optional object with addition params for query
      * @param {number} [params.limit] - number of records to be returned
      * @param {number} [params.offset] - record offset
-     * @returns {Promise<Array<number>>} An array of executions
+     * @returns An array of executions
      */
     searchByCumulusIds(knexOrTrx: Knex | Knex.Transaction, executionCumulusIds: Array<number> | number, params: {
         limit: number;
         offset: number;
-    }): Promise<Array<number>>;
+    }): Promise<Array<PostgresExecutionRecord>>;
 }
 export { ExecutionPgModel };
 //# sourceMappingURL=execution.d.ts.map

package/dist/models/execution.js

@@ -42,7 +42,7 @@ class ExecutionPgModel extends base_1.BasePgModel {
      * @param {Object} [params] - Optional object with addition params for query
      * @param {number} [params.limit] - number of records to be returned
      * @param {number} [params.offset] - record offset
-     * @returns {Promise<Array<number>>} An array of executions
+     * @returns An array of executions
      */
     async searchByCumulusIds(knexOrTrx, executionCumulusIds, params) {
         const { limit, offset, ...sortQueries } = params || {};

package/dist/search/BaseSearch.d.ts

@@ -0,0 +1,187 @@
+import { Knex } from 'knex';
+import { BaseRecord } from '../types/base';
+import { DbQueryParameters, QueryEvent, QueryStringParameters } from '../types/search';
+export declare const typeToTable: {
+    [key: string]: string;
+};
+/**
+ * Class to build and execute db search query
+ */
+declare class BaseSearch {
+    readonly type: string;
+    readonly tableName: string;
+    readonly queryStringParameters: QueryStringParameters;
+    dbQueryParameters: DbQueryParameters;
+    constructor(event: QueryEvent, type: string);
+    /**
+     * check if joined collections table search is needed
+     *
+     * @returns whether collection search is needed
+     */
+    protected searchCollection(): boolean;
+    /**
+     * check if joined pdrs table search is needed
+     *
+     * @returns whether pdr search is needed
+     */
+    protected searchPdr(): boolean;
+    /**
+     * check if joined providers table search is needed
+     *
+     * @returns whether provider search is needed
+     */
+    protected searchProvider(): boolean;
+    /**
+     * Determine if an estimated row count should be returned
+     *
+     * @param countSql - sql statement for count
+     * @returns whether an estimated row count should be returned
+     */
+    protected shouldEstimateRowcount(countSql: string): boolean;
+    /**
+     * Build the search query
+     *
+     * @param knex - DB client
+     * @returns queries for getting count and search result
+     */
+    protected buildSearch(knex: Knex): {
+        countQuery?: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+    };
+    /**
+     * Get metadata template for query result
+     *
+     * @returns metadata template
+     */
+    private _metaTemplate;
+    /**
+     * Build basic query
+     *
+     * @param knex - DB client
+     * @throws - function is not implemented
+     */
+    protected buildBasicQuery(knex: Knex): {
+        countQuery?: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+    };
+    /**
+     * Build queries for infix and prefix
+     *
+     * @param params
+     * @param [params.countQuery] - query builder for getting count
+     * @param params.searchQuery - query builder for search
+     * @param [params.dbQueryParameters] - db query parameters
+     */
+    protected buildInfixPrefixQuery(params: {
+        countQuery?: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+        dbQueryParameters?: DbQueryParameters;
+    }): void;
+    /**
+     * Build queries for checking if field 'exists'
+     *
+     * @param params
+     * @param [params.countQuery] - query builder for getting count
+     * @param params.searchQuery - query builder for search
+     * @param [params.dbQueryParameters] - db query parameters
+     */
+    protected buildExistsQuery(params: {
+        countQuery?: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+        dbQueryParameters?: DbQueryParameters;
+    }): void;
+    /**
+     * Build queries for range fields
+     *
+     * @param params
+     * @param params.knex - db client
+     * @param [params.countQuery] - query builder for getting count
+     * @param params.searchQuery - query builder for search
+     * @param [params.dbQueryParameters] - db query parameters
+     */
+    protected buildRangeQuery(params: {
+        knex?: Knex;
+        countQuery?: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+        dbQueryParameters?: DbQueryParameters;
+    }): void;
+    /**
+     * Build queries for term fields
+     *
+     * @param params
+     * @param [params.countQuery] - query builder for getting count
+     * @param params.searchQuery - query builder for search
+     * @param [params.dbQueryParameters] - db query parameters
+     */
+    protected buildTermQuery(params: {
+        countQuery?: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+        dbQueryParameters?: DbQueryParameters;
+    }): void;
+    /**
+     * Build queries for terms fields
+     *
+     * @param params
+     * @param [params.countQuery] - query builder for getting count
+     * @param params.searchQuery - query builder for search
+     * @param [params.dbQueryParameters] - db query parameters
+     */
+    protected buildTermsQuery(params: {
+        countQuery?: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+        dbQueryParameters?: DbQueryParameters;
+    }): void;
+    /**
+     * Build queries for checking if field doesn't match the given value
+     *
+     * @param params
+     * @param [params.countQuery] - query builder for getting count
+     * @param params.searchQuery - query builder for search
+     * @param [params.dbQueryParameters] - db query parameters
+     */
+    protected buildNotMatchQuery(params: {
+        countQuery?: Knex.QueryBuilder;
+        searchQuery: Knex.QueryBuilder;
+        dbQueryParameters?: DbQueryParameters;
+    }): void;
+    /**
+     * Build queries for sort keys and fields
+     *
+     * @param params
+     * @param params.searchQuery - query builder for search
+     * @param [params.dbQueryParameters] - db query parameters
+     */
+    protected buildSortQuery(params: {
+        searchQuery: Knex.QueryBuilder;
+        dbQueryParameters?: DbQueryParameters;
+    }): void;
+    /**
+     * Translate postgres records to api records
+     *
+     * @param pgRecords - postgres records returned from query
+     * @param [knex] - knex client for additional queries if neccessary
+     * @throws - function is not implemented
+     */
+    protected translatePostgresRecordsToApiRecords(pgRecords: BaseRecord[], knex?: Knex): void;
+    /**
+     * Get estimated table rowcount
+     *
+     * @param params
+     * @param params.knex - DB client
+     * @param [params.tableName] - table name
+     * @returns rowcount
+     */
+    protected getEstimatedRowcount(params: {
+        knex: Knex;
+        tableName?: string;
+    }): Promise<number>;
+    /**
+     * Build and execute search query
+     *
+     * @param testKnex - knex for testing
+     * @returns search result
+     */
+    query(testKnex?: Knex): Promise<any>;
+}
+export { BaseSearch };
+//# sourceMappingURL=BaseSearch.d.ts.map