@cumulus/db 18.3.4 → 19.0.0

package/README.md

@@ -31,6 +31,11 @@ Typically, there are two TypeScript interfaces describing each Cumulus data type
 - `PostgresProvider`: Describes the data structure ready for insertion into the Cumulus Postgres database
 - `PostgresProviderRecord`: Describes the data structure after retrieval from the Cumulus database. This data type usually includes extra required properties (such as the auto-incremented primary key field), since those properties will exist once a record has been created.
 
+### BigInt cumulus_id columns
+
+For the BigInt columns, knex returns postgres as "string" type. In order to use cumulus_id as a number, knex hook
+postProcessResponse is configured to convert the return string from columns ending with "cumulus_id" to number.
+
 ## About Cumulus
 
 Cumulus is a cloud-based data ingest, archive, distribution and management

package/dist/config.d.ts

@@ -35,6 +35,24 @@ export declare const getConnectionConfig: ({ env, secretsManager, }: {
     env: NodeJS.ProcessEnv;
     secretsManager?: SecretsManager | undefined;
 }) => Promise<Knex.PgConnectionConfig>;
+/**
+ * Check if a string can be converted to a number
+ *
+ * @param bigIntValue - string to be converted to number
+ * @returns true if the string can be converted
+ * @throws - Throws error if the string can not be converted
+ */
+export declare const canSafelyConvertBigInt: (bigIntValue: string) => boolean;
+declare type RecordTypeWithNumberIdField<T> = {
+    [P in keyof T]: number | T[P];
+};
+/**
+ * Convert cumulus id fields to number
+ *
+ * @param record - record to be converted
+ * @returns the converted record
+ */
+export declare const convertIdColumnsToNumber: <T extends Record<string, any>>(record: T) => RecordTypeWithNumberIdField<T>;
 /**
  * Given a NodeJS.ProcessEnv with configuration values, build and return Knex
  * configuration
@@ -79,4 +97,5 @@ export declare const getKnexConfig: ({ env, secretsManager, }?: {
     env?: NodeJS.ProcessEnv | undefined;
     secretsManager?: SecretsManager | undefined;
 }) => Promise<Knex.Config>;
+export {};
 //# sourceMappingURL=config.d.ts.map

package/dist/config.js

@@ -1,7 +1,14 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getKnexConfig = exports.getConnectionConfig = exports.getConnectionConfigEnv = exports.getSecretConnectionConfig = exports.isKnexDebugEnabled = exports.localStackConnectionEnv = void 0;
+exports.getKnexConfig = exports.convertIdColumnsToNumber = exports.canSafelyConvertBigInt = exports.getConnectionConfig = exports.getConnectionConfigEnv = exports.getSecretConnectionConfig = exports.isKnexDebugEnabled = exports.localStackConnectionEnv = void 0;
 const aws_client_1 = require("@cumulus/aws-client");
+const mapValues_1 = __importDefault(require("lodash/mapValues"));
+const isNull_1 = __importDefault(require("lodash/isNull"));
+const isObject_1 = __importDefault(require("lodash/isObject"));
+const isString_1 = __importDefault(require("lodash/isString"));
 const common_1 = require("@cumulus/common");
 exports.localStackConnectionEnv = {
     PG_DATABASE: 'postgres',
@@ -76,6 +83,30 @@ const getConnectionConfig = async ({ env, secretsManager = aws_client_1.services
     return await (0, exports.getConnectionConfigEnv)(env);
 };
 exports.getConnectionConfig = getConnectionConfig;
+/**
+ * Check if a string can be converted to a number
+ *
+ * @param bigIntValue - string to be converted to number
+ * @returns true if the string can be converted
+ * @throws - Throws error if the string can not be converted
+ */
+const canSafelyConvertBigInt = (bigIntValue) => {
+    if (BigInt(bigIntValue) > BigInt(Number.MAX_SAFE_INTEGER)) {
+        throw new Error(`Failed to convert to number: ${bigIntValue} exceeds max safe integer ${Number.MAX_SAFE_INTEGER}`);
+    }
+    return true;
+};
+exports.canSafelyConvertBigInt = canSafelyConvertBigInt;
+/**
+ * Convert cumulus id fields to number
+ *
+ * @param record - record to be converted
+ * @returns the converted record
+ */
+const convertIdColumnsToNumber = (record) => (0, mapValues_1.default)(record, (value, key) => ((key.endsWith('cumulus_id') && !(0, isNull_1.default)(value) && (0, isString_1.default)(value) && (0, exports.canSafelyConvertBigInt)(value))
+    ? Number(value)
+    : value));
+exports.convertIdColumnsToNumber = convertIdColumnsToNumber;
 /**
  * Given a NodeJS.ProcessEnv with configuration values, build and return Knex
  * configuration
@@ -136,6 +167,14 @@ const getKnexConfig = async ({ env = process.env, secretsManager = aws_client_1.
             reapIntervalMillis: Number.parseInt(env.reapIntervalMillis ?? '1000', 10),
             propagateCreateError: false,
         },
+        // modify any knex query response to convert columns ending with "cumulus_id" from
+        // string | number to number
+        postProcessResponse: (result) => {
+            if (result && Array.isArray(result)) {
+                return result.map((row) => ((0, isObject_1.default)(row) ? (0, exports.convertIdColumnsToNumber)(row) : row));
+            }
+            return ((0, isObject_1.default)(result) ? (0, exports.convertIdColumnsToNumber)(result) : result);
+        },
     };
     knexConfig.acquireConnectionTimeout = env.acquireTimeoutMillis
         ? Number(env.acquireTimeoutMillis + 1000)

package/dist/index.d.ts

@@ -26,10 +26,15 @@ export { translateApiExecutionToPostgresExecution, translateApiExecutionToPostgr
 export { translateApiGranuleToPostgresGranule, translateApiGranuleToPostgresGranuleWithoutNilsRemoved, translatePostgresGranuleToApiGranule, translatePostgresGranuleResultToApiGranule, } from './translate/granules';
 export { translateApiPdrToPostgresPdr, translatePostgresPdrToApiPdr, } from './translate/pdr';
 export { getCollectionsByGranuleIds, } from './lib/collection';
-export { executionArnsFromGranuleIdsAndWorkflowNames, newestExecutionArnFromGranuleIdWorkflowName, getWorkflowNameIntersectFromGranuleIds, getApiExecutionCumulusIds, getApiGranuleExecutionCumulusIdsByExecution, getExecutionInfoByGranuleCumulusId, } from './lib/execution';
+export { batchDeleteExecutionFromDatabaseByCumulusCollectionId, executionArnsFromGranuleIdsAndWorkflowNames, getApiExecutionCumulusIds, getApiGranuleExecutionCumulusIdsByExecution, getExecutionInfoByGranuleCumulusId, getWorkflowNameIntersectFromGranuleIds, newestExecutionArnFromGranuleIdWorkflowName, } from './lib/execution';
 export { getFilesAndGranuleInfoQuery, } from './lib/file';
 export { getApiGranuleCumulusIds, getApiGranuleExecutionCumulusIds, getGranuleCollectionId, getUniqueGranuleByGranuleId, getGranuleByUniqueColumns, upsertGranuleWithExecutionJoinRecord, getGranulesByApiPropertiesQuery, getGranulesByGranuleId, getGranuleAndCollection, } from './lib/granule';
 export { QuerySearchClient, } from './lib/QuerySearchClient';
+export { BaseSearch, } from './search/BaseSearch';
+export { ExecutionSearch, } from './search/ExecutionSearch';
+export { GranuleSearch, } from './search/GranuleSearch';
+export { StatsSearch, } from './search/StatsSearch';
+export { CollectionSearch, } from './search/CollectionSearch';
 export { AsyncOperationPgModel } from './models/async_operation';
 export { BasePgModel } from './models/base';
 export { CollectionPgModel } from './models/collection';

package/dist/index.js

@@ -23,8 +23,8 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getFilesAndGranuleInfoQuery = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.getCollectionsByGranuleIds = exports.translatePostgresPdrToApiPdr = exports.translateApiPdrToPostgresPdr = exports.translatePostgresGranuleResultToApiGranule = exports.translatePostgresGranuleToApiGranule = exports.translateApiGranuleToPostgresGranuleWithoutNilsRemoved = exports.translateApiGranuleToPostgresGranule = exports.translatePostgresExecutionToApiExecution = exports.translateApiExecutionToPostgresExecutionWithoutNilsRemoved = exports.translateApiExecutionToPostgresExecution = exports.translateApiRuleToPostgresRuleRaw = exports.translateApiRuleToPostgresRule = exports.translatePostgresRuleToApiRule = exports.translatePostgresProviderToApiProvider = exports.translateApiProviderToPostgresProvider = exports.translatePostgresCollectionToApiCollection = exports.translateApiCollectionToPostgresCollection = exports.translatePostgresFileToApiFile = exports.translateApiFiletoPostgresFile = exports.translatePostgresAsyncOperationToApiAsyncOperation = exports.translateApiAsyncOperationToPostgresAsyncOperation = exports.nullifyUndefinedProviderValues = exports.validateProviderHost = exports.migrationDir = exports.TableNames = exports.createRejectableTransaction = exports.localStackConnectionEnv = exports.getKnexConfig = exports.getKnexClient = exports.isCollisionError = exports.generateLocalTestDb = exports.fakeRuleRecordFactory = exports.fakeProviderRecordFactory = exports.fakePdrRecordFactory = exports.fakeGranuleRecordFactory = exports.fakeFileRecordFactory = exports.fakeExecutionRecordFactory = exports.fakeCollectionRecordFactory = exports.fakeAsyncOperationRecordFactory = exports.destroyLocalTestDb = exports.deleteTestDatabase = exports.createTestDatabase = exports.Knex = void 0;
-exports.RulePgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.QuerySearchClient = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = void 0;
+exports.newestExecutionArnFromGranuleIdWorkflowName = exports.getWorkflowNameIntersectFromGranuleIds = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getCollectionsByGranuleIds = exports.translatePostgresPdrToApiPdr = exports.translateApiPdrToPostgresPdr = exports.translatePostgresGranuleResultToApiGranule = exports.translatePostgresGranuleToApiGranule = exports.translateApiGranuleToPostgresGranuleWithoutNilsRemoved = exports.translateApiGranuleToPostgresGranule = exports.translatePostgresExecutionToApiExecution = exports.translateApiExecutionToPostgresExecutionWithoutNilsRemoved = exports.translateApiExecutionToPostgresExecution = exports.translateApiRuleToPostgresRuleRaw = exports.translateApiRuleToPostgresRule = exports.translatePostgresRuleToApiRule = exports.translatePostgresProviderToApiProvider = exports.translateApiProviderToPostgresProvider = exports.translatePostgresCollectionToApiCollection = exports.translateApiCollectionToPostgresCollection = exports.translatePostgresFileToApiFile = exports.translateApiFiletoPostgresFile = exports.translatePostgresAsyncOperationToApiAsyncOperation = exports.translateApiAsyncOperationToPostgresAsyncOperation = exports.nullifyUndefinedProviderValues = exports.validateProviderHost = exports.migrationDir = exports.TableNames = exports.createRejectableTransaction = exports.localStackConnectionEnv = exports.getKnexConfig = exports.getKnexClient = exports.isCollisionError = exports.generateLocalTestDb = exports.fakeRuleRecordFactory = exports.fakeProviderRecordFactory = exports.fakePdrRecordFactory = exports.fakeGranuleRecordFactory = exports.fakeFileRecordFactory = exports.fakeExecutionRecordFactory = exports.fakeCollectionRecordFactory = exports.fakeAsyncOperationRecordFactory = exports.destroyLocalTestDb = exports.deleteTestDatabase = exports.createTestDatabase = exports.Knex = void 0;
+exports.RulePgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.CollectionSearch = exports.StatsSearch = exports.GranuleSearch = exports.ExecutionSearch = exports.BaseSearch = exports.QuerySearchClient = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getFilesAndGranuleInfoQuery = void 0;
 const path = __importStar(require("path"));
 var knex_1 = require("knex");
 Object.defineProperty(exports, "Knex", { enumerable: true, get: function () { return knex_1.Knex; } });
@@ -87,12 +87,13 @@ Object.defineProperty(exports, "translatePostgresPdrToApiPdr", { enumerable: tru
 var collection_1 = require("./lib/collection");
 Object.defineProperty(exports, "getCollectionsByGranuleIds", { enumerable: true, get: function () { return collection_1.getCollectionsByGranuleIds; } });
 var execution_1 = require("./lib/execution");
+Object.defineProperty(exports, "batchDeleteExecutionFromDatabaseByCumulusCollectionId", { enumerable: true, get: function () { return execution_1.batchDeleteExecutionFromDatabaseByCumulusCollectionId; } });
 Object.defineProperty(exports, "executionArnsFromGranuleIdsAndWorkflowNames", { enumerable: true, get: function () { return execution_1.executionArnsFromGranuleIdsAndWorkflowNames; } });
-Object.defineProperty(exports, "newestExecutionArnFromGranuleIdWorkflowName", { enumerable: true, get: function () { return execution_1.newestExecutionArnFromGranuleIdWorkflowName; } });
-Object.defineProperty(exports, "getWorkflowNameIntersectFromGranuleIds", { enumerable: true, get: function () { return execution_1.getWorkflowNameIntersectFromGranuleIds; } });
 Object.defineProperty(exports, "getApiExecutionCumulusIds", { enumerable: true, get: function () { return execution_1.getApiExecutionCumulusIds; } });
 Object.defineProperty(exports, "getApiGranuleExecutionCumulusIdsByExecution", { enumerable: true, get: function () { return execution_1.getApiGranuleExecutionCumulusIdsByExecution; } });
 Object.defineProperty(exports, "getExecutionInfoByGranuleCumulusId", { enumerable: true, get: function () { return execution_1.getExecutionInfoByGranuleCumulusId; } });
+Object.defineProperty(exports, "getWorkflowNameIntersectFromGranuleIds", { enumerable: true, get: function () { return execution_1.getWorkflowNameIntersectFromGranuleIds; } });
+Object.defineProperty(exports, "newestExecutionArnFromGranuleIdWorkflowName", { enumerable: true, get: function () { return execution_1.newestExecutionArnFromGranuleIdWorkflowName; } });
 var file_2 = require("./lib/file");
 Object.defineProperty(exports, "getFilesAndGranuleInfoQuery", { enumerable: true, get: function () { return file_2.getFilesAndGranuleInfoQuery; } });
 var granule_1 = require("./lib/granule");
@@ -107,6 +108,16 @@ Object.defineProperty(exports, "getGranulesByGranuleId", { enumerable: true, get
 Object.defineProperty(exports, "getGranuleAndCollection", { enumerable: true, get: function () { return granule_1.getGranuleAndCollection; } });
 var QuerySearchClient_1 = require("./lib/QuerySearchClient");
 Object.defineProperty(exports, "QuerySearchClient", { enumerable: true, get: function () { return QuerySearchClient_1.QuerySearchClient; } });
+var BaseSearch_1 = require("./search/BaseSearch");
+Object.defineProperty(exports, "BaseSearch", { enumerable: true, get: function () { return BaseSearch_1.BaseSearch; } });
+var ExecutionSearch_1 = require("./search/ExecutionSearch");
+Object.defineProperty(exports, "ExecutionSearch", { enumerable: true, get: function () { return ExecutionSearch_1.ExecutionSearch; } });
+var GranuleSearch_1 = require("./search/GranuleSearch");
+Object.defineProperty(exports, "GranuleSearch", { enumerable: true, get: function () { return GranuleSearch_1.GranuleSearch; } });
+var StatsSearch_1 = require("./search/StatsSearch");
+Object.defineProperty(exports, "StatsSearch", { enumerable: true, get: function () { return StatsSearch_1.StatsSearch; } });
+var CollectionSearch_1 = require("./search/CollectionSearch");
+Object.defineProperty(exports, "CollectionSearch", { enumerable: true, get: function () { return CollectionSearch_1.CollectionSearch; } });
 var async_operation_1 = require("./models/async_operation");
 Object.defineProperty(exports, "AsyncOperationPgModel", { enumerable: true, get: function () { return async_operation_1.AsyncOperationPgModel; } });
 var base_1 = require("./models/base");

package/dist/lib/execution.d.ts

@@ -92,4 +92,9 @@ export declare const getApiExecutionCumulusIds: (knexOrTransaction: Knex | Knex.
 export declare const getApiGranuleExecutionCumulusIdsByExecution: (knexOrTransaction: Knex | Knex.Transaction, executions: Array<{
     arn: string;
 }>, executionPgModel?: ExecutionPgModel, granulesExecutionsPgModel?: GranulesExecutionsPgModel) => Promise<Array<number>>;
+export declare const batchDeleteExecutionFromDatabaseByCumulusCollectionId: (params: {
+    knex: Knex | Knex.Transaction;
+    collectionCumulusId: number;
+    batchSize: number;
+}) => Promise<number>;
 //# sourceMappingURL=execution.d.ts.map

package/dist/lib/execution.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.getExecutionArnsByGranuleCumulusId = exports.getExecutionInfoByGranuleCumulusId = void 0;
+exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.getExecutionArnsByGranuleCumulusId = exports.getExecutionInfoByGranuleCumulusId = void 0;
 const errors_1 = require("@cumulus/errors");
 const execution_1 = require("../models/execution");
 const granules_executions_1 = require("../models/granules-executions");
@@ -164,4 +164,19 @@ const getApiGranuleExecutionCumulusIdsByExecution = async (knexOrTransaction, ex
     return granuleCumulusIds;
 };
 exports.getApiGranuleExecutionCumulusIdsByExecution = getApiGranuleExecutionCumulusIdsByExecution;
+const batchDeleteExecutionFromDatabaseByCumulusCollectionId = async (params) => {
+    const { knex, collectionCumulusId, batchSize = 1 } = params;
+    try {
+        return await knex('executions')
+            .whereIn('cumulus_id', knex.select('cumulus_id')
+            .from('executions')
+            .where('collection_cumulus_id', collectionCumulusId)
+            .limit(batchSize))
+            .delete();
+    }
+    catch (error) {
+        throw new Error(`Failed to delete from database: ${error.message}`);
+    }
+};
+exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = batchDeleteExecutionFromDatabaseByCumulusCollectionId;
 //# sourceMappingURL=execution.js.map

package/dist/migrations/20240125171703_update_granule_execution_cumulus_id_type.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: () => Promise<void>;
+//# sourceMappingURL=20240125171703_update_granule_execution_cumulus_id_type.d.ts.map

package/dist/migrations/20240125171703_update_granule_execution_cumulus_id_type.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    await knex.raw('ALTER TABLE files ALTER COLUMN granule_cumulus_id TYPE BIGINT');
+    await knex.raw('ALTER TABLE executions ALTER COLUMN cumulus_id TYPE BIGINT, ALTER COLUMN parent_cumulus_id TYPE BIGINT');
+    await knex.raw('ALTER TABLE granules_executions ALTER COLUMN granule_cumulus_id TYPE BIGINT, ALTER COLUMN execution_cumulus_id TYPE BIGINT');
+    await knex.raw('ALTER TABLE pdrs ALTER COLUMN execution_cumulus_id TYPE BIGINT');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) executions');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) files');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) granules_executions');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) pdrs');
+};
+exports.up = up;
+const down = async () => {
+    console.log('Warning - this migration cannot be rolled back');
+};
+exports.down = down;
+exports.config = {
+    transaction: false,
+};
+//# sourceMappingURL=20240125171703_update_granule_execution_cumulus_id_type.js.map

package/dist/migrations/20240126135619_granules_add_indexes.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20240126135619_granules_add_indexes.d.ts.map

package/dist/migrations/20240126135619_granules_add_indexes.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    await knex.raw('CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS granules_collection_cumulus_id_granule_id_unique ON granules(collection_cumulus_id, granule_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_granule_id_index ON granules(granule_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_provider_collection_cumulus_id_granule_id_index ON granules(provider_cumulus_id, collection_cumulus_id, granule_id)');
+};
+exports.up = up;
+const down = async (knex) => {
+    await knex.raw('DROP INDEX IF EXISTS granules_collection_cumulus_id_granule_id_unique');
+    await knex.raw('DROP INDEX IF EXISTS granules_granule_id_index');
+    await knex.raw('DROP INDEX IF EXISTS granules_provider_collection_cumulus_id_granule_id_index');
+};
+exports.down = down;
+exports.config = {
+    transaction: false,
+};
+//# sourceMappingURL=20240126135619_granules_add_indexes.js.map

package/dist/migrations/20240126165019_granules_update_constraints.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20240126165019_granules_update_constraints.d.ts.map

package/dist/migrations/20240126165019_granules_update_constraints.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const addConstraintIfNotExists = async ({ knex, tableName, constraintName, constraintSql, }) => {
+    const constraintExists = await knex.raw(`
+    SELECT constraint_name
+    FROM information_schema.constraint_column_usage
+    WHERE constraint_name = ?
+    AND table_name = ?
+  `, [constraintName, tableName]);
+    if (constraintExists.rows.length === 0) {
+        await knex.raw(`
+      ALTER TABLE ${tableName} ${constraintSql}
+    `);
+        console.log(`Constraint ${constraintName} added to table ${tableName}`);
+    }
+    else {
+        console.log(`Constraint ${constraintName} already exists on table ${tableName}`);
+    }
+};
+const up = async (knex) => {
+    // ALTER TABLE granules ADD UNIQUE USING INDEX granules_collection_cumulus_id_granule_id_unique
+    await addConstraintIfNotExists({
+        knex,
+        tableName: 'granules',
+        constraintName: 'granules_collection_cumulus_id_granule_id_unique',
+        constraintSql: 'ADD UNIQUE USING INDEX granules_collection_cumulus_id_granule_id_unique',
+    });
+    await knex.raw('ALTER TABLE granules DROP constraint IF EXISTS granules_granule_id_collection_cumulus_id_unique');
+};
+exports.up = up;
+const down = async (knex) => {
+    // ALTER TABLE granules ADD CONSTRAINT
+    // granules_granule_id_collection_cumulus_id_unique UNIQUE (granule_id, collection_cumulus_id)
+    await addConstraintIfNotExists({
+        knex,
+        tableName: 'granules',
+        constraintName: 'granules_granule_id_collection_cumulus_id_unique',
+        constraintSql: 'ADD CONSTRAINT granules_granule_id_collection_cumulus_id_unique UNIQUE (granule_id, collection_cumulus_id)',
+    });
+    await knex.raw('ALTER TABLE granules DROP CONSTRAINT IF EXISTS granules_collection_cumulus_id_granule_id_unique');
+};
+exports.down = down;
+//# sourceMappingURL=20240126165019_granules_update_constraints.js.map

package/dist/migrations/20240606060726_alter_async_operations_add_operation_type_bulk_execution_delete.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20240606060726_alter_async_operations_add_operation_type_bulk_execution_delete.d.ts.map

package/dist/migrations/20240606060726_alter_async_operations_add_operation_type_bulk_execution_delete.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const formatAlterTableEnumSql = (tableName, columnName, enums) => {
+    const constraintName = `${tableName}_${columnName}_check`;
+    return [
+        `ALTER TABLE ${tableName}`,
+        `DROP CONSTRAINT IF EXISTS ${constraintName};`,
+        `ALTER TABLE ${tableName} ADD CONSTRAINT ${constraintName} CHECK (${columnName} = ANY (ARRAY['${enums.join("'::text, '")}'::text]));`,
+    ].join('\n');
+};
+const up = async (knex) => {
+    await knex.raw(formatAlterTableEnumSql('async_operations', 'operation_type', [
+        'Bulk Granule Delete',
+        'Bulk Granule Reingest',
+        'Bulk Granules',
+        'Bulk Execution Delete',
+        'Data Migration',
+        'Dead-Letter Processing',
+        'DLA Migration',
+        'ES Index',
+        'Kinesis Replay',
+        'Reconciliation Report',
+        'SQS Replay',
+    ]));
+};
+exports.up = up;
+const down = async (knex) => {
+    await knex.raw(formatAlterTableEnumSql('async_operations', 'operation_type', [
+        'ES Index',
+        'Bulk Granules',
+        'Bulk Granule Reingest',
+        'Bulk Granule Delete',
+        'Dead-Letter Processing',
+        'DLA Migration',
+        'Kinesis Replay',
+        'Reconciliation Report',
+        'Data Migration',
+        'SQS Replay',
+    ]));
+};
+exports.down = down;
+//# sourceMappingURL=20240606060726_alter_async_operations_add_operation_type_bulk_execution_delete.js.map

package/dist/migrations/20240613174614_add_execution_parent_and_collection_indexes.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20240613174614_add_execution_parent_and_collection_indexes.d.ts.map

package/dist/migrations/20240613174614_add_execution_parent_and_collection_indexes.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS executions_parent_cumulus_id_index ON executions(parent_cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS executions_collection_cumulus_id_index ON executions(collection_cumulus_id)');
+};
+exports.up = up;
+const down = async (knex) => {
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS executions_parent_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS executions_collection_cumulus_id_index');
+};
+exports.down = down;
+exports.config = {
+    transaction: false,
+};
+//# sourceMappingURL=20240613174614_add_execution_parent_and_collection_indexes.js.map

package/dist/migrations/20240617204826_update_executions_deletion_constraint.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20240617204826_update_executions_deletion_constraint.d.ts.map

package/dist/migrations/20240617204826_update_executions_deletion_constraint.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    await knex.raw('ALTER TABLE executions DROP CONSTRAINT IF EXISTS executions_parent_cumulus_id_foreign');
+    await knex.raw('ALTER TABLE executions ADD CONSTRAINT executions_parent_cumulus_id_foreign FOREIGN KEY (parent_cumulus_id) REFERENCES executions(cumulus_id) ON DELETE SET NULL');
+};
+exports.up = up;
+const down = async (knex) => {
+    await knex.raw('ALTER TABLE executions DROP CONSTRAINT IF EXISTS executions_parent_cumulus_id_foreign');
+    await knex.raw('ALTER TABLE executions ADD CONSTRAINT executions_parent_cumulus_id_foreign FOREIGN KEY (parent_cumulus_id) REFERENCES executions(cumulus_id)');
+};
+exports.down = down;
+//# sourceMappingURL=20240617204826_update_executions_deletion_constraint.js.map

package/dist/migrations/20240728101230_add_table_indexes.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20240728101230_add_table_indexes.d.ts.map

package/dist/migrations/20240728101230_add_table_indexes.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS async_operations_updated_at_index ON async_operations(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS async_operations_status_operation_type_cumulus_id_index ON async_operations(status, operation_type, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS collections_updated_at_index ON collections(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS executions_updated_at_index ON executions(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS executions_status_collection_cumulus_id_index ON executions(status, collection_cumulus_id, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS files_updated_at_index ON files(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_updated_at_index ON granules(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_coll_status_processendtime_cumulus_id_index ON granules(collection_cumulus_id, status, processing_end_date_time, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_status_provider_collection_cumulus_id_index ON granules(status, provider_cumulus_id, collection_cumulus_id, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_updated_at_index ON pdrs(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_status_provider_collection_cumulus_id_index ON pdrs(status, provider_cumulus_id, collection_cumulus_id, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_execution_cumulus_id_index ON pdrs(execution_cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_coll_status_cumulus_id_index ON pdrs(collection_cumulus_id, status, cumulus_id)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS pdrs_provider_collection_cumulus_id_name_index ON pdrs(provider_cumulus_id, collection_cumulus_id, name)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS providers_updated_at_index ON providers(updated_at)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS rules_updated_at_index ON rules(updated_at)');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) async_operations');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) collections');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) executions');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) files');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) granules');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) pdrs');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) providers');
+    await knex.raw('VACUUM (ANALYZE, VERBOSE) rules');
+};
+exports.up = up;
+const down = async (knex) => {
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS async_operations_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS async_operations_status_operation_type_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS collections_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS executions_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS executions_status_collection_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS files_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS granules_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS granules_coll_status_processendtime_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS granules_status_provider_collection_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_status_provider_collection_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_execution_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_coll_status_cumulus_id_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS pdrs_provider_collection_cumulus_id_name_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS providers_updated_at_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS rules_updated_at_index');
+};
+exports.down = down;
+exports.config = {
+    transaction: false,
+};
+//# sourceMappingURL=20240728101230_add_table_indexes.js.map

package/dist/models/execution.d.ts

@@ -16,12 +16,12 @@ declare class ExecutionPgModel extends BasePgModel<PostgresExecution, PostgresEx
      * @param {Object} [params] - Optional object with addition params for query
      * @param {number} [params.limit] - number of records to be returned
      * @param {number} [params.offset] - record offset
-     * @returns {Promise<Array<number>>} An array of executions
+     * @returns An array of executions
      */
     searchByCumulusIds(knexOrTrx: Knex | Knex.Transaction, executionCumulusIds: Array<number> | number, params: {
         limit: number;
         offset: number;
-    }): Promise<Array<number>>;
+    }): Promise<Array<PostgresExecutionRecord>>;
 }
 export { ExecutionPgModel };
 //# sourceMappingURL=execution.d.ts.map

package/dist/models/execution.js

@@ -42,7 +42,7 @@ class ExecutionPgModel extends base_1.BasePgModel {
      * @param {Object} [params] - Optional object with addition params for query
      * @param {number} [params.limit] - number of records to be returned
      * @param {number} [params.offset] - record offset
-     * @returns {Promise<Array<number>>} An array of executions
+     * @returns An array of executions
      */
     async searchByCumulusIds(knexOrTrx, executionCumulusIds, params) {
         const { limit, offset, ...sortQueries } = params || {};