@cumulus/db 10.0.1 → 10.1.1

package/dist/config.d.ts

@@ -8,6 +8,7 @@ export declare const localStackConnectionEnv: {
     PG_DATABASE: string;
     PG_PORT: string;
 };
+export declare const isKnexDebugEnabled: (env?: NodeJS.ProcessEnv) => boolean;
 export declare const getSecretConnectionConfig: (SecretId: string, secretsManager: AWS.SecretsManager) => Promise<Knex.PgConnectionConfig>;
 export declare const getConnectionConfigEnv: (env: NodeJS.ProcessEnv) => Knex.PgConnectionConfig;
 /**

package/dist/config.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getKnexConfig = exports.getConnectionConfig = exports.getConnectionConfigEnv = exports.getSecretConnectionConfig = exports.localStackConnectionEnv = void 0;
+exports.getKnexConfig = exports.getConnectionConfig = exports.getConnectionConfigEnv = exports.getSecretConnectionConfig = exports.isKnexDebugEnabled = exports.localStackConnectionEnv = void 0;
 const aws_sdk_1 = __importDefault(require("aws-sdk"));
 const common_1 = require("@cumulus/common");
 exports.localStackConnectionEnv = {
@@ -13,6 +13,8 @@ exports.localStackConnectionEnv = {
     PG_DATABASE: 'postgres',
     PG_PORT: '5432',
 };
+const isKnexDebugEnabled = (env = {}) => env.KNEX_DEBUG === 'true';
+exports.isKnexDebugEnabled = isKnexDebugEnabled;
 const getSecretConnectionConfig = async (SecretId, secretsManager) => {
     var _a;
     const response = await secretsManager.getSecretValue({ SecretId }).promise();
@@ -110,7 +112,7 @@ const getKnexConfig = async ({ env = process.env, secretsManager = new aws_sdk_1
     const knexConfig = {
         client: 'pg',
         connection: await (0, exports.getConnectionConfig)({ env, secretsManager }),
-        debug: env.KNEX_DEBUG === 'true',
+        debug: (0, exports.isKnexDebugEnabled)(env),
         asyncStackTraces: env.KNEX_ASYNC_STACK_TRACES === 'true',
         pool: {
             min: 0,

package/dist/connection.js

@@ -44,16 +44,37 @@ const log = new logger_1.default({ sender: '@cumulus/db/connection' });
  * @returns {Promise<Knex>} a Knex configuration object that has returned at least one query
  */
 const getKnexClient = async ({ env = process.env, secretsManager = new aws_sdk_1.default.SecretsManager(), knexLogger = log, } = {}) => {
+    if ((0, config_1.isKnexDebugEnabled)(env)) {
+        knexLogger.info('Initializing connection pool...');
+    }
     const knexConfig = await (0, config_1.getKnexConfig)({ env, secretsManager });
     const knexClient = (0, knex_1.knex)(knexConfig);
-    //@ts-ignore
-    // context is an internal object that isn't typed
-    // this is needed to force tarn to log per-retry failures
-    // and allow propagateCreateError to be set `false`
-    knexClient.context.client.pool.on('createFail', (_, error) => {
-        knexLogger.warn('knex failed on attempted connection', error);
-        throw error;
-    });
+    if ((0, config_1.isKnexDebugEnabled)(env)) {
+        //@ts-ignore
+        // context is an internal object that isn't typed
+        // this is needed to force tarn to log per-retry failures
+        // and allow propagateCreateError to be set `false`
+        knexClient.context.client.pool.on('createFail', (_, error) => {
+            knexLogger.warn('knex failed on attempted connection', error);
+            throw error;
+        });
+        //@ts-ignore
+        knexClient.context.client.pool.on('createSuccess', (_, resource) => {
+            knexLogger.info(`added connection to pool: ${JSON.stringify(resource)}`);
+        });
+        //@ts-ignore
+        knexClient.context.client.pool.on('acquireSuccess', (_, resource) => {
+            knexLogger.info(`acquired connection from pool: ${JSON.stringify(resource)}`);
+        });
+        //@ts-ignore
+        knexClient.context.client.pool.on('release', (resource) => {
+            knexLogger.info(`released connection from pool: ${JSON.stringify(resource)}`);
+        });
+        //@ts-ignore
+        knexClient.context.client.pool.on('poolDestroySuccess', () => {
+            knexLogger.info('pool is destroyed');
+        });
+    }
     return knexClient;
 };
 exports.getKnexClient = getKnexClient;

package/dist/index.d.ts

@@ -18,7 +18,7 @@ export { translateApiAsyncOperationToPostgresAsyncOperation } from './translate/
 export { translateApiFiletoPostgresFile, translatePostgresFileToApiFile, } from './translate/file';
 export { translateApiCollectionToPostgresCollection } from './translate/collections';
 export { translateApiProviderToPostgresProvider, } from './translate/providers';
-export { translateApiRuleToPostgresRule } from './translate/rules';
+export { translateApiRuleToPostgresRule, translateApiRuleToPostgresRuleRaw, } from './translate/rules';
 export { translateApiExecutionToPostgresExecution, translatePostgresExecutionToApiExecution, } from './translate/executions';
 export { translateApiGranuleToPostgresGranule, translatePostgresGranuleToApiGranule, } from './translate/granules';
 export { translateApiPdrToPostgresPdr } from './translate/pdrs';

package/dist/index.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
@@ -19,8 +23,8 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.upsertGranuleWithExecutionJoinRecord = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getExecutionArnsByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.translateApiPdrToPostgresPdr = exports.translatePostgresGranuleToApiGranule = exports.translateApiGranuleToPostgresGranule = exports.translatePostgresExecutionToApiExecution = exports.translateApiExecutionToPostgresExecution = exports.translateApiRuleToPostgresRule = exports.translateApiProviderToPostgresProvider = exports.translateApiCollectionToPostgresCollection = exports.translatePostgresFileToApiFile = exports.translateApiFiletoPostgresFile = exports.translateApiAsyncOperationToPostgresAsyncOperation = exports.nullifyUndefinedProviderValues = exports.validateProviderHost = exports.migrationDir = exports.TableNames = exports.createRejectableTransaction = exports.localStackConnectionEnv = exports.getKnexConfig = exports.getKnexClient = exports.generateLocalTestDb = exports.fakeRuleRecordFactory = exports.fakeProviderRecordFactory = exports.fakePdrRecordFactory = exports.fakeGranuleRecordFactory = exports.fakeFileRecordFactory = exports.fakeExecutionRecordFactory = exports.fakeCollectionRecordFactory = exports.fakeAsyncOperationRecordFactory = exports.destroyLocalTestDb = exports.deleteTestDatabase = exports.createTestDatabase = exports.Knex = void 0;
-exports.RulePgModel = void 0;
+exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.upsertGranuleWithExecutionJoinRecord = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getExecutionArnsByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = exports.getApiExecutionCumulusIds = exports.getWorkflowNameIntersectFromGranuleIds = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.translateApiPdrToPostgresPdr = exports.translatePostgresGranuleToApiGranule = exports.translateApiGranuleToPostgresGranule = exports.translatePostgresExecutionToApiExecution = exports.translateApiExecutionToPostgresExecution = exports.translateApiRuleToPostgresRuleRaw = exports.translateApiRuleToPostgresRule = exports.translateApiProviderToPostgresProvider = exports.translateApiCollectionToPostgresCollection = exports.translatePostgresFileToApiFile = exports.translateApiFiletoPostgresFile = exports.translateApiAsyncOperationToPostgresAsyncOperation = exports.nullifyUndefinedProviderValues = exports.validateProviderHost = exports.migrationDir = exports.TableNames = exports.createRejectableTransaction = exports.localStackConnectionEnv = exports.getKnexConfig = exports.getKnexClient = exports.generateLocalTestDb = exports.fakeRuleRecordFactory = exports.fakeProviderRecordFactory = exports.fakePdrRecordFactory = exports.fakeGranuleRecordFactory = exports.fakeFileRecordFactory = exports.fakeExecutionRecordFactory = exports.fakeCollectionRecordFactory = exports.fakeAsyncOperationRecordFactory = exports.destroyLocalTestDb = exports.deleteTestDatabase = exports.createTestDatabase = exports.Knex = void 0;
+exports.RulePgModel = exports.ProviderPgModel = void 0;
 const path = __importStar(require("path"));
 var knex_1 = require("knex");
 Object.defineProperty(exports, "Knex", { enumerable: true, get: function () { return knex_1.Knex; } });
@@ -61,6 +65,7 @@ var providers_1 = require("./translate/providers");
 Object.defineProperty(exports, "translateApiProviderToPostgresProvider", { enumerable: true, get: function () { return providers_1.translateApiProviderToPostgresProvider; } });
 var rules_1 = require("./translate/rules");
 Object.defineProperty(exports, "translateApiRuleToPostgresRule", { enumerable: true, get: function () { return rules_1.translateApiRuleToPostgresRule; } });
+Object.defineProperty(exports, "translateApiRuleToPostgresRuleRaw", { enumerable: true, get: function () { return rules_1.translateApiRuleToPostgresRuleRaw; } });
 var executions_1 = require("./translate/executions");
 Object.defineProperty(exports, "translateApiExecutionToPostgresExecution", { enumerable: true, get: function () { return executions_1.translateApiExecutionToPostgresExecution; } });
 Object.defineProperty(exports, "translatePostgresExecutionToApiExecution", { enumerable: true, get: function () { return executions_1.translatePostgresExecutionToApiExecution; } });

package/dist/migrations/20220126172008_files_granule_id_index.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20220126172008_files_granule_id_index.d.ts.map

package/dist/migrations/20220126172008_files_granule_id_index.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    await knex.schema.table('files', (table) => {
+        table.index('granule_cumulus_id');
+    });
+};
+exports.up = up;
+const down = async (knex) => {
+    await knex.schema.table('files', (table) => {
+        table.dropIndex('granule_cumulus_id');
+    });
+};
+exports.down = down;
+//# sourceMappingURL=20220126172008_files_granule_id_index.js.map

package/dist/models/base.d.ts

@@ -22,7 +22,7 @@ declare class BasePgModel<ItemType, RecordType extends {
      * Fetches multiple items from Postgres
      *
      * @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
-     * @param {Partial<RecordType>} params - An object or any portion of an object of type RecordType
+     * @param {Object} params - An object or any portion of an object of type RecordType
      * @returns {Promise<RecordType[]>} List of returned records
      */
     search(knexOrTransaction: Knex | Knex.Transaction, params: Partial<RecordType>): Promise<RecordType[]>;
@@ -41,7 +41,7 @@ declare class BasePgModel<ItemType, RecordType extends {
      *
      * @param {Knex | Knex.Transaction} knexOrTransaction -
      *  DB client or transaction
-     * @param {Array<keyof RecordType>} columnNames - column names for whereIn query
+     * @param {Array<string>} columnNames - column names for whereIn query
      * @param {Array<string>} values - record values for whereIn query
      * @returns {Promise<Array<number>>} An array of cumulus_ids for the returned records
      */
@@ -59,9 +59,11 @@ declare class BasePgModel<ItemType, RecordType extends {
      *
      * @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
      * @param {ItemType} item - A record to insert into the DB
+     * @param {string | Array<string>} returningFields - A string or array of strings
+     *   of columns to return. Defaults to 'cumulus_id'.
      * @returns {Promise<number[]>} List of IDs of the inserted records
      */
-    create(knexOrTransaction: Knex | Knex.Transaction, item: ItemType): Promise<number[]>;
+    create(knexOrTransaction: Knex | Knex.Transaction, item: ItemType, returningFields?: string | string[]): Promise<number[]>;
     /**
      * Deletes the item from Postgres
      *

package/dist/models/base.js

@@ -40,7 +40,7 @@ class BasePgModel {
      * Fetches multiple items from Postgres
      *
      * @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
-     * @param {Partial<RecordType>} params - An object or any portion of an object of type RecordType
+     * @param {Object} params - An object or any portion of an object of type RecordType
      * @returns {Promise<RecordType[]>} List of returned records
      */
     async search(knexOrTransaction, params) {
@@ -72,7 +72,7 @@ class BasePgModel {
      *
      * @param {Knex | Knex.Transaction} knexOrTransaction -
      *  DB client or transaction
-     * @param {Array<keyof RecordType>} columnNames - column names for whereIn query
+     * @param {Array<string>} columnNames - column names for whereIn query
      * @param {Array<string>} values - record values for whereIn query
      * @returns {Promise<Array<number>>} An array of cumulus_ids for the returned records
      */
@@ -106,12 +106,14 @@ class BasePgModel {
      *
      * @param {Knex | Knex.Transaction} knexOrTransaction - DB client or transaction
      * @param {ItemType} item - A record to insert into the DB
+     * @param {string | Array<string>} returningFields - A string or array of strings
+     *   of columns to return. Defaults to 'cumulus_id'.
      * @returns {Promise<number[]>} List of IDs of the inserted records
      */
-    async create(knexOrTransaction, item) {
+    async create(knexOrTransaction, item, returningFields = 'cumulus_id') {
         return await knexOrTransaction(this.tableName)
             .insert(item)
-            .returning('cumulus_id');
+            .returning(returningFields);
     }
     /**
      * Deletes the item from Postgres

package/dist/translate/executions.js

@@ -95,8 +95,8 @@ const translateApiExecutionToPostgresExecution = async (dynamoRecord, knex, coll
         updated_at: dynamoRecord.updatedAt ? new Date(dynamoRecord.updatedAt) : undefined,
     };
     if (dynamoRecord.collectionId !== undefined) {
-        const collectionNameVersionArray = dynamoRecord.collectionId.split('___');
-        translatedRecord.collection_cumulus_id = await collectionPgModel.getRecordCumulusId(knex, { name: collectionNameVersionArray[0], version: collectionNameVersionArray[1] });
+        const { name, version } = (0, Collections_1.deconstructCollectionId)(dynamoRecord.collectionId);
+        translatedRecord.collection_cumulus_id = await collectionPgModel.getRecordCumulusId(knex, { name, version });
     }
     // If we have a parentArn, try a lookup in Postgres. If there's a match, set the parent_cumulus_id
     if (dynamoRecord.parentArn !== undefined) {

package/dist/translate/rules.d.ts

@@ -12,5 +12,15 @@ import { PostgresRule } from '../types/rule';
  * @param {Object} providerPgModel - Instance of the provider database model
  * @returns {Object} A rule record
  */
+export declare const translateApiRuleToPostgresRuleRaw: (record: RuleRecord, dbClient: Knex, collectionPgModel?: CollectionPgModel, providerPgModel?: ProviderPgModel) => Promise<PostgresRule>;
+/**
+ * Generate a Postgres rule record from a DynamoDB record and remove nil properties.
+ *
+ * @param {Object} record - A rule
+ * @param {Object} dbClient - Knex client for reading from RDS database
+ * @param {Object} collectionPgModel - Instance of the collection database model
+ * @param {Object} providerPgModel - Instance of the provider database model
+ * @returns {Object} A rule record
+ */
 export declare const translateApiRuleToPostgresRule: (record: RuleRecord, dbClient: Knex, collectionPgModel?: CollectionPgModel, providerPgModel?: ProviderPgModel) => Promise<PostgresRule>;
 //# sourceMappingURL=rules.d.ts.map

package/dist/translate/rules.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.translateApiRuleToPostgresRule = void 0;
+exports.translateApiRuleToPostgresRule = exports.translateApiRuleToPostgresRuleRaw = void 0;
+const util_1 = require("@cumulus/common/util");
 const collection_1 = require("../models/collection");
 const provider_1 = require("../models/provider");
 /**
@@ -12,26 +13,37 @@ const provider_1 = require("../models/provider");
  * @param {Object} providerPgModel - Instance of the provider database model
  * @returns {Object} A rule record
  */
+const translateApiRuleToPostgresRuleRaw = async (record, dbClient, collectionPgModel = new collection_1.CollectionPgModel(), providerPgModel = new provider_1.ProviderPgModel()) => ({
+    name: record.name,
+    workflow: record.workflow,
+    provider_cumulus_id: record.provider ? await providerPgModel.getRecordCumulusId(dbClient, { name: record.provider }) : undefined,
+    collection_cumulus_id: record.collection ? await collectionPgModel.getRecordCumulusId(dbClient, { name: record.collection.name, version: record.collection.version }) : undefined,
+    meta: record.meta,
+    payload: record.payload,
+    queue_url: record.queueUrl,
+    arn: record.rule.arn,
+    type: record.rule.type,
+    value: record.rule.value,
+    log_event_arn: record.rule.logEventArn,
+    enabled: (record.state === undefined) || (record.state === 'ENABLED'),
+    tags: (record.tags ? JSON.stringify(record.tags) : undefined),
+    execution_name_prefix: record.executionNamePrefix,
+    created_at: (record.createdAt ? new Date(record.createdAt) : undefined),
+    updated_at: (record.updatedAt ? new Date(record.updatedAt) : undefined),
+});
+exports.translateApiRuleToPostgresRuleRaw = translateApiRuleToPostgresRuleRaw;
+/**
+ * Generate a Postgres rule record from a DynamoDB record and remove nil properties.
+ *
+ * @param {Object} record - A rule
+ * @param {Object} dbClient - Knex client for reading from RDS database
+ * @param {Object} collectionPgModel - Instance of the collection database model
+ * @param {Object} providerPgModel - Instance of the provider database model
+ * @returns {Object} A rule record
+ */
 const translateApiRuleToPostgresRule = async (record, dbClient, collectionPgModel = new collection_1.CollectionPgModel(), providerPgModel = new provider_1.ProviderPgModel()) => {
-    const ruleRecord = {
-        name: record.name,
-        workflow: record.workflow,
-        provider_cumulus_id: record.provider ? await providerPgModel.getRecordCumulusId(dbClient, { name: record.provider }) : undefined,
-        collection_cumulus_id: record.collection ? await collectionPgModel.getRecordCumulusId(dbClient, { name: record.collection.name, version: record.collection.version }) : undefined,
-        meta: record.meta,
-        payload: record.payload,
-        queue_url: record.queueUrl,
-        arn: record.rule.arn,
-        type: record.rule.type,
-        value: record.rule.value,
-        log_event_arn: record.rule.logEventArn,
-        enabled: (record.state === undefined) || (record.state === 'ENABLED'),
-        tags: (record.tags ? JSON.stringify(record.tags) : undefined),
-        execution_name_prefix: record.executionNamePrefix,
-        created_at: (record.createdAt ? new Date(record.createdAt) : undefined),
-        updated_at: (record.updatedAt ? new Date(record.updatedAt) : undefined),
-    };
-    return ruleRecord;
+    const ruleRecord = await (0, exports.translateApiRuleToPostgresRuleRaw)(record, dbClient, collectionPgModel, providerPgModel);
+    return (0, util_1.removeNilProperties)(ruleRecord);
 };
 exports.translateApiRuleToPostgresRule = translateApiRuleToPostgresRule;
 //# sourceMappingURL=rules.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/db",
-  "version": "10.0.1",
+  "version": "10.1.1",
   "description": "Utilities for working with the Cumulus DB",
   "license": "Apache-2.0",
   "main": "./dist/index.js",
@@ -29,16 +29,16 @@
     "node": ">=12.18.0"
   },
   "dependencies": {
-    "@cumulus/aws-client": "10.0.1",
-    "@cumulus/common": "10.0.1",
-    "@cumulus/errors": "10.0.1",
-    "@cumulus/logger": "10.0.1",
-    "@cumulus/message": "10.0.1",
-    "@cumulus/types": "10.0.1",
+    "@cumulus/aws-client": "10.1.1",
+    "@cumulus/common": "10.1.1",
+    "@cumulus/errors": "10.1.1",
+    "@cumulus/logger": "10.1.1",
+    "@cumulus/message": "10.1.1",
+    "@cumulus/types": "10.1.1",
     "crypto-random-string": "^3.2.0",
     "is-valid-hostname": "0.0.1",
     "knex": "0.95.15",
-    "lodash": "^4.17.20",
+    "lodash": "^4.17.21",
     "pg": "^8.3.0",
     "snake-camel": "^1.0.6",
     "uuid": "8.3.2"
@@ -46,5 +46,5 @@
   "devDependencies": {
     "@types/uuid": "^8.0.0"
   },
-  "gitHead": "49c3c88336838184f22f35fbce298c71cd269138"
+  "gitHead": "b34bda5035263c74c8341b25abfb033226a27387"
 }