@cumulus/db 20.3.0 → 21.0.0

package/dist/index.d.ts

@@ -29,7 +29,7 @@ export { translateApiPdrToPostgresPdr, translatePostgresPdrToApiPdr, } from './t
 export { translateApiReconReportToPostgresReconReport, translatePostgresReconReportToApiReconReport, } from './translate/reconciliation_reports';
 export { getCollectionsByGranuleIds, getUniqueCollectionsByGranuleFilter, } from './lib/collection';
 export { batchDeleteExecutionFromDatabaseByCumulusCollectionId, executionArnsFromGranuleIdsAndWorkflowNames, getApiExecutionCumulusIds, getApiGranuleExecutionCumulusIdsByExecution, getExecutionInfoByGranuleCumulusId, getWorkflowNameIntersectFromGranuleIds, newestExecutionArnFromGranuleIdWorkflowName, } from './lib/execution';
-export { getFilesAndGranuleInfoQuery, } from './lib/file';
+export { getFilesAndGranuleInfoQuery, getGranuleIdAndCollectionIdFromFile, } from './lib/file';
 export { getApiGranuleCumulusIds, getApiGranuleExecutionCumulusIds, getGranuleCollectionId, getUniqueGranuleByGranuleId, getGranuleByUniqueColumns, upsertGranuleWithExecutionJoinRecord, getGranulesByApiPropertiesQuery, getGranulesByGranuleId, getGranuleAndCollection, updateBatchGranulesCollection, } from './lib/granule';
 export { QuerySearchClient, } from './lib/QuerySearchClient';
 export { AsyncOperationSearch, } from './search/AsyncOperationSearch';

package/dist/index.js

@@ -24,7 +24,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getApiExecutionCumulusIds = exports.executionArnsFromGranuleIdsAndWorkflowNames = exports.batchDeleteExecutionFromDatabaseByCumulusCollectionId = exports.getUniqueCollectionsByGranuleFilter = exports.getCollectionsByGranuleIds = exports.translatePostgresReconReportToApiReconReport = exports.translateApiReconReportToPostgresReconReport = exports.translatePostgresPdrToApiPdr = exports.translateApiPdrToPostgresPdr = exports.translatePostgresGranuleResultToApiGranule = exports.translatePostgresGranuleToApiGranule = exports.translateApiGranuleToPostgresGranuleWithoutNilsRemoved = exports.translateApiGranuleToPostgresGranule = exports.translatePostgresExecutionToApiExecution = exports.translateApiExecutionToPostgresExecutionWithoutNilsRemoved = exports.translateApiExecutionToPostgresExecution = exports.translateApiRuleToPostgresRuleRaw = exports.translateApiRuleToPostgresRule = exports.translatePostgresRuleToApiRule = exports.translatePostgresProviderToApiProvider = exports.translateApiProviderToPostgresProvider = exports.translatePostgresCollectionToApiCollection = exports.translateApiCollectionToPostgresCollection = exports.translatePostgresFileToApiFile = exports.translateApiFiletoPostgresFile = exports.translatePostgresAsyncOperationToApiAsyncOperation = exports.translateApiAsyncOperationToPostgresAsyncOperation = exports.nullifyUndefinedProviderValues = exports.validateProviderHost = exports.migrationDir = exports.TableNames = exports.createRejectableTransaction = exports.localStackConnectionEnv = exports.getKnexConfig = exports.getKnexClient = exports.isCollisionError = exports.generateLocalTestDb = exports.fakeRuleRecordFactory = exports.fakeReconciliationReportRecordFactory = exports.fakeProviderRecordFactory = exports.fakePdrRecordFactory = exports.fakeGranuleRecordFactory = exports.fakeFileRecordFactory = exports.fakeExecutionRecordFactory = exports.fakeCollectionRecordFactory = exports.fakeAsyncOperationRecordFactory = exports.destroyLocalTestDb = exports.deleteTestDatabase = exports.createTestDatabase = exports.Knex = void 0;
-exports.RulePgModel = exports.ReconciliationReportPgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.ReconciliationReportSearch = exports.StatsSearch = exports.RuleSearch = exports.ProviderSearch = exports.PdrSearch = exports.GranuleSearch = exports.ExecutionSearch = exports.CollectionSearch = exports.AsyncOperationSearch = exports.QuerySearchClient = exports.updateBatchGranulesCollection = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getFilesAndGranuleInfoQuery = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.getWorkflowNameIntersectFromGranuleIds = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = void 0;
+exports.RulePgModel = exports.ReconciliationReportPgModel = exports.ProviderPgModel = exports.PdrPgModel = exports.GranulesExecutionsPgModel = exports.GranulePgModel = exports.FilePgModel = exports.ExecutionPgModel = exports.CollectionPgModel = exports.BasePgModel = exports.AsyncOperationPgModel = exports.ReconciliationReportSearch = exports.StatsSearch = exports.RuleSearch = exports.ProviderSearch = exports.PdrSearch = exports.GranuleSearch = exports.ExecutionSearch = exports.CollectionSearch = exports.AsyncOperationSearch = exports.QuerySearchClient = exports.updateBatchGranulesCollection = exports.getGranuleAndCollection = exports.getGranulesByGranuleId = exports.getGranulesByApiPropertiesQuery = exports.upsertGranuleWithExecutionJoinRecord = exports.getGranuleByUniqueColumns = exports.getUniqueGranuleByGranuleId = exports.getGranuleCollectionId = exports.getApiGranuleExecutionCumulusIds = exports.getApiGranuleCumulusIds = exports.getGranuleIdAndCollectionIdFromFile = exports.getFilesAndGranuleInfoQuery = exports.newestExecutionArnFromGranuleIdWorkflowName = exports.getWorkflowNameIntersectFromGranuleIds = exports.getExecutionInfoByGranuleCumulusId = exports.getApiGranuleExecutionCumulusIdsByExecution = void 0;
 const path = __importStar(require("path"));
 var knex_1 = require("knex");
 Object.defineProperty(exports, "Knex", { enumerable: true, get: function () { return knex_1.Knex; } });
@@ -101,6 +101,7 @@ Object.defineProperty(exports, "getWorkflowNameIntersectFromGranuleIds", { enume
 Object.defineProperty(exports, "newestExecutionArnFromGranuleIdWorkflowName", { enumerable: true, get: function () { return execution_1.newestExecutionArnFromGranuleIdWorkflowName; } });
 var file_2 = require("./lib/file");
 Object.defineProperty(exports, "getFilesAndGranuleInfoQuery", { enumerable: true, get: function () { return file_2.getFilesAndGranuleInfoQuery; } });
+Object.defineProperty(exports, "getGranuleIdAndCollectionIdFromFile", { enumerable: true, get: function () { return file_2.getGranuleIdAndCollectionIdFromFile; } });
 var granule_1 = require("./lib/granule");
 Object.defineProperty(exports, "getApiGranuleCumulusIds", { enumerable: true, get: function () { return granule_1.getApiGranuleCumulusIds; } });
 Object.defineProperty(exports, "getApiGranuleExecutionCumulusIds", { enumerable: true, get: function () { return granule_1.getApiGranuleExecutionCumulusIds; } });

package/dist/lib/file.d.ts

@@ -1,6 +1,31 @@
 import { Knex } from 'knex';
 import { PostgresFileRecord } from '../types/file';
 import { PostgresGranuleRecord } from '../types/granule';
+/**
+ * Retrieves the granule ID, collection name, and collection version associated
+ * with a specific file by joining data across the files, granules, and collections tables.
+ *
+ * @param {Object} params - The parameters for the query.
+ * @param {Knex} params.knex - The Knex client object for database interaction.
+ * @param {string} params.bucket - The S3 bucket of the file.
+ * @param {string} params.key - The S3 key (path) of the file.
+ * @returns {Knex.QueryBuilder} A Knex query builder object that, when executed,
+ * will return the granule_cumulus_id, collection_name, and collection_version
+ * for the specified file.
+ */
+export declare const getGranuleIdAndCollectionIdFromFile: ({ knex, bucket, key, }: {
+    knex: Knex;
+    bucket: string;
+    key: string;
+}) => Knex.QueryBuilder<any, {
+    _base: any;
+    _hasSelection: true;
+    _keys: "granules.granule_id" | "collections.name as collection_name" | "collections.version as collection_version";
+    _aliases: {};
+    _single: false;
+    _intersectProps: {};
+    _unionProps: undefined;
+}>;
 /**
  * Helper to build a query that returns records from the files table with data
  * joined in from the granules table optionally filtered by collectionIds,

package/dist/lib/file.js

@@ -1,8 +1,31 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getFilesAndGranuleInfoQuery = void 0;
+exports.getFilesAndGranuleInfoQuery = exports.getGranuleIdAndCollectionIdFromFile = void 0;
 const Collections_1 = require("@cumulus/message/Collections");
 const tables_1 = require("../tables");
+/**
+ * Retrieves the granule ID, collection name, and collection version associated
+ * with a specific file by joining data across the files, granules, and collections tables.
+ *
+ * @param {Object} params - The parameters for the query.
+ * @param {Knex} params.knex - The Knex client object for database interaction.
+ * @param {string} params.bucket - The S3 bucket of the file.
+ * @param {string} params.key - The S3 key (path) of the file.
+ * @returns {Knex.QueryBuilder} A Knex query builder object that, when executed,
+ * will return the granule_cumulus_id, collection_name, and collection_version
+ * for the specified file.
+ */
+const getGranuleIdAndCollectionIdFromFile = ({ knex, bucket, key, }) => {
+    const { files: filesTable, granules: granulesTable, collections: collectionsTable } = tables_1.TableNames;
+    return knex(filesTable)
+        .select(`${granulesTable}.granule_id`, `${collectionsTable}.name as collection_name`, `${collectionsTable}.version as collection_version`)
+        .innerJoin(granulesTable, `${filesTable}.granule_cumulus_id`, `${granulesTable}.cumulus_id`)
+        .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`)
+        .where(`${filesTable}.bucket`, bucket)
+        .andWhere(`${filesTable}.key`, key)
+        .first();
+};
+exports.getGranuleIdAndCollectionIdFromFile = getGranuleIdAndCollectionIdFromFile;
 /**
  * Helper to build a query that returns records from the files table with data
  * joined in from the granules table optionally filtered by collectionIds,

package/dist/migrations/20250425134823_granules_add_producer_granule_id.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20250425134823_granules_add_producer_granule_id.d.ts.map

package/dist/migrations/20250425134823_granules_add_producer_granule_id.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    if (!await knex.schema.hasColumn('granules', 'producer_granule_id')) {
+        await knex.schema.table('granules', (table) => {
+            table
+                .text('producer_granule_id')
+                .comment('Producer Granule Id');
+        });
+        await knex('granules').update('producer_granule_id', knex.raw('granule_id'));
+        await knex.schema.table('granules', (table) => {
+            table.text('producer_granule_id').notNullable().alter();
+        });
+        await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_producer_granule_id_index ON granules(producer_granule_id)');
+    }
+};
+exports.up = up;
+const down = async (knex) => {
+    if (await knex.schema.hasColumn('granules', 'producer_granule_id')) {
+        await knex.schema.table('granules', (table) => {
+            table.dropColumn('producer_granule_id');
+        });
+    }
+};
+exports.down = down;
+exports.config = {
+    transaction: false,
+};
+//# sourceMappingURL=20250425134823_granules_add_producer_granule_id.js.map

package/dist/migrations/20250617190412_add_archived_and_index.d.ts

@@ -0,0 +1,4 @@
+import { Knex } from 'knex';
+export declare const up: (knex: Knex) => Promise<void>;
+export declare const down: (knex: Knex) => Promise<void>;
+//# sourceMappingURL=20250617190412_add_archived_and_index.d.ts.map

package/dist/migrations/20250617190412_add_archived_and_index.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const up = async (knex) => {
+    if (!(await knex.schema.hasColumn('granules', 'archived'))) {
+        await knex.schema.table('granules', (table) => {
+            table.boolean('archived')
+                .comment('granule has been "archived"')
+                .defaultTo('false')
+                .notNullable();
+        });
+    }
+    if (!(await knex.schema.hasColumn('executions', 'archived'))) {
+        await knex.schema.table('executions', (table) => {
+            table.boolean('archived')
+                .comment('execution has been "archived"')
+                .defaultTo('false')
+                .notNullable();
+        });
+    }
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS executions_archived_index ON executions (archived)');
+    await knex.raw('CREATE INDEX CONCURRENTLY IF NOT EXISTS granules_archived_index ON granules (archived)');
+};
+exports.up = up;
+const down = async (knex) => {
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS granules_archived_index');
+    await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS executions_archived_index');
+    if (await knex.schema.hasColumn('granules', 'archived')) {
+        await knex.schema.table('granules', (table) => {
+            table.dropColumn('archived');
+        });
+    }
+    if (await knex.schema.hasColumn('executions', 'archived')) {
+        await knex.schema.table('executions', (table) => {
+            table.dropColumn('archived');
+        });
+    }
+};
+exports.down = down;
+exports.config = {
+    transaction: false,
+};
+//# sourceMappingURL=20250617190412_add_archived_and_index.js.map

package/dist/search/field-mapping.js

@@ -9,6 +9,9 @@ const logger_1 = __importDefault(require("@cumulus/logger"));
 const log = new logger_1.default({ sender: '@cumulus/db/field-mapping' });
 // functions to map the api search string field name and value to postgres db field
 const granuleMapping = {
+    archived: (value) => ({
+        archived: value,
+    }),
     beginningDateTime: (value) => ({
         beginning_date_time: value,
     }),
@@ -39,6 +42,9 @@ const granuleMapping = {
     processingStartDateTime: (value) => ({
         processing_start_date_time: value,
     }),
+    producerGranuleId: (value) => ({
+        producer_granule_id: value,
+    }),
     productionDateTime: (value) => ({
         production_date_time: value,
     }),
@@ -205,6 +211,9 @@ const executionMapping = {
             collectionVersion: version,
         };
     },
+    archived: (value) => ({
+        archived: value,
+    }),
 };
 const pdrMapping = {
     address: (value) => ({

package/dist/test-utils.js

@@ -69,6 +69,7 @@ const fakeExecutionRecordFactory = (params) => {
     const executionId = (0, uuid_1.v4)();
     const executionARN = `arn:aws:states:us-east-1:12345:execution:test-TestExecution:${executionId}`;
     return {
+        archived: false,
         arn: executionARN,
         url: (0, Executions_1.getExecutionUrlFromArn)(executionARN),
         status: 'running',
@@ -90,6 +91,7 @@ const fakeProviderRecordFactory = (params) => ({
 exports.fakeProviderRecordFactory = fakeProviderRecordFactory;
 const fakeGranuleRecordFactory = (params) => ({
     granule_id: (0, crypto_random_string_1.default)({ length: 5 }),
+    producer_granule_id: (0, crypto_random_string_1.default)({ length: 5 }),
     status: 'completed',
     created_at: new Date(),
     ...params,

package/dist/translate/executions.js

@@ -34,6 +34,7 @@ const translatePostgresExecutionToApiExecutionWithoutDbQuery = ({ executionRecor
         asyncOperationId,
         collectionId,
         parentArn,
+        archived: executionRecord.archived,
         createdAt: executionRecord.created_at.getTime(),
         updatedAt: executionRecord.updated_at.getTime(),
         timestamp: executionRecord.timestamp?.getTime(),
@@ -111,6 +112,7 @@ const translateApiExecutionToPostgresExecutionWithoutNilsRemoved = async (apiRec
     // Map old record to new schema.
     const translatedRecord = {
         async_operation_cumulus_id: (apiRecord.asyncOperationId ? await asyncOperationPgModel.getRecordCumulusId(knex, { id: apiRecord.asyncOperationId }) : ((0, isNull_1.default)(apiRecord.asyncOperationId) ? null : undefined)),
+        archived: apiRecord.archived,
         status: apiRecord.status,
         arn: apiRecord.arn,
         duration: apiRecord.duration,

package/dist/translate/granules.js

@@ -29,6 +29,7 @@ const file_2 = require("./file");
  * @returns An API Granule with associated Files
  */
 const translatePostgresGranuleToApiGranuleWithoutDbQuery = ({ granulePgRecord, collectionPgRecord, executionUrls = [], files = [], pdr, providerPgRecord, }) => (0, util_1.removeNilProperties)({
+    archived: granulePgRecord.archived,
     beginningDateTime: granulePgRecord.beginning_date_time?.toISOString(),
     cmrLink: granulePgRecord.cmr_link,
     collectionId: (0, Collections_1.constructCollectionId)(collectionPgRecord.name, collectionPgRecord.version),
@@ -43,6 +44,7 @@ const translatePostgresGranuleToApiGranuleWithoutDbQuery = ({ granulePgRecord, c
     pdrName: pdr ? pdr.name : undefined,
     processingEndDateTime: granulePgRecord.processing_end_date_time?.toISOString(),
     processingStartDateTime: granulePgRecord.processing_start_date_time?.toISOString(),
+    producerGranuleId: granulePgRecord.producer_granule_id,
     productionDateTime: granulePgRecord.production_date_time?.toISOString(),
     productVolume: granulePgRecord.product_volume,
     provider: providerPgRecord ? providerPgRecord.name : undefined,
@@ -118,8 +120,11 @@ const validateApiToPostgresGranuleObject = (apiGranule) => {
     if ((0, isNil_1.default)(apiGranule.granuleId)) {
         throw new errors_1.ValidationError('granuleId cannot be undefined on a granule, granules must have a collection and a granule ID');
     }
+    if ((0, isNil_1.default)(apiGranule.producerGranuleId)) {
+        throw new errors_1.ValidationError('producerGranuleId cannot be undefined on a granule, granules must have a producerGranuleId');
+    }
     if ((0, isNull_1.default)(apiGranule.status)) {
-        throw new errors_1.ValidationError('status cannot be null on a granule, granules must have a collection and a granule ID');
+        throw new errors_1.ValidationError('status cannot be null on a granule, granules must have a status');
     }
 };
 /**
@@ -159,9 +164,11 @@ const translateApiGranuleToPostgresGranuleWithoutNilsRemoved = async ({ dynamoRe
         provider_cumulus_id = await providerPgModel.getRecordCumulusId(knexOrTransaction, { name: dynamoRecord.provider });
     }
     const granuleRecord = {
+        archived: dynamoRecord.archived,
         granule_id: dynamoRecord.granuleId,
         status: dynamoRecord.status,
         collection_cumulus_id: await collectionPgModel.getRecordCumulusId(knexOrTransaction, { name, version }),
+        producer_granule_id: dynamoRecord.producerGranuleId,
         published: dynamoRecord.published,
         duration: dynamoRecord.duration,
         time_to_archive: dynamoRecord.timeToArchive,

package/dist/types/execution.d.ts

@@ -16,10 +16,12 @@ export interface PostgresExecution {
     updated_at?: Date | null;
     url?: string | null;
     workflow_name?: string | null;
+    archived: boolean;
 }
 export interface PostgresExecutionRecord extends PostgresExecution {
     created_at: Date;
     cumulus_id: number;
     updated_at: Date;
+    archived: boolean;
 }
 //# sourceMappingURL=execution.d.ts.map

package/dist/types/granule.d.ts

@@ -4,6 +4,8 @@ export interface PostgresGranuleUniqueColumns {
     collection_cumulus_id: number;
 }
 export interface PostgresGranule extends PostgresGranuleUniqueColumns {
+    archived: boolean;
+    producer_granule_id: string;
     status?: GranuleStatus;
     cmr_link?: string | null;
     error?: object | null;
@@ -26,7 +28,9 @@ export interface PostgresGranule extends PostgresGranuleUniqueColumns {
     query_fields?: unknown | null;
 }
 export interface PostgresGranuleRecord extends Omit<PostgresGranule, 'product_volume'> {
+    archived: boolean;
     cumulus_id: number;
+    producer_granule_id: string;
     product_volume?: string;
     created_at: Date;
     updated_at: Date;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/db",
-  "version": "20.3.0",
+  "version": "21.0.0",
   "description": "Utilities for working with the Cumulus DB",
   "license": "Apache-2.0",
   "main": "./dist/index.js",
@@ -33,12 +33,12 @@
   },
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.621.0",
-    "@cumulus/aws-client": "20.3.0",
-    "@cumulus/common": "20.3.0",
-    "@cumulus/errors": "20.3.0",
-    "@cumulus/logger": "20.3.0",
-    "@cumulus/message": "20.3.0",
-    "@cumulus/types": "20.3.0",
+    "@cumulus/aws-client": "21.0.0",
+    "@cumulus/common": "21.0.0",
+    "@cumulus/errors": "21.0.0",
+    "@cumulus/logger": "21.0.0",
+    "@cumulus/message": "21.0.0",
+    "@cumulus/types": "21.0.0",
     "crypto-random-string": "^3.2.0",
     "is-valid-hostname": "1.0.2",
     "knex": "2.4.1",
@@ -50,5 +50,5 @@
   "devDependencies": {
     "@types/uuid": "^8.0.0"
   },
-  "gitHead": "0d3a61cd0185f62360b3e8ab6e59f69e93ac7410"
+  "gitHead": "19bb3477969662a9e0b300f10f6df23b6c0654db"
 }