@cumulus/db 21.3.1 → 21.3.2-testlerna.0

package/dist/s3search/GranuleS3Search.js

@@ -0,0 +1,100 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GranuleS3Search = void 0;
+const pick_1 = __importDefault(require("lodash/pick"));
+const logger_1 = __importDefault(require("@cumulus/logger"));
+const duckdbHelpers_1 = require("./duckdbHelpers");
+const DuckDBSearchExecutor_1 = require("./DuckDBSearchExecutor");
+const GranuleSearch_1 = require("../search/GranuleSearch");
+const granules_1 = require("../translate/granules");
+const log = new logger_1.default({ sender: '@cumulus/db/GranuleS3Search' });
+/**
+ * Class to build and execute DuckDB search query for granules
+ */
+class GranuleS3Search extends GranuleSearch_1.GranuleSearch {
+    constructor(event, dbConnection) {
+        super(event, false); // disables estimateTableRowCount
+        this.dbConnection = dbConnection;
+        this.duckDBSearchExecutor = new DuckDBSearchExecutor_1.DuckDBSearchExecutor({
+            dbConnection,
+            dbQueryParameters: this.dbQueryParameters,
+            getMetaTemplate: this._metaTemplate.bind(this),
+            translateRecords: this.translatePostgresRecordsToApiRecords.bind(this),
+        });
+    }
+    /**
+     * Translate postgres records to api records
+     *
+     * @param pgRecords - postgres records returned from query
+     * @param knexClient - DB client
+     * @returns translated api records
+     */
+    async translatePostgresRecordsToApiRecords(pgRecords, knexClient) {
+        log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `);
+        const { fields, includeFullRecord } = this.dbQueryParameters;
+        const fileMapping = {};
+        const executionMapping = {};
+        const cumulusIds = pgRecords.map((record) => record.cumulus_id);
+        if (includeFullRecord) {
+            //get files
+            const files = await (0, duckdbHelpers_1.getFilesByGranuleCumulusIds)({
+                connection: this.dbConnection,
+                granuleCumulusIds: cumulusIds,
+                knexBuilder: knexClient,
+            });
+            files.forEach((file) => {
+                if (!(file.granule_cumulus_id in fileMapping)) {
+                    fileMapping[file.granule_cumulus_id] = [];
+                }
+                fileMapping[file.granule_cumulus_id].push(file);
+            });
+            //get Executions
+            const executions = await (0, duckdbHelpers_1.getExecutionInfoByGranuleCumulusIds)({
+                connection: this.dbConnection,
+                granuleCumulusIds: cumulusIds,
+                knexBuilder: knexClient,
+            });
+            executions.forEach((execution) => {
+                if (!(execution.granule_cumulus_id in executionMapping)) {
+                    executionMapping[execution.granule_cumulus_id] = execution;
+                }
+            });
+        }
+        const apiRecords = pgRecords.map((item) => {
+            const collectionPgRecord = {
+                cumulus_id: item.collection_cumulus_id,
+                name: item.collectionName,
+                version: item.collectionVersion,
+            };
+            const executionUrls = executionMapping[item.cumulus_id]?.url
+                ? [{ url: executionMapping[item.cumulus_id].url }]
+                : [];
+            const pdr = item.pdrName ? { name: item.pdrName } : undefined;
+            const providerPgRecord = item.providerName ? { name: item.providerName } : undefined;
+            const fileRecords = fileMapping[item.cumulus_id] || [];
+            const apiRecord = (0, granules_1.translatePostgresGranuleToApiGranuleWithoutDbQuery)({
+                granulePgRecord: item,
+                collectionPgRecord,
+                pdr,
+                providerPgRecord,
+                files: fileRecords,
+                executionUrls,
+            });
+            return fields ? (0, pick_1.default)(apiRecord, fields) : apiRecord;
+        });
+        return apiRecords;
+    }
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    async query() {
+        return this.duckDBSearchExecutor.query((knexBuilder) => this.buildSearch(knexBuilder));
+    }
+}
+exports.GranuleS3Search = GranuleS3Search;
+//# sourceMappingURL=GranuleS3Search.js.map

package/dist/s3search/PdrS3Search.d.ts

@@ -0,0 +1,20 @@
+import { DuckDBConnection } from '@duckdb/node-api';
+import { PdrSearch } from '../search/PdrSearch';
+import { QueryEvent } from '../types/search';
+/**
+ * Class to build and execute DuckDB search query for PDRs
+ */
+export declare class PdrS3Search extends PdrSearch {
+    private duckDBSearchExecutor;
+    constructor(event: QueryEvent, dbConnection: DuckDBConnection);
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    query(): Promise<{
+        meta: import("../search/BaseSearch").Meta;
+        results: any[];
+    }>;
+}
+//# sourceMappingURL=PdrS3Search.d.ts.map

package/dist/s3search/PdrS3Search.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PdrS3Search = void 0;
+const DuckDBSearchExecutor_1 = require("./DuckDBSearchExecutor");
+const PdrSearch_1 = require("../search/PdrSearch");
+/**
+ * Class to build and execute DuckDB search query for PDRs
+ */
+class PdrS3Search extends PdrSearch_1.PdrSearch {
+    constructor(event, dbConnection) {
+        super(event);
+        this.duckDBSearchExecutor = new DuckDBSearchExecutor_1.DuckDBSearchExecutor({
+            dbConnection,
+            dbQueryParameters: this.dbQueryParameters,
+            getMetaTemplate: this._metaTemplate.bind(this),
+            translateRecords: this.translatePostgresRecordsToApiRecords.bind(this),
+        });
+    }
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    async query() {
+        return this.duckDBSearchExecutor.query((knexBuilder) => this.buildSearch(knexBuilder));
+    }
+}
+exports.PdrS3Search = PdrS3Search;
+//# sourceMappingURL=PdrS3Search.js.map

package/dist/s3search/ProviderS3Search.d.ts

@@ -0,0 +1,20 @@
+import { DuckDBConnection } from '@duckdb/node-api';
+import { ProviderSearch } from '../search/ProviderSearch';
+import { QueryEvent } from '../types/search';
+/**
+ * Class to build and execute DuckDB search query for providers
+ */
+export declare class ProviderS3Search extends ProviderSearch {
+    private duckDBSearchExecutor;
+    constructor(event: QueryEvent, dbConnection: DuckDBConnection);
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    query(): Promise<{
+        meta: import("../search/BaseSearch").Meta;
+        results: any[];
+    }>;
+}
+//# sourceMappingURL=ProviderS3Search.d.ts.map

package/dist/s3search/ProviderS3Search.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProviderS3Search = void 0;
+const DuckDBSearchExecutor_1 = require("./DuckDBSearchExecutor");
+const ProviderSearch_1 = require("../search/ProviderSearch");
+/**
+ * Class to build and execute DuckDB search query for providers
+ */
+class ProviderS3Search extends ProviderSearch_1.ProviderSearch {
+    constructor(event, dbConnection) {
+        super(event);
+        this.duckDBSearchExecutor = new DuckDBSearchExecutor_1.DuckDBSearchExecutor({
+            dbConnection,
+            dbQueryParameters: this.dbQueryParameters,
+            getMetaTemplate: this._metaTemplate.bind(this),
+            translateRecords: this.translatePostgresRecordsToApiRecords.bind(this),
+        });
+    }
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    async query() {
+        return this.duckDBSearchExecutor.query((knexBuilder) => this.buildSearch(knexBuilder));
+    }
+}
+exports.ProviderS3Search = ProviderS3Search;
+//# sourceMappingURL=ProviderS3Search.js.map

package/dist/s3search/ReconciliationReportS3Search.d.ts

@@ -0,0 +1,20 @@
+import { DuckDBConnection } from '@duckdb/node-api';
+import { ReconciliationReportSearch } from '../search/ReconciliationReportSearch';
+import { QueryEvent } from '../types/search';
+/**
+ * Class to build and execute DuckDB search query for Reconciliation Report
+ */
+export declare class ReconciliationReportS3Search extends ReconciliationReportSearch {
+    private duckDBSearchExecutor;
+    constructor(event: QueryEvent, dbConnection: DuckDBConnection);
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    query(): Promise<{
+        meta: import("../search/BaseSearch").Meta;
+        results: any[];
+    }>;
+}
+//# sourceMappingURL=ReconciliationReportS3Search.d.ts.map

package/dist/s3search/ReconciliationReportS3Search.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ReconciliationReportS3Search = void 0;
+const DuckDBSearchExecutor_1 = require("./DuckDBSearchExecutor");
+const ReconciliationReportSearch_1 = require("../search/ReconciliationReportSearch");
+/**
+ * Class to build and execute DuckDB search query for Reconciliation Report
+ */
+class ReconciliationReportS3Search extends ReconciliationReportSearch_1.ReconciliationReportSearch {
+    constructor(event, dbConnection) {
+        super(event);
+        this.duckDBSearchExecutor = new DuckDBSearchExecutor_1.DuckDBSearchExecutor({
+            dbConnection,
+            dbQueryParameters: this.dbQueryParameters,
+            getMetaTemplate: this._metaTemplate.bind(this),
+            translateRecords: this.translatePostgresRecordsToApiRecords.bind(this),
+        });
+    }
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    async query() {
+        return this.duckDBSearchExecutor.query((knexBuilder) => this.buildSearch(knexBuilder));
+    }
+}
+exports.ReconciliationReportS3Search = ReconciliationReportS3Search;
+//# sourceMappingURL=ReconciliationReportS3Search.js.map

package/dist/s3search/RuleS3Search.d.ts

@@ -0,0 +1,20 @@
+import { DuckDBConnection } from '@duckdb/node-api';
+import { RuleSearch } from '../search/RuleSearch';
+import { QueryEvent } from '../types/search';
+/**
+ * Class to build and execute DuckDB search query for rules
+ */
+export declare class RuleS3Search extends RuleSearch {
+    private duckDBSearchExecutor;
+    constructor(event: QueryEvent, dbConnection: DuckDBConnection);
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    query(): Promise<{
+        meta: import("../search/BaseSearch").Meta;
+        results: any[];
+    }>;
+}
+//# sourceMappingURL=RuleS3Search.d.ts.map

package/dist/s3search/RuleS3Search.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RuleS3Search = void 0;
+const DuckDBSearchExecutor_1 = require("./DuckDBSearchExecutor");
+const RuleSearch_1 = require("../search/RuleSearch");
+/**
+ * Class to build and execute DuckDB search query for rules
+ */
+class RuleS3Search extends RuleSearch_1.RuleSearch {
+    constructor(event, dbConnection) {
+        super(event);
+        this.duckDBSearchExecutor = new DuckDBSearchExecutor_1.DuckDBSearchExecutor({
+            dbConnection,
+            dbQueryParameters: this.dbQueryParameters,
+            getMetaTemplate: this._metaTemplate.bind(this),
+            translateRecords: this.translatePostgresRecordsToApiRecords.bind(this),
+        });
+    }
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    async query() {
+        return this.duckDBSearchExecutor.query((knexBuilder) => this.buildSearch(knexBuilder));
+    }
+}
+exports.RuleS3Search = RuleS3Search;
+//# sourceMappingURL=RuleS3Search.js.map

package/dist/s3search/StatsS3Search.d.ts

@@ -0,0 +1,25 @@
+import { DuckDBConnection } from '@duckdb/node-api';
+import { QueryEvent } from '../types/search';
+import { ApiAggregateResult, StatsSearch, SummaryResult } from '../search/StatsSearch';
+/**
+ * A class to query postgres for the STATS and STATS/AGGREGATE endpoints
+ */
+declare class StatsS3Search extends StatsSearch {
+    private dbConnection;
+    private knexBuilder;
+    constructor(event: QueryEvent, type: string, dbConnection: DuckDBConnection);
+    /**
+     * Queries postgres for a summary of statistics around the granules in the system
+     *
+     * @returns the postgres aggregations based on query
+     */
+    summary(): Promise<SummaryResult>;
+    /**
+     * Executes the aggregate search query
+     *
+     * @returns the aggregate query results in api format
+     */
+    aggregate(): Promise<ApiAggregateResult>;
+}
+export { StatsS3Search };
+//# sourceMappingURL=StatsS3Search.d.ts.map

package/dist/s3search/StatsS3Search.js

@@ -0,0 +1,51 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StatsS3Search = void 0;
+const knex_1 = require("knex");
+const logger_1 = __importDefault(require("@cumulus/logger"));
+const StatsSearch_1 = require("../search/StatsSearch");
+const duckdbHelpers_1 = require("./duckdbHelpers");
+const log = new logger_1.default({ sender: '@cumulus/db/StatsSearch' });
+/**
+ * A class to query postgres for the STATS and STATS/AGGREGATE endpoints
+ */
+class StatsS3Search extends StatsSearch_1.StatsSearch {
+    constructor(event, type, dbConnection) {
+        super(event, type);
+        this.dbConnection = dbConnection;
+        // Use 'pg' dialect to generate DuckDB-compatible SQL ($1, $2, etc.)
+        this.knexBuilder = (0, knex_1.knex)({ client: 'pg' });
+    }
+    /**
+     * Queries postgres for a summary of statistics around the granules in the system
+     *
+     * @returns the postgres aggregations based on query
+     */
+    async summary() {
+        const aggregateQuery = this.buildSummaryQuery(this.knexBuilder);
+        log.debug(`summary about to execute query: ${aggregateQuery?.toSQL().sql}`);
+        const { sql, bindings } = aggregateQuery.toSQL().toNative();
+        const reader = await this.dbConnection.runAndReadAll(sql, (0, duckdbHelpers_1.prepareBindings)([...bindings]) // prepareBindings must be imported/defined in scope
+        );
+        const aggregateQueryRes = reader.getRowObjectsJson();
+        return this.formatSummaryResult(aggregateQueryRes[0]);
+    }
+    /**
+     * Executes the aggregate search query
+     *
+     * @returns the aggregate query results in api format
+     */
+    async aggregate() {
+        const { searchQuery } = this.buildSearch(this.knexBuilder);
+        const { sql, bindings } = searchQuery.toSQL().toNative();
+        const reader = await this.dbConnection.runAndReadAll(sql, (0, duckdbHelpers_1.prepareBindings)([...bindings]) // prepareBindings must be imported/defined in scope
+        );
+        const records = reader.getRowObjectsJson();
+        return this.formatAggregateResult(records);
+    }
+}
+exports.StatsS3Search = StatsS3Search;
+//# sourceMappingURL=StatsS3Search.js.map

package/dist/s3search/duckdbHelpers.d.ts

@@ -0,0 +1,43 @@
+import { DuckDBValue, DuckDBConnection } from '@duckdb/node-api';
+import { Knex } from 'knex';
+import { PostgresFileRecord } from '../types/file';
+export declare function prepareBindings(bindings: ReadonlyArray<any>): DuckDBValue[];
+/**
+ * Returns execution records sorted by most recent first for an input
+ * set of Granule Cumulus IDs.
+ *
+ * @param {object} params - The function parameters.
+ * @param {DuckDBConnection} params.connection - The active DuckDB connection.
+ * @param {number[]} params.granuleCumulusIds - Array of granule IDs to filter by.
+ * @param {Knex} [params.knexBuilder] - Optional Knex instance (defaults to 'pg' client).
+ * @param {number} [params.limit] - Optional limit for the number of records returned.
+ * @returns {Promise<{ granule_cumulus_id: number, url: string }[]>}
+ *   Array of objects containing granule_cumulus_id and execution url, sorted by timestamp desc.
+ */
+export declare const getExecutionInfoByGranuleCumulusIds: ({ connection, granuleCumulusIds, knexBuilder, limit, }: {
+    connection: DuckDBConnection;
+    granuleCumulusIds: number[];
+    knexBuilder: Knex;
+    limit?: number | undefined;
+}) => Promise<{
+    granule_cumulus_id: number;
+    url: string;
+}[]>;
+/**
+ * Searches for file records by granule cumulus IDs using a DuckDB connection.
+ *
+ * @param params - Function parameters
+ * @param params.connection - Active DuckDB connection used to execute the query
+ * @param params.granuleCumulusIds - Array of granule Cumulus IDs to filter by
+ * @param [params.columns='*'] - Columns to select (string or string array)
+ * @param [params.knexBuilder] - Optional Knex instance (defaults to PostgreSQL dialect)
+ * @returns Promise resolving to an array of normalized `PostgresFileRecord` objects
+ * @throws If the DuckDB query execution fails
+ */
+export declare const getFilesByGranuleCumulusIds: ({ connection, granuleCumulusIds, columns, knexBuilder, }: {
+    connection: DuckDBConnection;
+    granuleCumulusIds: number[];
+    columns?: string | string[] | undefined;
+    knexBuilder?: Knex<any, any[]> | undefined;
+}) => Promise<PostgresFileRecord[]>;
+//# sourceMappingURL=duckdbHelpers.d.ts.map

package/dist/s3search/duckdbHelpers.js

@@ -0,0 +1,83 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFilesByGranuleCumulusIds = exports.getExecutionInfoByGranuleCumulusIds = exports.prepareBindings = void 0;
+const knex_1 = require("knex");
+const logger_1 = __importDefault(require("@cumulus/logger"));
+const tables_1 = require("../tables");
+const log = new logger_1.default({ sender: '@cumulus/db/duckdbHelpers' });
+function prepareBindings(bindings) {
+    return bindings.map((value) => {
+        if (value instanceof Date)
+            return value.toISOString();
+        if (value !== null && typeof value === 'object')
+            return JSON.stringify(value);
+        return value;
+    });
+}
+exports.prepareBindings = prepareBindings;
+/**
+ * Returns execution records sorted by most recent first for an input
+ * set of Granule Cumulus IDs.
+ *
+ * @param {object} params - The function parameters.
+ * @param {DuckDBConnection} params.connection - The active DuckDB connection.
+ * @param {number[]} params.granuleCumulusIds - Array of granule IDs to filter by.
+ * @param {Knex} [params.knexBuilder] - Optional Knex instance (defaults to 'pg' client).
+ * @param {number} [params.limit] - Optional limit for the number of records returned.
+ * @returns {Promise<{ granule_cumulus_id: number, url: string }[]>}
+ *   Array of objects containing granule_cumulus_id and execution url, sorted by timestamp desc.
+ */
+const getExecutionInfoByGranuleCumulusIds = async ({ connection, granuleCumulusIds, knexBuilder = (0, knex_1.knex)({ client: 'pg' }), limit, }) => {
+    const knexQuery = knexBuilder(tables_1.TableNames.executions)
+        .select([
+        `${tables_1.TableNames.executions}.url`,
+        `${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`,
+    ])
+        .join(tables_1.TableNames.granulesExecutions, `${tables_1.TableNames.executions}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.execution_cumulus_id`)
+        .whereIn(`${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusIds)
+        .orderBy(`${tables_1.TableNames.executions}.timestamp`, 'desc');
+    if (limit)
+        knexQuery.limit(limit);
+    const { sql, bindings } = knexQuery.toSQL().toNative();
+    log.debug(`getExecutionInfoByGranuleCumulusIds query: ${sql}`);
+    // Use spread operator to convert ReadonlyArray to mutable array
+    const reader = await connection.runAndReadAll(sql, prepareBindings([...bindings]));
+    return reader.getRowObjectsJson();
+};
+exports.getExecutionInfoByGranuleCumulusIds = getExecutionInfoByGranuleCumulusIds;
+/**
+ * Searches for file records by granule cumulus IDs using a DuckDB connection.
+ *
+ * @param params - Function parameters
+ * @param params.connection - Active DuckDB connection used to execute the query
+ * @param params.granuleCumulusIds - Array of granule Cumulus IDs to filter by
+ * @param [params.columns='*'] - Columns to select (string or string array)
+ * @param [params.knexBuilder] - Optional Knex instance (defaults to PostgreSQL dialect)
+ * @returns Promise resolving to an array of normalized `PostgresFileRecord` objects
+ * @throws If the DuckDB query execution fails
+ */
+const getFilesByGranuleCumulusIds = async ({ connection, granuleCumulusIds, columns = '*', knexBuilder = (0, knex_1.knex)({ client: 'pg' }), }) => {
+    const knexQuery = knexBuilder(tables_1.TableNames.files)
+        .select(columns)
+        .whereIn('granule_cumulus_id', granuleCumulusIds);
+    const { sql, bindings } = knexQuery.toSQL().toNative();
+    // Execute using DuckDB connection
+    const reader = await connection.runAndReadAll(sql, prepareBindings([...bindings]) // prepareBindings must be imported/defined in scope
+    );
+    const rows = reader.getRowObjectsJson();
+    // Mapping resolves TS errors and handles DuckDB type conversions
+    return rows.map((row) => ({
+        ...row,
+        // Ensure IDs are numbers (handles DuckDB BigInt/String return types)
+        cumulus_id: Number(row.cumulus_id),
+        granule_cumulus_id: Number(row.granule_cumulus_id),
+        // Convert ISO timestamp strings back to JS Date objects
+        created_at: row.created_at ? new Date(row.created_at) : undefined,
+        updated_at: row.updated_at ? new Date(row.updated_at) : undefined,
+    }));
+};
+exports.getFilesByGranuleCumulusIds = getFilesByGranuleCumulusIds;
+//# sourceMappingURL=duckdbHelpers.js.map

package/dist/s3search/s3TableSchemas.d.ts

@@ -0,0 +1,11 @@
+export declare const asyncOperationsS3TableSql: (tableName?: string) => string;
+export declare const collectionsS3TableSql: (tableName?: string) => string;
+export declare const executionsS3TableSql: (tableName?: string) => string;
+export declare const filesS3TableSql: (tableName?: string) => string;
+export declare const granulesS3TableSql: (tableName?: string) => string;
+export declare const granulesExecutionsS3TableSql: (tableName?: string) => string;
+export declare const pdrsS3TableSql: (tableName?: string) => string;
+export declare const providersS3TableSql: (tableName?: string) => string;
+export declare const reconciliationReportsS3TableSql: (tableName?: string) => string;
+export declare const rulesS3TableSql: (tableName?: string) => string;
+//# sourceMappingURL=s3TableSchemas.d.ts.map