@cumulus/db 18.3.4 → 19.0.0

package/dist/search/StatsSearch.js

@@ -0,0 +1,214 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StatsSearch = void 0;
+const omit_1 = __importDefault(require("lodash/omit"));
+const logger_1 = __importDefault(require("@cumulus/logger"));
+const connection_1 = require("../connection");
+const tables_1 = require("../tables");
+const BaseSearch_1 = require("./BaseSearch");
+const log = new logger_1.default({ sender: '@cumulus/db/StatsSearch' });
+const infixMapping = {
+    granules: 'granule_id',
+    collections: 'name',
+    providers: 'name',
+    executions: 'arn',
+    pdrs: 'name',
+};
+/**
+ * A class to query postgres for the STATS and STATS/AGGREGATE endpoints
+ */
+class StatsSearch extends BaseSearch_1.BaseSearch {
+    constructor(event, type) {
+        const { field, ...queryStringParameters } = event.queryStringParameters || {};
+        super({ queryStringParameters }, type);
+        this.field = field ?? 'status';
+        this.dbQueryParameters = (0, omit_1.default)(this.dbQueryParameters, ['limit', 'offset']);
+    }
+    /**
+     * Formats the postgres records into an API stats/aggregate response
+     *
+     * @param result - the postgres query results
+     * @returns the api object with the aggregate statistics
+     */
+    formatAggregateResult(result) {
+        let totalCount = 0;
+        const responses = [];
+        for (const row of Object.keys(result)) {
+            responses.push({
+                key: result[row].aggregatedfield,
+                count: Number.parseInt(result[row].count, 10),
+            });
+            totalCount += Number(result[row].count);
+        }
+        return {
+            meta: {
+                name: 'cumulus-api',
+                count: totalCount,
+                field: this.field,
+            },
+            count: responses,
+        };
+    }
+    /**
+     * Formats the postgres results into an API stats/summary response
+     *
+     * @param result - the knex summary query results
+     * @returns the api object with the summary statistics
+     */
+    formatSummaryResult(result) {
+        const timestampTo = this.dbQueryParameters.range?.updated_at?.lte ?? new Date();
+        const timestampFrom = this.dbQueryParameters.range?.updated_at?.gte ?? new Date(0);
+        const dateto = timestampTo.toISOString();
+        const datefrom = timestampFrom.toISOString();
+        return {
+            errors: {
+                dateFrom: datefrom,
+                dateTo: dateto,
+                value: Number(result.count_errors),
+                aggregation: 'count',
+                unit: 'error',
+            },
+            collections: {
+                dateFrom: datefrom,
+                dateTo: dateto,
+                value: Number(result.count_collections),
+                aggregation: 'count',
+                unit: 'collection',
+            },
+            processingTime: {
+                dateFrom: datefrom,
+                dateTo: dateto,
+                value: Number(result.avg_processing_time),
+                aggregation: 'average',
+                unit: 'second',
+            },
+            granules: {
+                dateFrom: datefrom,
+                dateTo: dateto,
+                value: Number(result.count_granules),
+                aggregation: 'count',
+                unit: 'granule',
+            },
+        };
+    }
+    /**
+     * Queries postgres for a summary of statistics around the granules in the system
+     *
+     * @param testKnex - the knex client to be used
+     * @returns the postgres aggregations based on query
+     */
+    async summary(testKnex) {
+        const knex = testKnex ?? await (0, connection_1.getKnexClient)();
+        const aggregateQuery = knex(this.tableName);
+        this.buildRangeQuery({ searchQuery: aggregateQuery });
+        aggregateQuery.select(knex.raw(`COUNT(CASE WHEN ${this.tableName}.error ->> 'Error' is not null THEN 1 END) AS count_errors`), knex.raw('COUNT(*) AS count_granules'), knex.raw(`AVG(${this.tableName}.duration) AS avg_processing_time`), knex.raw(`COUNT(DISTINCT ${this.tableName}.collection_cumulus_id) AS count_collections`));
+        log.debug(`summary about to execute query: ${aggregateQuery?.toSQL().sql}`);
+        const aggregateQueryRes = await aggregateQuery;
+        return this.formatSummaryResult(aggregateQueryRes[0]);
+    }
+    /**
+     * Performs joins on the collections/pdrs/providers table if neccessary
+     *
+     * @param query - the knex query to be joined or not
+     */
+    joinTables(query) {
+        const { collections: collectionsTable, providers: providersTable, pdrs: pdrsTable, } = tables_1.TableNames;
+        if (this.searchCollection()) {
+            query.join(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`);
+        }
+        if (this.searchProvider()) {
+            query.join(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`);
+        }
+        if (this.searchPdr()) {
+            query.join(pdrsTable, `${this.tableName}.pdr_cumulus_id`, `${pdrsTable}.cumulus_id`);
+        }
+    }
+    /**
+     * Aggregates the search query based on queryStringParameters
+     *
+     * @param query - the knex query to be aggregated
+     * @param knex - the knex client to be used
+     */
+    aggregateQueryField(query, knex) {
+        if (this.field?.includes('error.Error')) {
+            query.select(knex.raw("error ->> 'Error' as aggregatedfield"));
+        }
+        else {
+            query.select(`${this.tableName}.${this.field} as aggregatedfield`);
+        }
+        query.modify((queryBuilder) => this.joinTables(queryBuilder))
+            .count('* as count')
+            .groupBy('aggregatedfield')
+            .orderBy([{ column: 'count', order: 'desc' }, { column: 'aggregatedfield' }]);
+    }
+    /**
+     * Builds basic query
+     *
+     * @param knex - the knex client
+     * @returns the search query
+     */
+    buildBasicQuery(knex) {
+        const searchQuery = knex(this.tableName);
+        this.aggregateQueryField(searchQuery, knex);
+        return { searchQuery };
+    }
+    /**
+     * Builds queries for infix and prefix
+     *
+     * @param params
+     * @param params.searchQuery - the search query
+     * @param [params.dbQueryParameters] - the db query parameters
+     */
+    buildInfixPrefixQuery(params) {
+        const { searchQuery, dbQueryParameters } = params;
+        const { infix, prefix } = dbQueryParameters || this.dbQueryParameters;
+        const fieldName = infixMapping[this.tableName];
+        if (infix) {
+            searchQuery.whereLike(`${this.tableName}.${fieldName}`, `%${infix}%`);
+        }
+        if (prefix) {
+            searchQuery.whereLike(`${this.tableName}.${fieldName}`, `%${prefix}%`);
+        }
+    }
+    /**
+     * Builds queries for term fields
+     *
+     * @param params
+     * @param params.searchQuery - the search query
+     * @param [params.dbQueryParameters] - the db query parameters
+     * @returns the updated search query based on queryStringParams
+     */
+    buildTermQuery(params) {
+        const { dbQueryParameters, searchQuery } = params;
+        const { term = {} } = dbQueryParameters ?? this.dbQueryParameters;
+        if (this.field?.includes('error.Error')) {
+            searchQuery.whereRaw(`${this.tableName}.error ->> 'Error' is not null`);
+        }
+        return super.buildTermQuery({
+            ...params,
+            dbQueryParameters: { term: (0, omit_1.default)(term, 'error.Error') },
+        });
+    }
+    /**
+     * Executes the aggregate search query
+     *
+     * @param testKnex - the knex client to be used
+     * @returns the aggregate query results in api format
+     */
+    async aggregate(testKnex) {
+        const knex = testKnex ?? await (0, connection_1.getKnexClient)();
+        const { searchQuery } = this.buildSearch(knex);
+        try {
+            const pgRecords = await searchQuery;
+            return this.formatAggregateResult(pgRecords);
+        }
+        catch (error) {
+            return error;
+        }
+    }
+}
+exports.StatsSearch = StatsSearch;
+//# sourceMappingURL=StatsSearch.js.map

package/dist/search/field-mapping.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Map query string field to db field
+ *
+ * @param type - query record type
+ * @param queryField - query field
+ * @param queryField.name - query field value
+ * @param [queryField.value] - query field value
+ * @returns db field
+ */
+export declare const mapQueryStringFieldToDbField: (type: string, queryField: {
+    name: string;
+    value?: string;
+}) => {
+    [key: string]: any;
+} | undefined;
+//# sourceMappingURL=field-mapping.d.ts.map

package/dist/search/field-mapping.js

@@ -0,0 +1,304 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapQueryStringFieldToDbField = void 0;
+const Collections_1 = require("@cumulus/message/Collections");
+const logger_1 = __importDefault(require("@cumulus/logger"));
+const log = new logger_1.default({ sender: '@cumulus/db/field-mapping' });
+// functions to map the api search string field name and value to postgres db field
+const granuleMapping = {
+    beginningDateTime: (value) => ({
+        beginning_date_time: value,
+    }),
+    cmrLink: (value) => ({
+        cmr_link: value,
+    }),
+    createdAt: (value) => ({
+        created_at: value && new Date(Number(value)),
+    }),
+    duration: (value) => ({
+        duration: value && Number(value),
+    }),
+    endingDateTime: (value) => ({
+        ending_date_time: value,
+    }),
+    granuleId: (value) => ({
+        granule_id: value,
+    }),
+    _id: (value) => ({
+        granule_id: value,
+    }),
+    lastUpdateDateTime: (value) => ({
+        last_update_date_time: value,
+    }),
+    processingEndDateTime: (value) => ({
+        processing_end_date_time: value,
+    }),
+    processingStartDateTime: (value) => ({
+        processing_start_date_time: value,
+    }),
+    productionDateTime: (value) => ({
+        production_date_time: value,
+    }),
+    productVolume: (value) => ({
+        product_volume: value,
+    }),
+    published: (value) => ({
+        published: (value === 'true'),
+    }),
+    status: (value) => ({
+        status: value,
+    }),
+    timestamp: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    timeToArchive: (value) => ({
+        time_to_archive: Number(value),
+    }),
+    timeToPreprocess: (value) => ({
+        time_to_process: Number(value),
+    }),
+    updatedAt: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    error: (value) => ({
+        error: value,
+    }),
+    // nested error field
+    'error.Error': (value) => ({
+        'error.Error': value,
+    }),
+    'error.Error.keyword': (value) => ({
+        'error.Error': value,
+    }),
+    // The following fields require querying other tables
+    collectionId: (value) => {
+        const { name, version } = (value && (0, Collections_1.deconstructCollectionId)(value)) || {};
+        return {
+            collectionName: name,
+            collectionVersion: version,
+        };
+    },
+    provider: (value) => ({
+        providerName: value,
+    }),
+    pdrName: (value) => ({
+        pdrName: value,
+    }),
+};
+const asyncOperationMapping = {
+    createdAt: (value) => ({
+        created_at: value && new Date(Number(value)),
+    }),
+    id: (value) => ({
+        id: value,
+    }),
+    operationType: (value) => ({
+        operation_type: value,
+    }),
+    status: (value) => ({
+        status: value,
+    }),
+    taskArn: (value) => ({
+        task_arn: value,
+    }),
+    timestamp: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    updatedAt: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+};
+const collectionMapping = {
+    createdAt: (value) => ({
+        created_at: value && new Date(Number(value)),
+    }),
+    name: (value) => ({
+        name: value,
+    }),
+    version: (value) => ({
+        version: value,
+    }),
+    _id: (value) => {
+        const { name, version } = (value && (0, Collections_1.deconstructCollectionId)(value)) || {};
+        return {
+            collectionName: name,
+            collectionVersion: version,
+        };
+    },
+    duplicateHandling: (value) => ({
+        duplicate_handling: value,
+    }),
+    granuleId: (value) => ({
+        granule_id_validation_regex: value,
+    }),
+    granuleIdExtraction: (value) => ({
+        granule_id_extraction_regex: value,
+    }),
+    timestamp: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    updatedAt: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    reportToEms: (value) => ({
+        report_to_ems: (value === 'true'),
+    }),
+    process: (value) => ({
+        process: value,
+    }),
+    sampleFileName: (value) => ({
+        sample_file_name: value,
+    }),
+    url_path: (value) => ({
+        url_path: value,
+    }),
+};
+// TODO add and verify all queryable fields for the following record types
+const executionMapping = {
+    arn: (value) => ({
+        arn: value,
+    }),
+    createdAt: (value) => ({
+        created_at: value && new Date(Number(value)),
+    }),
+    duration: (value) => ({
+        duration: value && Number(value),
+    }),
+    // nested error field
+    'error.Error': (value) => ({
+        'error.Error': value,
+    }),
+    'error.Error.keyword': (value) => ({
+        'error.Error': value,
+    }),
+    execution: (value) => ({
+        url: value,
+    }),
+    type: (value) => ({
+        workflow_name: value,
+    }),
+    status: (value) => ({
+        status: value,
+    }),
+    timestamp: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    updatedAt: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    // The following fields require querying other tables
+    asyncOperationId: (value) => ({
+        asyncOperationId: value,
+    }),
+    parentArn: (value) => ({
+        parentArn: value,
+    }),
+    collectionId: (value) => {
+        const { name, version } = (value && (0, Collections_1.deconstructCollectionId)(value)) || {};
+        return {
+            collectionName: name,
+            collectionVersion: version,
+        };
+    },
+};
+const pdrMapping = {
+    createdAt: (value) => ({
+        created_at: value && new Date(Number(value)),
+    }),
+    pdrName: (value) => ({
+        name: value,
+    }),
+    status: (value) => ({
+        status: value,
+    }),
+    timestamp: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    updatedAt: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    // The following fields require querying other tables
+    collectionId: (value) => {
+        const { name, version } = (value && (0, Collections_1.deconstructCollectionId)(value)) || {};
+        return {
+            collectionName: name,
+            collectionVersion: version,
+        };
+    },
+    provider: (value) => ({
+        providerName: value,
+    }),
+};
+const providerMapping = {
+    createdAt: (value) => ({
+        created_at: value && new Date(Number(value)),
+    }),
+    id: (value) => ({
+        name: value,
+    }),
+    timestamp: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    updatedAt: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+};
+const ruleMapping = {
+    createdAt: (value) => ({
+        created_at: value && new Date(Number(value)),
+    }),
+    name: (value) => ({
+        name: value,
+    }),
+    state: (value) => ({
+        enabled: (value === 'ENABLED'),
+    }),
+    timestamp: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    updatedAt: (value) => ({
+        updated_at: value && new Date(Number(value)),
+    }),
+    // The following fields require querying other tables
+    collectionId: (value) => {
+        const { name, version } = (value && (0, Collections_1.deconstructCollectionId)(value)) || {};
+        return {
+            collectionName: name,
+            collectionVersion: version,
+        };
+    },
+    provider: (value) => ({
+        providerName: value,
+    }),
+};
+// type and its mapping
+const supportedMappings = {
+    granule: granuleMapping,
+    asyncOperation: asyncOperationMapping,
+    collection: collectionMapping,
+    execution: executionMapping,
+    pdr: pdrMapping,
+    provider: providerMapping,
+    rule: ruleMapping,
+};
+/**
+ * Map query string field to db field
+ *
+ * @param type - query record type
+ * @param queryField - query field
+ * @param queryField.name - query field value
+ * @param [queryField.value] - query field value
+ * @returns db field
+ */
+const mapQueryStringFieldToDbField = (type, queryField) => {
+    if (!(supportedMappings[type] && supportedMappings[type][queryField.name])) {
+        log.warn(`No db mapping field found for type: ${type}, field ${JSON.stringify(queryField)}`);
+        return undefined;
+    }
+    return supportedMappings[type] && supportedMappings[type][queryField.name](queryField.value);
+};
+exports.mapQueryStringFieldToDbField = mapQueryStringFieldToDbField;
+//# sourceMappingURL=field-mapping.js.map

package/dist/search/queries.d.ts

@@ -0,0 +1,10 @@
+import { DbQueryParameters, QueryStringParameters } from '../types/search';
+/**
+ * Convert api query string parameters to db query parameters
+ *
+ * @param type - query record type
+ * @param queryStringParameters - query string parameters
+ * @returns db query parameters
+ */
+export declare const convertQueryStringToDbQueryParameters: (type: string, queryStringParameters: QueryStringParameters) => DbQueryParameters;
+//# sourceMappingURL=queries.d.ts.map