@cumulus/db 21.3.1-alpha.0 → 21.3.2-testlerna.0

package/dist/s3search/ReconciliationReportS3Search.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ReconciliationReportS3Search = void 0;
+const DuckDBSearchExecutor_1 = require("./DuckDBSearchExecutor");
+const ReconciliationReportSearch_1 = require("../search/ReconciliationReportSearch");
+/**
+ * Class to build and execute DuckDB search query for Reconciliation Report
+ */
+class ReconciliationReportS3Search extends ReconciliationReportSearch_1.ReconciliationReportSearch {
+    constructor(event, dbConnection) {
+        super(event);
+        this.duckDBSearchExecutor = new DuckDBSearchExecutor_1.DuckDBSearchExecutor({
+            dbConnection,
+            dbQueryParameters: this.dbQueryParameters,
+            getMetaTemplate: this._metaTemplate.bind(this),
+            translateRecords: this.translatePostgresRecordsToApiRecords.bind(this),
+        });
+    }
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    async query() {
+        return this.duckDBSearchExecutor.query((knexBuilder) => this.buildSearch(knexBuilder));
+    }
+}
+exports.ReconciliationReportS3Search = ReconciliationReportS3Search;
+//# sourceMappingURL=ReconciliationReportS3Search.js.map

package/dist/s3search/RuleS3Search.d.ts

@@ -0,0 +1,20 @@
+import { DuckDBConnection } from '@duckdb/node-api';
+import { RuleSearch } from '../search/RuleSearch';
+import { QueryEvent } from '../types/search';
+/**
+ * Class to build and execute DuckDB search query for rules
+ */
+export declare class RuleS3Search extends RuleSearch {
+    private duckDBSearchExecutor;
+    constructor(event: QueryEvent, dbConnection: DuckDBConnection);
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    query(): Promise<{
+        meta: import("../search/BaseSearch").Meta;
+        results: any[];
+    }>;
+}
+//# sourceMappingURL=RuleS3Search.d.ts.map

package/dist/s3search/RuleS3Search.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RuleS3Search = void 0;
+const DuckDBSearchExecutor_1 = require("./DuckDBSearchExecutor");
+const RuleSearch_1 = require("../search/RuleSearch");
+/**
+ * Class to build and execute DuckDB search query for rules
+ */
+class RuleS3Search extends RuleSearch_1.RuleSearch {
+    constructor(event, dbConnection) {
+        super(event);
+        this.duckDBSearchExecutor = new DuckDBSearchExecutor_1.DuckDBSearchExecutor({
+            dbConnection,
+            dbQueryParameters: this.dbQueryParameters,
+            getMetaTemplate: this._metaTemplate.bind(this),
+            translateRecords: this.translatePostgresRecordsToApiRecords.bind(this),
+        });
+    }
+    /**
+     * Build and execute search query
+     *
+     * @returns search result
+     */
+    async query() {
+        return this.duckDBSearchExecutor.query((knexBuilder) => this.buildSearch(knexBuilder));
+    }
+}
+exports.RuleS3Search = RuleS3Search;
+//# sourceMappingURL=RuleS3Search.js.map

package/dist/s3search/StatsS3Search.d.ts

@@ -0,0 +1,25 @@
+import { DuckDBConnection } from '@duckdb/node-api';
+import { QueryEvent } from '../types/search';
+import { ApiAggregateResult, StatsSearch, SummaryResult } from '../search/StatsSearch';
+/**
+ * A class to query postgres for the STATS and STATS/AGGREGATE endpoints
+ */
+declare class StatsS3Search extends StatsSearch {
+    private dbConnection;
+    private knexBuilder;
+    constructor(event: QueryEvent, type: string, dbConnection: DuckDBConnection);
+    /**
+     * Queries postgres for a summary of statistics around the granules in the system
+     *
+     * @returns the postgres aggregations based on query
+     */
+    summary(): Promise<SummaryResult>;
+    /**
+     * Executes the aggregate search query
+     *
+     * @returns the aggregate query results in api format
+     */
+    aggregate(): Promise<ApiAggregateResult>;
+}
+export { StatsS3Search };
+//# sourceMappingURL=StatsS3Search.d.ts.map

package/dist/s3search/StatsS3Search.js

@@ -0,0 +1,51 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StatsS3Search = void 0;
+const knex_1 = require("knex");
+const logger_1 = __importDefault(require("@cumulus/logger"));
+const StatsSearch_1 = require("../search/StatsSearch");
+const duckdbHelpers_1 = require("./duckdbHelpers");
+const log = new logger_1.default({ sender: '@cumulus/db/StatsSearch' });
+/**
+ * A class to query postgres for the STATS and STATS/AGGREGATE endpoints
+ */
+class StatsS3Search extends StatsSearch_1.StatsSearch {
+    constructor(event, type, dbConnection) {
+        super(event, type);
+        this.dbConnection = dbConnection;
+        // Use 'pg' dialect to generate DuckDB-compatible SQL ($1, $2, etc.)
+        this.knexBuilder = (0, knex_1.knex)({ client: 'pg' });
+    }
+    /**
+     * Queries postgres for a summary of statistics around the granules in the system
+     *
+     * @returns the postgres aggregations based on query
+     */
+    async summary() {
+        const aggregateQuery = this.buildSummaryQuery(this.knexBuilder);
+        log.debug(`summary about to execute query: ${aggregateQuery?.toSQL().sql}`);
+        const { sql, bindings } = aggregateQuery.toSQL().toNative();
+        const reader = await this.dbConnection.runAndReadAll(sql, (0, duckdbHelpers_1.prepareBindings)([...bindings]) // prepareBindings must be imported/defined in scope
+        );
+        const aggregateQueryRes = reader.getRowObjectsJson();
+        return this.formatSummaryResult(aggregateQueryRes[0]);
+    }
+    /**
+     * Executes the aggregate search query
+     *
+     * @returns the aggregate query results in api format
+     */
+    async aggregate() {
+        const { searchQuery } = this.buildSearch(this.knexBuilder);
+        const { sql, bindings } = searchQuery.toSQL().toNative();
+        const reader = await this.dbConnection.runAndReadAll(sql, (0, duckdbHelpers_1.prepareBindings)([...bindings]) // prepareBindings must be imported/defined in scope
+        );
+        const records = reader.getRowObjectsJson();
+        return this.formatAggregateResult(records);
+    }
+}
+exports.StatsS3Search = StatsS3Search;
+//# sourceMappingURL=StatsS3Search.js.map

package/dist/s3search/duckdbHelpers.d.ts

@@ -0,0 +1,43 @@
+import { DuckDBValue, DuckDBConnection } from '@duckdb/node-api';
+import { Knex } from 'knex';
+import { PostgresFileRecord } from '../types/file';
+export declare function prepareBindings(bindings: ReadonlyArray<any>): DuckDBValue[];
+/**
+ * Returns execution records sorted by most recent first for an input
+ * set of Granule Cumulus IDs.
+ *
+ * @param {object} params - The function parameters.
+ * @param {DuckDBConnection} params.connection - The active DuckDB connection.
+ * @param {number[]} params.granuleCumulusIds - Array of granule IDs to filter by.
+ * @param {Knex} [params.knexBuilder] - Optional Knex instance (defaults to 'pg' client).
+ * @param {number} [params.limit] - Optional limit for the number of records returned.
+ * @returns {Promise<{ granule_cumulus_id: number, url: string }[]>}
+ *   Array of objects containing granule_cumulus_id and execution url, sorted by timestamp desc.
+ */
+export declare const getExecutionInfoByGranuleCumulusIds: ({ connection, granuleCumulusIds, knexBuilder, limit, }: {
+    connection: DuckDBConnection;
+    granuleCumulusIds: number[];
+    knexBuilder: Knex;
+    limit?: number | undefined;
+}) => Promise<{
+    granule_cumulus_id: number;
+    url: string;
+}[]>;
+/**
+ * Searches for file records by granule cumulus IDs using a DuckDB connection.
+ *
+ * @param params - Function parameters
+ * @param params.connection - Active DuckDB connection used to execute the query
+ * @param params.granuleCumulusIds - Array of granule Cumulus IDs to filter by
+ * @param [params.columns='*'] - Columns to select (string or string array)
+ * @param [params.knexBuilder] - Optional Knex instance (defaults to PostgreSQL dialect)
+ * @returns Promise resolving to an array of normalized `PostgresFileRecord` objects
+ * @throws If the DuckDB query execution fails
+ */
+export declare const getFilesByGranuleCumulusIds: ({ connection, granuleCumulusIds, columns, knexBuilder, }: {
+    connection: DuckDBConnection;
+    granuleCumulusIds: number[];
+    columns?: string | string[] | undefined;
+    knexBuilder?: Knex<any, any[]> | undefined;
+}) => Promise<PostgresFileRecord[]>;
+//# sourceMappingURL=duckdbHelpers.d.ts.map

package/dist/s3search/duckdbHelpers.js

@@ -0,0 +1,83 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFilesByGranuleCumulusIds = exports.getExecutionInfoByGranuleCumulusIds = exports.prepareBindings = void 0;
+const knex_1 = require("knex");
+const logger_1 = __importDefault(require("@cumulus/logger"));
+const tables_1 = require("../tables");
+const log = new logger_1.default({ sender: '@cumulus/db/duckdbHelpers' });
+function prepareBindings(bindings) {
+    return bindings.map((value) => {
+        if (value instanceof Date)
+            return value.toISOString();
+        if (value !== null && typeof value === 'object')
+            return JSON.stringify(value);
+        return value;
+    });
+}
+exports.prepareBindings = prepareBindings;
+/**
+ * Returns execution records sorted by most recent first for an input
+ * set of Granule Cumulus IDs.
+ *
+ * @param {object} params - The function parameters.
+ * @param {DuckDBConnection} params.connection - The active DuckDB connection.
+ * @param {number[]} params.granuleCumulusIds - Array of granule IDs to filter by.
+ * @param {Knex} [params.knexBuilder] - Optional Knex instance (defaults to 'pg' client).
+ * @param {number} [params.limit] - Optional limit for the number of records returned.
+ * @returns {Promise<{ granule_cumulus_id: number, url: string }[]>}
+ *   Array of objects containing granule_cumulus_id and execution url, sorted by timestamp desc.
+ */
+const getExecutionInfoByGranuleCumulusIds = async ({ connection, granuleCumulusIds, knexBuilder = (0, knex_1.knex)({ client: 'pg' }), limit, }) => {
+    const knexQuery = knexBuilder(tables_1.TableNames.executions)
+        .select([
+        `${tables_1.TableNames.executions}.url`,
+        `${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`,
+    ])
+        .join(tables_1.TableNames.granulesExecutions, `${tables_1.TableNames.executions}.cumulus_id`, `${tables_1.TableNames.granulesExecutions}.execution_cumulus_id`)
+        .whereIn(`${tables_1.TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusIds)
+        .orderBy(`${tables_1.TableNames.executions}.timestamp`, 'desc');
+    if (limit)
+        knexQuery.limit(limit);
+    const { sql, bindings } = knexQuery.toSQL().toNative();
+    log.debug(`getExecutionInfoByGranuleCumulusIds query: ${sql}`);
+    // Use spread operator to convert ReadonlyArray to mutable array
+    const reader = await connection.runAndReadAll(sql, prepareBindings([...bindings]));
+    return reader.getRowObjectsJson();
+};
+exports.getExecutionInfoByGranuleCumulusIds = getExecutionInfoByGranuleCumulusIds;
+/**
+ * Searches for file records by granule cumulus IDs using a DuckDB connection.
+ *
+ * @param params - Function parameters
+ * @param params.connection - Active DuckDB connection used to execute the query
+ * @param params.granuleCumulusIds - Array of granule Cumulus IDs to filter by
+ * @param [params.columns='*'] - Columns to select (string or string array)
+ * @param [params.knexBuilder] - Optional Knex instance (defaults to PostgreSQL dialect)
+ * @returns Promise resolving to an array of normalized `PostgresFileRecord` objects
+ * @throws If the DuckDB query execution fails
+ */
+const getFilesByGranuleCumulusIds = async ({ connection, granuleCumulusIds, columns = '*', knexBuilder = (0, knex_1.knex)({ client: 'pg' }), }) => {
+    const knexQuery = knexBuilder(tables_1.TableNames.files)
+        .select(columns)
+        .whereIn('granule_cumulus_id', granuleCumulusIds);
+    const { sql, bindings } = knexQuery.toSQL().toNative();
+    // Execute using DuckDB connection
+    const reader = await connection.runAndReadAll(sql, prepareBindings([...bindings]) // prepareBindings must be imported/defined in scope
+    );
+    const rows = reader.getRowObjectsJson();
+    // Mapping resolves TS errors and handles DuckDB type conversions
+    return rows.map((row) => ({
+        ...row,
+        // Ensure IDs are numbers (handles DuckDB BigInt/String return types)
+        cumulus_id: Number(row.cumulus_id),
+        granule_cumulus_id: Number(row.granule_cumulus_id),
+        // Convert ISO timestamp strings back to JS Date objects
+        created_at: row.created_at ? new Date(row.created_at) : undefined,
+        updated_at: row.updated_at ? new Date(row.updated_at) : undefined,
+    }));
+};
+exports.getFilesByGranuleCumulusIds = getFilesByGranuleCumulusIds;
+//# sourceMappingURL=duckdbHelpers.js.map

package/dist/s3search/s3TableSchemas.d.ts

@@ -0,0 +1,11 @@
+export declare const asyncOperationsS3TableSql: (tableName?: string) => string;
+export declare const collectionsS3TableSql: (tableName?: string) => string;
+export declare const executionsS3TableSql: (tableName?: string) => string;
+export declare const filesS3TableSql: (tableName?: string) => string;
+export declare const granulesS3TableSql: (tableName?: string) => string;
+export declare const granulesExecutionsS3TableSql: (tableName?: string) => string;
+export declare const pdrsS3TableSql: (tableName?: string) => string;
+export declare const providersS3TableSql: (tableName?: string) => string;
+export declare const reconciliationReportsS3TableSql: (tableName?: string) => string;
+export declare const rulesS3TableSql: (tableName?: string) => string;
+//# sourceMappingURL=s3TableSchemas.d.ts.map

package/dist/s3search/s3TableSchemas.js

@@ -0,0 +1,272 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rulesS3TableSql = exports.reconciliationReportsS3TableSql = exports.providersS3TableSql = exports.pdrsS3TableSql = exports.granulesExecutionsS3TableSql = exports.granulesS3TableSql = exports.filesS3TableSql = exports.executionsS3TableSql = exports.collectionsS3TableSql = exports.asyncOperationsS3TableSql = void 0;
+const asyncOperationsS3TableSql = (tableName = 'async_operations') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id INTEGER PRIMARY KEY,
+    id UUID NOT NULL,
+    description TEXT NOT NULL,
+    operation_type TEXT NOT NULL,
+    output JSON,
+    status TEXT NOT NULL,
+    task_arn TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CONSTRAINT async_operations_id_unique UNIQUE (id),
+    CONSTRAINT async_operations_status_check
+        CHECK (status IN (
+            'RUNNING',
+            'SUCCEEDED',
+            'RUNNER_FAILED',
+            'TASK_FAILED'
+        )),
+    CONSTRAINT async_operations_operation_type_check
+        CHECK (operation_type IN (
+            'Bulk Execution Archive',
+            'Bulk Execution Delete',
+            'Bulk Granules',
+            'Bulk Granule Archive',
+            'Bulk Granule Delete',
+            'Bulk Granule Reingest',
+            'Data Migration',
+            'Dead-Letter Processing',
+            'DLA Migration',
+            'ES Index',
+            'Kinesis Replay',
+            'Migration Count Report',
+            'Reconciliation Report',
+            'SQS Replay'
+        ))
+);`;
+exports.asyncOperationsS3TableSql = asyncOperationsS3TableSql;
+const collectionsS3TableSql = (tableName = 'collections') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    version TEXT NOT NULL,
+    sample_file_name TEXT NOT NULL,
+    granule_id_validation_regex TEXT NOT NULL,
+    granule_id_extraction_regex TEXT NOT NULL,
+    files JSON NOT NULL,
+    process TEXT,
+    url_path TEXT,
+    duplicate_handling TEXT,
+    report_to_ems BOOLEAN,
+    ignore_files_config_for_discovery BOOLEAN,
+    meta JSON,
+    tags JSON,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE (name, version),
+    CHECK (duplicate_handling IN ('error', 'replace', 'skip', 'version'))
+);`;
+exports.collectionsS3TableSql = collectionsS3TableSql;
+const executionsS3TableSql = (tableName = 'executions') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id BIGINT PRIMARY KEY,
+    arn TEXT NOT NULL,
+    async_operation_cumulus_id INTEGER,
+    collection_cumulus_id INTEGER,
+    parent_cumulus_id BIGINT,
+    cumulus_version TEXT,
+    url TEXT,
+    status TEXT NOT NULL,
+    tasks JSON,
+    error JSON,
+    workflow_name TEXT,
+    duration REAL,
+    original_payload JSON,
+    final_payload JSON,
+    "timestamp" TIMESTAMPTZ,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    archived BOOLEAN NOT NULL DEFAULT FALSE,
+    CONSTRAINT executions_arn_unique UNIQUE (arn),
+    CONSTRAINT executions_url_unique UNIQUE (url),
+    CONSTRAINT executions_async_operation_cumulus_id_foreign
+        FOREIGN KEY (async_operation_cumulus_id)
+        REFERENCES async_operations (cumulus_id),
+    CONSTRAINT executions_collection_cumulus_id_foreign
+        FOREIGN KEY (collection_cumulus_id)
+        REFERENCES collections (cumulus_id),
+    CONSTRAINT executions_parent_cumulus_id_foreign
+        FOREIGN KEY (parent_cumulus_id)
+        REFERENCES ${tableName} (cumulus_id),
+    CONSTRAINT executions_status_check
+        CHECK (status IN ('running', 'completed', 'failed', 'unknown'))
+);`;
+exports.executionsS3TableSql = executionsS3TableSql;
+const filesS3TableSql = (tableName = 'files') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id BIGINT PRIMARY KEY,
+    granule_cumulus_id BIGINT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    file_size BIGINT,
+    bucket TEXT NOT NULL,
+    checksum_type TEXT,
+    checksum_value TEXT,
+    file_name TEXT,
+    key TEXT NOT NULL,
+    path TEXT,
+    source TEXT,
+    type TEXT,
+    CONSTRAINT files_bucket_key_unique UNIQUE (bucket, key),
+    CONSTRAINT files_granule_cumulus_id_foreign
+        FOREIGN KEY (granule_cumulus_id)
+        REFERENCES granules (cumulus_id)
+);`;
+exports.filesS3TableSql = filesS3TableSql;
+const granulesS3TableSql = (tableName = 'granules') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id BIGINT PRIMARY KEY,
+    granule_id TEXT NOT NULL,
+    status TEXT NOT NULL,
+    collection_cumulus_id INTEGER NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    published BOOLEAN,
+    duration REAL,
+    time_to_archive REAL,
+    time_to_process REAL,
+    product_volume BIGINT,
+    error JSON,
+    cmr_link TEXT,
+    pdr_cumulus_id INTEGER,
+    provider_cumulus_id INTEGER,
+    beginning_date_time TIMESTAMPTZ,
+    ending_date_time TIMESTAMPTZ,
+    last_update_date_time TIMESTAMPTZ,
+    processing_end_date_time TIMESTAMPTZ,
+    processing_start_date_time TIMESTAMPTZ,
+    production_date_time TIMESTAMPTZ,
+    query_fields JSON,
+    "timestamp" TIMESTAMPTZ,
+    producer_granule_id TEXT NOT NULL,
+    archived BOOLEAN NOT NULL DEFAULT FALSE,
+    UNIQUE (collection_cumulus_id, granule_id),
+    CHECK (status IN ('running', 'completed', 'failed', 'queued'))
+);`;
+exports.granulesS3TableSql = granulesS3TableSql;
+const granulesExecutionsS3TableSql = (tableName = 'granules_executions') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    granule_cumulus_id BIGINT NOT NULL,
+    execution_cumulus_id BIGINT NOT NULL,
+    CONSTRAINT granules_executions_granule_execution_unique
+        UNIQUE (granule_cumulus_id, execution_cumulus_id),
+    CONSTRAINT granules_executions_execution_cumulus_id_foreign
+        FOREIGN KEY (execution_cumulus_id)
+        REFERENCES executions (cumulus_id),
+    CONSTRAINT granules_executions_granule_cumulus_id_foreign
+        FOREIGN KEY (granule_cumulus_id)
+        REFERENCES granules (cumulus_id)
+);`;
+exports.granulesExecutionsS3TableSql = granulesExecutionsS3TableSql;
+const pdrsS3TableSql = (tableName = 'pdrs') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id INTEGER PRIMARY KEY,
+    collection_cumulus_id INTEGER NOT NULL,
+    provider_cumulus_id INTEGER NOT NULL,
+    execution_cumulus_id BIGINT,
+    status TEXT NOT NULL,
+    name TEXT NOT NULL,
+    progress REAL,
+    pan_sent BOOLEAN,
+    pan_message TEXT,
+    stats JSON,
+    address TEXT,
+    original_url TEXT,
+    duration REAL,
+    "timestamp" TIMESTAMPTZ,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CONSTRAINT pdrs_name_unique UNIQUE (name),
+    CONSTRAINT pdrs_collection_cumulus_id_foreign
+        FOREIGN KEY (collection_cumulus_id)
+        REFERENCES collections (cumulus_id),
+    CONSTRAINT pdrs_execution_cumulus_id_foreign
+        FOREIGN KEY (execution_cumulus_id)
+        REFERENCES executions (cumulus_id),
+    CONSTRAINT pdrs_provider_cumulus_id_foreign
+        FOREIGN KEY (provider_cumulus_id)
+        REFERENCES providers (cumulus_id),
+    CONSTRAINT pdrs_status_check
+        CHECK (status IN ('running', 'failed', 'completed'))
+);`;
+exports.pdrsS3TableSql = pdrsS3TableSql;
+const providersS3TableSql = (tableName = 'providers') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    protocol TEXT NOT NULL DEFAULT 'http',
+    host TEXT NOT NULL,
+    port INTEGER,
+    username TEXT,
+    password TEXT,
+    global_connection_limit INTEGER,
+    private_key TEXT,
+    cm_key_id TEXT,
+    certificate_uri TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    allowed_redirects TEXT[],
+    max_download_time INTEGER,
+    CONSTRAINT providers_name_unique UNIQUE (name),
+    CONSTRAINT providers_protocol_check
+        CHECK (protocol IN ('http', 'https', 'ftp', 'sftp', 's3'))
+);`;
+exports.providersS3TableSql = providersS3TableSql;
+const reconciliationReportsS3TableSql = (tableName = 'reconciliation_reports') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    type TEXT NOT NULL,
+    status TEXT NOT NULL,
+    location TEXT,
+    error JSON,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CONSTRAINT reconciliation_reports_name_unique UNIQUE (name),
+    CONSTRAINT reconciliation_reports_type_check
+        CHECK (type IN (
+            'Granule Inventory',
+            'Granule Not Found',
+            'Internal',
+            'Inventory',
+            'ORCA Backup'
+        )),
+    CONSTRAINT reconciliation_reports_status_check
+        CHECK (status IN ('Generated', 'Pending', 'Failed'))
+);`;
+exports.reconciliationReportsS3TableSql = reconciliationReportsS3TableSql;
+const rulesS3TableSql = (tableName = 'rules') => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    cumulus_id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    workflow TEXT NOT NULL,
+    collection_cumulus_id INTEGER,
+    provider_cumulus_id INTEGER,
+    type TEXT NOT NULL,
+    enabled BOOLEAN NOT NULL,
+    value TEXT,
+    arn TEXT,
+    log_event_arn TEXT,
+    execution_name_prefix TEXT,
+    payload JSON,
+    meta JSON,
+    tags JSON,
+    queue_url TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CONSTRAINT rules_name_unique UNIQUE (name),
+    CONSTRAINT rules_collection_cumulus_id_foreign
+        FOREIGN KEY (collection_cumulus_id)
+        REFERENCES collections (cumulus_id),
+    CONSTRAINT rules_provider_cumulus_id_foreign
+        FOREIGN KEY (provider_cumulus_id)
+        REFERENCES providers (cumulus_id),
+    CONSTRAINT rules_type_check
+        CHECK (type IN ('onetime', 'scheduled', 'sns', 'kinesis', 'sqs'))
+);`;
+exports.rulesS3TableSql = rulesS3TableSql;
+//# sourceMappingURL=s3TableSchemas.js.map

package/dist/search/BaseSearch.d.ts

@@ -1,11 +1,44 @@
 import { Knex } from 'knex';
 import { BaseRecord } from '../types/base';
 import { DbQueryParameters, QueryEvent, QueryStringParameters } from '../types/search';
+export declare type Meta = {
+    name: string;
+    stack?: string;
+    table?: string;
+    limit?: number;
+    page?: number;
+    count?: number;
+};
 export declare const typeToTable: {
     [key: string]: string;
 };
 /**
- * Class to build and execute db search query
+ * BaseSearch
+ *
+ * Abstract base class for building and executing database search queries.
+ *
+ * Responsibilities:
+ *  - Parse and normalize incoming query string parameters.
+ *  - Build database queries using Knex.
+ *  - Execute queries against PostgreSQL by default.
+ *  - Return standardized search API response format including metadata.
+ *
+ * Default Behavior:
+ *  - The `query()` method executes against PostgreSQL using a Knex client.
+ *
+ * DuckDB Support:
+ *  - Subclasses that query DuckDB (e.g., *S3Search classes) must override
+ *    the `query()` and related methods
+ *  - DuckDB subclasses are responsible for:
+ *      - Executing queries using a DuckDB connection.
+ *      - Handling sequential execution (to avoid prepared statement conflicts).
+ *      - Translating DuckDB result types (e.g., string dates/JSON) into proper API types.
+ *
+ * Design Notes:
+ *  - Query construction logic (e.g., `buildSearch`) is shared across Postgres
+ *    and DuckDB implementations.
+ *  - Execution strategy is delegated to subclasses when a different database
+ *    engine is required.
  */
 declare abstract class BaseSearch {
     readonly type: string;
@@ -44,6 +77,17 @@ declare abstract class BaseSearch {
      * @returns whether an estimated row count should be returned
      */
     protected shouldEstimateRowcount(countSql: string): boolean;
+    /**
+     * Build a JSON query expression string for nested fields.
+     *
+     *
+     * @param fullFieldName - Dot-separated JSON path, e.g., 'query_fields.cnm.receivedTime'
+     * @returns The JSON query path string
+     * @example
+     * buildJsonQueryExpression('query_fields.cnm.receivedTime')
+     * // returns: query_fields -> 'cnm' ->> 'receivedTime'
+     */
+    protected buildJsonQueryExpression: (fullFieldName: string) => string;
     /**
      * Build the search query
      *
@@ -59,7 +103,7 @@ declare abstract class BaseSearch {
      *
      * @returns metadata template
      */
-    private _metaTemplate;
+    protected _metaTemplate(): Meta;
     /**
      * Build basic query
      *