@adobe/spacecat-shared-athena-client 1.0.3 → 1.0.4

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-athena-client-v1.0.4](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-athena-client-v1.0.3...@adobe/spacecat-shared-athena-client-v1.0.4) (2025-07-15)
+
+
+### Bug Fixes
+
+* add pagination for athena ([#848](https://github.com/adobe/spacecat-shared/issues/848)) ([2007cfa](https://github.com/adobe/spacecat-shared/commit/2007cfad79f1921ad015c9c0301cf790f9b9405d))
+
 # [@adobe/spacecat-shared-athena-client-v1.0.3](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-athena-client-v1.0.2...@adobe/spacecat-shared-athena-client-v1.0.3) (2025-07-12)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-athena-client",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "Shared modules of the Spacecat Services - AWS Athena Client",
   "type": "module",
   "engines": {

package/src/index.js

@@ -14,8 +14,8 @@ import {
   AthenaClient,
   StartQueryExecutionCommand,
   GetQueryExecutionCommand,
-  GetQueryResultsCommand,
   QueryExecutionState,
+  paginateGetQueryResults,
 } from '@aws-sdk/client-athena';
 import { instrumentAWSClient, hasText } from '@adobe/spacecat-shared-utils';
 
@@ -220,29 +220,38 @@ export class AWSAthenaClient {
    * @returns {Promise<Array>} – Parsed query results
    */
   async query(sql, database, description = 'Athena query', opts = {}) {
-    const {
-      backoffMs = this.backoffMs,
-      maxRetries = this.maxRetries,
-      pollIntervalMs = this.pollIntervalMs,
-      maxPollAttempts = this.maxPollAttempts,
-    } = opts;
+    const queryExecutionId = await this.execute(sql, database, description, opts);
 
-    this.log.info(`[Athena Client] Executing: ${description}`);
+    this.log.debug(`[Athena Client] Fetching paginated results for QueryExecutionId=${queryExecutionId}`);
 
-    const queryExecutionId = await this.#startQueryWithRetry(
-      sql,
-      database,
-      description,
-      backoffMs,
-      maxRetries,
-    );
+    const paginationConfig = {
+      client: this.client,
+    };
 
-    await this.#pollToCompletion(queryExecutionId, description, pollIntervalMs, maxPollAttempts);
+    const input = {
+      QueryExecutionId: queryExecutionId,
+    };
 
-    const results = await this.client.send(
-      new GetQueryResultsCommand({ QueryExecutionId: queryExecutionId }),
-    );
+    const paginator = paginateGetQueryResults(paginationConfig, input);
+
+    const allResults = [];
+    let pageCount = 0;
+    let totalRows = 0;
+
+    /* c8 ignore start */
+    for await (const page of paginator) {
+      pageCount += 1;
+      const pageRows = page.ResultSet?.Rows?.length || 0;
+      totalRows += pageRows;
+
+      this.log.debug(`[Athena Client] Processing page ${pageCount} with ${pageRows} rows`);
+
+      const pageResults = AWSAthenaClient.#parseAthenaResults(page);
+      allResults.push(...pageResults);
+    }
+    /* c8 ignore stop */
 
-    return AWSAthenaClient.#parseAthenaResults(results);
+    this.log.info(`[Athena Client] Fetched ${totalRows} total rows across ${pageCount} pages`);
+    return allResults;
   }
 }