@adobe/spacecat-shared-data-access 2.62.2 → 2.63.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-data-access-v2.63.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.62.2...@adobe/spacecat-shared-data-access-v2.63.0) (2025-09-19)
+
+
+### Features
+
+* System-Wide Pagination Fix for DynamoDB Data Truncation ([#964](https://github.com/adobe/spacecat-shared/issues/964)) ([8f82e5f](https://github.com/adobe/spacecat-shared/commit/8f82e5fe2d5ed9ecd8ad2ff4e139b0588b3d8c01))
+
 # [@adobe/spacecat-shared-data-access-v2.62.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.62.1...@adobe/spacecat-shared-data-access-v2.62.2) (2025-09-16)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "2.62.2",
+  "version": "2.63.0",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "engines": {

package/src/models/base/base.collection.js

@@ -252,15 +252,34 @@ class BaseCollection {
       let result = await query.go(queryOptions);
       let allData = result.data;
 
-      // if the caller requests ALL pages and we're not using limit: 1,
-      // continue to fetch until there is no cursor.
-      if (options.fetchAllPages && options.limit !== 1) {
+      // Smart pagination behavior:
+      // - fetchAllPages: true → Always paginate through all results
+      // - fetchAllPages: false → Only fetch first page
+      // - undefined → Auto-paginate when no limit specified, respect limits otherwise
+      const shouldFetchAllPages = options.fetchAllPages === true
+                                 || (options.fetchAllPages !== false && !options.limit);
+
+      if (shouldFetchAllPages) {
+        let pageCount = 1;
+        const startTime = Date.now();
+
         while (result.cursor) {
+          pageCount += 1;
           queryOptions.cursor = result.cursor;
           // eslint-disable-next-line no-await-in-loop
           result = await query.go(queryOptions);
           allData = allData.concat(result.data);
         }
+
+        // Debug logging to track pagination behavior
+        if (pageCount > 1) {
+          this.log.debug(`Pagination completed for ${this.entityName}`, {
+            entityName: this.entityName,
+            totalPages: pageCount,
+            totalRecords: allData.length,
+            queryDuration: Date.now() - startTime,
+          });
+        }
       }
 
       if (options.limit === 1) {

package/src/models/base/index.d.ts

@@ -34,6 +34,13 @@ export interface QueryOptions {
   limit?: number;
   order?: string;
   attributes?: string[];
+  /** 
+   * Whether to automatically fetch all pages of results. 
+   * - `true`: Always paginate through all results
+   * - `false`: Only fetch first page
+   * - `undefined`: Auto-paginate when no limit specified, respect limits otherwise
+   */
+  fetchAllPages?: boolean;
 }
 
 export interface BaseCollection<T extends BaseModel> {

package/src/util/accessor.utils.js

@@ -130,10 +130,16 @@ export function createAccessor(config) { /* eslint-disable no-underscore-dangle
     } else {
       const { keys, options } = parseAccessorArgs(collection, requiredKeys, args);
       const allKeys = { ...foreignKeys, ...keys };
+      const finalOptions = all
+        ? {
+          ...options,
+          fetchAllPages: options?.fetchAllPages ?? !options?.limit,
+        }
+        : options;
 
       result = all
-        ? collection.allByIndexKeys(allKeys, options)
-        : collection.findByIndexKeys(allKeys, options);
+        ? collection.allByIndexKeys(allKeys, finalOptions)
+        : collection.findByIndexKeys(allKeys, finalOptions);
     }
 
     result = await result;