@squiz/db-lib 1.77.0 → 1.77.1

package/src/externalized/ExternalizedDynamoDbRepository.ts

@@ -54,6 +54,20 @@ export abstract class ExternalizedDynamoDbRepository<
     super(tableName, dbManager, entityName, entityDefinition, classRef);
   }
 
+  /**
+   * Removes storageLocation from the response.
+   * storageLocation is an internal S3 storage implementation detail that should never be exposed to API consumers.
+   * @param {DATA_CLASS} item - The item to process.
+   * @returns {DATA_CLASS} The item without storageLocation.
+   */
+  private removeStorageLocationFromResponse(item: DATA_CLASS): DATA_CLASS {
+    if ((item as any).storageLocation !== undefined) {
+      const { storageLocation: _storageLocation, ...itemWithoutStorage } = item as any;
+      return new this.classRef(itemWithoutStorage);
+    }
+    return item;
+  }
+
   /**
    * Creates a new item in the repository.
    * If the item exceeds DynamoDB's size limit, it will automatically externalize the content to S3.
@@ -70,7 +84,8 @@ export abstract class ExternalizedDynamoDbRepository<
     options: { overrideExisting?: boolean } = { overrideExisting: false },
   ): Promise<DATA_CLASS> {
     try {
-      return await this.createItemInternal(value, transaction, additionalValue, options);
+      const result = await this.createItemInternal(value, transaction, additionalValue, options);
+      return this.removeStorageLocationFromResponse(result);
     } catch (error) {
       // If the item exceeds DynamoDB's size limit, externalize the content to S3 and retry
       if (this.isDynamoItemSizeLimitError(error)) {
@@ -78,7 +93,8 @@ export abstract class ExternalizedDynamoDbRepository<
           `ExternalizedDynamoDbRepository: Item exceeded DynamoDB size limit for ${this.entityName}. Retrying with external storage.`,
         );
         const externalizedValue = await this.prepareValueForStorage(value);
-        return await this.createItemInternal(externalizedValue, transaction, additionalValue, options);
+        const result = await this.createItemInternal(externalizedValue, transaction, additionalValue, options);
+        return this.removeStorageLocationFromResponse(result);
       }
       throw error;
     }
@@ -180,8 +196,14 @@ export abstract class ExternalizedDynamoDbRepository<
       await this.storage.delete(previousLocation);
     }
 
-    // If item was externalized, hydrate it from S3 to return full content
-    return (await this.hydrateFromExternalStorage(createdItem)) || createdItem;
+    // Check if we should hydrate from S3
+    // Only hydrate if content is actually stored in S3 (not just stale storageLocation)
+    if (this.isContentInS3(createdItem)) {
+      // Content fields are empty, so full content is in S3
+      return (await this.hydrateFromExternalStorage(createdItem)) || createdItem;
+    }
+
+    return createdItem;
   }
 
   /**
@@ -192,7 +214,8 @@ export abstract class ExternalizedDynamoDbRepository<
    */
   public async updateItem(value: Partial<SHAPE>): Promise<DATA_CLASS | undefined> {
     try {
-      return await this.updateItemInternal(value);
+      const result = await this.updateItemInternal(value);
+      return result ? this.removeStorageLocationFromResponse(result) : undefined;
     } catch (error) {
       // If the item exceeds DynamoDB's size limit, externalize the content to S3 and retry
       if (this.isDynamoItemSizeLimitError(error)) {
@@ -200,7 +223,8 @@ export abstract class ExternalizedDynamoDbRepository<
           `ExternalizedDynamoDbRepository: Update exceeded DynamoDB size limit for ${this.entityName}. Retrying with external storage.`,
         );
         const externalizedValue = await this.prepareValueForStorage(value as DATA_CLASS);
-        return await this.updateItemInternal(externalizedValue);
+        const result = await this.updateItemInternal(externalizedValue);
+        return result ? this.removeStorageLocationFromResponse(result) : undefined;
       }
       throw error;
     }
@@ -281,9 +305,14 @@ export abstract class ExternalizedDynamoDbRepository<
 
         const result = await this.client.update(updateCommandInput);
         updatedItem = result.Attributes ? this.hydrateItem(result.Attributes) : undefined;
+
+        // Preserve storageLocation from DynamoDB response (hydrateItem might strip it)
+        if (updatedItem && result.Attributes?.storageLocation) {
+          (updatedItem as any).storageLocation = result.Attributes.storageLocation;
+        }
       }
     } else {
-      // No storageLocation, use parent's method
+      // No storageLocation in update value, use parent's method
       // Strip storageLocation before passing to parent to avoid "Excess properties" error
       const { storageLocation: _, ...valueWithoutStorage } = value as any;
       updatedItem = await super.updateItem(valueWithoutStorage);
@@ -293,14 +322,39 @@ export abstract class ExternalizedDynamoDbRepository<
       return undefined;
     }
 
+    // Check for stale storageLocation - this happens when content transitions from large (S3) to small (inline DynamoDB)
+    // If storageLocation exists but content is NOT in S3 (content is inline), the storageLocation is stale
+    let newLocation = updatedItem.storageLocation;
+    if (newLocation && !this.isContentInS3(updatedItem)) {
+      // Remove the stale storageLocation from DynamoDB
+      await this.client.update({
+        TableName: this.tableName,
+        Key: {
+          [this.keys.pk.attributeName]: this.getPk(value),
+          [this.keys.sk.attributeName]: this.getSk(value),
+        },
+        UpdateExpression: 'REMOVE storageLocation',
+      });
+
+      // Mark that there's no longer a new S3 location (we just removed it)
+      newLocation = undefined;
+      delete (updatedItem as any).storageLocation;
+    }
+
     // Clean up old S3 file if it exists and is different from new location
-    const newLocation = updatedItem.storageLocation;
     if (previousLocation && previousLocation.key !== newLocation?.key) {
       await this.storage.delete(previousLocation);
     }
 
-    // If item was externalized, hydrate it from S3 to return full content
-    return (await this.hydrateFromExternalStorage(updatedItem)) || updatedItem;
+    // Check if we should hydrate from S3
+    // Only hydrate if content is actually stored in S3 (not just stale storageLocation)
+    // For large->small updates, DynamoDB might still have storageLocation but content is in DynamoDB
+    if (this.isContentInS3(updatedItem)) {
+      // Content fields are empty, so full content is in S3
+      return (await this.hydrateFromExternalStorage(updatedItem)) || updatedItem;
+    }
+
+    return updatedItem;
   }
 
   /**
@@ -364,6 +418,7 @@ export abstract class ExternalizedDynamoDbRepository<
    * Prepares a value for storage by externalizing large content to S3.
    * @param {DATA_CLASS} value - The value to prepare for storage.
    * @returns {Promise<DATA_CLASS>} A promise that resolves to the prepared value for DynamoDB.
+   * @throws {Error} If S3 storage is not configured or if saving to S3 fails.
    */
   public async prepareValueForStorage(value: DATA_CLASS): Promise<DATA_CLASS> {
     const serialized = JSON.stringify(value);
@@ -450,6 +505,31 @@ export abstract class ExternalizedDynamoDbRepository<
     return hydrated;
   }
 
+  /**
+   * Checks if content is actually stored in S3 by examining if large content fields are empty.
+   * When content is externalized to S3, fieldsAsJsonString fields are set to empty objects/arrays.
+   * @param {DATA_CLASS} item - The item to check.
+   * @returns {boolean} True if content is in S3, false if content is in DynamoDB.
+   */
+  private isContentInS3(item: DATA_CLASS): boolean {
+    if (!item.storageLocation) {
+      return false;
+    }
+
+    // Check if large content fields are empty (indicating content is in S3)
+    for (const fieldName of this.fieldsAsJsonString) {
+      const fieldValue = (item as any)[fieldName];
+      if (fieldValue !== undefined) {
+        const isEmpty = typeof fieldValue === 'object' && Object.keys(fieldValue).length === 0;
+        if (isEmpty) {
+          return true;
+        }
+      }
+    }
+
+    return false;
+  }
+
   /**
    * Hydrates a record from external storage.
    * @param {DATA_CLASS} record - The record to hydrate.