quarantine 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fc1311c40c02cd62c89672cad8ef3504a0abe8cb0e9c6eb4a3ccdf4a7acc664
-  data.tar.gz: a8afa4b5d8de5c26e5a03790d0d053c77044caf08fde8cdd4dde2165a33e48e2
+  metadata.gz: aaede44e051f9b183da284c82c86ae341ac4361bb8c36ba12f7cb11c4fc47350
+  data.tar.gz: dce85fa1bac7d33224b907e53ed570763b87cbd783ae983d88e7dc563b0b0db0
 SHA512:
-  metadata.gz: 84bf594f24acca0a92cb49da883368de553a816ac3d0f4f58b3bf5c147db86019c858f1f1f9a11d38e9e2af5de859a07e29d6b46c239972c33ae23a008f12ad2
-  data.tar.gz: 2e9f6d30eff342421f12a99bedf64888cdca6406b20496fa481309acb1df7eb1acc70e0ada1d8821780a23fc9296e1254727f343c0c1d716db1679117d4e2471
+  metadata.gz: dc724bcd18323c374bd1e8c5ae472db1579d9334e4d050c352058902981488a5659b542c26ba464e88d47bb102efb20ce9faa5990ba098225200cab11324792f
+  data.tar.gz: 63d3dbeb70b087ba783a925cd57d1cf76488fbaae6819b53509fafe3b6609e2fb0e126f3ae03e70f00ebfe358cd93e42e4341bd44790b4773e6cf7c3d1921655

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+### 1.0.6
+  Update DynamoDB batch_write_item implementation to check for duplicates based on different keys before uploading
+
 ### 1.0.5
   Add aws_credentials argument during dynamodb initialization to override the AWS SDK credential chain
 

data/lib/quarantine/databases/dynamo_db.rb

@@ -16,7 +16,7 @@ class Quarantine
 
       def scan(table_name)
         begin
-          result = dynamodb.scan({ table_name: table_name })
+          result = dynamodb.scan(table_name: table_name)
         rescue Aws::DynamoDB::Errors::ServiceError
           raise Quarantine::DatabaseError
         end
@@ -24,17 +24,32 @@ class Quarantine
         result&.items
       end
 
-      def batch_write_item(table_name, items, additional_attributes = {})
+      def batch_write_item(table_name, items, additional_attributes = {}, dedup_keys = %w[id full_description])
+        return if items.empty?
+
+        # item_a is a duplicate of item_b if all values for each dedup_key in both item_a and item_b match
+        is_a_duplicate = ->(item_a, item_b) { dedup_keys.all? { |key| item_a[key] == item_b[key] } }
+
+        scanned_items = scan(table_name)
+
+        deduped_items = items.reject do |item|
+          scanned_items.any? do |scanned_item|
+            is_a_duplicate.call(item.to_string_hash, scanned_item)
+          end
+        end
+
+        return if deduped_items.empty?
+
         dynamodb.batch_write_item(
-          { request_items: {
-            table_name => items.map do |item|
+          request_items: {
+            table_name => deduped_items.map do |item|
               {
                 put_request: {
                   item: { **item.to_hash, **additional_attributes }
                 }
               }
             end
-          } }
+          }
         )
       rescue Aws::DynamoDB::Errors::ServiceError
         raise Quarantine::DatabaseError
@@ -42,11 +57,9 @@ class Quarantine
 
       def delete_item(table_name, keys)
         dynamodb.delete_item(
-          {
-            table_name: table_name,
-            key: {
-              **keys
-            }
+          table_name: table_name,
+          key: {
+            **keys
           }
         )
       rescue Aws::DynamoDB::Errors::ServiceError

data/lib/quarantine/test.rb

@@ -20,5 +20,14 @@ class Quarantine
         build_number: build_number
       }
     end
+
+    def to_string_hash
+      {
+        'id' => id,
+        'full_description' => full_description,
+        'location' => location,
+        'build_number' => build_number
+      }
+    end
   end
 end

data/lib/quarantine/version.rb

@@ -1,3 +1,3 @@
 class Quarantine
-  VERSION = '1.0.5'.freeze
+  VERSION = '1.0.6'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quarantine
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - Flexport Engineering, Eric Zhu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-30 00:00:00.000000000 Z
+date: 2020-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-dynamodb
@@ -91,7 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Quarantine flaky Ruby Rspec tests