remote_files 3.6.0 → 3.7.0.pre.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59ec945098b2e44d854eefecff7a29217c09c7ea3fd81da6617f8e13ec940589
-  data.tar.gz: e3ff2aeb923049f688e34a851d951ecacfad9f057d3c3711d3b49cfa040593d5
+  metadata.gz: 00b8fe0548d27ed5c3e7f604be263e0c70f0b3fce867c6528886d45372122aa6
+  data.tar.gz: a3064a9ef511550b94a4965e55eb2f2b15b45dc2244965a1fedb8e7b5f8e45a8
 SHA512:
-  metadata.gz: 406694b1d2d9abc40f261402f385c6b2e152a6aeb18c6408616883dca850a1852d5f02317d60278391f4121116ea2eb33ce288bc089f70f4d8679e00bac54514
-  data.tar.gz: fc1d0a961b56cc2acafcfa1e17c33b0a0d573a93c6cec814ff4fa23c9db7e604bc64e1eb6b8a8e8bbe14603929b219c8d147297ed365d0c010235a3640904283
+  metadata.gz: '068011e2c0d56dc7244fa15840d19751cadbefa0e940153f71c11c23b90877d0b6ed49721b79b281c2bb8cf3fcf807c4c78947be632e508f84e01ffe4b3123a0'
+  data.tar.gz: 9ef6af33362c317887fc25ba988bcfc58414306b83f085890b6041387bddda6c60bb11c917b79a2a6259417fbc54194764d74eceed36c0f62a0a8822b1285109

data/lib/remote_files/configuration.rb

@@ -1,3 +1,5 @@
+require 'concurrent-ruby'
+
 module RemoteFiles
   class Configuration
     attr_reader :name
@@ -6,6 +8,7 @@ module RemoteFiles
       @name       = name
       @stores     = []
       @stores_map = {}
+      @max_delete_in_parallel = config.delete(:max_delete_in_parallel) || 10
       from_hash(config)
     end
 
@@ -125,27 +128,53 @@ module RemoteFiles
       RemoteFiles.delete_file(file)
     end
 
-    def delete_now!(file)
+    def delete_now!(file, parallel: false)
       exceptions = []
       stores = file.read_write_stores
 
       raise "No stores configured" if stores.empty?
 
-      stores.each do |store|
-        begin
-          store.delete!(file.identifier)
-        rescue NotFoundError => e
-          exceptions << e
+      if parallel
+        delete_in_parallel!(file, stores, exceptions)
+      else
+        stores.each do |store|
+          begin
+            store.delete!(file.identifier)
+          rescue NotFoundError => e
+            exceptions << e
+          end
         end
       end
 
-      if exceptions.size == stores.size # they all failed
-        raise exceptions.first
-      end
+      raise exceptions.first if exceptions.size == stores.size # they all failed
 
       true
     end
 
+    # This method is used to delete a file from all stores in parallel
+    # exceptions are passed back to the caller
+    def delete_in_parallel!(file, stores, exceptions)
+      pool = Concurrent::FixedThreadPool.new(@max_delete_in_parallel)
+
+      futures = stores.map do |store|
+        Concurrent::Promises.future_on(pool) do
+          begin
+            store.delete!(file.identifier)
+          rescue NotFoundError => e
+            e
+          end
+        end
+      end
+
+      futures.each do |future|
+        result = future.value
+        exceptions << result if result.is_a?(Exception)
+      end
+
+      pool.shutdown
+      pool.wait_for_termination
+    end
+
     def synchronize!(file)
       file.missing_stores.each do |store|
         next if store.read_only?

data/lib/remote_files/file.rb

@@ -118,8 +118,8 @@ module RemoteFiles
       end
     end
 
-    def delete_now!
-      configuration.delete_now!(self)
+    def delete_now!(parallel: false)
+      configuration.delete_now!(self, parallel: parallel)
     end
   end
 end

data/lib/remote_files/version.rb

@@ -1,3 +1,3 @@
 module RemoteFiles
-  VERSION = '3.6.0'
+  VERSION = '3.7.0-alpha.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remote_files
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.7.0.pre.alpha.1
 platform: ruby
 authors:
 - Mick Staugaard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-05-17 00:00:00.000000000 Z
+date: 2024-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-aws
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 description: A library for uploading files to multiple remote storage backends like
   Amazon S3 and Rackspace CloudFiles.
 email:
@@ -76,7 +90,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
+rubygems_version: 3.5.16
 signing_key: 
 specification_version: 4
 summary: The purpose of the library is to implement a simple interface for uploading