water_wheel 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0518601af3c35d322c7f14602f5dd469f8de55bb0c5da665aac785e2e347fc6f'
-  data.tar.gz: 568716059678669010c4f1013a6a65e65d42212e502a27e5b5175b53d3ac7a34
+  metadata.gz: d75be9afccde503215262b32fc3bcdbae69ae6653d249a58a37eb206e897a3b7
+  data.tar.gz: 0dcb13d9a9185e697156b6ad0acaa5d3bcc97d7472e420562709993d3b93b3e1
 SHA512:
-  metadata.gz: 04f6b8c096ffcc76dabf5fe8b4a6de5dde617832afe7199577f5b378b37b8324ea48e2aa364f2a9430f5cc0772f625580b5fb0f8fc32e20cdb9d9c85ee29be94
-  data.tar.gz: 6eb77d7fff319bd40f1aebae07e65404a46ef22b191285c94de530d7b1731ad50c39de500ae95d00fa990cd54eba4f0a657c49149647f2c6b78aff0c83c527e4
+  metadata.gz: 871aad2fc9a25bfc8911525930eb1cbcceb20605e041f520061fa7f78650a28e9e879fe83ff0719f11b155f1a78e40d0d322c1e95f0660c18eef7bf68902cdc3
+  data.tar.gz: d6a35db07b29eaa57f057aa0c56e4f5b6dd13fdeccebbac143d2b8aeb68db20ca1612966de6a3b0072f18b7b194a0ef03a1725ce905716b70fb088a21b2869c6

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
+* None
 
-## [0.1.0] - 2021-12-30
+## [0.2.0] - 2022-01-29
+### Added
+- Introduced `parallel_count` setting, which can be used to set the number of parallel threads to use.
 
-- Initial release
+### Changed
+- Changed retry range from not only upload section to but also pre-upload section.
+
+## [0.1.0] - 2022-01-22
+### Added
+- Initial release.
+- Using `water_wheel` gem, you can backup your local files or directories to AWS S3.

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    water_wheel (0.1.0)
+    water_wheel (0.2.0)
       fog-aws (~> 3.0)
+      parallel (~> 1.21)
       retriable (~> 3.1)
 
 GEM
@@ -11,7 +12,7 @@ GEM
     ast (2.4.2)
     builder (3.2.4)
     diff-lcs (1.5.0)
-    excon (0.89.0)
+    excon (0.90.0)
     fog-aws (3.12.0)
       fog-core (~> 2.1)
       fog-json (~> 1.1)
@@ -32,11 +33,11 @@ GEM
     ipaddress (0.8.3)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.1115)
+    mime-types-data (3.2022.0105)
     multi_json (1.15.0)
-    nokogiri (1.12.5-x86_64-darwin)
+    nokogiri (1.13.1-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.12.5-x86_64-linux)
+    nokogiri (1.13.1-x86_64-linux)
       racc (~> 1.4)
     parallel (1.21.0)
     parser (3.0.3.2)

data/README.md

@@ -1,5 +1,6 @@
 # WaterWheel
 
+[![Gem Version](https://badge.fury.io/rb/water_wheel.svg)](https://badge.fury.io/rb/water_wheel)
 [![RSpec](https://github.com/shoutatani/water_wheel/actions/workflows/main.yml/badge.svg)](https://github.com/shoutatani/water_wheel/actions/workflows/main.yml)
 
 Backup your local files or directories into cloud storages.
@@ -11,7 +12,7 @@ Now AWS S3 (Simple Storage Service) is only available.
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'water_wheel'
+gem "water_wheel"
 ```
 
 And then execute:
@@ -26,13 +27,13 @@ Or install it yourself as:
 
 1. Create a new S3 bucket for backup.
 
-1. add `require 'water_wheel'` to your application or ruby script.
+1. add `require "water_wheel"` to your application or ruby script.
 
 1. Setting water_wheel gem with your AWS credential.
 
     ```ruby
     WaterWheel.configure do |config|
-      config.provider = 'AWS'
+      config.provider = "AWS"
       config.aws_access_key_id = "your_access_key_id"
       config.aws_secret_access_key = "your_secret_access_key"
       config.aws_region = "your_region"
@@ -40,14 +41,15 @@ Or install it yourself as:
       config.absolute_path_on_files = ["/path/to/your/files", "/path/to/your/other/files"]
       config.absolute_path_on_directories = ["/path/to/your/directories", "/path/to/your/other/directories"]
       config.ordered_omit_path_prefixes = ["/path/prefix/to/omit"]
-      config.storage_class = 'STANDARD' # STANDARD, REDUCED_REDUNDANCY, STANDARD_IA...
+      config.storage_class = "STANDARD" # STANDARD, REDUCED_REDUNDANCY, STANDARD_IA...
+      config.parallel_count = 3
       config.dry_run = false
     end
     ```
 
+    * You can change upload parallel thread count by `parallel_count` setting. Default is 3.
     * if you'd like to do a dry run, set `config.dry_run = true`
 
-
 1. Run, and you will see the result.
 
     ```ruby

data/lib/water_wheel/backup.rb

@@ -2,6 +2,7 @@
 
 require "fog/aws"
 require "retriable"
+require "parallel"
 
 module WaterWheel
   class Backup
@@ -61,7 +62,7 @@ module WaterWheel
 
       def upload_directory(absolute_directory_path)
         included_files = Dir.glob(File.join(absolute_directory_path, "**", "*"))
-        included_files.each do |included_file|
+        Parallel.each(included_files, in_threads: configuration.parallel_count) do |included_file|
           next if File.directory?(included_file)
 
           upload_file(included_file)
@@ -71,14 +72,14 @@ module WaterWheel
       def upload_file(absolute_file_path)
         bucket_file_key = create_bucket_key(absolute_file_path)
 
-        return unless shouldUploadFile?(bucket_file_key, absolute_file_path)
+        begin
+          return unless shouldUploadFile?(bucket_file_key, absolute_file_path)
 
-        if configuration.dry_run
-          WaterWheel.logger.info "Dry run: Uploading #{absolute_file_path} to #{bucket_file_key}"
-          return
-        end
+          if configuration.dry_run
+            WaterWheel.logger.info "Dry run: Uploading #{absolute_file_path} to #{bucket_file_key}"
+            return
+          end
 
-        begin
           Retriable.retriable do
             bucket.files.create(
               key: bucket_file_key,
@@ -103,7 +104,7 @@ module WaterWheel
         bucket_file_key = absolute_file_path
         configuration.ordered_omit_path_prefixes.each do |prefix|
           if absolute_file_path.start_with?(prefix)
-            bucket_file_key = absolute_file_path.sub(prefix, '')
+            bucket_file_key = absolute_file_path.sub(prefix, "")
             break
           end
         end

data/lib/water_wheel/configuration.rb

@@ -11,6 +11,7 @@ module WaterWheel
                   :absolute_path_on_directories,
                   :ordered_omit_path_prefixes,
                   :storage_class,
+                  :parallel_count,
                   :dry_run
 
     def initialize
@@ -23,6 +24,7 @@ module WaterWheel
       @absolute_path_on_directories = []
       @ordered_omit_path_prefixes = ["/"]
       @storage_class = "STANDARD"
+      @parallel_count = 3
       @dry_run = false
     end
 
@@ -41,6 +43,7 @@ module WaterWheel
     def validate!
       validate_aws_configuration if provider == "AWS"
       validate_backup_target
+      validate_parallel_count
     end
 
     private
@@ -51,19 +54,20 @@ module WaterWheel
          aws_region
          aws_bucket_name
       ).each do |key|
-       if self.public_send(key).nil? || self.public_send(key).empty?
-         error_message = <<~ERROR
-           WaterWheel configuration #{key} is missing.
-           Please set it as
-            `WaterWheel.configure do |config|
-               config.#{key} = "value"
-             end`
-         ERROR
-         raise error_message
-       end
-     end
+        if self.public_send(key).nil? || self.public_send(key).empty?
+          error_message = <<~ERROR
+            WaterWheel configuration #{key} is missing.
+            Please set it as
+              `WaterWheel.configure do |config|
+                 config.#{key} = "value"
+               end`
+          ERROR
+          raise error_message
+        end
+      end
+    end
 
-     def validate_backup_target
+    def validate_backup_target
       both_upload_target_are_not_set = %i(absolute_path_on_files absolute_path_on_directories).all? do |key|
         self.public_send(key).nil? || self.public_send(key).empty?
       end
@@ -78,7 +82,19 @@ module WaterWheel
         ERROR
         raise error_message
       end
-     end
+    end
+
+    def validate_parallel_count
+      if parallel_count.nil? || parallel_count < 1
+        error_message = <<~ERROR
+          WaterWheel parallel count is invalid.
+          Please set it as
+            `WaterWheel.configure do |config|
+               config.parallel_count = 10
+             end`
+        ERROR
+        raise error_message
+      end
     end
   end
 end

data/lib/water_wheel/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WaterWheel
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/water_wheel.gemspec

@@ -29,14 +29,12 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # Uncomment to register a new dependency of your gem
   spec.add_dependency "fog-aws", "~> 3.0"
   spec.add_dependency "retriable", "~> 3.1"
+  spec.add_dependency "parallel", "~> 1.21"
 
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rubocop", "~> 1.21"
 
-  # For more information and examples about making a new gem, check out our
-  # guide at: https://bundler.io/guides/creating_gem.html
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: water_wheel
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - shoutatani
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-01-22 00:00:00.000000000 Z
+date: 2022-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-aws
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement