s3_website 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6d978b88e689f1a6f7463c39696829fbdbc3a0b
-  data.tar.gz: e43484971117687e58977914ceb57e0e0841158f
+  metadata.gz: de54cc1b98c305031806e7486cfb3c4e62027db1
+  data.tar.gz: 4d95bafa1686532d6e3c8223418694b8699b2ad0
 SHA512:
-  metadata.gz: b58f0a2ba92a4a4d082fa82aee0720f2003d3591cff245e523428de9c3d8ccd3ce14fbb3d399c95f13df37c4f3dad5211ab70bcd992b1624297c9dad051e979a
-  data.tar.gz: d818c530894bca0b630eb77b14a73a5e5addb5e7f8d724a9411b188427e716f47f94da395fdaffc64b62cf53654773abd8a2f08b1a242e454575bd5dea4f26ab
+  metadata.gz: 45905b400e8439f0040d180d051066407beccb219870a9ea68077c6a2bdbb1e78a5dbfb2cdbc8d5472ebc4db3c04c83751f27b37e0e341713033c5d0e8674b6a
+  data.tar.gz: a49e6beb52409c4ee47fa620efa540c5014a320f67ec518231d0632ca97fcf8e23f19fcc5f9a9d706f3dc71baa2c0ae1d8e9a4f083427b1a415b22b06c1d828b

data/README.md

@@ -286,6 +286,25 @@ S3Website::Uploader.run('/path/to/your/website/_site/', config, in_headless)
 The code above will assume that you have the `s3_website.yml` in the directory
 `/path/to/your/website`.
 
+### Specifying custom concurrency level
+
+By default, `s3_website` does 25 operations in parallel. An operation can be an
+HTTP PUT operation against the S3 API, for example.
+
+You can increase the concurrency level by adding the following setting into the
+`s3_website.yml` file:
+
+```
+concurrency_level: <integer>
+```
+
+If your site has 100 files, it's a good idea to set the concurrency level to
+100. As a result, `s3_website` will process each of your 100 files in parallel.
+
+If you experience the "too many open files" error, either increase the amount of
+maximum open files (on Unix-like systems, see `man ulimit`) or decrease the
+`concurrency_level` setting.
+
 ## Example configurations
 
 See

data/changelog.md

@@ -2,6 +2,10 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
+## 1.1.0
+
+* Add possibility to define the concurrency level in *s3_website.yml*
+
 ## 1.0.3
 
 * Reject blacklisted files in a more appropriate place

data/lib/s3_website/diff_helper.rb

@@ -1,7 +1,7 @@
 module S3Website
   class DiffHelper
-    def self.resolve_files_to_upload(s3_bucket, site_dir)
-      s3_data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket)
+    def self.resolve_files_to_upload(s3_bucket, site_dir, config)
+      s3_data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket, config)
       fs_data_source = Filey::DataSources::FileSystem.new(site_dir)
       changed_local_files =
         Filey::Comparison.list_changed(fs_data_source, s3_data_source)

data/lib/s3_website/keyboard.rb

@@ -1,6 +1,6 @@
 module S3Website
   class Keyboard
-    def self.if_user_confirms_delete(to_delete, standard_input=STDIN)
+    def self.if_user_confirms_delete(to_delete, config, standard_input=STDIN)
       delete_all = false
       keep_all = false
       confirmed_deletes = to_delete.map do |f|
@@ -19,7 +19,7 @@ module S3Website
           f
         end
       end.select { |f| f }
-      Parallelism.each_in_parallel_or_sequentially(confirmed_deletes) { |f|
+      Parallelism.each_in_parallel_or_sequentially(confirmed_deletes, config) { |f|
         yield f
       }
     end

data/lib/s3_website/parallelism.rb

@@ -1,12 +1,13 @@
 module S3Website
   class Parallelism
-    def self.each_in_parallel_or_sequentially(items, &operation)
+    def self.each_in_parallel_or_sequentially(items, config, &operation)
       if ENV['disable_parallel_processing']
         items.each do |item|
           operation.call item
         end
       else
-        items.each_slice(DEFAULT_CONCURRENCY_LEVEL) { |items|
+        slice_size = config['concurrency_level'] || DEFAULT_CONCURRENCY_LEVEL
+        items.each_slice(slice_size) { |items|
           threads = items.map do |item|
             Thread.new(item) { |item|
               operation.call item
@@ -19,6 +20,6 @@ module S3Website
 
     private
 
-    DEFAULT_CONCURRENCY_LEVEL = 100
+    DEFAULT_CONCURRENCY_LEVEL = 25
   end
 end

data/lib/s3_website/uploader.rb

@@ -18,11 +18,17 @@ module S3Website
       redirects = config['redirects'] || {}
       changed_redirects = setup_redirects redirects, config, s3
 
-      deleted_files_count = remove_superfluous_files(s3, { :s3_bucket => config['s3_bucket'],
-                                                           :site_dir => site_dir,
-                                                           :redirects => redirects,
-                                                           :in_headless_mode => in_headless_mode,
-                                                           :ignore_on_server => config["ignore_on_server"] })
+      deleted_files_count = remove_superfluous_files(
+        s3,
+        config,
+        {
+          :s3_bucket => config['s3_bucket'],
+          :site_dir => site_dir,
+          :redirects => redirects,
+          :in_headless_mode => in_headless_mode,
+          :ignore_on_server => config["ignore_on_server"]
+        }
+      )
 
       print_done_report config
 
@@ -41,7 +47,10 @@ module S3Website
 
     def self.upload_files(s3, config, site_dir)
       changed_files, new_files = DiffHelper.resolve_files_to_upload(
-        s3.buckets[config['s3_bucket']], site_dir)
+        s3.buckets[config['s3_bucket']],
+        site_dir,
+        config
+      )
       to_upload = (changed_files + new_files).reject { |f| Upload.is_blacklisted f }
       if to_upload.empty?
         puts "No new or changed files to upload"
@@ -59,7 +68,7 @@ module S3Website
     end
 
     def self.upload_in_parallel_or_sequentially(files_to_upload, s3, config, site_dir)
-      Parallelism.each_in_parallel_or_sequentially(files_to_upload) { |f|
+      Parallelism.each_in_parallel_or_sequentially(files_to_upload, config) { |f|
         upload_file(f, s3, config, site_dir)
       }
     end
@@ -114,7 +123,7 @@ module S3Website
       end
     end
 
-    def self.remove_superfluous_files(s3, options)
+    def self.remove_superfluous_files(s3, config, options)
       s3_bucket_name = options.fetch(:s3_bucket)
       site_dir = options.fetch(:site_dir)
       in_headless_mode = options.fetch(:in_headless_mode)
@@ -130,7 +139,7 @@ module S3Website
           deleted_files_count += 1
         }
       else
-        Keyboard.if_user_confirms_delete(files_to_delete) { |s3_object_key|
+        Keyboard.if_user_confirms_delete(files_to_delete, config) { |s3_object_key|
           delete_s3_object s3, s3_bucket_name, s3_object_key
           deleted_files_count += 1
         }

data/s3_website.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "s3_website"
-  s.version     = "1.0.3"
+  s.version     = "1.1.0"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Lauri Lehmijoki"]
   s.email       = ["lauri.lehmijoki@iki.fi"]
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
   s.default_executable = %q{s3_website}
 
   s.add_dependency 'aws-sdk', '~> 1.8.5'
-  s.add_dependency 'filey-diff', '~> 1'
+  s.add_dependency 'filey-diff', '~> 1.3'
   s.add_dependency 'simple-cloudfront-invalidator', '~> 1'
   s.add_dependency 'erubis', '~> 2.7.0'
   s.add_dependency 'mime-types', '= 1.19'

data/spec/lib/keyboard_spec.rb

@@ -49,8 +49,11 @@ describe S3Website::Keyboard do
 
     def call_keyboard(s3_object_keys, standard_input)
       deleted_keys = []
-      S3Website::Keyboard.if_user_confirms_delete(s3_object_keys, 
-                                                   standard_input) { |key|
+      S3Website::Keyboard.if_user_confirms_delete(
+        s3_object_keys,
+        config = {},
+        standard_input
+      ) { |key|
         deleted_keys << key
       }
       deleted_keys

data/spec/lib/parallelism_spec.rb

@@ -2,6 +2,10 @@ require 'spec_helper'
 
 describe S3Website::Parallelism do
   context 'user has disabled parallelism' do
+    let(:config) {
+      {}
+    }
+
     before(:all) {
       @original_disable_state = ENV['disable_parallel_processing']
       ENV['disable_parallel_processing'] = 'true'
@@ -14,7 +18,7 @@ describe S3Website::Parallelism do
     it 'runs things sequentially' do
       ints = (0..100).to_a
       after_processing = []
-      S3Website::Parallelism.each_in_parallel_or_sequentially(ints) { |int|
+      S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
         after_processing << int
       }
       ints.should eq(after_processing)
@@ -22,6 +26,10 @@ describe S3Website::Parallelism do
   end
 
   context 'user has not disabled parallelism' do
+    let(:config) {
+      {}
+    }
+
     before(:all) {
       @original_disable_state = ENV['disable_parallel_processing']
       ENV.delete 'disable_parallel_processing'
@@ -34,7 +42,7 @@ describe S3Website::Parallelism do
     it 'runs things in parallel' do
       ints = (0..100).to_a
       after_processing = []
-      S3Website::Parallelism.each_in_parallel_or_sequentially(ints) { |int|
+      S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
         after_processing << int
       }
       ints.should_not eq(after_processing) # Parallel processing introduces non-determinism
@@ -42,21 +50,32 @@ describe S3Website::Parallelism do
   end
 
   context 'limiting parallelism' do
-    before(:each) {
-      ints = (0..199).to_a
-      @after_processing = []
-      S3Website::Parallelism.each_in_parallel_or_sequentially(ints) { |int|
-        @after_processing << int
+    shared_examples 'parallel processing' do |config|
+      let(:concurrency_level) {
+        config['concurrency_level'] || S3Website::Parallelism::DEFAULT_CONCURRENCY_LEVEL
       }
-    }
 
-    it "does at most #{S3Website::Parallelism::DEFAULT_CONCURRENCY_LEVEL} operations in parallel" do
-      @after_processing.slice(0, 99).all? do |int|
-        int <= 99
-      end.should be true
-      @after_processing.slice(100, 199).all? do |int|
-        int >= 100 and int <= 199
-      end.should be true
+      before(:each) {
+        ints = (0..199).to_a
+        @after_processing = []
+        S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
+          @after_processing << int
+        }
+      }
+
+      it "does at most <concurrency_level> operations in parallel" do
+        @after_processing.slice(0, concurrency_level).all? do |int|
+          int <= concurrency_level
+        end.should be true
+        @after_processing.slice(100, concurrency_level).all? do |int|
+          int >= 100 and int <= 100 + concurrency_level
+        end.should be true
+      end
     end
+
+
+    include_examples 'parallel processing', config = {}
+
+    include_examples 'parallel processing', config = { 'concurrency_level' => 100 }
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_website
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: ruby
 authors:
 - Lauri Lehmijoki
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: simple-cloudfront-invalidator
   requirement: !ruby/object:Gem::Requirement