s3_website 1.0.3 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a6d978b88e689f1a6f7463c39696829fbdbc3a0b
4
- data.tar.gz: e43484971117687e58977914ceb57e0e0841158f
3
+ metadata.gz: de54cc1b98c305031806e7486cfb3c4e62027db1
4
+ data.tar.gz: 4d95bafa1686532d6e3c8223418694b8699b2ad0
5
5
  SHA512:
6
- metadata.gz: b58f0a2ba92a4a4d082fa82aee0720f2003d3591cff245e523428de9c3d8ccd3ce14fbb3d399c95f13df37c4f3dad5211ab70bcd992b1624297c9dad051e979a
7
- data.tar.gz: d818c530894bca0b630eb77b14a73a5e5addb5e7f8d724a9411b188427e716f47f94da395fdaffc64b62cf53654773abd8a2f08b1a242e454575bd5dea4f26ab
6
+ metadata.gz: 45905b400e8439f0040d180d051066407beccb219870a9ea68077c6a2bdbb1e78a5dbfb2cdbc8d5472ebc4db3c04c83751f27b37e0e341713033c5d0e8674b6a
7
+ data.tar.gz: a49e6beb52409c4ee47fa620efa540c5014a320f67ec518231d0632ca97fcf8e23f19fcc5f9a9d706f3dc71baa2c0ae1d8e9a4f083427b1a415b22b06c1d828b
data/README.md CHANGED
@@ -286,6 +286,25 @@ S3Website::Uploader.run('/path/to/your/website/_site/', config, in_headless)
286
286
  The code above will assume that you have the `s3_website.yml` in the directory
287
287
  `/path/to/your/website`.
288
288
 
289
+ ### Specifying custom concurrency level
290
+
291
+ By default, `s3_website` does 25 operations in parallel. An operation can be an
292
+ HTTP PUT operation against the S3 API, for example.
293
+
294
+ You can increase the concurrency level by adding the following setting into the
295
+ `s3_website.yml` file:
296
+
297
+ ```
298
+ concurrency_level: <integer>
299
+ ```
300
+
301
+ If your site has 100 files, it's a good idea to set the concurrency level to
302
+ 100. As a result, `s3_website` will process each of your 100 files in parallel.
303
+
304
+ If you experience the "too many open files" error, either increase the amount of
305
+ maximum open files (on Unix-like systems, see `man ulimit`) or decrease the
306
+ `concurrency_level` setting.
307
+
289
308
  ## Example configurations
290
309
 
291
310
  See
data/changelog.md CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  This project uses [Semantic Versioning](http://semver.org).
4
4
 
5
+ ## 1.1.0
6
+
7
+ * Add possibility to define the concurrency level in *s3_website.yml*
8
+
5
9
  ## 1.0.3
6
10
 
7
11
  * Reject blacklisted files in a more appropriate place
@@ -1,7 +1,7 @@
1
1
  module S3Website
2
2
  class DiffHelper
3
- def self.resolve_files_to_upload(s3_bucket, site_dir)
4
- s3_data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket)
3
+ def self.resolve_files_to_upload(s3_bucket, site_dir, config)
4
+ s3_data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket, config)
5
5
  fs_data_source = Filey::DataSources::FileSystem.new(site_dir)
6
6
  changed_local_files =
7
7
  Filey::Comparison.list_changed(fs_data_source, s3_data_source)
@@ -1,6 +1,6 @@
1
1
  module S3Website
2
2
  class Keyboard
3
- def self.if_user_confirms_delete(to_delete, standard_input=STDIN)
3
+ def self.if_user_confirms_delete(to_delete, config, standard_input=STDIN)
4
4
  delete_all = false
5
5
  keep_all = false
6
6
  confirmed_deletes = to_delete.map do |f|
@@ -19,7 +19,7 @@ module S3Website
19
19
  f
20
20
  end
21
21
  end.select { |f| f }
22
- Parallelism.each_in_parallel_or_sequentially(confirmed_deletes) { |f|
22
+ Parallelism.each_in_parallel_or_sequentially(confirmed_deletes, config) { |f|
23
23
  yield f
24
24
  }
25
25
  end
@@ -1,12 +1,13 @@
1
1
  module S3Website
2
2
  class Parallelism
3
- def self.each_in_parallel_or_sequentially(items, &operation)
3
+ def self.each_in_parallel_or_sequentially(items, config, &operation)
4
4
  if ENV['disable_parallel_processing']
5
5
  items.each do |item|
6
6
  operation.call item
7
7
  end
8
8
  else
9
- items.each_slice(DEFAULT_CONCURRENCY_LEVEL) { |items|
9
+ slice_size = config['concurrency_level'] || DEFAULT_CONCURRENCY_LEVEL
10
+ items.each_slice(slice_size) { |items|
10
11
  threads = items.map do |item|
11
12
  Thread.new(item) { |item|
12
13
  operation.call item
@@ -19,6 +20,6 @@ module S3Website
19
20
 
20
21
  private
21
22
 
22
- DEFAULT_CONCURRENCY_LEVEL = 100
23
+ DEFAULT_CONCURRENCY_LEVEL = 25
23
24
  end
24
25
  end
@@ -18,11 +18,17 @@ module S3Website
18
18
  redirects = config['redirects'] || {}
19
19
  changed_redirects = setup_redirects redirects, config, s3
20
20
 
21
- deleted_files_count = remove_superfluous_files(s3, { :s3_bucket => config['s3_bucket'],
22
- :site_dir => site_dir,
23
- :redirects => redirects,
24
- :in_headless_mode => in_headless_mode,
25
- :ignore_on_server => config["ignore_on_server"] })
21
+ deleted_files_count = remove_superfluous_files(
22
+ s3,
23
+ config,
24
+ {
25
+ :s3_bucket => config['s3_bucket'],
26
+ :site_dir => site_dir,
27
+ :redirects => redirects,
28
+ :in_headless_mode => in_headless_mode,
29
+ :ignore_on_server => config["ignore_on_server"]
30
+ }
31
+ )
26
32
 
27
33
  print_done_report config
28
34
 
@@ -41,7 +47,10 @@ module S3Website
41
47
 
42
48
  def self.upload_files(s3, config, site_dir)
43
49
  changed_files, new_files = DiffHelper.resolve_files_to_upload(
44
- s3.buckets[config['s3_bucket']], site_dir)
50
+ s3.buckets[config['s3_bucket']],
51
+ site_dir,
52
+ config
53
+ )
45
54
  to_upload = (changed_files + new_files).reject { |f| Upload.is_blacklisted f }
46
55
  if to_upload.empty?
47
56
  puts "No new or changed files to upload"
@@ -59,7 +68,7 @@ module S3Website
59
68
  end
60
69
 
61
70
  def self.upload_in_parallel_or_sequentially(files_to_upload, s3, config, site_dir)
62
- Parallelism.each_in_parallel_or_sequentially(files_to_upload) { |f|
71
+ Parallelism.each_in_parallel_or_sequentially(files_to_upload, config) { |f|
63
72
  upload_file(f, s3, config, site_dir)
64
73
  }
65
74
  end
@@ -114,7 +123,7 @@ module S3Website
114
123
  end
115
124
  end
116
125
 
117
- def self.remove_superfluous_files(s3, options)
126
+ def self.remove_superfluous_files(s3, config, options)
118
127
  s3_bucket_name = options.fetch(:s3_bucket)
119
128
  site_dir = options.fetch(:site_dir)
120
129
  in_headless_mode = options.fetch(:in_headless_mode)
@@ -130,7 +139,7 @@ module S3Website
130
139
  deleted_files_count += 1
131
140
  }
132
141
  else
133
- Keyboard.if_user_confirms_delete(files_to_delete) { |s3_object_key|
142
+ Keyboard.if_user_confirms_delete(files_to_delete, config) { |s3_object_key|
134
143
  delete_s3_object s3, s3_bucket_name, s3_object_key
135
144
  deleted_files_count += 1
136
145
  }
data/s3_website.gemspec CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "s3_website"
6
- s.version = "1.0.3"
6
+ s.version = "1.1.0"
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.authors = ["Lauri Lehmijoki"]
9
9
  s.email = ["lauri.lehmijoki@iki.fi"]
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
17
17
  s.default_executable = %q{s3_website}
18
18
 
19
19
  s.add_dependency 'aws-sdk', '~> 1.8.5'
20
- s.add_dependency 'filey-diff', '~> 1'
20
+ s.add_dependency 'filey-diff', '~> 1.3'
21
21
  s.add_dependency 'simple-cloudfront-invalidator', '~> 1'
22
22
  s.add_dependency 'erubis', '~> 2.7.0'
23
23
  s.add_dependency 'mime-types', '= 1.19'
@@ -49,8 +49,11 @@ describe S3Website::Keyboard do
49
49
 
50
50
  def call_keyboard(s3_object_keys, standard_input)
51
51
  deleted_keys = []
52
- S3Website::Keyboard.if_user_confirms_delete(s3_object_keys,
53
- standard_input) { |key|
52
+ S3Website::Keyboard.if_user_confirms_delete(
53
+ s3_object_keys,
54
+ config = {},
55
+ standard_input
56
+ ) { |key|
54
57
  deleted_keys << key
55
58
  }
56
59
  deleted_keys
@@ -2,6 +2,10 @@ require 'spec_helper'
2
2
 
3
3
  describe S3Website::Parallelism do
4
4
  context 'user has disabled parallelism' do
5
+ let(:config) {
6
+ {}
7
+ }
8
+
5
9
  before(:all) {
6
10
  @original_disable_state = ENV['disable_parallel_processing']
7
11
  ENV['disable_parallel_processing'] = 'true'
@@ -14,7 +18,7 @@ describe S3Website::Parallelism do
14
18
  it 'runs things sequentially' do
15
19
  ints = (0..100).to_a
16
20
  after_processing = []
17
- S3Website::Parallelism.each_in_parallel_or_sequentially(ints) { |int|
21
+ S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
18
22
  after_processing << int
19
23
  }
20
24
  ints.should eq(after_processing)
@@ -22,6 +26,10 @@ describe S3Website::Parallelism do
22
26
  end
23
27
 
24
28
  context 'user has not disabled parallelism' do
29
+ let(:config) {
30
+ {}
31
+ }
32
+
25
33
  before(:all) {
26
34
  @original_disable_state = ENV['disable_parallel_processing']
27
35
  ENV.delete 'disable_parallel_processing'
@@ -34,7 +42,7 @@ describe S3Website::Parallelism do
34
42
  it 'runs things in parallel' do
35
43
  ints = (0..100).to_a
36
44
  after_processing = []
37
- S3Website::Parallelism.each_in_parallel_or_sequentially(ints) { |int|
45
+ S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
38
46
  after_processing << int
39
47
  }
40
48
  ints.should_not eq(after_processing) # Parallel processing introduces non-determinism
@@ -42,21 +50,32 @@ describe S3Website::Parallelism do
42
50
  end
43
51
 
44
52
  context 'limiting parallelism' do
45
- before(:each) {
46
- ints = (0..199).to_a
47
- @after_processing = []
48
- S3Website::Parallelism.each_in_parallel_or_sequentially(ints) { |int|
49
- @after_processing << int
53
+ shared_examples 'parallel processing' do |config|
54
+ let(:concurrency_level) {
55
+ config['concurrency_level'] || S3Website::Parallelism::DEFAULT_CONCURRENCY_LEVEL
50
56
  }
51
- }
52
57
 
53
- it "does at most #{S3Website::Parallelism::DEFAULT_CONCURRENCY_LEVEL} operations in parallel" do
54
- @after_processing.slice(0, 99).all? do |int|
55
- int <= 99
56
- end.should be true
57
- @after_processing.slice(100, 199).all? do |int|
58
- int >= 100 and int <= 199
59
- end.should be true
58
+ before(:each) {
59
+ ints = (0..199).to_a
60
+ @after_processing = []
61
+ S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
62
+ @after_processing << int
63
+ }
64
+ }
65
+
66
+ it "does at most <concurrency_level> operations in parallel" do
67
+ @after_processing.slice(0, concurrency_level).all? do |int|
68
+ int <= concurrency_level
69
+ end.should be true
70
+ @after_processing.slice(100, concurrency_level).all? do |int|
71
+ int >= 100 and int <= 100 + concurrency_level
72
+ end.should be true
73
+ end
60
74
  end
75
+
76
+
77
+ include_examples 'parallel processing', config = {}
78
+
79
+ include_examples 'parallel processing', config = { 'concurrency_level' => 100 }
61
80
  end
62
81
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: s3_website
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Lauri Lehmijoki
@@ -30,14 +30,14 @@ dependencies:
30
30
  requirements:
31
31
  - - ~>
32
32
  - !ruby/object:Gem::Version
33
- version: '1'
33
+ version: '1.3'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ~>
39
39
  - !ruby/object:Gem::Version
40
- version: '1'
40
+ version: '1.3'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: simple-cloudfront-invalidator
43
43
  requirement: !ruby/object:Gem::Requirement