s3_website 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +5 -0
  3. data/.travis.yml +5 -0
  4. data/Gemfile +3 -0
  5. data/Gemfile.lock +81 -0
  6. data/LICENSE +42 -0
  7. data/README.md +332 -0
  8. data/Rakefile +18 -0
  9. data/bin/s3_website +38 -0
  10. data/changelog.md +7 -0
  11. data/example-configurations.md +60 -0
  12. data/features/as-library.feature +29 -0
  13. data/features/cassettes/cucumber_tags/create-redirect.yml +384 -0
  14. data/features/cassettes/cucumber_tags/new-and-changed-files.yml +303 -0
  15. data/features/cassettes/cucumber_tags/new-files-for-sydney.yml +211 -0
  16. data/features/cassettes/cucumber_tags/new-files.yml +355 -0
  17. data/features/cassettes/cucumber_tags/no-new-or-changed-files.yml +359 -0
  18. data/features/cassettes/cucumber_tags/one-file-to-delete.yml +390 -0
  19. data/features/cassettes/cucumber_tags/only-changed-files.yml +411 -0
  20. data/features/cassettes/cucumber_tags/s3-and-cloudfront-when-updating-a-file.yml +435 -0
  21. data/features/cassettes/cucumber_tags/s3-and-cloudfront.yml +290 -0
  22. data/features/cloudfront.feature +35 -0
  23. data/features/delete.feature +18 -0
  24. data/features/instructions-for-new-user.feature +94 -0
  25. data/features/redirects.feature +16 -0
  26. data/features/step_definitions/steps.rb +67 -0
  27. data/features/support/env.rb +26 -0
  28. data/features/support/test_site_dirs/cdn-powered.blog.fi/_site/css/styles.css +3 -0
  29. data/features/support/test_site_dirs/cdn-powered.blog.fi/_site/index.html +5 -0
  30. data/features/support/test_site_dirs/cdn-powered.blog.fi/s3_website.yml +4 -0
  31. data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/_site/css/styles.css +3 -0
  32. data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/_site/index.html +10 -0
  33. data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/s3_website.yml +4 -0
  34. data/features/support/test_site_dirs/create-redirects/_site/.gitkeep +0 -0
  35. data/features/support/test_site_dirs/create-redirects/s3_website.yml +6 -0
  36. data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/assets/picture.gif +0 -0
  37. data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/css/styles.css +3 -0
  38. data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/index.html +5 -0
  39. data/features/support/test_site_dirs/index-and-assets.blog.fi/s3_website.yml +3 -0
  40. data/features/support/test_site_dirs/my.blog.com/_site/css/styles.css +3 -0
  41. data/features/support/test_site_dirs/my.blog.com/_site/index.html +5 -0
  42. data/features/support/test_site_dirs/my.blog.com/s3_website.yml +3 -0
  43. data/features/support/test_site_dirs/my.sydney.blog.au/_site/css/styles.css +3 -0
  44. data/features/support/test_site_dirs/my.sydney.blog.au/_site/index.html +5 -0
  45. data/features/support/test_site_dirs/my.sydney.blog.au/s3_website.yml +4 -0
  46. data/features/support/test_site_dirs/new-and-changed-files.com/_site/css/styles.css +4 -0
  47. data/features/support/test_site_dirs/new-and-changed-files.com/_site/index.html +8 -0
  48. data/features/support/test_site_dirs/new-and-changed-files.com/s3_website.yml +3 -0
  49. data/features/support/test_site_dirs/no-new-or-changed-files.com/_site/css/styles.css +3 -0
  50. data/features/support/test_site_dirs/no-new-or-changed-files.com/_site/index.html +5 -0
  51. data/features/support/test_site_dirs/no-new-or-changed-files.com/s3_website.yml +3 -0
  52. data/features/support/test_site_dirs/only-changed-files.com/_site/css/styles.css +3 -0
  53. data/features/support/test_site_dirs/only-changed-files.com/_site/index.html +9 -0
  54. data/features/support/test_site_dirs/only-changed-files.com/s3_website.yml +3 -0
  55. data/features/support/test_site_dirs/site.with.css-maxage.com/_site/css/styles.css +3 -0
  56. data/features/support/test_site_dirs/site.with.css-maxage.com/_site/index.html +5 -0
  57. data/features/support/test_site_dirs/site.with.css-maxage.com/s3_website.yml +5 -0
  58. data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/_site/css/styles.css +3 -0
  59. data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/_site/index.html +5 -0
  60. data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/s3_website.yml +5 -0
  61. data/features/support/test_site_dirs/site.with.gzipped-html.com/_site/css/styles.css +3 -0
  62. data/features/support/test_site_dirs/site.with.gzipped-html.com/_site/index.html +5 -0
  63. data/features/support/test_site_dirs/site.with.gzipped-html.com/s3_website.yml +5 -0
  64. data/features/support/test_site_dirs/site.with.maxage.com/_site/css/styles.css +3 -0
  65. data/features/support/test_site_dirs/site.with.maxage.com/_site/index.html +5 -0
  66. data/features/support/test_site_dirs/site.with.maxage.com/s3_website.yml +4 -0
  67. data/features/support/test_site_dirs/unpublish-a-post.com/_site/css/styles.css +3 -0
  68. data/features/support/test_site_dirs/unpublish-a-post.com/s3_website.yml +3 -0
  69. data/features/support/vcr.rb +18 -0
  70. data/features/sync.feature +80 -0
  71. data/features/website-performance.feature +57 -0
  72. data/lib/cloudfront/invalidator.rb +22 -0
  73. data/lib/s3_website.rb +20 -0
  74. data/lib/s3_website/config_loader.rb +56 -0
  75. data/lib/s3_website/diff_helper.rb +21 -0
  76. data/lib/s3_website/endpoint.rb +30 -0
  77. data/lib/s3_website/errors.rb +28 -0
  78. data/lib/s3_website/keyboard.rb +27 -0
  79. data/lib/s3_website/parallelism.rb +18 -0
  80. data/lib/s3_website/retry.rb +19 -0
  81. data/lib/s3_website/tasks.rb +42 -0
  82. data/lib/s3_website/upload.rb +103 -0
  83. data/lib/s3_website/uploader.rb +160 -0
  84. data/s3-website.gemspec +39 -0
  85. data/spec/lib/config_loader_spec.rb +20 -0
  86. data/spec/lib/endpoint_spec.rb +27 -0
  87. data/spec/lib/keyboard_spec.rb +59 -0
  88. data/spec/lib/parallelism_spec.rb +43 -0
  89. data/spec/lib/retry_spec.rb +34 -0
  90. data/spec/lib/upload_spec.rb +205 -0
  91. data/spec/lib/uploader_spec.rb +30 -0
  92. data/spec/sample_files/hyde_site/_site/.vimrc +5 -0
  93. data/spec/sample_files/hyde_site/_site/css/styles.css +3 -0
  94. data/spec/sample_files/hyde_site/_site/index.html +1 -0
  95. data/spec/sample_files/hyde_site/s3_website.yml +3 -0
  96. data/spec/sample_files/tokyo_site/_site/.vimrc +5 -0
  97. data/spec/sample_files/tokyo_site/_site/css/styles.css +3 -0
  98. data/spec/sample_files/tokyo_site/_site/index.html +1 -0
  99. data/spec/sample_files/tokyo_site/s3_website.yml +4 -0
  100. data/spec/spec_helper.rb +1 -0
  101. metadata +416 -0
@@ -0,0 +1,35 @@
1
+ Feature: Invalidate the Cloudfront distribution
2
+
3
+ In order to publish my posts
4
+ As a blogger who delivers his blog via an S3-based Cloudfront distribution
5
+ I want to run s3_website
6
+ And see, that the items in the distribution were invalidated
7
+ So that my latest updates will be immediately available to readers
8
+
9
+ @s3-and-cloudfront
10
+ Scenario: Upload to S3 and then invalidate the Cloudfront distribution
11
+ When my S3 website is in "features/support/test_site_dirs/cdn-powered.blog.fi"
12
+ Then s3_website will push my blog to S3 and invalidate the Cloudfront distribution
13
+ And the output should contain
14
+ """
15
+ Invalidating Cloudfront items...
16
+ /
17
+ succeeded
18
+ """
19
+
20
+ @s3-and-cloudfront-when-updating-a-file
21
+ Scenario: Update a blog entry and then upload
22
+ When my S3 website is in "features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi"
23
+ Then s3_website will push my blog to S3 and invalidate the Cloudfront distribution
24
+ And the output should equal
25
+ """
26
+ Deploying _site/* to s3-website-test.net
27
+ Uploading 1 changed file(s)
28
+ Upload index.html: Success!
29
+ Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
30
+ Invalidating Cloudfront items...
31
+ /index.html
32
+ /
33
+ succeeded
34
+
35
+ """
@@ -0,0 +1,18 @@
1
+ Feature: remove an S3 website page from S3
2
+
3
+ In order to remove a webpage from S3
4
+ As a blogger
5
+ I want to run s3_website and see that my webpage was deleted from S3
6
+
7
+ @one-file-to-delete
8
+ Scenario: The user deletes a blog post
9
+ When my S3 website is in "features/support/test_site_dirs/unpublish-a-post.com"
10
+ Then s3_website will push my blog to S3
11
+ And the output should equal
12
+ """
13
+ Deploying _site/* to s3-website-test.net
14
+ No new or changed files to upload
15
+ Delete index.html: Success!
16
+ Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
17
+
18
+ """
@@ -0,0 +1,94 @@
1
+ Feature: Instructions for a new user
2
+
3
+ As a new s3_website user
4
+ I would like to get helpful feedback when running `s3_website`
5
+ So that I can upload my S3 website to S3 without headache
6
+
7
+ Scenario: Run s3_website in the wrong directory
8
+ When I run `s3_website push`
9
+ Then the output should contain:
10
+ """
11
+ I can't find any directory called _site. Are you in the right directory?
12
+ """
13
+
14
+ Scenario: Create placeholder config file
15
+ Given a directory named "_site"
16
+ When I run `s3_website cfg create`
17
+ Then the output should contain:
18
+ """
19
+ I've just generated a file called s3_website.yml. Go put your details in it!
20
+ """
21
+ Then the file "s3_website.yml" should contain:
22
+ """
23
+ s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
24
+ s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
25
+ s3_bucket: your.blog.bucket.com
26
+ """
27
+
28
+ Scenario: Run s3_website push for the first time
29
+ Given a directory named "_site"
30
+ When I run `s3_website push`
31
+ Then the output should contain:
32
+ """
33
+ I've just generated a file called s3_website.yml. Go put your details in it!
34
+ """
35
+ Then the file "s3_website.yml" should contain:
36
+ """
37
+ s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
38
+ s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
39
+ s3_bucket: your.blog.bucket.com
40
+ """
41
+
42
+ Scenario: Run s3_website with an empty configuration file
43
+ Given a directory named "_site"
44
+ And an empty file named "s3_website.yml"
45
+ When I run `s3_website push`
46
+ Then the output should contain:
47
+ """
48
+ I can't parse the file s3_website.yml. It should look like this:
49
+ s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
50
+ s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
51
+ s3_bucket: your.blog.bucket.com
52
+ """
53
+
54
+ Scenario: Run s3_website with a malformed configuration file
55
+ Given a directory named "_site"
56
+ And a file named "s3_website.yml" with:
57
+ """
58
+ s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
59
+ this is not yaml
60
+ """
61
+ When I run `s3_website push`
62
+ Then the output should contain:
63
+ """
64
+ I can't parse the file s3_website.yml. It should look like this:
65
+ s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
66
+ s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
67
+ s3_bucket: your.blog.bucket.com
68
+ """
69
+
70
+ Scenario: Run s3_website with a configuration file that does not contain a bucket
71
+ Given a directory named "_site"
72
+ And a file named "s3_website.yml" with:
73
+ """
74
+ s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
75
+ s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
76
+ s3_bucket:
77
+ """
78
+ When I run `s3_website push`
79
+ Then the output should contain:
80
+ """
81
+ I can't parse the file s3_website.yml. It should look like this:
82
+ s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
83
+ s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
84
+ s3_bucket: your.blog.bucket.com
85
+ """
86
+
87
+ @new-files
88
+ Scenario: Print the URL of the to the user
89
+ When my S3 website is in "features/support/test_site_dirs/my.blog.com"
90
+ Then s3_website will push my blog to S3
91
+ And the output should contain
92
+ """
93
+ Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
94
+ """
@@ -0,0 +1,16 @@
1
+ Feature: configure redirects
2
+
3
+ @create-redirect
4
+ Scenario: The user wants to configure new redirects for HTTP resources
5
+ When my S3 website is in "features/support/test_site_dirs/create-redirects"
6
+ Then s3_website will push my blog to S3
7
+ And the output should equal
8
+ """
9
+ Deploying _site/* to s3-website-test.net
10
+ No new or changed files to upload
11
+ Creating new redirects ...
12
+ Redirect welcome.php to /welcome: Success!
13
+ Redirect pets/dogs to /cats-and-dogs/wuf: Success!
14
+ Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
15
+
16
+ """
@@ -0,0 +1,67 @@
1
+ require 'rspec'
2
+
3
+ When /^my S3 website is in "(.*?)"$/ do |blog_dir|
4
+ @blog_dir = blog_dir
5
+ end
6
+
7
+ When /^s3_website will push my blog to S3$/ do
8
+ push_files
9
+ end
10
+
11
+ Then /^s3_website will push my blog to S(\d+) and invalidate the Cloudfront distribution$/ do |args|
12
+ push_files
13
+ end
14
+
15
+ Then /^the output should equal$/ do |expected_console_output|
16
+ @console_output.should eq(expected_console_output)
17
+ end
18
+
19
+ Then /^the output should contain$/ do |expected_console_output|
20
+ @console_output.should include(expected_console_output)
21
+ end
22
+
23
+ Then /^report that it uploaded (\d+) new and (\d+) changed files into S3$/ do
24
+ |new_count, changed_count|
25
+ @amount_of_new_files.should == new_count.to_i
26
+ @amount_of_changed_files.should == changed_count.to_i
27
+ end
28
+
29
+ Then /^report that it invalidated (\d+) Cloudfront item$/ do |expected|
30
+ @amount_of_invalidated_items.should == expected.to_i
31
+ end
32
+
33
+ Then /^report that it created (\d+) new redirects$/ do |expected|
34
+ @amount_of_new_redirects.should == expected.to_i
35
+ end
36
+
37
+ Then /^report that it deleted (\d+) file from S3$/ do |amount_of_deleted_files|
38
+ @amount_of_deleted_files.should == amount_of_deleted_files.to_i
39
+ end
40
+
41
+ def push_files
42
+ @console_output = capture_stdout {
43
+ in_headless_mode = true
44
+ result = S3Website::Tasks.push(
45
+ "#{@blog_dir}/_site",
46
+ in_headless_mode
47
+ )
48
+ @amount_of_new_files = result[:new_files_count]
49
+ @amount_of_changed_files = result[:changed_files_count]
50
+ @amount_of_deleted_files = result[:deleted_files_count]
51
+ @amount_of_invalidated_items = result[:invalidated_items_count]
52
+ @amount_of_new_redirects = result[:changed_redirects_count]
53
+ }
54
+ end
55
+
56
+ module Kernel
57
+ require 'stringio'
58
+
59
+ def capture_stdout
60
+ out = StringIO.new
61
+ $stdout = out
62
+ yield
63
+ out.string
64
+ ensure
65
+ $stdout = STDOUT
66
+ end
67
+ end
@@ -0,0 +1,26 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+
4
+ Bundler.require
5
+
6
+ require 'aruba/cucumber'
7
+ require 'cucumber/rspec/doubles'
8
+
9
+ # Following from 'aruba/cucumber'
10
+ Before do
11
+ @__aruba_original_paths = (ENV['PATH'] || '').split(File::PATH_SEPARATOR)
12
+ ENV['PATH'] = ([File.expand_path('bin')] + @__aruba_original_paths).join(File::PATH_SEPARATOR)
13
+ @aruba_timeout_seconds = 5
14
+ end
15
+
16
+ After do
17
+ ENV['PATH'] = @__aruba_original_paths.join(File::PATH_SEPARATOR)
18
+ end
19
+ # End of following from 'aruba/cucumber'
20
+
21
+ # Disable colored gem. Its difficult to test output when it contains colored strings.
22
+ module Colored
23
+ def colorize(string, options = {})
24
+ string
25
+ end
26
+ end
@@ -0,0 +1,5 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ </html>
@@ -0,0 +1,4 @@
1
+ s3_id: foo
2
+ s3_secret: foo
3
+ s3_bucket: s3-website-test.net
4
+ cloudfront_distribution_id: foo
@@ -0,0 +1,10 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ <body>
6
+ This is something for a change!
7
+
8
+
9
+ </body>
10
+ </html>
@@ -0,0 +1,4 @@
1
+ s3_id: foo
2
+ s3_secret: foo
3
+ s3_bucket: s3-website-test.net
4
+ cloudfront_distribution_id: EIWJAF0IEWPBB
@@ -0,0 +1,6 @@
1
+ s3_id: foo
2
+ s3_secret: foo
3
+ s3_bucket: s3-website-test.net
4
+ redirects:
5
+ welcome.php: welcome
6
+ pets/dogs: cats-and-dogs/wuf
@@ -0,0 +1,5 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ </html>
@@ -0,0 +1,3 @@
1
+ s3_id: key
2
+ s3_secret: pass
3
+ s3_bucket: s3-website-test.net
@@ -0,0 +1,5 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ </html>
@@ -0,0 +1,3 @@
1
+ s3_id: key
2
+ s3_secret: pass
3
+ s3_bucket: s3-website-test.net
@@ -0,0 +1,5 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ </html>
@@ -0,0 +1,4 @@
1
+ s3_id: key
2
+ s3_secret: pass
3
+ s3_bucket: s3-website-test.net
4
+ s3_endpoint: ap-southeast-2
@@ -0,0 +1,8 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ <body>
6
+ i am a new blog post
7
+ </body>
8
+ </html>
@@ -0,0 +1,3 @@
1
+ s3_id: foo
2
+ s3_secret: foo
3
+ s3_bucket: s3-website-test.net
@@ -0,0 +1,5 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ </html>
@@ -0,0 +1,3 @@
1
+ s3_id: foo
2
+ s3_secret: foo
3
+ s3_bucket: s3-website-test.net
@@ -0,0 +1,9 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ <body>
6
+ i got some new content for you, arnie
7
+ z
8
+ </body>
9
+ </html>
@@ -0,0 +1,3 @@
1
+ s3_id: foo
2
+ s3_secret: foo
3
+ s3_bucket: s3-website-test.net
@@ -0,0 +1,5 @@
1
+ <html>
2
+ <head>
3
+ <title>hello!</title>
4
+ </head>
5
+ </html>