s3_website_monadic 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.travis.yml +3 -0
- data/Gemfile +3 -0
- data/LICENSE +42 -0
- data/README.md +451 -0
- data/Rakefile +24 -0
- data/additional-docs/example-configurations.md +62 -0
- data/additional-docs/setting-up-aws-credentials.md +51 -0
- data/assembly.sbt +3 -0
- data/bin/s3_website +80 -0
- data/build.sbt +33 -0
- data/changelog.md +215 -0
- data/features/as-library.feature +29 -0
- data/features/cassettes/cucumber_tags/create-redirect.yml +384 -0
- data/features/cassettes/cucumber_tags/empty-bucket.yml +89 -0
- data/features/cassettes/cucumber_tags/new-and-changed-files.yml +303 -0
- data/features/cassettes/cucumber_tags/new-files-for-sydney.yml +211 -0
- data/features/cassettes/cucumber_tags/new-files.yml +355 -0
- data/features/cassettes/cucumber_tags/no-new-or-changed-files.yml +359 -0
- data/features/cassettes/cucumber_tags/one-file-to-delete.yml +390 -0
- data/features/cassettes/cucumber_tags/only-changed-files.yml +411 -0
- data/features/cassettes/cucumber_tags/s3-and-cloudfront-after-deleting-a-file.yml +434 -0
- data/features/cassettes/cucumber_tags/s3-and-cloudfront-when-updating-a-file.yml +435 -0
- data/features/cassettes/cucumber_tags/s3-and-cloudfront.yml +290 -0
- data/features/cloudfront.feature +54 -0
- data/features/command-line-help.feature +54 -0
- data/features/delete.feature +19 -0
- data/features/error_reporting.feature +24 -0
- data/features/instructions-for-new-user.feature +154 -0
- data/features/jekyll-support.feature +20 -0
- data/features/nanoc-support.feature +20 -0
- data/features/push.feature +115 -0
- data/features/redirects.feature +14 -0
- data/features/security.feature +15 -0
- data/features/step_definitions/steps.rb +86 -0
- data/features/support/env.rb +26 -0
- data/features/support/test_site_dirs/cdn-powered.blog.fi/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/cdn-powered.blog.fi/_site/index.html +5 -0
- data/features/support/test_site_dirs/cdn-powered.blog.fi/s3_website.yml +4 -0
- data/features/support/test_site_dirs/cdn-powered.when-deleted-a-file.blog.fi/_site/index.html +10 -0
- data/features/support/test_site_dirs/cdn-powered.when-deleted-a-file.blog.fi/s3_website.yml +5 -0
- data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/_site/index.html +10 -0
- data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/s3_website.yml +4 -0
- data/features/support/test_site_dirs/create-redirects/_site/.gitkeep +0 -0
- data/features/support/test_site_dirs/create-redirects/s3_website.yml +6 -0
- data/features/support/test_site_dirs/ignored-files.com/_site/css/styles.css +4 -0
- data/features/support/test_site_dirs/ignored-files.com/_site/index.html +8 -0
- data/features/support/test_site_dirs/ignored-files.com/s3_website.yml +5 -0
- data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/assets/picture.gif +0 -0
- data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/index.html +5 -0
- data/features/support/test_site_dirs/index-and-assets.blog.fi/s3_website.yml +3 -0
- data/features/support/test_site_dirs/jekyllrb.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/jekyllrb.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/jekyllrb.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/my.blog-with-clean-urls.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/my.blog-with-clean-urls.com/_site/index +5 -0
- data/features/support/test_site_dirs/my.blog-with-clean-urls.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/my.blog.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/my.blog.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/my.blog.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/my.sydney.blog.au/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/my.sydney.blog.au/_site/index.html +5 -0
- data/features/support/test_site_dirs/my.sydney.blog.au/s3_website.yml +4 -0
- data/features/support/test_site_dirs/nanoc.ws/public/output/css/styles.css +3 -0
- data/features/support/test_site_dirs/nanoc.ws/public/output/index.html +5 -0
- data/features/support/test_site_dirs/nanoc.ws/s3_website.yml +3 -0
- data/features/support/test_site_dirs/new-and-changed-files.com/_site/css/styles.css +4 -0
- data/features/support/test_site_dirs/new-and-changed-files.com/_site/index.html +8 -0
- data/features/support/test_site_dirs/new-and-changed-files.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/no-new-or-changed-files.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/no-new-or-changed-files.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/no-new-or-changed-files.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/only-changed-files.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/only-changed-files.com/_site/index.html +9 -0
- data/features/support/test_site_dirs/only-changed-files.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/site-that-contains-s3-website-file.com/_site/s3_website.yml +3 -0
- data/features/support/test_site_dirs/site-that-contains-s3-website-file.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/site-with-text-doc.com/_site/file.txt +1 -0
- data/features/support/test_site_dirs/site.with.css-maxage.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/site.with.css-maxage.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/site.with.css-maxage.com/s3_website.yml +5 -0
- data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/s3_website.yml +5 -0
- data/features/support/test_site_dirs/site.with.gzipped-html.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/site.with.gzipped-html.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/site.with.gzipped-html.com/s3_website.yml +5 -0
- data/features/support/test_site_dirs/site.with.maxage.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/site.with.maxage.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/site.with.maxage.com/s3_website.yml +4 -0
- data/features/support/test_site_dirs/unpublish-a-post.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/unpublish-a-post.com/s3_website.yml +3 -0
- data/features/support/vcr.rb +20 -0
- data/features/website-performance.feature +57 -0
- data/lib/cloudfront/invalidator.rb +37 -0
- data/lib/s3_website/config_loader.rb +55 -0
- data/lib/s3_website/diff_helper.rb +113 -0
- data/lib/s3_website/endpoint.rb +37 -0
- data/lib/s3_website/errors.rb +42 -0
- data/lib/s3_website/jekyll.rb +5 -0
- data/lib/s3_website/keyboard.rb +27 -0
- data/lib/s3_website/nanoc.rb +5 -0
- data/lib/s3_website/parallelism.rb +25 -0
- data/lib/s3_website/paths.rb +39 -0
- data/lib/s3_website/retry.rb +19 -0
- data/lib/s3_website/tasks.rb +36 -0
- data/lib/s3_website/upload.rb +137 -0
- data/lib/s3_website/uploader.rb +177 -0
- data/lib/s3_website.rb +34 -0
- data/project/assembly.sbt +1 -0
- data/project/build.properties +0 -0
- data/project/plugins.sbt +1 -0
- data/project/sbt-launch-0.13.2.jar +0 -0
- data/resources/configuration_file_template.yml +56 -0
- data/s3_website.gemspec +41 -0
- data/sbt +4 -0
- data/spec/lib/cloudfront/invalidator_spec.rb +60 -0
- data/spec/lib/config_loader_spec.rb +20 -0
- data/spec/lib/endpoint_spec.rb +31 -0
- data/spec/lib/error_spec.rb +21 -0
- data/spec/lib/keyboard_spec.rb +62 -0
- data/spec/lib/parallelism_spec.rb +81 -0
- data/spec/lib/paths_spec.rb +7 -0
- data/spec/lib/retry_spec.rb +34 -0
- data/spec/lib/upload_spec.rb +303 -0
- data/spec/lib/uploader_spec.rb +37 -0
- data/spec/sample_files/hyde_site/_site/.vimrc +5 -0
- data/spec/sample_files/hyde_site/_site/css/styles.css +3 -0
- data/spec/sample_files/hyde_site/_site/index.html +1 -0
- data/spec/sample_files/hyde_site/s3_website.yml +3 -0
- data/spec/sample_files/tokyo_site/_site/.vimrc +5 -0
- data/spec/sample_files/tokyo_site/_site/css/styles.css +3 -0
- data/spec/sample_files/tokyo_site/_site/index.html +1 -0
- data/spec/sample_files/tokyo_site/s3_website.yml +4 -0
- data/spec/spec_helper.rb +1 -0
- data/src/main/scala/s3/website/CloudFront.scala +96 -0
- data/src/main/scala/s3/website/Diff.scala +42 -0
- data/src/main/scala/s3/website/Implicits.scala +7 -0
- data/src/main/scala/s3/website/Push.scala +191 -0
- data/src/main/scala/s3/website/Ruby.scala +12 -0
- data/src/main/scala/s3/website/S3.scala +139 -0
- data/src/main/scala/s3/website/model/Config.scala +152 -0
- data/src/main/scala/s3/website/model/S3Endpoint.scala +22 -0
- data/src/main/scala/s3/website/model/Site.scala +68 -0
- data/src/main/scala/s3/website/model/errors.scala +11 -0
- data/src/main/scala/s3/website/model/push.scala +192 -0
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +445 -0
- metadata +508 -0
@@ -0,0 +1,115 @@
|
|
1
|
+
Feature: upload S3 website to S3
|
2
|
+
|
3
|
+
In order to push my website to S3
|
4
|
+
As a blogger
|
5
|
+
I want to run s3_website and say OMG it just worked!
|
6
|
+
|
7
|
+
@new-files
|
8
|
+
Scenario: Push a new S3 website to S3
|
9
|
+
When my S3 website is in "features/support/test_site_dirs/my.blog.com"
|
10
|
+
And I call the push command
|
11
|
+
Then the output should contain
|
12
|
+
"""
|
13
|
+
Deploying features/support/test_site_dirs/my.blog.com/_site/* to s3-website-test.net
|
14
|
+
Calculating diff ... done
|
15
|
+
Uploading 2 new file(s)
|
16
|
+
"""
|
17
|
+
And the output should contain
|
18
|
+
"""
|
19
|
+
Upload css/styles.css: Success!
|
20
|
+
"""
|
21
|
+
And the output should contain
|
22
|
+
"""
|
23
|
+
Upload index.html: Success!
|
24
|
+
"""
|
25
|
+
|
26
|
+
@new-files
|
27
|
+
@network-io
|
28
|
+
@starts-new-os-process
|
29
|
+
Scenario: The website resides in a non-standard directory
|
30
|
+
Given a directory named "this-is-a-non-standard-website-dir"
|
31
|
+
And a file named "s3_website.yml" with:
|
32
|
+
"""
|
33
|
+
s3_id: id
|
34
|
+
s3_secret: secret
|
35
|
+
s3_bucket: website.net
|
36
|
+
"""
|
37
|
+
And I run `s3_website push --site this-is-a-non-standard-website-dir`
|
38
|
+
Then the output should contain:
|
39
|
+
"""
|
40
|
+
Deploying this-is-a-non-standard-website-dir/* to website.net
|
41
|
+
"""
|
42
|
+
|
43
|
+
@new-files-for-sydney
|
44
|
+
Scenario: Push a new S3 website to an S3 bucket in Sydney
|
45
|
+
When my S3 website is in "features/support/test_site_dirs/my.sydney.blog.au"
|
46
|
+
And I call the push command
|
47
|
+
Then the output should contain
|
48
|
+
"""
|
49
|
+
Done! Go visit: http://s3-website-test.net.s3-website-ap-southeast-2.amazonaws.com/index.html
|
50
|
+
"""
|
51
|
+
|
52
|
+
@new-and-changed-files
|
53
|
+
Scenario: Upload a new blog post and change an old post
|
54
|
+
When my S3 website is in "features/support/test_site_dirs/new-and-changed-files.com"
|
55
|
+
And I call the push command
|
56
|
+
Then the output should contain
|
57
|
+
"""
|
58
|
+
Deploying features/support/test_site_dirs/new-and-changed-files.com/_site/* to s3-website-test.net
|
59
|
+
Calculating diff ... done
|
60
|
+
Uploading 1 new and 1 changed file(s)
|
61
|
+
"""
|
62
|
+
And the output should contain
|
63
|
+
"""
|
64
|
+
Upload css/styles.css: Success!
|
65
|
+
"""
|
66
|
+
And the output should contain
|
67
|
+
"""
|
68
|
+
Upload index.html: Success!
|
69
|
+
"""
|
70
|
+
And the output should contain
|
71
|
+
"""
|
72
|
+
Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
|
73
|
+
|
74
|
+
"""
|
75
|
+
|
76
|
+
@only-changed-files
|
77
|
+
Scenario: Update an existing blog post
|
78
|
+
When my S3 website is in "features/support/test_site_dirs/only-changed-files.com"
|
79
|
+
And I call the push command
|
80
|
+
Then the output should equal
|
81
|
+
"""
|
82
|
+
Deploying features/support/test_site_dirs/only-changed-files.com/_site/* to s3-website-test.net
|
83
|
+
Calculating diff ... done
|
84
|
+
Uploading 1 changed file(s)
|
85
|
+
Upload index.html: Success!
|
86
|
+
Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
|
87
|
+
|
88
|
+
"""
|
89
|
+
|
90
|
+
@no-new-or-changed-files
|
91
|
+
Scenario: The user runs s3_website even though he doesn't have new or changed posts
|
92
|
+
When my S3 website is in "features/support/test_site_dirs/no-new-or-changed-files.com"
|
93
|
+
And I call the push command
|
94
|
+
Then the output should equal
|
95
|
+
"""
|
96
|
+
Deploying features/support/test_site_dirs/no-new-or-changed-files.com/_site/* to s3-website-test.net
|
97
|
+
Calculating diff ... done
|
98
|
+
No new or changed files to upload
|
99
|
+
Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
|
100
|
+
|
101
|
+
"""
|
102
|
+
|
103
|
+
@new-and-changed-files
|
104
|
+
Scenario: The blogger user does not want to upload certain files
|
105
|
+
When my S3 website is in "features/support/test_site_dirs/ignored-files.com"
|
106
|
+
And I call the push command
|
107
|
+
Then the output should equal
|
108
|
+
"""
|
109
|
+
Deploying features/support/test_site_dirs/ignored-files.com/_site/* to s3-website-test.net
|
110
|
+
Calculating diff ... done
|
111
|
+
Uploading 1 changed file(s)
|
112
|
+
Upload css/styles.css: Success!
|
113
|
+
Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
|
114
|
+
|
115
|
+
"""
|
@@ -0,0 +1,14 @@
|
|
1
|
+
Feature: configure redirects
|
2
|
+
|
3
|
+
@create-redirect
|
4
|
+
Scenario: The user wants to configure new redirects for HTTP resources
|
5
|
+
When my S3 website is in "features/support/test_site_dirs/create-redirects"
|
6
|
+
And I call the push command
|
7
|
+
Then the output should contain
|
8
|
+
"""
|
9
|
+
Creating new redirects ...
|
10
|
+
Redirect welcome.php to /welcome: Success!
|
11
|
+
Redirect pets/dogs to /cats-and-dogs/wuf: Success!
|
12
|
+
Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
|
13
|
+
|
14
|
+
"""
|
@@ -0,0 +1,15 @@
|
|
1
|
+
Feature: Security
|
2
|
+
|
3
|
+
@empty-bucket
|
4
|
+
Scenario: The user does not want to upload the s3_website.yml file
|
5
|
+
When my S3 website is in "features/support/test_site_dirs/site-that-contains-s3-website-file.com"
|
6
|
+
And I call the push command
|
7
|
+
Then the output should not contain "s3_website.yml"
|
8
|
+
And the output should equal
|
9
|
+
"""
|
10
|
+
Deploying features/support/test_site_dirs/site-that-contains-s3-website-file.com/_site/* to another-s3-website-test.net
|
11
|
+
Calculating diff ... done
|
12
|
+
No new or changed files to upload
|
13
|
+
Done! Go visit: http://another-s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
|
14
|
+
|
15
|
+
"""
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
|
3
|
+
When /^my S3 website is in "(.*?)"$/ do |blog_dir|
|
4
|
+
@blog_dir = blog_dir
|
5
|
+
end
|
6
|
+
|
7
|
+
When /^I call the push command$/ do
|
8
|
+
push_files
|
9
|
+
end
|
10
|
+
|
11
|
+
Then /^the output should equal$/ do |expected_console_output|
|
12
|
+
@console_output.should eq(expected_console_output)
|
13
|
+
end
|
14
|
+
|
15
|
+
Then /^the output should contain$/ do |expected_console_output|
|
16
|
+
@console_output.should include(expected_console_output)
|
17
|
+
end
|
18
|
+
|
19
|
+
Then /^s3_website should report that it uploaded (\d+) new and (\d+) changed files into S3$/ do
|
20
|
+
|new_count, changed_count|
|
21
|
+
@amount_of_new_files.should == new_count.to_i
|
22
|
+
@amount_of_changed_files.should == changed_count.to_i
|
23
|
+
end
|
24
|
+
|
25
|
+
Then /^s3_website should report that it invalidated (\d+) Cloudfront item$/ do |expected|
|
26
|
+
@amount_of_invalidated_items.should == expected.to_i
|
27
|
+
end
|
28
|
+
|
29
|
+
Then /^s3_website should report that it created (\d+) new redirects$/ do |expected|
|
30
|
+
@amount_of_new_redirects.should == expected.to_i
|
31
|
+
end
|
32
|
+
|
33
|
+
Then /^s3_website should report that it deleted (\d+) file from S3$/ do |amount_of_deleted_files|
|
34
|
+
@amount_of_deleted_files.should == amount_of_deleted_files.to_i
|
35
|
+
end
|
36
|
+
|
37
|
+
def push_files
|
38
|
+
@console_output = capture_stdout {
|
39
|
+
in_headless_mode = true
|
40
|
+
site_path = S3Website::Paths.infer_site_path(
|
41
|
+
'infer automatically',
|
42
|
+
@blog_dir
|
43
|
+
)
|
44
|
+
result = S3Website::Tasks.push(
|
45
|
+
@blog_dir,
|
46
|
+
site_path,
|
47
|
+
in_headless_mode
|
48
|
+
)
|
49
|
+
@amount_of_new_files = result[:new_files_count]
|
50
|
+
@amount_of_changed_files = result[:changed_files_count]
|
51
|
+
@amount_of_deleted_files = result[:deleted_files_count]
|
52
|
+
@amount_of_invalidated_items = result[:invalidated_items_count]
|
53
|
+
@amount_of_new_redirects = result[:changed_redirects_count]
|
54
|
+
}
|
55
|
+
end
|
56
|
+
|
57
|
+
module S3Website
|
58
|
+
class DiffHelper
|
59
|
+
class DiffProgressIndicator
|
60
|
+
def initialize(start_msg, end_msg)
|
61
|
+
puts "#{start_msg} #{end_msg}"
|
62
|
+
end
|
63
|
+
|
64
|
+
def render_next_step
|
65
|
+
# Simplify testing of stdout by doing nothing here.
|
66
|
+
end
|
67
|
+
|
68
|
+
def finish
|
69
|
+
# Simplify testing of stdout by doing nothing here.
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
module Kernel
|
76
|
+
require 'stringio'
|
77
|
+
|
78
|
+
def capture_stdout
|
79
|
+
out = StringIO.new
|
80
|
+
$stdout = out
|
81
|
+
yield
|
82
|
+
out.string
|
83
|
+
ensure
|
84
|
+
$stdout = STDOUT
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
|
4
|
+
Bundler.require
|
5
|
+
|
6
|
+
require 'aruba/cucumber'
|
7
|
+
require 'cucumber/rspec/doubles'
|
8
|
+
|
9
|
+
# Following from 'aruba/cucumber'
|
10
|
+
Before do
|
11
|
+
@__aruba_original_paths = (ENV['PATH'] || '').split(File::PATH_SEPARATOR)
|
12
|
+
ENV['PATH'] = ([File.expand_path('bin')] + @__aruba_original_paths).join(File::PATH_SEPARATOR)
|
13
|
+
@aruba_timeout_seconds = 5
|
14
|
+
end
|
15
|
+
|
16
|
+
After do
|
17
|
+
ENV['PATH'] = @__aruba_original_paths.join(File::PATH_SEPARATOR)
|
18
|
+
end
|
19
|
+
# End of following from 'aruba/cucumber'
|
20
|
+
|
21
|
+
# Disable colored gem. Its difficult to test output when it contains colored strings.
|
22
|
+
module Colored
|
23
|
+
def colorize(string, options = {})
|
24
|
+
string
|
25
|
+
end
|
26
|
+
end
|
File without changes
|
File without changes
|
@@ -0,0 +1 @@
|
|
1
|
+
hello world
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'vcr'
|
2
|
+
|
3
|
+
VCR.configure do |c|
|
4
|
+
c.hook_into :webmock
|
5
|
+
c.cassette_library_dir = 'features/cassettes'
|
6
|
+
end
|
7
|
+
|
8
|
+
VCR.cucumber_tags do |t|
|
9
|
+
t.tag '@new-files'
|
10
|
+
t.tag '@new-files-for-sydney'
|
11
|
+
t.tag '@new-and-changed-files'
|
12
|
+
t.tag '@only-changed-files'
|
13
|
+
t.tag '@no-new-or-changed-files'
|
14
|
+
t.tag '@s3-and-cloudfront'
|
15
|
+
t.tag '@s3-and-cloudfront-when-updating-a-file'
|
16
|
+
t.tag '@s3-and-cloudfront-after-deleting-a-file'
|
17
|
+
t.tag '@one-file-to-delete'
|
18
|
+
t.tag '@create-redirect'
|
19
|
+
t.tag '@empty-bucket'
|
20
|
+
end
|