RubyGems - s3_website - Versions diffs - 0.1.0 - Mend

s3_website 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (101) hide show

data/features/support/test_site_dirs/site.with.css-maxage.com/s3_website.yml ADDED Viewed

@@ -0,0 +1,5 @@
+s3_id: key
+s3_secret: pass
+s3_bucket: s3-website-test.net
+max_age:
+  "css/*": 100

data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/_site/css/styles.css ADDED Viewed

@@ -0,0 +1,3 @@
+body {
+}

data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/_site/index.html ADDED Viewed

@@ -0,0 +1,5 @@
+<html>
+  <head>
+    <title>hello!</title>
+  </head>
+</html>

data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/s3_website.yml ADDED Viewed

@@ -0,0 +1,5 @@
+s3_id: key
+s3_secret: pass
+s3_bucket: s3-website-test.net
+gzip: true
+max_age: 300

data/features/support/test_site_dirs/site.with.gzipped-html.com/_site/css/styles.css ADDED Viewed

@@ -0,0 +1,3 @@
+body {
+}

data/features/support/test_site_dirs/site.with.gzipped-html.com/_site/index.html ADDED Viewed

@@ -0,0 +1,5 @@
+<html>
+  <head>
+    <title>hello!</title>
+  </head>
+</html>

data/features/support/test_site_dirs/site.with.gzipped-html.com/s3_website.yml ADDED Viewed

@@ -0,0 +1,5 @@
+s3_id: key
+s3_secret: pass
+s3_bucket: s3-website-test.net
+gzip:
+  - .html

data/features/support/test_site_dirs/site.with.maxage.com/_site/css/styles.css ADDED Viewed

@@ -0,0 +1,3 @@
+body {
+}

data/features/support/test_site_dirs/site.with.maxage.com/_site/index.html ADDED Viewed

@@ -0,0 +1,5 @@
+<html>
+  <head>
+    <title>hello!</title>
+  </head>
+</html>

data/features/support/test_site_dirs/site.with.maxage.com/s3_website.yml ADDED Viewed

@@ -0,0 +1,4 @@
+s3_id: key
+s3_secret: pass
+s3_bucket: s3-website-test.net
+max_age: 120

data/features/support/test_site_dirs/unpublish-a-post.com/_site/css/styles.css ADDED Viewed

@@ -0,0 +1,3 @@
+body {
+}

data/features/support/test_site_dirs/unpublish-a-post.com/s3_website.yml ADDED Viewed

@@ -0,0 +1,3 @@
+s3_id: foo
+s3_secret: foo
+s3_bucket: s3-website-test.net

data/features/support/vcr.rb ADDED Viewed

@@ -0,0 +1,18 @@
+require 'vcr'
+VCR.configure do |c|
+  c.hook_into :webmock
+  c.cassette_library_dir = 'features/cassettes'
+end
+VCR.cucumber_tags do |t|
+  t.tag '@new-files'
+  t.tag '@new-files-for-sydney'
+  t.tag '@new-and-changed-files'
+  t.tag '@only-changed-files'
+  t.tag '@no-new-or-changed-files'
+  t.tag '@s3-and-cloudfront'
+  t.tag '@s3-and-cloudfront-when-updating-a-file'
+  t.tag '@one-file-to-delete'
+  t.tag '@create-redirect'
+end

data/features/sync.feature ADDED Viewed

@@ -0,0 +1,80 @@
+Feature: upload S3 website to S3
+  In order to push my website to S3
+  As a blogger
+  I want to run s3_website and say OMG it just worked!
+  @new-files
+  Scenario: Push a new S3 website to S3
+    When my S3 website is in "features/support/test_site_dirs/my.blog.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Deploying _site/* to s3-website-test.net
+      Uploading 2 new file(s)
+      """
+    And the output should contain
+      """
+      Upload css/styles.css: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html: Success!
+      """
+  @new-files-for-sydney
+  Scenario: Push a new S3 website to an S3 bucket in Sydney
+    When my S3 website is in "features/support/test_site_dirs/my.sydney.blog.au"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Done! Go visit: http://s3-website-test.net.s3-website-ap-southeast-2.amazonaws.com/index.html
+      """
+  @new-and-changed-files
+  Scenario: Upload a new blog post and change an old post
+    When my S3 website is in "features/support/test_site_dirs/new-and-changed-files.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Deploying _site/* to s3-website-test.net
+      Uploading 1 new and 1 changed file(s)
+      """
+    And the output should contain
+      """
+      Upload css/styles.css: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html: Success!
+      """
+    And the output should contain
+      """
+      Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
+      """
+  @only-changed-files
+  Scenario: Update an existing blog post
+    When my S3 website is in "features/support/test_site_dirs/only-changed-files.com"
+    Then s3_website will push my blog to S3
+    And the output should equal
+      """
+      Deploying _site/* to s3-website-test.net
+      Uploading 1 changed file(s)
+      Upload index.html: Success!
+      Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
+      """
+  @no-new-or-changed-files
+  Scenario: The user runs s3_website even though he doesn't have new or changed posts
+    When my S3 website is in "features/support/test_site_dirs/no-new-or-changed-files.com"
+    Then s3_website will push my blog to S3
+    And the output should equal
+      """
+      Deploying _site/* to s3-website-test.net
+      No new or changed files to upload
+      Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
+      """

data/features/website-performance.feature ADDED Viewed

@@ -0,0 +1,57 @@
+Feature: improve response times of your S3 website website
+  As a blogger
+  I want to benefit from HTTP performance optimisations
+  So that my readers would not have to wait long for my website to load
+  @new-files
+  Scenario: Set Cache-Control: max-age for all uploaded files
+    When my S3 website is in "features/support/test_site_dirs/site.with.maxage.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Upload css/styles.css [max-age=120]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [max-age=120]: Success!
+      """
+  @new-files
+  Scenario: Set Cache-Control: max-age for CSS files only
+    When my S3 website is in "features/support/test_site_dirs/site.with.css-maxage.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Upload css/styles.css [max-age=100]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [max-age=0]: Success!
+      """
+  @new-files
+  Scenario: Set Content-Encoding: gzip HTTP header for HTML files
+    When my S3 website is in "features/support/test_site_dirs/site.with.gzipped-html.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Upload css/styles.css: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [gzipped]: Success!
+      """
+  @new-files
+  Scenario: Set both the Content-Encoding: gzip and Cache-Control: max-age headers
+    When my S3 website is in "features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Upload css/styles.css [gzipped] [max-age=300]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [gzipped] [max-age=300]: Success!
+      """

data/lib/cloudfront/invalidator.rb ADDED Viewed

@@ -0,0 +1,22 @@
+module S3Website
+  module Cloudfront
+    class Invalidator
+      def self.invalidate(config, changed_files)
+        aws_key = config['s3_id']
+        aws_secret = config['s3_secret']
+        s3_bucket_name = config['s3_bucket']
+        cloudfront_distribution_id = config['cloudfront_distribution_id']
+        s3 = AWS::S3.new(
+          :access_key_id => aws_key,
+          :secret_access_key => aws_secret)
+        s3_object_keys = changed_files
+        s3_object_keys << ""
+        report = SimpleCloudfrontInvalidator::CloudfrontClient.new(
+          aws_key, aws_secret, cloudfront_distribution_id).invalidate(
+            s3_object_keys)
+        puts report[:text_report]
+        report[:invalidated_items_count]
+      end
+    end
+  end
+end

data/lib/s3_website.rb ADDED Viewed

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'yaml'
+require 'erubis'
+require 'aws-sdk'
+require 'simple-cloudfront-invalidator'
+require 'filey-diff'
+require 'mime/types'
+require 'thor'
+module S3Website
+  DEFAULT_GZIP_EXTENSIONS = %w(.html .css .js .svg .txt)
+end
+%w{errors upload uploader tasks config_loader retry keyboard diff_helper endpoint parallelism}.each do |file|
+  require File.dirname(__FILE__) + "/s3_website/#{file}"
+end
+%w{invalidator}.each do |file|
+  require File.dirname(__FILE__) + "/cloudfront/#{file}"
+end

data/lib/s3_website/config_loader.rb ADDED Viewed

@@ -0,0 +1,56 @@
+module S3Website
+  class ConfigLoader
+    CONFIGURATION_FILE = 's3_website.yml'
+    CONFIGURATION_FILE_TEMPLATE = <<-EOF
+s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
+s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
+s3_bucket: your.blog.bucket.com
+    EOF
+    def self.check_project(site_dir)
+      raise NotAJekyllProjectError unless File.directory?(site_dir)
+    end
+    # Raise NoConfigurationFileError if the configuration file does not exists
+    def self.check_s3_configuration(site_dir)
+      unless File.exists?(get_configuration_file(site_dir))
+        create_template_configuration_file site_dir
+        raise NoConfigurationFileError
+      end
+    end
+    private
+    # Load configuration from s3_website.yml
+    # Raise MalformedConfigurationFileError if the configuration file does not contain the keys we expect
+    def self.load_configuration(site_dir)
+      config = load_yaml_file_and_validate site_dir
+      return config
+    end
+    def self.create_template_configuration_file(site_dir)
+      File.open(get_configuration_file(site_dir), 'w') { |f|
+        f.write(CONFIGURATION_FILE_TEMPLATE)
+      }
+    end
+    def self.load_yaml_file_and_validate(site_dir)
+      begin
+        config = YAML.load(Erubis::Eruby.new(File.read(get_configuration_file(site_dir))).result)
+      rescue Exception
+        raise MalformedConfigurationFileError
+      end
+      raise MalformedConfigurationFileError unless config
+      raise MalformedConfigurationFileError if
+        ['s3_bucket'].any? { |key|
+          mandatory_config_value = config[key]
+          mandatory_config_value.nil? || mandatory_config_value == ''
+        }
+      config
+    end
+    def self.get_configuration_file(site_dir)
+      "#{site_dir}/../#{CONFIGURATION_FILE}"
+    end
+  end
+end

data/lib/s3_website/diff_helper.rb ADDED Viewed

@@ -0,0 +1,21 @@
+module S3Website
+  class DiffHelper
+    def self.resolve_files_to_upload(s3_bucket, site_dir)
+      s3_data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket)
+      fs_data_source = Filey::DataSources::FileSystem.new(site_dir)
+      changed_local_files =
+        Filey::Comparison.list_changed(fs_data_source, s3_data_source)
+      new_local_files =
+        Filey::Comparison.list_missing(fs_data_source, s3_data_source)
+      [ normalise(changed_local_files), normalise(new_local_files) ]
+    end
+    private
+    def self.normalise(fileys)
+      fileys.map { |filey|
+        filey.full_path.sub(/\.\//, '')
+      }
+    end
+  end
+end

data/lib/s3_website/endpoint.rb ADDED Viewed

@@ -0,0 +1,30 @@
+module S3Website
+  class Endpoint
+    DEFAULT_LOCATION_CONSTRAINT = 'us-east-1'
+    attr_reader :region, :location_constraint, :hostname, :website_hostname
+    def initialize(location_constraint=nil)
+      location_constraint = DEFAULT_LOCATION_CONSTRAINT if location_constraint.nil?
+      raise "Invalid S3 location constraint #{location_constraint}" unless
+        location_constraints.has_key?location_constraint
+      @region = location_constraints.fetch(location_constraint)[:region]
+      @hostname = location_constraints.fetch(location_constraint)[:endpoint]
+      @website_hostname = location_constraints.fetch(location_constraint)[:website_hostname]
+      @location_constraint = location_constraint
+    end
+    # http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
+    def location_constraints
+      {
+        'us-east-1'      => { :region => 'US Standard',                   :website_hostname => 's3-website-us-east-1.amazonaws.com',      :endpoint => 's3.amazonaws.com' },
+        'us-west-2'      => { :region => 'US West (Oregon)',              :website_hostname => 's3-website-us-west-2.amazonaws.com',      :endpoint => 's3-us-west-2.amazonaws.com' },
+        'us-west-1'      => { :region => 'US West (Northern California)', :website_hostname => 's3-website-us-west-1.amazonaws.com',      :endpoint => 's3-us-west-1.amazonaws.com' },
+        'EU'             => { :region => 'EU (Ireland)',                  :website_hostname => 's3-website-eu-west-1.amazonaws.com',      :endpoint => 's3-eu-west-1.amazonaws.com' },
+        'ap-southeast-1' => { :region => 'Asia Pacific (Singapore)',      :website_hostname => 's3-website-ap-southeast-1.amazonaws.com', :endpoint => 's3-ap-southeast-1.amazonaws.com' },
+        'ap-southeast-2' => { :region => 'Asia Pacific (Sydney)',         :website_hostname => 's3-website-ap-southeast-2.amazonaws.com', :endpoint => 's3-ap-southeast-2.amazonaws.com' },
+        'ap-northeast-1' => { :region => 'Asia Pacific (Tokyo)',          :website_hostname => 's3-website-ap-northeast-1.amazonaws.com', :endpoint => 's3-ap-northeast-1.amazonaws.com' },
+        'sa-east-1'      => { :region => 'South America (Sao Paulo)',     :website_hostname => 's3-website-sa-east-1.amazonaws.com',      :endpoint => 's3-sa-east-1.amazonaws.com' }
+      }
+    end
+  end
+end

data/lib/s3_website/errors.rb ADDED Viewed

@@ -0,0 +1,28 @@
+module S3Website
+  class S3WebsiteError < StandardError
+  end
+  class NotAJekyllProjectError < S3WebsiteError
+    def initialize(message = "I can't find any directory called _site. Are you in the right directory?")
+      super(message)
+    end
+  end
+  class NoConfigurationFileError < S3WebsiteError
+    def initialize(message = "I've just generated a file called s3_website.yml. Go put your details in it!")
+      super(message)
+    end
+  end
+  class MalformedConfigurationFileError < S3WebsiteError
+    def initialize(message = "I can't parse the file s3_website.yml. It should look like this:\n#{ConfigLoader::CONFIGURATION_FILE_TEMPLATE}")
+      super(message)
+    end
+  end
+  class RetryAttemptsExhaustedError < S3WebsiteError
+    def initialize(message = "Operation failed even though we tried to recover from it")
+      super(message)
+    end
+  end
+end

data/lib/s3_website/keyboard.rb ADDED Viewed

@@ -0,0 +1,27 @@
+module S3Website
+  class Keyboard
+    def self.if_user_confirms_delete(to_delete, standard_input=STDIN)
+      delete_all = false
+      keep_all = false
+      confirmed_deletes = to_delete.map do |f|
+        delete = false
+        keep = false
+        until delete || delete_all || keep || keep_all
+          puts "#{f} is on S3 but not in your _site directory anymore. Do you want to [d]elete, [D]elete all, [k]eep, [K]eep all?"
+          case standard_input.gets.chomp
+          when 'd' then delete = true
+          when 'D' then delete_all = true
+          when 'k' then keep = true
+          when 'K' then keep_all = true
+          end
+        end
+        if (delete_all || delete) && !(keep_all || keep)
+          f
+        end
+      end.select { |f| f }
+      Parallelism.each_in_parallel_or_sequentially(confirmed_deletes) { |f|
+        yield f
+      }
+    end
+  end
+end