RubyGems - s3_website - Versions diffs - 0.1.0 - Mend

@@ -0,0 +1,18 @@
+require 'vcr'
+VCR.configure do |c|
+  c.hook_into :webmock
+  c.cassette_library_dir = 'features/cassettes'
+end
+VCR.cucumber_tags do |t|
+  t.tag '@new-files'
+  t.tag '@new-files-for-sydney'
+  t.tag '@new-and-changed-files'
+  t.tag '@only-changed-files'
+  t.tag '@no-new-or-changed-files'
+  t.tag '@s3-and-cloudfront'
+  t.tag '@s3-and-cloudfront-when-updating-a-file'
+  t.tag '@one-file-to-delete'
+  t.tag '@create-redirect'
+end

data/features/sync.feature ADDED Viewed

@@ -0,0 +1,80 @@
+Feature: upload S3 website to S3
+  In order to push my website to S3
+  As a blogger
+  I want to run s3_website and say OMG it just worked!
+  @new-files
+  Scenario: Push a new S3 website to S3
+    When my S3 website is in "features/support/test_site_dirs/my.blog.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Deploying _site/* to s3-website-test.net
+      Uploading 2 new file(s)
+      """
+    And the output should contain
+      """
+      Upload css/styles.css: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html: Success!
+      """
+  @new-files-for-sydney
+  Scenario: Push a new S3 website to an S3 bucket in Sydney
+    When my S3 website is in "features/support/test_site_dirs/my.sydney.blog.au"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Done! Go visit: http://s3-website-test.net.s3-website-ap-southeast-2.amazonaws.com/index.html
+      """
+  @new-and-changed-files
+  Scenario: Upload a new blog post and change an old post
+    When my S3 website is in "features/support/test_site_dirs/new-and-changed-files.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Deploying _site/* to s3-website-test.net
+      Uploading 1 new and 1 changed file(s)
+      """
+    And the output should contain
+      """
+      Upload css/styles.css: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html: Success!
+      """
+    And the output should contain
+      """
+      Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
+      """
+  @only-changed-files
+  Scenario: Update an existing blog post
+    When my S3 website is in "features/support/test_site_dirs/only-changed-files.com"
+    Then s3_website will push my blog to S3
+    And the output should equal
+      """
+      Deploying _site/* to s3-website-test.net
+      Uploading 1 changed file(s)
+      Upload index.html: Success!
+      Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
+      """
+  @no-new-or-changed-files
+  Scenario: The user runs s3_website even though he doesn't have new or changed posts
+    When my S3 website is in "features/support/test_site_dirs/no-new-or-changed-files.com"
+    Then s3_website will push my blog to S3
+    And the output should equal
+      """
+      Deploying _site/* to s3-website-test.net
+      No new or changed files to upload
+      Done! Go visit: http://s3-website-test.net.s3-website-us-east-1.amazonaws.com/index.html
+      """

data/features/website-performance.feature ADDED Viewed

@@ -0,0 +1,57 @@
+Feature: improve response times of your S3 website website
+  As a blogger
+  I want to benefit from HTTP performance optimisations
+  So that my readers would not have to wait long for my website to load
+  @new-files
+  Scenario: Set Cache-Control: max-age for all uploaded files
+    When my S3 website is in "features/support/test_site_dirs/site.with.maxage.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Upload css/styles.css [max-age=120]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [max-age=120]: Success!
+      """
+  @new-files
+  Scenario: Set Cache-Control: max-age for CSS files only
+    When my S3 website is in "features/support/test_site_dirs/site.with.css-maxage.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Upload css/styles.css [max-age=100]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [max-age=0]: Success!
+      """
+  @new-files
+  Scenario: Set Content-Encoding: gzip HTTP header for HTML files
+    When my S3 website is in "features/support/test_site_dirs/site.with.gzipped-html.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Upload css/styles.css: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [gzipped]: Success!
+      """
+  @new-files
+  Scenario: Set both the Content-Encoding: gzip and Cache-Control: max-age headers
+    When my S3 website is in "features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com"
+    Then s3_website will push my blog to S3
+    And the output should contain
+      """
+      Upload css/styles.css [gzipped] [max-age=300]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [gzipped] [max-age=300]: Success!
+      """

data/lib/cloudfront/invalidator.rb ADDED Viewed

@@ -0,0 +1,22 @@
+module S3Website
+  module Cloudfront
+    class Invalidator
+      def self.invalidate(config, changed_files)
+        aws_key = config['s3_id']
+        aws_secret = config['s3_secret']
+        s3_bucket_name = config['s3_bucket']
+        cloudfront_distribution_id = config['cloudfront_distribution_id']
+        s3 = AWS::S3.new(
+          :access_key_id => aws_key,
+          :secret_access_key => aws_secret)
+        s3_object_keys = changed_files
+        s3_object_keys << ""
+        report = SimpleCloudfrontInvalidator::CloudfrontClient.new(
+          aws_key, aws_secret, cloudfront_distribution_id).invalidate(
+            s3_object_keys)
+        puts report[:text_report]
+        report[:invalidated_items_count]
+      end
+    end
+  end
+end

data/lib/s3_website.rb ADDED Viewed

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'yaml'
+require 'erubis'
+require 'aws-sdk'
+require 'simple-cloudfront-invalidator'
+require 'filey-diff'
+require 'mime/types'
+require 'thor'
+module S3Website
+  DEFAULT_GZIP_EXTENSIONS = %w(.html .css .js .svg .txt)
+end
+%w{errors upload uploader tasks config_loader retry keyboard diff_helper endpoint parallelism}.each do |file|
+  require File.dirname(__FILE__) + "/s3_website/#{file}"
+end
+%w{invalidator}.each do |file|
+  require File.dirname(__FILE__) + "/cloudfront/#{file}"
+end

data/lib/s3_website/config_loader.rb ADDED Viewed

@@ -0,0 +1,56 @@
+module S3Website
+  class ConfigLoader
+    CONFIGURATION_FILE = 's3_website.yml'
+    CONFIGURATION_FILE_TEMPLATE = <<-EOF
+s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
+s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
+s3_bucket: your.blog.bucket.com
+    EOF
+    def self.check_project(site_dir)
+      raise NotAJekyllProjectError unless File.directory?(site_dir)
+    end
+    # Raise NoConfigurationFileError if the configuration file does not exists
+    def self.check_s3_configuration(site_dir)
+      unless File.exists?(get_configuration_file(site_dir))
+        create_template_configuration_file site_dir
+        raise NoConfigurationFileError
+      end
+    end
+    private
+    # Load configuration from s3_website.yml
+    # Raise MalformedConfigurationFileError if the configuration file does not contain the keys we expect
+    def self.load_configuration(site_dir)
+      config = load_yaml_file_and_validate site_dir
+      return config
+    end
+    def self.create_template_configuration_file(site_dir)
+      File.open(get_configuration_file(site_dir), 'w') { |f|
+        f.write(CONFIGURATION_FILE_TEMPLATE)
+      }
+    end
+    def self.load_yaml_file_and_validate(site_dir)
+      begin
+        config = YAML.load(Erubis::Eruby.new(File.read(get_configuration_file(site_dir))).result)
+      rescue Exception
+        raise MalformedConfigurationFileError
+      end
+      raise MalformedConfigurationFileError unless config
+      raise MalformedConfigurationFileError if
+        ['s3_bucket'].any? { |key|
+          mandatory_config_value = config[key]
+          mandatory_config_value.nil? || mandatory_config_value == ''
+        }
+      config
+    end
+    def self.get_configuration_file(site_dir)
+      "#{site_dir}/../#{CONFIGURATION_FILE}"
+    end
+  end
+end

data/lib/s3_website/diff_helper.rb ADDED Viewed

@@ -0,0 +1,21 @@
+module S3Website
+  class DiffHelper
+    def self.resolve_files_to_upload(s3_bucket, site_dir)
+      s3_data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket)
+      fs_data_source = Filey::DataSources::FileSystem.new(site_dir)
+      changed_local_files =
+        Filey::Comparison.list_changed(fs_data_source, s3_data_source)
+      new_local_files =
+        Filey::Comparison.list_missing(fs_data_source, s3_data_source)
+      [ normalise(changed_local_files), normalise(new_local_files) ]
+    end
+    private
+    def self.normalise(fileys)
+      fileys.map { |filey|
+        filey.full_path.sub(/\.\//, '')
+      }
+    end
+  end
+end

data/lib/s3_website/endpoint.rb ADDED Viewed

@@ -0,0 +1,30 @@
+module S3Website
+  class Endpoint
+    DEFAULT_LOCATION_CONSTRAINT = 'us-east-1'
+    attr_reader :region, :location_constraint, :hostname, :website_hostname
+    def initialize(location_constraint=nil)
+      location_constraint = DEFAULT_LOCATION_CONSTRAINT if location_constraint.nil?
+      raise "Invalid S3 location constraint #{location_constraint}" unless
+        location_constraints.has_key?location_constraint
+      @region = location_constraints.fetch(location_constraint)[:region]
+      @hostname = location_constraints.fetch(location_constraint)[:endpoint]
+      @website_hostname = location_constraints.fetch(location_constraint)[:website_hostname]
+      @location_constraint = location_constraint
+    end
+    # http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
+    def location_constraints
+      {
+        'us-east-1'      => { :region => 'US Standard',                   :website_hostname => 's3-website-us-east-1.amazonaws.com',      :endpoint => 's3.amazonaws.com' },
+        'us-west-2'      => { :region => 'US West (Oregon)',              :website_hostname => 's3-website-us-west-2.amazonaws.com',      :endpoint => 's3-us-west-2.amazonaws.com' },
+        'us-west-1'      => { :region => 'US West (Northern California)', :website_hostname => 's3-website-us-west-1.amazonaws.com',      :endpoint => 's3-us-west-1.amazonaws.com' },
+        'EU'             => { :region => 'EU (Ireland)',                  :website_hostname => 's3-website-eu-west-1.amazonaws.com',      :endpoint => 's3-eu-west-1.amazonaws.com' },
+        'ap-southeast-1' => { :region => 'Asia Pacific (Singapore)',      :website_hostname => 's3-website-ap-southeast-1.amazonaws.com', :endpoint => 's3-ap-southeast-1.amazonaws.com' },
+        'ap-southeast-2' => { :region => 'Asia Pacific (Sydney)',         :website_hostname => 's3-website-ap-southeast-2.amazonaws.com', :endpoint => 's3-ap-southeast-2.amazonaws.com' },
+        'ap-northeast-1' => { :region => 'Asia Pacific (Tokyo)',          :website_hostname => 's3-website-ap-northeast-1.amazonaws.com', :endpoint => 's3-ap-northeast-1.amazonaws.com' },
+        'sa-east-1'      => { :region => 'South America (Sao Paulo)',     :website_hostname => 's3-website-sa-east-1.amazonaws.com',      :endpoint => 's3-sa-east-1.amazonaws.com' }
+      }
+    end
+  end
+end

data/lib/s3_website/errors.rb ADDED Viewed

@@ -0,0 +1,28 @@
+module S3Website
+  class S3WebsiteError < StandardError
+  end
+  class NotAJekyllProjectError < S3WebsiteError
+    def initialize(message = "I can't find any directory called _site. Are you in the right directory?")
+      super(message)
+    end
+  end
+  class NoConfigurationFileError < S3WebsiteError
+    def initialize(message = "I've just generated a file called s3_website.yml. Go put your details in it!")
+      super(message)
+    end
+  end
+  class MalformedConfigurationFileError < S3WebsiteError
+    def initialize(message = "I can't parse the file s3_website.yml. It should look like this:\n#{ConfigLoader::CONFIGURATION_FILE_TEMPLATE}")
+      super(message)
+    end
+  end
+  class RetryAttemptsExhaustedError < S3WebsiteError
+    def initialize(message = "Operation failed even though we tried to recover from it")
+      super(message)
+    end
+  end
+end

data/lib/s3_website/keyboard.rb ADDED Viewed

@@ -0,0 +1,27 @@
+module S3Website
+  class Keyboard
+    def self.if_user_confirms_delete(to_delete, standard_input=STDIN)
+      delete_all = false
+      keep_all = false
+      confirmed_deletes = to_delete.map do |f|
+        delete = false
+        keep = false
+        until delete || delete_all || keep || keep_all
+          puts "#{f} is on S3 but not in your _site directory anymore. Do you want to [d]elete, [D]elete all, [k]eep, [K]eep all?"
+          case standard_input.gets.chomp
+          when 'd' then delete = true
+          when 'D' then delete_all = true
+          when 'k' then keep = true
+          when 'K' then keep_all = true
+          end
+        end
+        if (delete_all || delete) && !(keep_all || keep)
+          f
+        end
+      end.select { |f| f }
+      Parallelism.each_in_parallel_or_sequentially(confirmed_deletes) { |f|
+        yield f
+      }
+    end
+  end
+end