s3_website_monadic 0.0.1

data/features/website-performance.feature

@@ -0,0 +1,57 @@
+Feature: improve response times of your S3 website website
+
+  As a blogger
+  I want to benefit from HTTP performance optimisations
+  So that my readers would not have to wait long for my website to load
+
+  @new-files
+  Scenario: Set Cache-Control: max-age for all uploaded files
+    When my S3 website is in "features/support/test_site_dirs/site.with.maxage.com"
+    And I call the push command
+    Then the output should contain
+      """
+      Upload css/styles.css [max-age=120]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [max-age=120]: Success!
+      """
+
+  @new-files
+  Scenario: Set Cache-Control: max-age for CSS files only
+    When my S3 website is in "features/support/test_site_dirs/site.with.css-maxage.com"
+    And I call the push command
+    Then the output should contain
+      """
+      Upload css/styles.css [max-age=100]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [max-age=0]: Success!
+      """
+
+  @new-files
+  Scenario: Set Content-Encoding: gzip HTTP header for HTML files
+    When my S3 website is in "features/support/test_site_dirs/site.with.gzipped-html.com"
+    And I call the push command
+    Then the output should contain
+      """
+      Upload css/styles.css: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [gzipped]: Success!
+      """
+
+  @new-files
+  Scenario: Set both the Content-Encoding: gzip and Cache-Control: max-age headers
+    When my S3 website is in "features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com"
+    And I call the push command
+    Then the output should contain
+      """
+      Upload css/styles.css [gzipped] [max-age=300]: Success!
+      """
+    And the output should contain
+      """
+      Upload index.html [gzipped] [max-age=300]: Success!
+      """

data/lib/cloudfront/invalidator.rb

@@ -0,0 +1,37 @@
+module S3Website
+  module Cloudfront
+    class Invalidator
+      def self.invalidate(config, changed_files)
+        aws_key = config['s3_id']
+        aws_secret = config['s3_secret']
+        cloudfront_distribution_id = config['cloudfront_distribution_id']
+        s3_object_keys = apply_config config, changed_files
+        s3_object_keys << ""
+        report = SimpleCloudfrontInvalidator::CloudfrontClient.new(
+          aws_key, aws_secret, cloudfront_distribution_id
+        ).invalidate(url_encode_keys s3_object_keys)
+        puts report[:text_report]
+        report[:invalidated_items_count]
+      end
+
+      private
+
+      def self.url_encode_keys(keys)
+        require 'uri'
+        keys.map do |key|
+          URI::encode(key, Regexp.union([URI::Parser.new.regexp[:UNSAFE],'~','@', "'"]))
+        end
+      end
+
+      def self.apply_config(config, changed_files)
+        if config['cloudfront_invalidate_root']
+          changed_files.map { |changed_file|
+            changed_file.sub /\/index.html$/, '/'
+          }
+        else
+          changed_files
+        end
+      end
+    end
+  end
+end

data/lib/s3_website/config_loader.rb

@@ -0,0 +1,55 @@
+module S3Website
+  class ConfigLoader
+    CONFIGURATION_FILE = 's3_website.yml'
+
+    def self.check_project(site_dir)
+      raise NoWebsiteDirectoryFound unless File.directory?(site_dir)
+    end
+
+    # Raise NoConfigurationFileError if the configuration file does not exists
+    def self.check_s3_configuration(dir)
+      config_file = dir + '/' + CONFIGURATION_FILE
+      unless File.exists?(config_file)
+        create_template_configuration_file config_file
+        raise NoConfigurationFileError
+      end
+    end
+
+    def self.read_configuration_file_template
+      path = File.dirname(__FILE__) + '/../../resources/configuration_file_template.yml'
+      cfg_template = File.open(path).read
+    end
+
+    private
+
+    # Load configuration from s3_website.yml
+    # Raise MalformedConfigurationFileError if the configuration file does not contain the keys we expect
+    def self.load_configuration(config_file_dir)
+      config = load_yaml_file_and_validate config_file_dir
+      return config
+    end
+
+    def self.create_template_configuration_file(file)
+      File.open(file, 'w') { |f|
+        f.write(read_configuration_file_template)
+      }
+    end
+
+    def self.load_yaml_file_and_validate(config_file_dir)
+      begin
+        config = YAML.load(Erubis::Eruby.new(
+          File.read(config_file_dir + '/' + CONFIGURATION_FILE)
+        ).result)
+      rescue Exception
+        raise MalformedConfigurationFileError
+      end
+      raise MalformedConfigurationFileError unless config
+      raise MalformedConfigurationFileError if
+        ['s3_bucket'].any? { |key|
+          mandatory_config_value = config[key]
+          mandatory_config_value.nil? || mandatory_config_value == ''
+        }
+      config
+    end
+  end
+end

data/lib/s3_website/diff_helper.rb

@@ -0,0 +1,113 @@
+module S3Website
+  class DiffHelper
+    def self.resolve_files_to_upload(s3_bucket, site_dir, config)
+      with_progress_indicator('Calculating diff') { |progress_indicator|
+        s3_data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket, config) { |filey|
+          progress_indicator.render_next_step
+        }
+        fs_data_source = Filey::DataSources::FileSystem.new(site_dir) { |filey|
+          progress_indicator.render_next_step
+        }
+        changed_local_files = Filey::Comparison.list_changed(
+          fs_data_source,
+          s3_data_source
+        )
+        new_local_files = Filey::Comparison.list_missing(
+          fs_data_source,
+          s3_data_source
+        )
+        [
+          reject_blacklisted(changed_local_files, config),
+          reject_blacklisted(new_local_files, config)
+        ]
+      }
+    end
+
+    private
+
+    def self.reject_blacklisted(file_paths, config)
+      (normalise file_paths).reject { |f| Upload.is_blacklisted(f, config) }
+    end
+
+    def self.with_progress_indicator(diff_msg)
+      progress_indicator = DiffProgressIndicator.new(diff_msg, "... done\n")
+      result = yield progress_indicator
+      progress_indicator.finish
+      result
+    end
+
+    def self.normalise(fileys)
+      fileys.map { |filey|
+        filey.full_path.sub(/\.\//, '')
+      }
+    end
+
+    class DiffProgressIndicator
+      def initialize(init_msg, end_msg)
+        @end_msg = end_msg
+        @ordinal_direction = 'n' # start from north
+        print init_msg
+        print '   '
+        render_next_step
+      end
+
+      def render_next_step
+        @ordinal_direction = DiffProgressIndicator.next_ordinal_direction @ordinal_direction
+        print("\b\b" + DiffProgressIndicator.render_ordinal_direction(@ordinal_direction) + ' ')
+      end
+
+      def finish
+        print "\b\b"
+        print @end_msg
+      end
+
+      private
+
+      def self.render_ordinal_direction(ordinal_direction)
+        case ordinal_direction
+        when 'n'
+          '|'
+        when 'ne'
+          '/'
+        when 'e'
+          '-'
+        when 'se'
+          '\\'
+        when 's'
+          '|'
+        when 'sw'
+          '/'
+        when 'w'
+          '-'
+        when 'nw'
+          '\\'
+        else
+          raise 'Unknown ordinal direction ' + ordinal_direction
+        end
+      end
+
+      def self.next_ordinal_direction(current_ordinal_direction)
+        case current_ordinal_direction
+        when 'n'
+          'ne'
+        when 'ne'
+          'e'
+        when 'e'
+          'se'
+        when 'se'
+          's'
+        when 's'
+          'sw'
+        when 'sw'
+          'w'
+        when 'w'
+          'nw'
+        when 'nw'
+          'n'
+        else
+          raise 'Unknown ordinal direction ' + current_ordinal_direction
+        end
+      end
+    end
+  end
+end

data/lib/s3_website/endpoint.rb

@@ -0,0 +1,37 @@
+module S3Website
+  class Endpoint
+    DEFAULT_LOCATION_CONSTRAINT = 'us-east-1'
+    attr_reader :region, :location_constraint, :hostname, :website_hostname
+
+    def initialize(location_constraint=nil)
+      location_constraint = DEFAULT_LOCATION_CONSTRAINT if location_constraint.nil?
+      raise "Invalid S3 location constraint #{location_constraint}" unless
+        location_constraints.has_key?location_constraint
+      @region = location_constraints.fetch(location_constraint)[:region]
+      @hostname = location_constraints.fetch(location_constraint)[:endpoint]
+      @website_hostname = location_constraints.fetch(location_constraint)[:website_hostname]
+      @location_constraint = location_constraint
+    end
+
+    # http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
+    def location_constraints
+      eu_west_1_region = {
+        :region           => 'EU (Ireland)',
+        :website_hostname => 's3-website-eu-west-1.amazonaws.com',
+        :endpoint         => 's3-eu-west-1.amazonaws.com'
+      }
+
+      {
+        'us-east-1'      => { :region => 'US Standard',                   :website_hostname => 's3-website-us-east-1.amazonaws.com',      :endpoint => 's3.amazonaws.com' },
+        'us-west-2'      => { :region => 'US West (Oregon)',              :website_hostname => 's3-website-us-west-2.amazonaws.com',      :endpoint => 's3-us-west-2.amazonaws.com' },
+        'us-west-1'      => { :region => 'US West (Northern California)', :website_hostname => 's3-website-us-west-1.amazonaws.com',      :endpoint => 's3-us-west-1.amazonaws.com' },
+        'EU'             => eu_west_1_region,
+        'eu-west-1'      => eu_west_1_region,
+        'ap-southeast-1' => { :region => 'Asia Pacific (Singapore)',      :website_hostname => 's3-website-ap-southeast-1.amazonaws.com', :endpoint => 's3-ap-southeast-1.amazonaws.com' },
+        'ap-southeast-2' => { :region => 'Asia Pacific (Sydney)',         :website_hostname => 's3-website-ap-southeast-2.amazonaws.com', :endpoint => 's3-ap-southeast-2.amazonaws.com' },
+        'ap-northeast-1' => { :region => 'Asia Pacific (Tokyo)',          :website_hostname => 's3-website-ap-northeast-1.amazonaws.com', :endpoint => 's3-ap-northeast-1.amazonaws.com' },
+        'sa-east-1'      => { :region => 'South America (Sao Paulo)',     :website_hostname => 's3-website-sa-east-1.amazonaws.com',      :endpoint => 's3-sa-east-1.amazonaws.com' }
+      }
+    end
+  end
+end

data/lib/s3_website/errors.rb

@@ -0,0 +1,42 @@
+module S3Website
+  class S3WebsiteError < StandardError
+  end
+
+  class NoWebsiteDirectoryFound < S3WebsiteError
+    def initialize(message = "I can't find any website. Are you in the right directory?")
+      super(message)
+    end
+  end
+
+  class NoPredefinedWebsiteDirectoryFound < NoWebsiteDirectoryFound
+    def initialize(message = "I can't find a website in any of the following directories: #{Paths.site_paths.join(', ')}. Please specify the location of the website with the --site option.")
+      super(message)
+    end
+  end
+
+  class NoConfigurationFileError < S3WebsiteError
+    def initialize(message = "I've just generated a file called s3_website.yml. Go put your details in it!")
+      super(message)
+    end
+  end
+
+  class MalformedConfigurationFileError < S3WebsiteError
+    def initialize(message = "I can't parse the file s3_website.yml. It should look like this:\n#{ConfigLoader.read_configuration_file_template}")
+      super(message)
+    end
+  end
+
+  class RetryAttemptsExhaustedError < S3WebsiteError
+    def initialize(message = "Operation failed even though we tried to recover from it")
+      super(message)
+    end
+  end
+
+  def self.error_report(error)
+    if error.is_a? S3WebsiteError
+      "#{error.message}"
+    else
+      "#{error.message} (#{error.class})"
+    end
+  end
+end

data/lib/s3_website/jekyll.rb

@@ -0,0 +1,5 @@
+module S3Website
+  module Jekyll
+    SITE_PATH = '_site'
+  end
+end

data/lib/s3_website/keyboard.rb

@@ -0,0 +1,27 @@
+module S3Website
+  class Keyboard
+    def self.if_user_confirms_delete(to_delete, config, standard_input=STDIN)
+      delete_all = false
+      keep_all = false
+      confirmed_deletes = to_delete.map do |f|
+        delete = false
+        keep = false
+        until delete || delete_all || keep || keep_all
+          puts "#{f} is on S3 but not in your website directory anymore. Do you want to [d]elete, [D]elete all, [k]eep, [K]eep all?"
+          case standard_input.gets.chomp
+          when 'd' then delete = true
+          when 'D' then delete_all = true
+          when 'k' then keep = true
+          when 'K' then keep_all = true
+          end
+        end
+        if (delete_all || delete) && !(keep_all || keep)
+          f
+        end
+      end.select { |f| f }
+      Parallelism.each_in_parallel_or_sequentially(confirmed_deletes, config) { |f|
+        yield f
+      }
+    end
+  end
+end

data/lib/s3_website/nanoc.rb

@@ -0,0 +1,5 @@
+module S3Website
+  module Nanoc
+    SITE_PATH = 'public/output'
+  end
+end

data/lib/s3_website/parallelism.rb

@@ -0,0 +1,25 @@
+module S3Website
+  class Parallelism
+    def self.each_in_parallel_or_sequentially(items, config, &operation)
+      if ENV['disable_parallel_processing']
+        items.each do |item|
+          operation.call item
+        end
+      else
+        slice_size = config['concurrency_level'] || DEFAULT_CONCURRENCY_LEVEL
+        items.each_slice(slice_size) { |items|
+          threads = items.map do |item|
+            Thread.new(item) { |item|
+              operation.call item
+            }
+          end
+          threads.each(&:join)
+        }
+      end
+    end
+
+    private
+
+    DEFAULT_CONCURRENCY_LEVEL = 3
+  end
+end

data/lib/s3_website/paths.rb

@@ -0,0 +1,39 @@
+module S3Website
+  class Paths
+    def self.site_paths
+      [Nanoc::SITE_PATH, Jekyll::SITE_PATH]
+    end
+
+    def self.infer_site_path(candidate_path, pwd)
+      if candidate_path == 'infer automatically'
+        infer_automatically pwd
+      else
+        candidate_path_if_exists candidate_path
+      end
+    end
+
+    private
+
+    def self.candidate_path_if_exists(candidate_path)
+      raise NoWebsiteDirectoryFound.new(
+        "Can't find a website in " + candidate_path
+      ) unless File.exists? candidate_path
+      candidate_path
+    end
+
+    def self.infer_automatically(pwd)
+      site_path = site_paths.
+        map do |site_path|
+        pwd + '/' + site_path
+        end.
+          find do |site_path|
+          File.exists? site_path
+          end
+        if site_path
+          site_path
+        else
+          raise NoPredefinedWebsiteDirectoryFound
+        end
+    end
+  end
+end

data/lib/s3_website/retry.rb

@@ -0,0 +1,19 @@
+module S3Website
+  class Retry
+    def self.run_with_retry(sleep_milliseconds = 3.000)
+      attempt = 0
+      begin
+        yield
+      rescue Exception => e
+        $stderr.puts "Exception Occurred:  #{e.message} (#{e.class})  Retrying in 3 seconds..."
+        sleep sleep_milliseconds
+        attempt += 1
+        if attempt <= 3
+          retry
+        else
+          raise RetryAttemptsExhaustedError
+        end
+      end
+    end
+  end
+end

data/lib/s3_website/tasks.rb

@@ -0,0 +1,36 @@
+module S3Website
+  class Tasks
+    def self.push(config_file_dir, site_dir, in_headless_mode = false)
+      ConfigLoader.check_project site_dir
+      ConfigLoader.check_s3_configuration config_file_dir
+      config = S3Website::ConfigLoader.load_configuration config_file_dir
+      new_files_count, changed_files_count, deleted_files, changed_files, changed_redirects =
+        Uploader.run(site_dir, config, in_headless_mode)
+      invalidated_items_count =
+        invalidate_cf_dist_if_configured(config, changed_files + deleted_files + changed_redirects)
+      {
+        :new_files_count => new_files_count,
+        :changed_files_count => changed_files_count,
+        :deleted_files_count => deleted_files.size,
+        :invalidated_items_count => invalidated_items_count,
+        :changed_redirects_count => changed_redirects.size
+      }
+    end
+
+    def self.config_create(dir)
+      ConfigLoader.check_s3_configuration dir
+    end
+
+    private
+
+    def self.invalidate_cf_dist_if_configured(config, changed_files)
+      cloudfront_configured = config['cloudfront_distribution_id'] &&
+        (not config['cloudfront_distribution_id'].empty?)
+      invalidated_items_count = if cloudfront_configured
+        Cloudfront::Invalidator.invalidate(config, changed_files)
+      else
+        0
+      end
+    end
+  end
+end

data/lib/s3_website/upload.rb

@@ -0,0 +1,137 @@
+require 'tempfile'
+require 'zlib'
+require 'zopfli'
+
+module S3Website
+  class Upload
+    attr_reader :config, :file, :path, :full_path, :s3
+    BLACKLISTED_FILES = ['s3_website.yml']
+
+    def initialize(path, s3, config, site_dir)
+      raise "May not upload #{path}, because it's blacklisted" if Upload.is_blacklisted(path, config)
+      @path = path
+      @full_path = "#{site_dir}/#{path}"
+      @file = File.open("#{site_dir}/#{path}")
+      @s3 = s3
+      @config = config
+    end
+
+    def perform!
+      success = s3.buckets[config['s3_bucket']].objects[path].write(upload_file, upload_options)
+      upload_file.close
+      success
+    end
+
+    def details
+      "#{path}#{" [gzipped]" if gzip?}#{" [max-age=#{max_age}]" if cache_control?}"
+    end
+
+    def self.is_blacklisted(path, config)
+      [
+        config['exclude_from_upload'],
+        BLACKLISTED_FILES
+      ].flatten.compact.any? do |blacklisted_file|
+        Regexp.new(blacklisted_file).match path
+      end
+    end
+
+    private
+
+    def upload_file
+      @upload_file ||= gzip? ? gzipped_file : file
+    end
+
+    def gzip?
+      return false unless !!config['gzip']
+
+      extensions = config['gzip'].is_a?(Array) ? config['gzip'] : S3Website::DEFAULT_GZIP_EXTENSIONS
+      extensions.include?(File.extname(path))
+    end
+
+    def gzipped_file
+      tempfile = Tempfile.new(File.basename(path))
+      tempfile.binmode
+
+      if config['gzip_zopfli']
+        gz_data = Zopfli.deflate file.read, format: :gzip
+        tempfile.write(gz_data)
+        tempfile.flush
+      else
+        gz = Zlib::GzipWriter.new(tempfile, Zlib::BEST_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+        gz.mtime = File.mtime(full_path)
+        gz.orig_name = File.basename(path)
+        gz.write(file.read)
+
+        gz.flush
+        tempfile.flush
+        gz.close
+      end
+      tempfile.open
+
+      tempfile
+    end
+
+    def upload_options
+      opts = {}
+      opts[:reduced_redundancy] = config['s3_reduced_redundancy']
+      opts[:content_type] = resolve_content_type
+      opts[:content_encoding] = "gzip" if gzip?
+      opts[:cache_control] = cache_control_value if cache_control?
+      opts
+    end
+
+    def resolve_content_type
+      is_text = mime_type.start_with?('text/') || mime_type == 'application/json'
+      if is_text
+        "#{mime_type}; charset=utf-8" # Let's consider UTF-8 as the de-facto encoding standard for text
+      else
+        mime_type
+      end
+    end
+
+    def cache_control_value
+      if max_age == 0
+        "no-cache, max-age=0"
+      else
+        "max-age=#{max_age}"
+      end
+    end
+
+    def cache_control?
+      !!config['max_age']
+    end
+
+    def max_age
+      if config['max_age'].is_a?(Hash)
+        max_age_entries_most_specific_first.each do |glob_and_age|
+          (glob, age) = glob_and_age
+          return age if File.fnmatch(glob, path)
+        end
+      else
+        return config['max_age']
+      end
+
+      return 0
+    end
+
+    # The most specific max-age glob == the longest glob
+    def max_age_entries_most_specific_first
+      sorted_by_glob_length = config['max_age'].
+        each_pair.
+        to_a.
+        sort_by do |glob_and_age|
+          (glob, age) = glob_and_age
+          sort_key = glob.length
+        end.
+        reverse
+    end
+
+    def mime_type
+      if !!config['extensionless_mime_type'] && File.extname(path) == ""
+        config['extensionless_mime_type']
+      else
+        MIME::Types.type_for(path).first.to_s
+      end
+    end
+  end
+end