s3_website_monadic 0.0.1

data/lib/s3_website/uploader.rb

@@ -0,0 +1,177 @@
+module S3Website
+  class Uploader
+    def self.run(site_dir, config, in_headless_mode = false)
+      puts "Deploying #{site_dir.sub(Dir.pwd + '/', '')}/* to #{config['s3_bucket']}"
+
+      s3_config = { :s3_endpoint => Endpoint.new(config['s3_endpoint']).hostname }
+      s3_id, s3_secret = config['s3_id'], config['s3_secret']
+      unless s3_id.nil? || s3_id == '' || s3_secret.nil? || s3_secret == ''
+        s3_config.merge! :access_key_id => s3_id, :secret_access_key => s3_secret
+      end
+
+      s3 = AWS::S3.new(s3_config)
+
+      new_files_count, changed_files_count, changed_files = upload_files(
+        s3, config, site_dir
+      )
+
+      redirects = config['redirects'] || {}
+      changed_redirects = setup_redirects redirects, config, s3
+
+      deleted_files = remove_superfluous_files(
+        s3,
+        config,
+        {
+          :s3_bucket => config['s3_bucket'],
+          :site_dir => site_dir,
+          :redirects => redirects,
+          :in_headless_mode => in_headless_mode,
+          :ignore_on_server => config["ignore_on_server"]
+        }
+      )
+
+      print_done_report config
+
+      [new_files_count, changed_files_count, deleted_files, changed_files, changed_redirects, deleted_files]
+    end
+
+    private
+
+    def self.print_done_report(config)
+      bucket_name = config['s3_bucket']
+      website_hostname_suffix = Endpoint.new(config['s3_endpoint']).website_hostname
+      website_hostname_with_bucket =
+        "%s.%s" % [bucket_name, website_hostname_suffix]
+      puts "Done! Go visit: http://#{website_hostname_with_bucket}/index.html"
+    end
+
+    def self.upload_files(s3, config, site_dir)
+      changed_files, new_files = DiffHelper.resolve_files_to_upload(
+        s3.buckets[config['s3_bucket']],
+        site_dir,
+        config
+      )
+      to_upload = changed_files + new_files
+      if to_upload.empty?
+        puts "No new or changed files to upload"
+      else
+        pre_upload_report = []
+        pre_upload_report << "Uploading"
+        pre_upload_report << "#{new_files.length} new" if new_files.length > 0
+        pre_upload_report << "and" if changed_files.length > 0 and new_files.length > 0
+        pre_upload_report << "#{changed_files.length} changed" if changed_files.length > 0
+        pre_upload_report << "file(s)"
+        puts pre_upload_report.join(' ')
+        upload_in_parallel_or_sequentially to_upload, s3, config, site_dir
+      end
+      [new_files.length, changed_files.length, changed_files]
+    end
+
+    def self.upload_in_parallel_or_sequentially(files_to_upload, s3, config, site_dir)
+      Parallelism.each_in_parallel_or_sequentially(files_to_upload, config) { |f|
+        upload_file(f, s3, config, site_dir)
+      }
+    end
+
+    def self.upload_file(file, s3, config, site_dir)
+      Retry.run_with_retry do
+        upload = Upload.new(file, s3, config, site_dir)
+
+        if upload.perform!
+          print "Upload #{upload.details}: Success!\n"
+        else
+          print "Upload #{upload.details}: FAILURE!\n"
+        end
+      end
+    end
+
+    def self.setup_redirects(redirects, config, s3)
+      operations = redirects.map do |path, target|
+        setup_redirect(path, target, s3, config)
+      end
+      performed_operations = operations.reject do |op|
+        op == :no_redirect_operation_performed
+      end
+      unless performed_operations.empty?
+        puts 'Creating new redirects ...'
+      end
+      performed_operations.each do |redirect_operation|
+        puts '  ' + redirect_operation[:report]
+      end
+      performed_operations.map do |redirect_operation|
+        redirect_operation[:path]
+      end
+    end
+
+    def self.setup_redirect(path, target, s3, config)
+      target = '/' + target unless target =~ %r{^(/|https?://)}
+      s3_object = s3.buckets[config['s3_bucket']].objects[path]
+
+      begin
+        current_head = s3_object.head
+      rescue AWS::S3::Errors::NoSuchKey
+      end
+
+      if current_head.nil? or current_head[:website_redirect_location] != target
+        s3_object.write('', :website_redirect_location => target)
+        {
+          :report => "Redirect #{path} to #{target}: Success!",
+          :path => path
+        }
+      else
+        :no_redirect_operation_performed
+      end
+    end
+
+    def self.remove_superfluous_files(s3, config, options)
+      s3_bucket_name = options.fetch(:s3_bucket)
+      site_dir = options.fetch(:site_dir)
+      in_headless_mode = options.fetch(:in_headless_mode)
+
+      remote_files = s3.buckets[s3_bucket_name].objects.map { |f| f.key }
+      local_files = load_all_local_files(site_dir) + options.fetch(:redirects).keys
+      files_to_delete = build_list_of_files_to_delete(remote_files, local_files, options[:ignore_on_server])
+
+      deleted_files = []
+      if in_headless_mode
+        files_to_delete.each { |s3_object_key|
+          delete_s3_object s3, s3_bucket_name, s3_object_key
+          deleted_files << s3_object_key
+        }
+      else
+        Keyboard.if_user_confirms_delete(files_to_delete, config) { |s3_object_key|
+          delete_s3_object s3, s3_bucket_name, s3_object_key
+          deleted_files << s3_object_key
+        }
+      end
+      deleted_files
+    end
+
+    def self.build_list_of_files_to_delete(remote_files, local_files, ignore_on_server = nil)
+      files_to_delete = remote_files - local_files
+      files_to_delete.reject { |file|
+        ignore_regexps(ignore_on_server).any? do |ignore_regexp|
+          Regexp.new(ignore_regexp).match file
+        end
+      }
+    end
+
+    def self.ignore_regexps(ignore_on_server)
+      ignore_regexps = ignore_on_server || "a_string_that_should_never_match_ever"
+      ignore_regexps.class == Array ? ignore_regexps : [ignore_regexps]
+    end
+
+    def self.delete_s3_object(s3, s3_bucket_name, s3_object_key)
+      Retry.run_with_retry do
+        s3.buckets[s3_bucket_name].objects[s3_object_key].delete
+        print "Delete #{s3_object_key}: Success!\n"
+      end
+    end
+
+    def self.load_all_local_files(site_dir)
+      Dir.glob(site_dir + '/**/*', File::FNM_DOTMATCH).
+        delete_if { |f| File.directory?(f) }.
+        map { |f| f.sub(site_dir + '/', '') }
+    end
+  end
+end

data/lib/s3_website.rb

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'yaml'
+require 'erubis'
+require 'aws-sdk'
+require 'simple-cloudfront-invalidator'
+require 'filey-diff'
+require 'mime/types'
+require 'thor'
+
+module S3Website
+  DEFAULT_GZIP_EXTENSIONS = %w(.html .css .js .svg .txt)
+end
+
+%w{
+  errors
+  upload
+  uploader
+  tasks
+  config_loader
+  retry
+  keyboard
+  diff_helper
+  endpoint
+  parallelism
+  jekyll
+  nanoc
+  paths
+}.each do |file| 
+  require File.dirname(__FILE__) + "/s3_website/#{file}"
+end
+
+%w{invalidator}.each do |file|
+  require File.dirname(__FILE__) + "/cloudfront/#{file}"
+end

data/project/assembly.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")

data/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0")

data/resources/configuration_file_template.yml

@@ -0,0 +1,56 @@
+s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
+s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
+s3_bucket: your.blog.bucket.com
+
+# Below are examples of all the available configurations.
+# See README for more detailed info on each of them.
+
+# max_age:
+#   "assets/*": 6000
+#   "*": 300
+
+# gzip:
+#   - .html
+#   - .css
+#   - .md
+# gzip_zopfli: true
+
+# See http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region for valid endpoints
+# s3_endpoint: ap-northeast-1
+
+# ignore_on_server: that_folder_of_stuff_i_dont_keep_locally
+
+# exclude_from_upload:
+#   - those_folders_of_stuff
+#   - i_wouldnt_want_to_upload
+
+# s3_reduced_redundancy: true
+
+# cloudfront_distribution_id: your-dist-id
+
+# cloudfront_distribution_config:
+#   default_cache_behavior:
+#     min_TTL: <%= 60 * 60 * 24 %>
+#   aliases:
+#     quantity: 1
+#     items:
+#       CNAME: your.website.com
+
+# cloudfront_invalidate_root: true
+
+# concurrency_level: 5
+
+# redirects:
+#   index.php: /
+#   about.php: about.html
+#   music-files/promo.mp4: http://www.youtube.com/watch?v=dQw4w9WgXcQ
+
+# routing_rules:
+#   - condition:
+#       key_prefix_equals: blog/some_path
+#     redirect:
+#       host_name: blog.example.com
+#       replace_key_prefix_with: some_new_path/
+#       http_redirect_code: 301
+
+# extensionless_mime_type: text/html

data/s3_website.gemspec

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "s3_website_monadic"
+  s.version     = "0.0.1"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Lauri Lehmijoki"]
+  s.email       = ["lauri.lehmijoki@iki.fi"]
+  s.homepage    = "https://github.com/laurilehmijoki/s3_website"
+  s.summary     = %q{Manage your S3 website}
+  s.description = %q{
+    Sync website files, set redirects, use HTTP performance optimisations, deliver via
+    CloudFront.
+  }
+  s.license     = 'MIT'
+
+  s.default_executable = %q{s3_website}
+
+  s.add_dependency 'aws-sdk', '~> 1'
+  s.add_dependency 'filey-diff', '~> 1.4.3'
+  s.add_dependency 'simple-cloudfront-invalidator', '~> 1'
+  s.add_dependency 'erubis', '~> 2.7.0'
+  s.add_dependency 'mime-types', '~> 1'
+  s.add_dependency 'thor', '= 0.18.1'
+  s.add_dependency 'configure-s3-website', '= 1.5.5'
+  s.add_dependency 'zopfli', '~> 0.0.3'
+
+  s.add_development_dependency 'rspec', '2.14.0'
+  s.add_development_dependency 'rspec-expectations', '2.14.4'
+  s.add_development_dependency 'cucumber', '1.3.10'
+  s.add_development_dependency 'aruba', '0.5.3'
+  s.add_development_dependency 'rake', '10.1.1'
+  s.add_development_dependency 'vcr', '2.8.0'
+  s.add_development_dependency 'webmock', '1.16.1'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/sbt

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+SBT_OPTS="-Xms512M -Xmx1536M -Xss1M -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=256M"
+java $SBT_OPTS -jar `dirname $0`/project/sbt-launch-0.13.2.jar "$@"

data/spec/lib/cloudfront/invalidator_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe S3Website::Cloudfront::Invalidator do
+  let(:config) {{
+    's3_id' => 'aws id',
+    's3_secret' => 'aws secret',
+    'cloudfront_distribution_id' => 'EFXX'
+  }}
+
+  describe 'default behaviour' do
+    it 'invalidates the root resource' do
+      invalidator = create_simple_cloudfront_invalidator(config)
+      invalidator.
+        should_receive(:invalidate).
+        with(['index.html', '']).
+        and_return(:text_report => 'report txt')
+
+      S3Website::Cloudfront::Invalidator.invalidate(config, ['index.html'])
+    end
+  end
+
+  context 'option cloudfront_invalidate_root = true' do
+    let(:config_with_root_invalidation) {
+      config.merge( {
+        'cloudfront_invalidate_root' => true
+      })
+    }
+
+    it 'invalidates all root resources' do
+      invalidator = create_simple_cloudfront_invalidator(config_with_root_invalidation)
+      invalidator.
+        should_receive(:invalidate).
+        with(['article/', '']).
+        and_return(:text_report => 'report txt')
+
+      S3Website::Cloudfront::Invalidator.invalidate(config_with_root_invalidation, ['article/index.html'])
+    end
+  end
+
+  context 'the file name contains special characters' do
+    it 'encodes the file paths according to rfc1738' do
+      invalidator = create_simple_cloudfront_invalidator config
+      invalidator.
+        should_receive(:invalidate).
+        with(['article/arnold%27s%20file.html', '']).
+        and_return(:text_report => 'report txt')
+
+      S3Website::Cloudfront::Invalidator.invalidate(config, ["article/arnold's file.html"])
+    end
+  end
+
+  def create_simple_cloudfront_invalidator(config)
+    invalidator = double('invalidator')
+    SimpleCloudfrontInvalidator::CloudfrontClient.
+      should_receive(:new).
+      with(config['s3_id'], config['s3_secret'], config['cloudfront_distribution_id']).
+      and_return(invalidator)
+    invalidator
+  end
+end

data/spec/lib/config_loader_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe S3Website::ConfigLoader do
+  it 'supports eRuby syntax in s3_website.yml' do
+    config = S3Website::ConfigLoader.load_configuration('spec/sample_files/hyde_site/')
+    config['s3_id'].should eq('hello')
+    config['s3_secret'].should eq('world')
+    config['s3_bucket'].should eq('galaxy')
+  end
+
+  it 'does not define default endpoint' do
+    config = S3Website::ConfigLoader.load_configuration('spec/sample_files/hyde_site/')
+    config['s3_endpoint'].should be_nil
+  end
+
+  it 'reads the S3 endpoint setting from s3_website.yml' do
+    config = S3Website::ConfigLoader.load_configuration('spec/sample_files/tokyo_site')
+    config['s3_endpoint'].should eq('ap-northeast-1')
+  end
+end

data/spec/lib/endpoint_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+require 'pp'
+
+describe S3Website::Endpoint do
+
+  it 'uses the DEFAULT_LOCATION_CONSTRAINT constant to set the default location constraint' do
+    endpoint = S3Website::Endpoint.new
+    endpoint.location_constraint.should eq(S3Website::Endpoint::DEFAULT_LOCATION_CONSTRAINT)
+  end
+
+  it 'uses the "us-east-1" as the default location' do
+    S3Website::Endpoint::DEFAULT_LOCATION_CONSTRAINT.should eq('us-east-1')
+  end
+
+  it 'takes a valid location constraint as a constructor parameter' do
+    endpoint = S3Website::Endpoint.new('EU')
+    endpoint.location_constraint.should eq('EU')
+  end
+
+  it 'takes eu-west-1 as an alias for EU' do
+    endpoint = S3Website::Endpoint.new('eu-west-1')
+    endpoint.location_constraint.should eq('eu-west-1')
+    S3Website::Endpoint.new.location_constraints['EU'].should eq(S3Website::Endpoint.new.location_constraints['eu-west-1'])
+  end
+
+  it 'fails if the location constraint is invalid' do
+    expect {
+      S3Website::Endpoint.new('andromeda')
+    }.to raise_error
+  end
+end

data/spec/lib/error_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe 'error reporting' do
+  it 'prints the class name of the error' do
+    S3Website::error_report(SocketError.new('network is down')).should include(
+      'SocketError'
+    )
+  end
+
+  it 'prints the message of the error' do
+    S3Website::error_report(SocketError.new('network is down')).should eq(
+      'network is down (SocketError)'
+    )
+  end
+
+  it "only prints the message if the error is an #{S3Website::S3WebsiteError}" do
+    S3Website::error_report(S3Website::NoWebsiteDirectoryFound.new()).should eq(
+      "I can't find any website. Are you in the right directory?"
+    )
+  end
+end

data/spec/lib/keyboard_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe S3Website::Keyboard do
+  describe '.keep_or_delete' do
+    let(:s3_object_keys) { ['a', 'b', 'c'] }
+    let(:standard_input) { stub('std_in') }
+
+    it 'can delete only the first item' do
+      standard_input.stub(:gets).and_return("d", "K")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['a'])
+    end
+
+    it 'can delete only the second item' do
+      standard_input.stub(:gets).and_return("k", "d", "k")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['b'])
+    end
+
+    it 'can delete all but the first item' do
+      standard_input.stub(:gets).and_return("k", "D")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['b', 'c'])
+    end
+
+    it 'can delete all s3 objects' do
+      standard_input.stub(:gets).and_return("D")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['a', 'b', 'c'])
+    end
+
+    it 'can keep one s3 object' do
+      standard_input.stub(:gets).and_return("k", "d", "d")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['b', 'c'])
+    end
+
+    it 'can keep all s3 objects' do
+      standard_input.stub(:gets).and_return("k", "k", "k")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq([])
+    end
+
+    it 'can keep all s3 objects' do
+      standard_input.stub(:gets).and_return("K")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq([])
+    end
+
+    def call_keyboard(s3_object_keys, standard_input)
+      deleted_keys = []
+      S3Website::Keyboard.if_user_confirms_delete(
+        s3_object_keys,
+        config = {},
+        standard_input
+      ) { |key|
+        deleted_keys << key
+      }
+      deleted_keys
+    end
+  end
+end

data/spec/lib/parallelism_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+describe S3Website::Parallelism do
+  context 'user has disabled parallelism' do
+    let(:config) {
+      {}
+    }
+
+    before(:all) {
+      @original_disable_state = ENV['disable_parallel_processing']
+      ENV['disable_parallel_processing'] = 'true'
+    }
+
+    after(:all) {
+      ENV['disable_parallel_processing'] = @original_disable_state
+    }
+
+    it 'runs things sequentially' do
+      ints = (0..100).to_a
+      after_processing = []
+      S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
+        after_processing << int
+      }
+      ints.should eq(after_processing)
+    end
+  end
+
+  context 'user has not disabled parallelism' do
+    let(:config) {
+      {}
+    }
+
+    before(:all) {
+      @original_disable_state = ENV['disable_parallel_processing']
+      ENV.delete 'disable_parallel_processing'
+    }
+
+    after(:all) {
+      ENV['disable_parallel_processing'] = @original_disable_state if @original_disable_state
+    }
+
+    it 'runs things in parallel' do
+      ints = (0..100).to_a
+      after_processing = []
+      S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
+        after_processing << int
+      }
+      ints.should_not eq(after_processing) # Parallel processing introduces non-determinism
+    end
+  end
+
+  context 'limiting parallelism' do
+    shared_examples 'parallel processing' do |config|
+      let(:concurrency_level) {
+        config['concurrency_level'] || S3Website::Parallelism::DEFAULT_CONCURRENCY_LEVEL
+      }
+
+      before(:each) {
+        ints = (0..199).to_a
+        @after_processing = []
+        S3Website::Parallelism.each_in_parallel_or_sequentially(ints, config) { |int|
+          @after_processing << int
+        }
+      }
+
+      it "does at most <concurrency_level> operations in parallel" do
+        @after_processing.slice(0, concurrency_level).all? do |int|
+          int <= concurrency_level
+        end.should be true
+        @after_processing.slice(100, concurrency_level).all? do |int|
+          int >= 100 and int <= 100 + concurrency_level
+        end.should be true
+      end
+    end
+
+
+    include_examples 'parallel processing', config = {}
+
+    include_examples 'parallel processing', config = { 'concurrency_level' => 100 }
+  end
+end

data/spec/lib/paths_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe S3Website::Paths do
+  it 'recognises Jekyll and Nanoc site paths' do
+    S3Website::Paths.site_paths.should eq(['public/output', '_site'])
+  end
+end

data/spec/lib/retry_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe S3Website::Retry do
+  describe ".run_with_retry" do
+    it "retry the operation 4 times" do
+      retries = 0
+      begin
+        S3Website::Retry.run_with_retry(0.001) {
+          retries += 1
+          raise Exception
+        }
+      rescue
+      end
+      retries.should be(4)
+    end
+
+    it "throws an error if all retries fail" do
+      expect {
+        S3Website::Retry.run_with_retry(0.001) {
+          raise Exception
+        }
+      }.to raise_error(S3Website::RetryAttemptsExhaustedError)
+    end
+
+    it "re-runs the block if the block throws an error" do
+      retries = 0
+      S3Website::Retry.run_with_retry(0.001) {
+        retries += 1
+        raise Exception if retries < 2
+      }
+      retries.should be(2)
+    end
+  end
+end