s3_website 0.1.0

data/lib/s3_website/parallelism.rb

@@ -0,0 +1,18 @@
+module S3Website
+  class Parallelism
+    def self.each_in_parallel_or_sequentially(items, &operation)
+      if ENV['disable_parallel_processing']
+        items.each do |item|
+          operation.call item
+        end
+      else
+        threads = items.map do |item|
+          Thread.new(item) { |item|
+            operation.call item
+          }
+        end
+        threads.each { |thread| thread.join }
+      end
+    end
+  end
+end

data/lib/s3_website/retry.rb

@@ -0,0 +1,19 @@
+module S3Website
+  class Retry
+    def self.run_with_retry(sleep_milliseconds = 3.000)
+      attempt = 0
+      begin
+        yield
+      rescue Exception => e
+        $stderr.puts "Exception Occurred:  #{e.message} (#{e.class})  Retrying in 3 seconds..."
+        sleep sleep_milliseconds
+        attempt += 1
+        if attempt <= 3
+          retry
+        else
+          raise RetryAttemptsExhaustedError
+        end
+      end
+    end
+  end
+end

data/lib/s3_website/tasks.rb

@@ -0,0 +1,42 @@
+module S3Website
+  class Tasks
+    def self.push(site_dir, in_headless_mode = false)
+      ConfigLoader.check_project site_dir
+      ConfigLoader.check_s3_configuration site_dir
+      config = S3Website::ConfigLoader.load_configuration site_dir
+      new_files_count, changed_files_count, deleted_files_count, changed_files, changed_redirects =
+        Uploader.run(site_dir, config, in_headless_mode)
+      invalidated_items_count =
+        invalidate_cf_dist_if_configured(config, changed_files + changed_redirects)
+      {
+        :new_files_count => new_files_count,
+        :changed_files_count => changed_files_count,
+        :deleted_files_count => deleted_files_count,
+        :invalidated_items_count => invalidated_items_count,
+        :changed_redirects_count => changed_redirects.size
+      }
+    rescue S3WebsiteError => e
+      puts e.message
+      exit 1
+    end
+
+    def self.config_create(site_dir)
+      ConfigLoader.check_s3_configuration site_dir
+    rescue S3WebsiteError => e
+      puts e.message
+      exit 1
+    end
+
+    private
+
+    def self.invalidate_cf_dist_if_configured(config, changed_files)
+      cloudfront_configured = config['cloudfront_distribution_id'] &&
+        (not config['cloudfront_distribution_id'].empty?)
+      invalidated_items_count = if cloudfront_configured
+        Cloudfront::Invalidator.invalidate(config, changed_files)
+      else
+        0
+      end
+    end
+  end
+end

data/lib/s3_website/upload.rb

@@ -0,0 +1,103 @@
+require 'tempfile'
+require 'zlib'
+
+module S3Website
+  class Upload
+    attr_reader :config, :file, :path, :full_path, :s3
+
+    def initialize(path, s3, config, site_dir)
+      @path = path
+      @full_path = "#{site_dir}/#{path}"
+      @file = File.open("#{site_dir}/#{path}")
+      @s3 = s3
+      @config = config
+    end
+
+    def perform!
+      success = s3.buckets[config['s3_bucket']].objects[path].write(upload_file, upload_options)
+      upload_file.close
+      success
+    end
+
+    def details
+      "#{path}#{" [gzipped]" if gzip?}#{" [max-age=#{max_age}]" if cache_control?}"
+    end
+
+    private
+
+    def upload_file
+      @upload_file ||= gzip? ? gzipped_file : file
+    end
+
+    def gzip?
+      return false unless !!config['gzip']
+
+      extensions = config['gzip'].is_a?(Array) ? config['gzip'] : S3Website::DEFAULT_GZIP_EXTENSIONS
+      extensions.include?(File.extname(path))
+    end
+
+    def gzipped_file
+      tempfile = Tempfile.new(File.basename(path))
+
+      gz = Zlib::GzipWriter.new(tempfile, Zlib::BEST_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+
+      gz.mtime = File.mtime(full_path)
+      gz.orig_name = File.basename(path)
+      gz.write(file.read)
+
+      gz.flush
+      tempfile.flush
+
+      gz.close
+      tempfile.open
+
+      tempfile
+    end
+
+    def upload_options
+      opts = {
+        :content_type => mime_type,
+        :reduced_redundancy => config['s3_reduced_redundancy']
+      }
+
+      opts[:content_type] = "text/html; charset=utf-8" if mime_type == 'text/html'
+      opts[:content_encoding] = "gzip" if gzip?
+      opts[:cache_control] = "max-age=#{max_age}" if cache_control?
+
+      opts
+    end
+
+    def cache_control?
+      !!config['max_age']
+    end
+
+    def max_age
+      if config['max_age'].is_a?(Hash)
+        max_age_entries_most_specific_first.each do |glob_and_age|
+          (glob, age) = glob_and_age
+          return age if File.fnmatch(glob, path)
+        end
+      else
+        return config['max_age']
+      end
+
+      return 0
+    end
+
+    # The most specific max-age glob == the longest glob
+    def max_age_entries_most_specific_first
+      sorted_by_glob_length = config['max_age'].
+        each_pair.
+        to_a.
+        sort_by do |glob_and_age|
+          (glob, age) = glob_and_age
+          sort_key = glob.length
+        end.
+        reverse
+    end
+
+    def mime_type
+      MIME::Types.type_for(path).first
+    end
+  end
+end

data/lib/s3_website/uploader.rb

@@ -0,0 +1,160 @@
+module S3Website
+  class Uploader
+    def self.run(site_dir, config, in_headless_mode = false)
+      puts "Deploying _site/* to #{config['s3_bucket']}"
+
+      s3_config = { :s3_endpoint => Endpoint.new(config['s3_endpoint']).hostname }
+      s3_id, s3_secret = config['s3_id'], config['s3_secret']
+      unless s3_id.nil? || s3_id == '' || s3_secret.nil? || s3_secret == ''
+        s3_config.merge! :access_key_id => s3_id, :secret_access_key => s3_secret
+      end
+
+      s3 = AWS::S3.new(s3_config)
+
+      new_files_count, changed_files_count, changed_files = upload_files(
+        s3, config, site_dir
+      )
+
+      redirects = config['redirects'] || {}
+      changed_redirects = setup_redirects redirects, config, s3
+
+      deleted_files_count = remove_superfluous_files(s3, { :s3_bucket => config['s3_bucket'],
+                                                           :site_dir => site_dir,
+                                                           :redirects => redirects,
+                                                           :in_headless_mode => in_headless_mode,
+                                                           :ignore_on_server => config["ignore_on_server"] })
+
+      print_done_report config
+
+      [new_files_count, changed_files_count, deleted_files_count, changed_files, changed_redirects]
+    end
+
+    private
+
+    def self.print_done_report(config)
+      bucket_name = config['s3_bucket']
+      website_hostname_suffix = Endpoint.new(config['s3_endpoint']).website_hostname
+      website_hostname_with_bucket =
+        "%s.%s" % [bucket_name, website_hostname_suffix]
+      puts "Done! Go visit: http://#{website_hostname_with_bucket}/index.html"
+    end
+
+    def self.upload_files(s3, config, site_dir)
+      changed_files, new_files = DiffHelper.resolve_files_to_upload(
+        s3.buckets[config['s3_bucket']], site_dir)
+      to_upload = changed_files + new_files
+      if to_upload.empty?
+        puts "No new or changed files to upload"
+      else
+        pre_upload_report = []
+        pre_upload_report << "Uploading"
+        pre_upload_report << "#{new_files.length} new" if new_files.length > 0
+        pre_upload_report << "and" if changed_files.length > 0 and new_files.length > 0
+        pre_upload_report << "#{changed_files.length} changed" if changed_files.length > 0
+        pre_upload_report << "file(s)"
+        puts pre_upload_report.join(' ')
+        upload_in_parallel_or_sequentially to_upload, s3, config, site_dir
+      end
+      [new_files.length, changed_files.length, changed_files]
+    end
+
+    def self.upload_in_parallel_or_sequentially(files_to_upload, s3, config, site_dir)
+      Parallelism.each_in_parallel_or_sequentially(files_to_upload) { |f|
+        upload_file(f, s3, config, site_dir)
+      }
+    end
+
+    def self.upload_file(file, s3, config, site_dir)
+      Retry.run_with_retry do
+        upload = Upload.new(file, s3, config, site_dir)
+
+        if upload.perform!
+          puts "Upload #{upload.details}: Success!"
+        else
+          puts "Upload #{upload.details}: FAILURE!"
+        end
+      end
+    end
+
+    def self.setup_redirects(redirects, config, s3)
+      operations = redirects.map do |path, target|
+        setup_redirect(path, target, s3, config)
+      end
+      performed_operations = operations.reject do |op|
+        op == :no_redirect_operation_performed
+      end
+      unless performed_operations.empty?
+        puts 'Creating new redirects ...'
+      end
+      performed_operations.each do |redirect_operation|
+        puts '  ' + redirect_operation[:report]
+      end
+      performed_operations.map do |redirect_operation|
+        redirect_operation[:path]
+      end
+    end
+
+    def self.setup_redirect(path, target, s3, config)
+      target = '/' + target unless target =~ %r{^(/|https?://)}
+      s3_object = s3.buckets[config['s3_bucket']].objects[path]
+
+      begin
+        current_head = s3_object.head
+      rescue AWS::S3::Errors::NoSuchKey
+      end
+
+      if current_head.nil? or current_head[:website_redirect_location] != target
+        s3_object.write('', :website_redirect_location => target)
+        {
+          :report => "Redirect #{path} to #{target}: Success!",
+          :path => path
+        }
+      else
+        :no_redirect_operation_performed
+      end
+    end
+
+    def self.remove_superfluous_files(s3, options)
+      s3_bucket_name = options.fetch(:s3_bucket)
+      site_dir = options.fetch(:site_dir)
+      in_headless_mode = options.fetch(:in_headless_mode)
+
+      remote_files = s3.buckets[s3_bucket_name].objects.map { |f| f.key }
+      local_files = load_all_local_files(site_dir) + options.fetch(:redirects).keys
+      files_to_delete = build_list_of_files_to_delete(remote_files, local_files, options[:ignore_on_server])
+
+      deleted_files_count = 0
+      if in_headless_mode
+        files_to_delete.each { |s3_object_key|
+          delete_s3_object s3, s3_bucket_name, s3_object_key
+          deleted_files_count += 1
+        }
+      else
+        Keyboard.if_user_confirms_delete(files_to_delete) { |s3_object_key|
+          delete_s3_object s3, s3_bucket_name, s3_object_key
+          deleted_files_count += 1
+        }
+      end
+      deleted_files_count
+    end
+
+    def self.build_list_of_files_to_delete(remote_files, local_files, ignore_on_server = nil)
+      ignore_on_server = Regexp.new(ignore_on_server || "a_string_that_should_never_match_ever")
+      files_to_delete = remote_files - local_files
+      files_to_delete.reject { |file| ignore_on_server.match(file) }
+    end
+
+    def self.delete_s3_object(s3, s3_bucket_name, s3_object_key)
+      Retry.run_with_retry do
+        s3.buckets[s3_bucket_name].objects[s3_object_key].delete
+        puts("Delete #{s3_object_key}: Success!")
+      end
+    end
+
+    def self.load_all_local_files(site_dir)
+      Dir.glob(site_dir + '/**/*', File::FNM_DOTMATCH).
+        delete_if { |f| File.directory?(f) }.
+        map { |f| f.gsub(site_dir + '/', '') }
+    end
+  end
+end

data/s3-website.gemspec

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "s3_website"
+  s.version     = "0.1.0"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Lauri Lehmijoki"]
+  s.email       = ["lauri.lehmijoki@iki.fi"]
+  s.homepage    = "https://github.com/laurilehmijoki/s3_website"
+  s.summary     = %q{Manage your S3 website}
+  s.description = %q{
+    Sync website files, set redirects, use HTTP performance optimisations, deliver via
+    CloudFront.
+  }
+
+  s.default_executable = %q{s3_website}
+
+  s.add_dependency 'aws-sdk', '~> 1.8.5'
+  s.add_dependency 'filey-diff', '~> 1'
+  s.add_dependency 'simple-cloudfront-invalidator', '~> 1'
+  s.add_dependency 'erubis', '~> 2.7.0'
+  s.add_dependency 'mime-types', '= 1.19'
+  s.add_dependency 'thor', '= 0.18.1'
+  s.add_dependency 'configure-s3-website', '~> 1'
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'rspec-expectations'
+  s.add_development_dependency 'cucumber'
+  s.add_development_dependency 'aruba', '~> 0.5.1'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'vcr', '~> 2.5.0'
+  s.add_development_dependency 'webmock', '~> 1.8.0'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/spec/lib/config_loader_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe S3Website::ConfigLoader do
+  it 'supports eRuby syntax in s3_website.yml' do
+    config = S3Website::ConfigLoader.load_configuration('spec/sample_files/hyde_site/_site')
+    config['s3_id'].should eq('hello')
+    config['s3_secret'].should eq('world')
+    config['s3_bucket'].should eq('galaxy')
+  end
+
+  it 'does not define default endpoint' do
+    config = S3Website::ConfigLoader.load_configuration('spec/sample_files/hyde_site/_site')
+    config['s3_endpoint'].should be_nil
+  end
+
+  it 'reads the S3 endpoint setting from s3_website.yml' do
+    config = S3Website::ConfigLoader.load_configuration('spec/sample_files/tokyo_site/_site')
+    config['s3_endpoint'].should eq('ap-northeast-1')
+  end
+end

data/spec/lib/endpoint_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+require 'pp'
+
+describe S3Website::Endpoint do
+
+  it 'uses the DEFAULT_LOCATION_CONSTRAINT constant to set the default location constraint' do
+    endpoint = S3Website::Endpoint.new
+    endpoint.location_constraint.should eq(S3Website::Endpoint::DEFAULT_LOCATION_CONSTRAINT)
+  end
+
+  it 'uses the "us-east-1" as the default location' do
+    S3Website::Endpoint::DEFAULT_LOCATION_CONSTRAINT.should eq('us-east-1')
+  end
+
+  it 'takes a valid location constraint as a constructor parameter' do
+    endpoint = S3Website::Endpoint.new('EU')
+    endpoint.location_constraint.should eq('EU')
+  end
+
+  it 'fails if the location constraint is invalid' do
+    expect {
+      S3Website::Endpoint.new('andromeda')
+    }.to raise_error
+  end
+end
+
+

data/spec/lib/keyboard_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe S3Website::Keyboard do
+  describe '.keep_or_delete' do
+    let(:s3_object_keys) { ['a', 'b', 'c'] }
+    let(:standard_input) { stub('std_in') }
+
+    it 'can delete only the first item' do
+      standard_input.stub(:gets).and_return("d", "K")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['a'])
+    end
+
+    it 'can delete only the second item' do
+      standard_input.stub(:gets).and_return("k", "d", "k")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['b'])
+    end
+
+    it 'can delete all but the first item' do
+      standard_input.stub(:gets).and_return("k", "D")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['b', 'c'])
+    end
+
+    it 'can delete all s3 objects' do
+      standard_input.stub(:gets).and_return("D")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['a', 'b', 'c'])
+    end
+
+    it 'can keep one s3 object' do
+      standard_input.stub(:gets).and_return("k", "d", "d")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq(['b', 'c'])
+    end
+
+    it 'can keep all s3 objects' do
+      standard_input.stub(:gets).and_return("k", "k", "k")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq([])
+    end
+
+    it 'can keep all s3 objects' do
+      standard_input.stub(:gets).and_return("K")
+      deleted_keys = call_keyboard(s3_object_keys, standard_input)
+      deleted_keys.should eq([])
+    end
+
+    def call_keyboard(s3_object_keys, standard_input)
+      deleted_keys = []
+      S3Website::Keyboard.if_user_confirms_delete(s3_object_keys, 
+                                                   standard_input) { |key|
+        deleted_keys << key
+      }
+      deleted_keys
+    end
+  end
+end