s3_website_monadic 0.0.15 → 0.0.16

data/lib/s3_website/endpoint.rb

@@ -1,37 +0,0 @@
-module S3Website
-  class Endpoint
-    DEFAULT_LOCATION_CONSTRAINT = 'us-east-1'
-    attr_reader :region, :location_constraint, :hostname, :website_hostname
-
-    def initialize(location_constraint=nil)
-      location_constraint = DEFAULT_LOCATION_CONSTRAINT if location_constraint.nil?
-      raise "Invalid S3 location constraint #{location_constraint}" unless
-        location_constraints.has_key?location_constraint
-      @region = location_constraints.fetch(location_constraint)[:region]
-      @hostname = location_constraints.fetch(location_constraint)[:endpoint]
-      @website_hostname = location_constraints.fetch(location_constraint)[:website_hostname]
-      @location_constraint = location_constraint
-    end
-
-    # http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
-    def location_constraints
-      eu_west_1_region = {
-        :region           => 'EU (Ireland)',
-        :website_hostname => 's3-website-eu-west-1.amazonaws.com',
-        :endpoint         => 's3-eu-west-1.amazonaws.com'
-      }
-
-      {
-        'us-east-1'      => { :region => 'US Standard',                   :website_hostname => 's3-website-us-east-1.amazonaws.com',      :endpoint => 's3.amazonaws.com' },
-        'us-west-2'      => { :region => 'US West (Oregon)',              :website_hostname => 's3-website-us-west-2.amazonaws.com',      :endpoint => 's3-us-west-2.amazonaws.com' },
-        'us-west-1'      => { :region => 'US West (Northern California)', :website_hostname => 's3-website-us-west-1.amazonaws.com',      :endpoint => 's3-us-west-1.amazonaws.com' },
-        'EU'             => eu_west_1_region,
-        'eu-west-1'      => eu_west_1_region,
-        'ap-southeast-1' => { :region => 'Asia Pacific (Singapore)',      :website_hostname => 's3-website-ap-southeast-1.amazonaws.com', :endpoint => 's3-ap-southeast-1.amazonaws.com' },
-        'ap-southeast-2' => { :region => 'Asia Pacific (Sydney)',         :website_hostname => 's3-website-ap-southeast-2.amazonaws.com', :endpoint => 's3-ap-southeast-2.amazonaws.com' },
-        'ap-northeast-1' => { :region => 'Asia Pacific (Tokyo)',          :website_hostname => 's3-website-ap-northeast-1.amazonaws.com', :endpoint => 's3-ap-northeast-1.amazonaws.com' },
-        'sa-east-1'      => { :region => 'South America (Sao Paulo)',     :website_hostname => 's3-website-sa-east-1.amazonaws.com',      :endpoint => 's3-sa-east-1.amazonaws.com' }
-      }
-    end
-  end
-end

data/lib/s3_website/errors.rb

@@ -1,42 +0,0 @@
-module S3Website
-  class S3WebsiteError < StandardError
-  end
-
-  class NoWebsiteDirectoryFound < S3WebsiteError
-    def initialize(message = "I can't find any website. Are you in the right directory?")
-      super(message)
-    end
-  end
-
-  class NoPredefinedWebsiteDirectoryFound < NoWebsiteDirectoryFound
-    def initialize(message = "I can't find a website in any of the following directories: #{Paths.site_paths.join(', ')}. Please specify the location of the website with the --site option.")
-      super(message)
-    end
-  end
-
-  class NoConfigurationFileError < S3WebsiteError
-    def initialize(message = "I've just generated a file called s3_website.yml. Go put your details in it!")
-      super(message)
-    end
-  end
-
-  class MalformedConfigurationFileError < S3WebsiteError
-    def initialize(message = "I can't parse the file s3_website.yml. It should look like this:\n#{ConfigLoader.read_configuration_file_template}")
-      super(message)
-    end
-  end
-
-  class RetryAttemptsExhaustedError < S3WebsiteError
-    def initialize(message = "Operation failed even though we tried to recover from it")
-      super(message)
-    end
-  end
-
-  def self.error_report(error)
-    if error.is_a? S3WebsiteError
-      "#{error.message}"
-    else
-      "#{error.message} (#{error.class})"
-    end
-  end
-end

data/lib/s3_website/keyboard.rb

@@ -1,27 +0,0 @@
-module S3Website
-  class Keyboard
-    def self.if_user_confirms_delete(to_delete, config, standard_input=STDIN)
-      delete_all = false
-      keep_all = false
-      confirmed_deletes = to_delete.map do |f|
-        delete = false
-        keep = false
-        until delete || delete_all || keep || keep_all
-          puts "#{f} is on S3 but not in your website directory anymore. Do you want to [d]elete, [D]elete all, [k]eep, [K]eep all?"
-          case standard_input.gets.chomp
-          when 'd' then delete = true
-          when 'D' then delete_all = true
-          when 'k' then keep = true
-          when 'K' then keep_all = true
-          end
-        end
-        if (delete_all || delete) && !(keep_all || keep)
-          f
-        end
-      end.select { |f| f }
-      Parallelism.each_in_parallel_or_sequentially(confirmed_deletes, config) { |f|
-        yield f
-      }
-    end
-  end
-end

data/lib/s3_website/parallelism.rb

@@ -1,25 +0,0 @@
-module S3Website
-  class Parallelism
-    def self.each_in_parallel_or_sequentially(items, config, &operation)
-      if ENV['disable_parallel_processing']
-        items.each do |item|
-          operation.call item
-        end
-      else
-        slice_size = config['concurrency_level'] || DEFAULT_CONCURRENCY_LEVEL
-        items.each_slice(slice_size) { |items|
-          threads = items.map do |item|
-            Thread.new(item) { |item|
-              operation.call item
-            }
-          end
-          threads.each(&:join)
-        }
-      end
-    end
-
-    private
-
-    DEFAULT_CONCURRENCY_LEVEL = 3
-  end
-end

data/lib/s3_website/retry.rb

@@ -1,19 +0,0 @@
-module S3Website
-  class Retry
-    def self.run_with_retry(sleep_milliseconds = 3.000)
-      attempt = 0
-      begin
-        yield
-      rescue Exception => e
-        $stderr.puts "Exception Occurred:  #{e.message} (#{e.class})  Retrying in 3 seconds..."
-        sleep sleep_milliseconds
-        attempt += 1
-        if attempt <= 3
-          retry
-        else
-          raise RetryAttemptsExhaustedError
-        end
-      end
-    end
-  end
-end

data/lib/s3_website/tasks.rb

@@ -1,36 +0,0 @@
-module S3Website
-  class Tasks
-    def self.push(config_file_dir, site_dir, in_headless_mode = false)
-      ConfigLoader.check_project site_dir
-      ConfigLoader.check_s3_configuration config_file_dir
-      config = S3Website::ConfigLoader.load_configuration config_file_dir
-      new_files_count, changed_files_count, deleted_files, changed_files, changed_redirects =
-        Uploader.run(site_dir, config, in_headless_mode)
-      invalidated_items_count =
-        invalidate_cf_dist_if_configured(config, changed_files + deleted_files + changed_redirects)
-      {
-        :new_files_count => new_files_count,
-        :changed_files_count => changed_files_count,
-        :deleted_files_count => deleted_files.size,
-        :invalidated_items_count => invalidated_items_count,
-        :changed_redirects_count => changed_redirects.size
-      }
-    end
-
-    def self.config_create(dir)
-      ConfigLoader.check_s3_configuration dir
-    end
-
-    private
-
-    def self.invalidate_cf_dist_if_configured(config, changed_files)
-      cloudfront_configured = config['cloudfront_distribution_id'] &&
-        (not config['cloudfront_distribution_id'].empty?)
-      invalidated_items_count = if cloudfront_configured
-        Cloudfront::Invalidator.invalidate(config, changed_files)
-      else
-        0
-      end
-    end
-  end
-end

data/lib/s3_website/upload.rb

@@ -1,137 +0,0 @@
-require 'tempfile'
-require 'zlib'
-require 'zopfli'
-
-module S3Website
-  class Upload
-    attr_reader :config, :file, :path, :full_path, :s3
-    BLACKLISTED_FILES = ['s3_website.yml']
-
-    def initialize(path, s3, config, site_dir)
-      raise "May not upload #{path}, because it's blacklisted" if Upload.is_blacklisted(path, config)
-      @path = path
-      @full_path = "#{site_dir}/#{path}"
-      @file = File.open("#{site_dir}/#{path}")
-      @s3 = s3
-      @config = config
-    end
-
-    def perform!
-      success = s3.buckets[config['s3_bucket']].objects[path].write(upload_file, upload_options)
-      upload_file.close
-      success
-    end
-
-    def details
-      "#{path}#{" [gzipped]" if gzip?}#{" [max-age=#{max_age}]" if cache_control?}"
-    end
-
-    def self.is_blacklisted(path, config)
-      [
-        config['exclude_from_upload'],
-        BLACKLISTED_FILES
-      ].flatten.compact.any? do |blacklisted_file|
-        Regexp.new(blacklisted_file).match path
-      end
-    end
-
-    private
-
-    def upload_file
-      @upload_file ||= gzip? ? gzipped_file : file
-    end
-
-    def gzip?
-      return false unless !!config['gzip']
-
-      extensions = config['gzip'].is_a?(Array) ? config['gzip'] : S3Website::DEFAULT_GZIP_EXTENSIONS
-      extensions.include?(File.extname(path))
-    end
-
-    def gzipped_file
-      tempfile = Tempfile.new(File.basename(path))
-      tempfile.binmode
-
-      if config['gzip_zopfli']
-        gz_data = Zopfli.deflate file.read, format: :gzip
-        tempfile.write(gz_data)
-        tempfile.flush
-      else
-        gz = Zlib::GzipWriter.new(tempfile, Zlib::BEST_COMPRESSION, Zlib::DEFAULT_STRATEGY)
-        gz.mtime = File.mtime(full_path)
-        gz.orig_name = File.basename(path)
-        gz.write(file.read)
-
-        gz.flush
-        tempfile.flush
-        gz.close
-      end
-      tempfile.open
-
-      tempfile
-    end
-
-    def upload_options
-      opts = {}
-      opts[:reduced_redundancy] = config['s3_reduced_redundancy']
-      opts[:content_type] = resolve_content_type
-      opts[:content_encoding] = "gzip" if gzip?
-      opts[:cache_control] = cache_control_value if cache_control?
-      opts
-    end
-
-    def resolve_content_type
-      is_text = mime_type.start_with?('text/') || mime_type == 'application/json'
-      if is_text
-        "#{mime_type}; charset=utf-8" # Let's consider UTF-8 as the de-facto encoding standard for text
-      else
-        mime_type
-      end
-    end
-
-    def cache_control_value
-      if max_age == 0
-        "no-cache, max-age=0"
-      else
-        "max-age=#{max_age}"
-      end
-    end
-
-    def cache_control?
-      !!config['max_age']
-    end
-
-    def max_age
-      if config['max_age'].is_a?(Hash)
-        max_age_entries_most_specific_first.each do |glob_and_age|
-          (glob, age) = glob_and_age
-          return age if File.fnmatch(glob, path)
-        end
-      else
-        return config['max_age']
-      end
-
-      return 0
-    end
-
-    # The most specific max-age glob == the longest glob
-    def max_age_entries_most_specific_first
-      sorted_by_glob_length = config['max_age'].
-        each_pair.
-        to_a.
-        sort_by do |glob_and_age|
-          (glob, age) = glob_and_age
-          sort_key = glob.length
-        end.
-        reverse
-    end
-
-    def mime_type
-      if !!config['extensionless_mime_type'] && File.extname(path) == ""
-        config['extensionless_mime_type']
-      else
-        MIME::Types.type_for(path).first.to_s
-      end
-    end
-  end
-end

data/lib/s3_website/uploader.rb

@@ -1,177 +0,0 @@
-module S3Website
-  class Uploader
-    def self.run(site_dir, config, in_headless_mode = false)
-      puts "Deploying #{site_dir.sub(Dir.pwd + '/', '')}/* to #{config['s3_bucket']}"
-
-      s3_config = { :s3_endpoint => Endpoint.new(config['s3_endpoint']).hostname }
-      s3_id, s3_secret = config['s3_id'], config['s3_secret']
-      unless s3_id.nil? || s3_id == '' || s3_secret.nil? || s3_secret == ''
-        s3_config.merge! :access_key_id => s3_id, :secret_access_key => s3_secret
-      end
-
-      s3 = AWS::S3.new(s3_config)
-
-      new_files_count, changed_files_count, changed_files = upload_files(
-        s3, config, site_dir
-      )
-
-      redirects = config['redirects'] || {}
-      changed_redirects = setup_redirects redirects, config, s3
-
-      deleted_files = remove_superfluous_files(
-        s3,
-        config,
-        {
-          :s3_bucket => config['s3_bucket'],
-          :site_dir => site_dir,
-          :redirects => redirects,
-          :in_headless_mode => in_headless_mode,
-          :ignore_on_server => config["ignore_on_server"]
-        }
-      )
-
-      print_done_report config
-
-      [new_files_count, changed_files_count, deleted_files, changed_files, changed_redirects, deleted_files]
-    end
-
-    private
-
-    def self.print_done_report(config)
-      bucket_name = config['s3_bucket']
-      website_hostname_suffix = Endpoint.new(config['s3_endpoint']).website_hostname
-      website_hostname_with_bucket =
-        "%s.%s" % [bucket_name, website_hostname_suffix]
-      puts "Done! Go visit: http://#{website_hostname_with_bucket}/index.html"
-    end
-
-    def self.upload_files(s3, config, site_dir)
-      changed_files, new_files = DiffHelper.resolve_files_to_upload(
-        s3.buckets[config['s3_bucket']],
-        site_dir,
-        config
-      )
-      to_upload = changed_files + new_files
-      if to_upload.empty?
-        puts "No new or changed files to upload"
-      else
-        pre_upload_report = []
-        pre_upload_report << "Uploading"
-        pre_upload_report << "#{new_files.length} new" if new_files.length > 0
-        pre_upload_report << "and" if changed_files.length > 0 and new_files.length > 0
-        pre_upload_report << "#{changed_files.length} changed" if changed_files.length > 0
-        pre_upload_report << "file(s)"
-        puts pre_upload_report.join(' ')
-        upload_in_parallel_or_sequentially to_upload, s3, config, site_dir
-      end
-      [new_files.length, changed_files.length, changed_files]
-    end
-
-    def self.upload_in_parallel_or_sequentially(files_to_upload, s3, config, site_dir)
-      Parallelism.each_in_parallel_or_sequentially(files_to_upload, config) { |f|
-        upload_file(f, s3, config, site_dir)
-      }
-    end
-
-    def self.upload_file(file, s3, config, site_dir)
-      Retry.run_with_retry do
-        upload = Upload.new(file, s3, config, site_dir)
-
-        if upload.perform!
-          print "Upload #{upload.details}: Success!\n"
-        else
-          print "Upload #{upload.details}: FAILURE!\n"
-        end
-      end
-    end
-
-    def self.setup_redirects(redirects, config, s3)
-      operations = redirects.map do |path, target|
-        setup_redirect(path, target, s3, config)
-      end
-      performed_operations = operations.reject do |op|
-        op == :no_redirect_operation_performed
-      end
-      unless performed_operations.empty?
-        puts 'Creating new redirects ...'
-      end
-      performed_operations.each do |redirect_operation|
-        puts '  ' + redirect_operation[:report]
-      end
-      performed_operations.map do |redirect_operation|
-        redirect_operation[:path]
-      end
-    end
-
-    def self.setup_redirect(path, target, s3, config)
-      target = '/' + target unless target =~ %r{^(/|https?://)}
-      s3_object = s3.buckets[config['s3_bucket']].objects[path]
-
-      begin
-        current_head = s3_object.head
-      rescue AWS::S3::Errors::NoSuchKey
-      end
-
-      if current_head.nil? or current_head[:website_redirect_location] != target
-        s3_object.write('', :website_redirect_location => target)
-        {
-          :report => "Redirect #{path} to #{target}: Success!",
-          :path => path
-        }
-      else
-        :no_redirect_operation_performed
-      end
-    end
-
-    def self.remove_superfluous_files(s3, config, options)
-      s3_bucket_name = options.fetch(:s3_bucket)
-      site_dir = options.fetch(:site_dir)
-      in_headless_mode = options.fetch(:in_headless_mode)
-
-      remote_files = s3.buckets[s3_bucket_name].objects.map { |f| f.key }
-      local_files = load_all_local_files(site_dir) + options.fetch(:redirects).keys
-      files_to_delete = build_list_of_files_to_delete(remote_files, local_files, options[:ignore_on_server])
-
-      deleted_files = []
-      if in_headless_mode
-        files_to_delete.each { |s3_object_key|
-          delete_s3_object s3, s3_bucket_name, s3_object_key
-          deleted_files << s3_object_key
-        }
-      else
-        Keyboard.if_user_confirms_delete(files_to_delete, config) { |s3_object_key|
-          delete_s3_object s3, s3_bucket_name, s3_object_key
-          deleted_files << s3_object_key
-        }
-      end
-      deleted_files
-    end
-
-    def self.build_list_of_files_to_delete(remote_files, local_files, ignore_on_server = nil)
-      files_to_delete = remote_files - local_files
-      files_to_delete.reject { |file|
-        ignore_regexps(ignore_on_server).any? do |ignore_regexp|
-          Regexp.new(ignore_regexp).match file
-        end
-      }
-    end
-
-    def self.ignore_regexps(ignore_on_server)
-      ignore_regexps = ignore_on_server || "a_string_that_should_never_match_ever"
-      ignore_regexps.class == Array ? ignore_regexps : [ignore_regexps]
-    end
-
-    def self.delete_s3_object(s3, s3_bucket_name, s3_object_key)
-      Retry.run_with_retry do
-        s3.buckets[s3_bucket_name].objects[s3_object_key].delete
-        print "Delete #{s3_object_key}: Success!\n"
-      end
-    end
-
-    def self.load_all_local_files(site_dir)
-      Dir.glob(site_dir + '/**/*', File::FNM_DOTMATCH).
-        delete_if { |f| File.directory?(f) }.
-        map { |f| f.sub(site_dir + '/', '') }
-    end
-  end
-end