s3-website-deploy 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ce12647eba2163ed2673a99eecdb8e1a7b564571
+  data.tar.gz: bcdbc3e1f9142971df66774fc550b484f7ec2267
+SHA512:
+  metadata.gz: 5a42e6fd22c79d2c3280af19d7c7ec09c9b14bab54b30d6b7905b5e22dd7c86c314120783f0f994bf90fa53659fa67d3dc1b75075dbe5dc5e5a596fda9391981
+  data.tar.gz: fb8b198737940737ab3e9e5908afa2f1d6ffc3d9d24474c0922b16a4673cc06840f9fd27a3c67b2e85fdf4d363770d2827c22b1e26438aa78160a86a13114406

data/.circleci/config.yml

@@ -0,0 +1,46 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.4
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile" }}
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile" }}
+
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+            
+            bundle exec rspec --format progress \
+                            --format RspecJunitFormatter \
+                            --out /tmp/test-results/rspec.xml \
+                            --format progress \
+                            $TEST_FILES
+
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.4
+before_install: gem install bundler -v 1.16.1

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in s3-website-deploy.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 hogelog
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# S3WebsiteDeploy [![CircleCI](https://circleci.com/gh/hogelog/s3-website-deploy.svg?style=svg)](https://circleci.com/gh/hogelog/s3-website-deploy)
+
+s3-website-deploy is deploy tool for AWS S3 Website.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 's3-website-deploy'
+```
+
+## Usage
+
+```console
+$ bundle exec s3-website-deploy --bucket some-bucket --source build/ --region ap-northeast-1
+I, [2018-05-22T00:18:01.580281 #37788]  INFO -- : Start deploying build/ -> s3://some-bucket/
+I, [2018-05-22T00:18:02.841962 #37788]  INFO -- : Creating: favicon.ico
+I, [2018-05-22T00:18:02.842058 #37788]  INFO -- : Creating: index.html
+I, [2018-05-22T00:18:02.842105 #37788]  INFO -- : Finish deploying build/ -> s3://some-bucket/
+```
+
+https://github.com/hogelog/s3-website-deploy/blob/master/spec/dummy/deploy.yml is example of deploy config.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/hogelog/s3-website-deploy.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+namespace :dummy do
+  require "s3-website-deploy"
+end

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "s3-website-deploy"
+
+require "pry"
+Pry.start

data/bin/rake

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 150) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 150) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/s3-website-deploy

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "s3-website-deploy"
+
+S3WebsiteDeploy::CLI.run(ARGV)

data/lib/s3-website-deploy.rb

@@ -0,0 +1 @@
+require "s3_website_deploy"

data/lib/s3_website_deploy.rb

@@ -0,0 +1,4 @@
+require "s3_website_deploy/cli"
+require "s3_website_deploy/client"
+require "s3_website_deploy/configuration"
+require "s3_website_deploy/cache_policy"

data/lib/s3_website_deploy/cache_policy.rb

@@ -0,0 +1,29 @@
+module S3WebsiteDeploy
+  class CachePolicy
+    attr_reader :config, :policies
+
+    class Policy
+      attr_reader :pattern, :regexp, :value
+
+      def initialize(pattern, value)
+        @pattern = pattern
+        @regexp = Regexp.compile("\\A#{ Regexp.escape(pattern).gsub("\\*", ".*") }\\z")
+        @value = value
+      end
+
+      def match?(path)
+        regexp.match?(path)
+      end
+    end
+
+    def initialize(config)
+      @policies = config.map {|pattern, value| Policy.new(pattern, value) }
+    end
+
+    def cache_control(path)
+      policy = @policies.find{|policy| policy.match?(path) }
+      return unless policy
+      policy.value["cache_control"]
+    end
+  end
+end

data/lib/s3_website_deploy/cli.rb

@@ -0,0 +1,34 @@
+require "optparse"
+require "yaml"
+
+module S3WebsiteDeploy
+  class CLI
+    def self.run(argv)
+      config = parse_argv(argv)
+      S3WebsiteDeploy::Client.new(config).run
+    end
+
+    def self.parse_argv(argv)
+      opts = {}
+      parser = OptionParser.new
+      parser.on("-c [CONFIG]", "--config", "config YAML") {|v| opts["config"] = v }
+      parser.on("-s [SOURCE]", "--source", "source directory") {|v| opts["source"] = v }
+      parser.on("-r [REGION]", "--region", "target region") {|v| opts["region"] = v }
+      parser.on("-b [BUCKET]", "--bucket", "target bucket") {|v| opts["bucket"] = v }
+      parser.on("-p [PREFIX]", "--prefix", "target prefix") {|v| opts["prefix"] = v }
+      parser.on("--cache-policy POLICY", "cache policy") {|v| opts["cache-policy"] = JSON.parse(v) if v }
+      parser.on("-n", "--dry-run", "dry run") {|v| opts["dryrun"] = v }
+      parser.parse(argv)
+
+      file_opts = opts["config"] ? YAML.load_file(opts["config"]) : {}
+      source = (opts["source"] || file_opts["source"]) or raise ArgumentError, "--source is required"
+      region = (opts["region"] || file_opts["region"] || ENV["AWS_DEFAULT_REGION"]) or raise ArgumentError, "--region is required"
+      bucket = (opts["bucket"] || file_opts["bucket"]) or raise ArgumentError, "--bucket is required"
+      prefix = opts["prefix"] || file_opts["prefix"] || ""
+      cache_policy = opts["cache-policy"] || file_opts["cache_policy"] || {}
+      dryrun = opts["dryrun"]
+
+      S3WebsiteDeploy::Configuration.new(source: source, region: region, bucket: bucket, prefix: prefix, cache_policy: cache_policy, dryrun: dryrun)
+    end
+  end
+end

data/lib/s3_website_deploy/client.rb

@@ -0,0 +1,122 @@
+require "logger"
+require "pathname"
+
+require "aws-sdk-s3"
+require "mini_mime"
+
+module S3WebsiteDeploy
+  class Client
+    LocalFile = Struct.new(:path, :local_path, :content_md5)
+    S3File = Struct.new(:path, :etag)
+
+    def initialize(config, logger: Logger.new(STDOUT))
+      @config = config
+      @cache_policy = config.cache_policy
+      @logger = logger
+    end
+
+    def run
+      @logger.info("---- DRY RUN ----") if @config.dryrun
+      @logger.info("Start deploying #{@config.source} -> s3://#{@config.bucket}/#{@config.prefix}")
+      file_stats = fetch_file_stats
+      deploy(file_stats)
+      @logger.info("Finish deploying #{@config.source} -> s3://#{@config.bucket}/#{@config.prefix}")
+    end
+
+    def fetch_file_stats
+      stats = {}
+
+      source_files.each do |file|
+        stats[file.path] = { local: file }
+      end
+      remote_files.each do |file|
+        stats[file.path] ||= { }
+        stats[file.path][:remote] = file
+      end
+      stats
+    end
+
+    def deploy(file_stats)
+      file_stats.each do |path, stat|
+        local_file = stat[:local]
+        remote_file = stat[:remote]
+        if local_file
+          if !remote_file
+            @logger.info("Creating: #{local_file.path}")
+            deploy_local_file(local_file)
+          elsif local_file.content_md5 != remote_file.etag
+            @logger.info("Updating: #{local_file.path}")
+            deploy_local_file(local_file)
+          else
+            @logger.info("Skip: #{local_file.path}")
+          end
+        elsif remote_file
+          @logger.info("Deleting: #{remote_file.path}")
+          delete_remote_file(remote_file)
+        end
+      end
+    end
+
+    def deploy_local_file(local_file)
+      return if @config.dryrun
+      key = "#{@config.prefix}#{local_file.path}"
+      content_type = MiniMime.lookup_by_filename(local_file.path).content_type
+      File.open(local_file.local_path, "rb") do |file|
+        s3.put_object(
+          body: file,
+          bucket: @config.bucket,
+          key: key,
+          content_type: content_type,
+          cache_control: @cache_policy.cache_control(local_file.path),
+        )
+      end
+    end
+
+    def delete_remote_file(remote_file)
+      return if @config.dryrun
+      key = "#{@config.prefix}#{remote_file.path}"
+      s3.delete_object(
+        bucket: @config.bucket,
+        key: key,
+      )
+    end
+
+    def source_files
+      source_directory = Pathname.new(@config.source)
+      files = []
+      Pathname.glob(source_directory.join("**", "*").to_s).map do |pathname|
+        next if pathname.directory?
+        path = pathname.to_s
+        content_md5 = Digest::MD5.hexdigest(File.binread(path))
+        files << LocalFile.new(pathname.relative_path_from(source_directory).to_s, path, content_md5)
+      end
+      files
+    end
+
+    def remote_files
+      files = []
+      prefix_pathname = Pathname.new(@config.prefix)
+      next_token = nil
+      loop do
+        res = s3.list_objects_v2(
+          bucket: @config.bucket,
+          prefix: @config.prefix,
+          continuation_token: next_token,
+        )
+        res.contents.each do |content|
+          pathname = Pathname.new(content.key)
+          files << S3File.new(pathname.relative_path_from(prefix_pathname).to_s, JSON.parse(content.etag))
+        end
+        next_token = res.next_continuation_token
+        break unless next_token
+      end
+      files
+    end
+
+    private
+
+    def s3
+      @s3 ||= Aws::S3::Client.new(region: @config.region)
+    end
+  end
+end

data/lib/s3_website_deploy/configuration.rb

@@ -0,0 +1,14 @@
+module S3WebsiteDeploy
+  class Configuration
+    attr_reader :source, :region, :bucket, :prefix, :cache_policy, :dryrun
+
+    def initialize(source:, region:, bucket:, prefix:, cache_policy:, dryrun:)
+      @source = source
+      @region = region
+      @bucket = bucket
+      @prefix = prefix
+      @cache_policy = S3WebsiteDeploy::CachePolicy.new(cache_policy)
+      @dryrun = dryrun
+    end
+  end
+end

data/lib/s3_website_deploy/version.rb

@@ -0,0 +1,3 @@
+module S3WebsiteDeploy
+  VERSION = "0.1.0"
+end

data/s3-website-deploy.gemspec

@@ -0,0 +1,33 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "s3_website_deploy/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "s3-website-deploy"
+  spec.version       = S3WebsiteDeploy::VERSION
+  spec.authors       = ["hogelog"]
+  spec.email         = ["konbu.komuro@gmail.com"]
+
+  spec.summary       = %q{S3 Website deploy tool.}
+  spec.description   = %q{S3 Website deploy tool.}
+  spec.homepage      = "https://github.com/hogelog/s3-website-deploy"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-sdk-s3", "< 2"
+  spec.add_dependency "mini_mime"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec-mocks"
+  spec.add_development_dependency "rspec_junit_formatter"
+  spec.add_development_dependency "pry"
+end

metadata

@@ -0,0 +1,178 @@
+--- !ruby/object:Gem::Specification
+name: s3-website-deploy
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- hogelog
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-10-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: mini_mime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: S3 Website deploy tool.
+email:
+- konbu.komuro@gmail.com
+executables:
+- s3-website-deploy
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/rake
+- bin/rspec
+- bin/setup
+- exe/s3-website-deploy
+- lib/s3-website-deploy.rb
+- lib/s3_website_deploy.rb
+- lib/s3_website_deploy/cache_policy.rb
+- lib/s3_website_deploy/cli.rb
+- lib/s3_website_deploy/client.rb
+- lib/s3_website_deploy/configuration.rb
+- lib/s3_website_deploy/version.rb
+- s3-website-deploy.gemspec
+homepage: https://github.com/hogelog/s3-website-deploy
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.1
+signing_key: 
+specification_version: 4
+summary: S3 Website deploy tool.
+test_files: []