s3_deployer 0.6.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 844de0917a0347758d662fe6dc799217dc7d16c5
+  data.tar.gz: 3ea7e007d1c486ed7186fc76703de849a138b984
+SHA512:
+  metadata.gz: 3b7f997173a27444996997c954c2d75c3c6dc9ee8766841d9ffb146da5d7fb588d15bcabf9420118a1a6f9548327fb7d536db96832227019b9fe38591a870453
+  data.tar.gz: 6d8cc55576a7c732503dfa014b6a69f3c3e3c880d6499a7208cc5578694c98afbbf6cc6007710915d0e15c5569ac1708ad38a67b4d37a51cc6f7aa265b92381a

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in s3_deployer.gemspec
+gemspec

data/README.md

@@ -0,0 +1,142 @@
+# S3Deployer
+
+Tool for versioned deploying of client-side apps (or literally anything) to S3
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 's3_deployer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install s3_deployer
+
+## Usage
+
+You need to specify s3_deployer_config.rb file in your home directory, which may look like this:
+
+```ruby
+S3Deployer.configure do
+  bucket "some-bucket"
+  region 'us-east-1'
+  app_name "devastator"
+  app_path "path/to/#{app_name}#{"-#{version}" if version && version != ""}"
+  dist_dir "dist"
+  gzip [/\.js$/, /\.css$/] # or just use 'true' to gzip everything
+  colorize true
+  time_zone "America/Los_Angeles" # Useful when you develop from different timezones (e.g. for distributed team,
+                                  # or when deploy from some build server), to be consistent with revision numbers
+
+  before_stage ->(version) do
+    # Some custom code to execute before deploy or stage
+  end
+
+  after_stage ->(version) do
+    # Some custom code to execute after deploy or stage
+  end
+
+  before_switch ->(version) do
+    # Some custom code to execute before deploy or switch
+  end
+
+  after_switch ->(version) do
+    # Some custom code to execute after deploy or switch
+  end
+
+  before_deploy ->(version) do
+    # Some custom code to execute before deploy
+  end
+
+  after_deploy ->(version) do
+    # Some custom code to execute after deploy
+  end
+
+  # You also can specify environment-specific settings, the default environment is 'production'
+  environment(:development) do
+    bucket "some-bucket-dev"
+  end
+
+  access_key_id 'your S3 access key id'
+  secret_access_key 'your S3 secret access key'
+end
+```
+
+Note the 'dist_dir' setting, you should put all the necessary files to there, which should be sent to S3, before deploy.
+
+Then, you need to include Deployer's tasks to your Rakefile, like:
+
+```ruby
+require 'rubygems'
+require 'bundler'
+Bundler.setup
+
+require 's3_deployer/tasks'
+require './s3_deployer_config'
+```
+
+There are 3 main tasks - for deploy, switch and stage. When you stage, it gets all the files from the dist_dir,
+and copies them to S3. It creates a directories structure on S3, like:
+
+```
+/path
+  /to
+    /app
+      /20130809134509
+      /20130809140328
+      ...
+      SHAS
+```
+
+These '20130809134509'-like directories are actually 'staged' versions of the app. Directory name is just
+a revision name, in the format "%Y%m%d%H%M%S".
+
+Then, you have to do 'switch', which just copies the selected revision directory into 'current'.
+So, after 'switch' e.g. to 20130809134509, the directory structure will be like
+
+```
+/path
+  /to
+    /app
+      /20130809134509
+      /20130809140328
+      ...
+      /current
+      CURRENT_REVISION
+      SHAS
+```
+
+'current' contains the currently used copy of the app. Your app should use files from this directory.
+
+You also could do "deploy", it is basically "stage", and then "switch" to just staged revision.
+
+So, use it like this:
+
+```bash
+$ rake s3_deployer:stage # only creates timestamp dir, like 20130809134509, but doesn't override the 'current' dir
+$ rake s3_deployer:switch REVISION=20130809140330
+$ rake s3_deployer:deploy
+$ rake s3_deployer:deploy VERSION=new-stuff # check the example of deployer.rb above to see how it is used
+$ rake s3_deployer:current # get the currently deployed revision
+$ rake s3_deployer:list # get the list of all deployed revisions and their SHAs and commit subjects
+```
+
+If you want to run s3_deployer in some specific environment, use ENV variable:
+
+```bash
+$ ENV=development rake s3_deployer:deploy
+```
+
+Default environment is 'production'
+
+## Contributing
+
+1. Clone it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request pointing to master

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/s3_deployer/color.rb

@@ -0,0 +1,22 @@
+class S3Deployer
+  class Color
+    class << self
+      def green(text)
+        self.new(32).wrap(text)
+      end
+
+      def yellow(text)
+        self.new(33).wrap(text)
+      end
+    end
+
+    def initialize(color)
+      @color = color
+    end
+
+    def wrap(text)
+      "\e[#{@color}m#{text}\e[0m"
+    end
+  end
+end
+

data/lib/s3_deployer/config.rb

@@ -0,0 +1,36 @@
+class S3Deployer
+  class Config
+    attr_reader :version, :revision, :env
+
+    def initialize
+      @version = ENV["VERSION"] || ""
+      @revision = ENV["REVISION"] || ""
+      @env = ENV["ENV"] || "production"
+      @env_settings = {}
+      colorize true
+      time_zone "GMT"
+      current_path "current"
+    end
+
+    %w{
+      region bucket app_name app_path mixbook_host dist_dir access_key_id secret_access_key
+      gzip colorize time_zone current_path cache_control
+      before_deploy after_deploy before_stage after_stage before_switch after_switch
+    }.each do |method|
+      define_method method do |value = :omitted|
+        instance_variable_set("@#{method}", value) unless value == :omitted
+        instance_variable_get("@#{method}")
+      end
+    end
+
+    def environment(name, &block)
+      @env_settings[name.to_s] = block
+    end
+
+    def apply_environment_settings!
+      if @env_settings[@env.to_s]
+        instance_eval(&@env_settings[@env.to_s])
+      end
+    end
+  end
+end

data/lib/s3_deployer/tasks.rb

@@ -0,0 +1,28 @@
+require 's3_deployer'
+
+namespace :s3_deployer do
+  desc "Deploy"
+  task :deploy do
+    S3Deployer.deploy!
+  end
+
+  desc "Deploy the revision, but don't change it to the 'current' revision"
+  task :stage do
+    S3Deployer.stage!
+  end
+
+  desc "Switch"
+  task :switch do
+    S3Deployer.switch!
+  end
+
+  desc "Get current revision number"
+  task :current do
+    S3Deployer.current
+  end
+
+  desc "Get the list of deployed revisions"
+  task :list do
+    S3Deployer.list
+  end
+end

data/lib/s3_deployer/version.rb

@@ -0,0 +1,3 @@
+class S3Deployer
+  VERSION = "0.6.0"
+end

data/lib/s3_deployer.rb

@@ -0,0 +1,251 @@
+require 'json'
+require 'zlib'
+require 'stringio'
+require 'tzinfo'
+require 'rexml/document'
+require 'parallel'
+require 'aws-sdk'
+
+require "s3_deployer/config"
+require "s3_deployer/color"
+require "s3_deployer/version"
+
+class S3Deployer
+  DATE_FORMAT = "%Y%m%d%H%M%S"
+  CURRENT_REVISION = "CURRENT_REVISION"
+  class << self
+    attr_reader :config
+
+    def configure(&block)
+      @config = Config.new
+      @config.instance_eval(&block)
+      @config.apply_environment_settings!
+
+      Aws.config.update({
+        region: config.region,
+        credentials: Aws::Credentials.new(config.access_key_id, config.secret_access_key),
+      })
+    end
+
+    def execute(cmd)
+      puts "Running '#{cmd}'"
+      system(cmd, out: $stdout, err: :out)
+    end
+
+    def deploy!
+      revision = time_zone.now.strftime(DATE_FORMAT)
+      config.before_deploy[revision] if config.before_deploy
+      stage!(revision)
+      switch!(revision)
+      config.after_deploy[revision] if config.after_deploy
+    end
+
+    def stage!(revision = time_zone.now.strftime(DATE_FORMAT))
+      puts "Staging #{colorize(:green, revision)}"
+      config.before_stage[revision] if config.before_stage
+      copy_files_to_s3(revision)
+      store_git_hash(revision)
+      config.after_stage[revision] if config.after_stage
+    end
+
+    def switch!(revision = config.revision)
+      current_revision = get_current_revision
+      current_sha = sha_of_revision(current_revision)
+      sha = sha_of_revision(revision)
+      puts "Switching from #{colorize(:green, current_revision)} (#{colorize(:yellow, current_sha && current_sha[0..7])}) " +
+        "to #{colorize(:green, revision)} (#{colorize(:yellow, sha && sha[0..7])})"
+      if !revision || revision.strip.empty?
+        warn "You must specify the revision by REVISION env variable"
+        exit(1)
+      end
+      revision = normalize_revision(revision)
+      config.before_switch[current_revision, revision] if config.before_switch
+      prefix = config.app_path.empty? ? revision : File.join(config.app_path, revision)
+      list_of_objects = []
+      Aws::S3::Resource.new.bucket(config.bucket).objects(prefix: prefix).each do |object_summary|
+        list_of_objects << object_summary
+      end
+      Parallel.each(list_of_objects, in_threads: 20) do |object_summary|
+        object = object_summary.object
+        target_path = config.app_path.empty? ? @config.current_path : File.join(config.app_path, @config.current_path)
+        path = object.key.gsub(prefix, target_path)
+        value = object.get.body.read
+        value = object.content_encoding == "gzip" ? decompress(value) : value
+        store_value(path, value)
+      end
+      store_current_revision(revision)
+      config.after_switch[current_revision, revision] if config.after_switch
+    end
+
+    def current
+      current_revision = get_current_revision
+      if current_revision
+        puts "Current revision: #{current_revision} - #{get_datetime_from_revision(current_revision)}"
+      else
+        puts "There is no information about the current revision"
+      end
+    end
+
+    def normalize_revision(revision)
+      if revision && !revision.empty?
+        datetime = get_datetime_from_revision(revision)
+        if datetime
+          revision
+        else
+          shas_by_revisions.detect { |k, v| v.start_with?(revision) }.first
+        end
+      end
+    end
+
+    def list
+      puts "Getting the list of deployed revisions..."
+      current_revision = get_current_revision
+      get_list_of_revisions.each do |rev|
+        datetime = get_datetime_from_revision(rev)
+        sha = shas_by_revisions[rev]
+        title = sha ? `git show -s --format=%s #{sha}`.strip : nil
+        string = "#{rev} - #{datetime} #{sha ? " - #{sha[0..7]}" : ""} #{title ? "(#{title})" : ""} #{" <= current" if rev == current_revision}"
+        puts string
+      end
+    end
+
+    def changes(from, to)
+      from_sha = sha_of_revision(from)
+      to_sha = sha_of_revision(to)
+      if from_sha && to_sha
+        `git log --oneline --reverse #{from_sha}...#{to_sha}`.split("\n").map(&:strip)
+      else
+        []
+      end
+    end
+
+    def sha_of_revision(revision)
+      shas_by_revisions[normalize_revision(revision)]
+    end
+
+    private
+
+      def copy_files_to_s3(rev)
+        dir = File.join(config.app_path, rev)
+        Parallel.each(source_files_list, in_threads: 20) do |file|
+          s3_file = Pathname.new(file).relative_path_from(Pathname.new(config.dist_dir)).to_s
+          store_value(File.join(dir, s3_file), File.read(file))
+        end
+      end
+
+      def get_list_of_revisions
+        prefix = File.join(config.app_path)
+        body = Aws::S3::Client.new.list_objects({bucket: config.bucket, delimiter: '/', prefix: prefix + "/"})
+        body.common_prefixes.map(&:prefix).map { |e| e.gsub(prefix, "").gsub("/", "") }.select do |dir|
+          !!(Time.strptime(dir, DATE_FORMAT) rescue nil)
+        end.sort
+      end
+
+      def app_path_with_bucket
+        File.join(config.bucket, config.app_path)
+      end
+
+      def get_datetime_from_revision(revision)
+        date = Time.strptime(revision, DATE_FORMAT) rescue nil
+        date.strftime("%m/%d/%Y %H:%M") if date
+      end
+
+      def shas_by_revisions
+        @shas_by_revisions ||= get_value(File.join(config.app_path, "SHAS")).split("\n").inject({}) do |memo, line|
+          revision, sha = line.split(" - ").map(&:strip)
+          memo[revision] = sha
+          memo
+        end
+      rescue Aws::S3::Errors::NoSuchKey
+        {}
+      end
+
+      def current_revision_path
+        File.join(config.app_path, CURRENT_REVISION)
+      end
+
+      def get_value(key)
+        puts "Retrieving value #{key} on S3"
+        Aws::S3::Resource.new.bucket(config.bucket).object(key).get.body.read
+      end
+
+      def store_current_revision(revision)
+        store_value(current_revision_path, revision)
+      end
+
+      def store_git_hash(time)
+        value = shas_by_revisions.
+          merge(time => `git rev-parse HEAD`.strip).
+          map { |sha, rev| "#{sha} - #{rev}" }.join("\n")
+        store_value(File.join(config.app_path, "SHAS"), value)
+        @shas_by_revisions = nil
+      end
+
+      def get_current_revision
+        get_value(current_revision_path)
+      rescue Aws::S3::Errors::NoSuchKey
+        nil
+      end
+
+      def store_value(key, value)
+        puts "Storing value #{colorize(:yellow, key)} on S3#{", #{colorize(:green, 'gzipped')}" if should_compress?(key)}"
+        options = {acl: "public-read"}
+        if config.cache_control && !config.cache_control.empty?
+          options[:cache_control] = config.cache_control
+        end
+        if should_compress?(key)
+          options[:content_encoding] = "gzip"
+          value = compress(value)
+        end
+        Aws::S3::Resource.new.bucket(config.bucket).object(key).put(options.merge(body: value))
+      end
+
+      def should_compress?(key)
+        if [true, false, nil].include?(config.gzip)
+          !!config.gzip
+        else
+          key != CURRENT_REVISION && Array(config.gzip).any? { |regexp| key.match(regexp) }
+        end
+      end
+
+      def compress(source)
+        output = Stream.new
+        gz = Zlib::GzipWriter.new(output, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+        gz.write(source)
+        gz.close
+        output.string
+      end
+
+      def decompress(source)
+        begin
+          Zlib::GzipReader.new(StringIO.new(source)).read
+        rescue Zlib::GzipFile::Error
+          source
+        end
+      end
+
+      def source_files_list
+        Dir.glob(File.join(config.dist_dir, "**/*")).select { |f| File.file?(f) }
+      end
+
+      def colorize(color, text)
+        config.colorize ? Color.send(color, text) : text
+      end
+
+      def time_zone
+        TZInfo::Timezone.get(config.time_zone)
+      end
+
+      class Stream < StringIO
+        def initialize(*)
+          super
+          set_encoding "BINARY"
+        end
+
+        def close
+          rewind
+        end
+      end
+
+  end
+end

data/s3_deployer.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 's3_deployer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "s3_deployer"
+  spec.version       = S3Deployer::VERSION
+  spec.authors       = ["Anton Astashov"]
+  spec.email         = ["anton.astashov@gmail.com"]
+  spec.description   = "Simple gem for deploying client apps to S3"
+  spec.summary       = "Simple gem for deploying client apps to S3"
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'tzinfo'
+  spec.add_dependency 'json'
+  spec.add_dependency 'parallel'
+  spec.add_dependency 'aws-sdk'
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: s3_deployer
+version: !ruby/object:Gem::Version
+  version: 0.6.0
+platform: ruby
+authors:
+- Anton Astashov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: tzinfo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple gem for deploying client apps to S3
+email:
+- anton.astashov@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- lib/s3_deployer.rb
+- lib/s3_deployer/color.rb
+- lib/s3_deployer/config.rb
+- lib/s3_deployer/tasks.rb
+- lib/s3_deployer/version.rb
+- s3_deployer.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Simple gem for deploying client apps to S3
+test_files: []