shipit-engine 0.5.2 → 0.6.0

data/lib/shipit/command.rb

@@ -0,0 +1,171 @@
+require 'pty'
+require 'shellwords'
+require 'fileutils'
+require 'timeout'
+
+module Shipit
+  class Command
+    MAX_READ = 64.kilobytes
+
+    Error = Class.new(StandardError)
+
+    attr_reader :out, :code, :chdir, :env, :args, :pid, :timeout
+
+    def initialize(*args, default_timeout: 5.minutes.to_i, env: {}, chdir:)
+      @args, options = parse_arguments(args)
+      @timeout = options['timeout'.freeze] || options[:timeout] || default_timeout
+      @env = env
+      @chdir = chdir.to_s
+    end
+
+    def to_s
+      @args.join(' ')
+    end
+
+    def interpolate_environment_variables(argument)
+      return argument.map { |a| interpolate_environment_variables(a) } if argument.is_a?(Array)
+
+      argument.gsub(/(\$\w+)/) do |variable|
+        variable.sub!('$', '')
+        Shellwords.escape(@env.fetch(variable) { ENV[variable] })
+      end
+    end
+
+    def success?
+      code == 0
+    end
+
+    def exit_message
+      "#{self} exited with status #{@code}"
+    end
+
+    def run
+      output = []
+      stream do |out|
+        output << out
+      end
+      output.join
+    end
+
+    def run!
+      output = []
+      stream! do |out|
+        output << out
+      end
+      output.join
+    end
+
+    def with_full_path
+      old_path = ENV['PATH']
+      ENV['PATH'] = "#{ENV['PATH']}:#{Shipit::Engine.root.join('lib', 'snippets')}"
+      yield
+    ensure
+      ENV['PATH'] = old_path
+    end
+
+    def interpolated_arguments
+      interpolate_environment_variables(@args)
+    end
+
+    def start
+      return if @started
+      child_in = @out = @pid = nil
+      FileUtils.mkdir_p(@chdir)
+      with_full_path do
+        @out, child_in, @pid = PTY.spawn(@env, *interpolated_arguments, chdir: @chdir)
+        child_in.close
+      end
+      @started = true
+      self
+    end
+
+    def stream(&block)
+      start
+      begin
+        read_stream(@out, &block)
+      rescue Timeout::Error => error
+        @code = 'timeout'
+        yield red("No output received in the last #{timeout} seconds.") + "\n"
+        terminate!(&block)
+        raise error
+      rescue Errno::EIO # Somewhat expected on Linux: http://stackoverflow.com/a/10306782
+      end
+
+      _, status = Process.waitpid2(@pid)
+      @code = status.exitstatus
+      yield exit_message + "\n" unless success?
+
+      self
+    end
+
+    def check_status
+    end
+
+    def red(text)
+      "\033[1;31m#{text}\033[0m"
+    end
+
+    def stream!(&block)
+      stream(&block)
+      raise Command::Error.new(exit_message) unless success?
+      self
+    end
+
+    def read_stream(io)
+      loop do
+        with_timeout do
+          yield io.readpartial(MAX_READ)
+        end
+      end
+    rescue EOFError
+    end
+
+    def with_timeout(&block)
+      return yield unless timeout
+
+      Timeout.timeout(timeout, &block)
+    end
+
+    def terminate!(&block)
+      kill_and_wait('INT', 5, &block)
+      kill_and_wait('INT', 2, &block)
+      kill_and_wait('TERM', 5, &block)
+      kill_and_wait('TERM', 2, &block)
+      kill('KILL', &block)
+    rescue Errno::ECHILD
+      true # much success
+    ensure
+      begin
+        read_stream(@out, &block)
+      rescue
+      end
+    end
+
+    def kill_and_wait(sig, wait, &block)
+      kill(sig, &block)
+      Timeout.timeout(wait) do
+        read_stream(@out, &block)
+      end
+    rescue Timeout::Error
+    end
+
+    def kill(sig)
+      yield red("Sending SIG#{sig} to PID #{@pid}\n")
+      Process.kill(sig, @pid)
+    end
+
+    def parse_arguments(arguments)
+      options = {}
+      args = arguments.flatten.map do |argument|
+        case argument
+        when Hash
+          options.merge!(argument.values.first)
+          argument.keys.first
+        else
+          argument
+        end
+      end
+      return args, options
+    end
+  end
+end

data/lib/shipit/commands.rb

@@ -0,0 +1,25 @@
+module Shipit
+  class Commands
+    def self.for(model)
+      "#{model.class.name}Commands".constantize.new(model)
+    end
+
+    def self.git_version
+      @git_version ||= begin
+        `git --version` =~ /([\d\.]+)/
+        raise 'git command not found' unless $1
+        Gem::Version.new($1)
+      end
+    end
+
+    delegate :git_version, to: :class
+
+    def env
+      @env ||= Shipit.env
+    end
+
+    def git(*args)
+      Command.new("git", *args)
+    end
+  end
+end

data/lib/shipit/deploy_commands.rb

@@ -0,0 +1,21 @@
+module Shipit
+  class DeployCommands < TaskCommands
+    def steps
+      deploy_spec.deploy_steps!
+    end
+
+    def env
+      commit = @task.until_commit
+      super.merge(
+        'SHA' => commit.sha,
+        'REVISION' => commit.sha,
+      )
+    end
+
+    protected
+
+    def permalink
+      Shipit::Engine.routes.url_helpers.stack_deploy_url(@stack, @task)
+    end
+  end
+end

data/lib/shipit/engine.rb

@@ -1,5 +1,7 @@
 module Shipit
   class Engine < ::Rails::Engine
+    isolate_namespace Shipit
+
     paths['app/models'] << 'app/serializers' << 'app/serializers/concerns'
 
     initializer 'shipit.config' do |app|
@@ -7,6 +9,7 @@ module Shipit
       Shipit::Engine.routes.default_url_options[:host] = Shipit.host
 
       app.config.assets.precompile += %w(
+        favicon.ico
         task.js
         shipit.js
         shipit.css

data/lib/shipit/rollback_commands.rb

@@ -0,0 +1,7 @@
+module Shipit
+  class RollbackCommands < DeployCommands
+    def steps
+      deploy_spec.rollback_steps!
+    end
+  end
+end

data/lib/shipit/stack_commands.rb

@@ -0,0 +1,60 @@
+require 'pathname'
+require 'fileutils'
+
+module Shipit
+  class StackCommands < Commands
+    def initialize(stack)
+      @stack = stack
+    end
+
+    def fetch
+      create_directories
+      if Dir.exist?(@stack.git_path)
+        git('fetch', 'origin', '--tags', @stack.branch, env: env, chdir: @stack.git_path)
+      else
+        git_clone(@stack.repo_git_url, @stack.git_path, branch: @stack.branch, env: env, chdir: @stack.deploys_path)
+      end
+    end
+
+    def fetch_deployed_revision
+      with_temporary_working_directory do |dir|
+        spec = DeploySpec::FileSystem.new(dir, @stack.environment)
+        outputs = spec.fetch_deployed_revision_steps!.map do |command_line|
+          Command.new(command_line, env: env, chdir: dir).run
+        end
+        outputs.find(&:present?).try(:strip)
+      end
+    end
+
+    def build_cacheable_deploy_spec
+      with_temporary_working_directory do |dir|
+        DeploySpec::FileSystem.new(dir, @stack.environment).cacheable
+      end
+    end
+
+    def with_temporary_working_directory(commit: nil)
+      @stack.acquire_git_cache_lock do
+        fetch.run!
+        git('checkout', '--force', "origin/#{@stack.branch}", env: env, chdir: @stack.git_path).run!
+      end
+      Dir.mktmpdir do |dir|
+        git('clone', @stack.git_path, @stack.repo_name, chdir: dir).run!
+        git('checkout', commit.sha, chdir: dir) if commit
+        yield Pathname.new(File.join(dir, @stack.repo_name))
+      end
+    end
+
+    def git_clone(url, path, branch: 'master', **kwargs)
+      git('clone', *modern_git_args, '--recursive', '--branch', branch, url, path, **kwargs)
+    end
+
+    def modern_git_args
+      return [] unless git_version >= Gem::Version.new('1.7.10')
+      %w(--single-branch)
+    end
+
+    def create_directories
+      FileUtils.mkdir_p(@stack.deploys_path)
+    end
+  end
+end

data/lib/shipit/task_commands.rb

@@ -0,0 +1,68 @@
+module Shipit
+  class TaskCommands < Commands
+    delegate :fetch, to: :stack_commands
+
+    def initialize(task)
+      @task = task
+      @stack = task.stack
+    end
+
+    def deploy_spec
+      @deploy_spec ||= DeploySpec::FileSystem.new(@task.working_directory, @stack.environment)
+    end
+
+    def install_dependencies
+      deploy_spec.dependencies_steps!.map do |command_line|
+        Command.new(command_line, env: env, chdir: steps_directory)
+      end
+    end
+
+    def perform
+      steps.map do |command_line|
+        Command.new(command_line, env: env, chdir: steps_directory)
+      end
+    end
+
+    def steps
+      @task.definition.steps
+    end
+
+    def env
+      normalized_name = ActiveSupport::Inflector.transliterate(@task.author.name)
+      super.merge(
+        'ENVIRONMENT' => @stack.environment,
+        'USER' => "#{@task.author.login} (#{normalized_name}) via Shipit",
+        'EMAIL' => @task.author.email,
+        'BUNDLE_PATH' => Rails.root.join('data', 'bundler').to_s,
+        'SHIPIT_LINK' => permalink,
+        'LAST_DEPLOYED_SHA' => @stack.last_deployed_commit.sha,
+      ).merge(deploy_spec.machine_env).merge(@task.env)
+    end
+
+    def checkout(commit)
+      git('checkout', commit.sha, chdir: @task.working_directory)
+    end
+
+    def clone
+      git('clone', '--local', @stack.git_path, @task.working_directory, chdir: @stack.deploys_path)
+    end
+
+    def stack_commands
+      @stack_commands = StackCommands.new(@stack)
+    end
+
+    protected
+
+    def steps_directory
+      if sub_directory = deploy_spec.directory.presence
+        File.join(@task.working_directory, sub_directory)
+      else
+        @task.working_directory
+      end
+    end
+
+    def permalink
+      Shipit::Engine.routes.url_helpers.stack_task_url(@stack, @task)
+    end
+  end
+end

data/lib/shipit/version.rb

@@ -1,3 +1,3 @@
 module Shipit
-  VERSION = '0.5.2'
+  VERSION = '0.6.0'
 end

data/lib/tasks/cron.rake

@@ -1,15 +1,15 @@
 namespace :cron do
   desc "Updates deployed revisions"
   task minutely: :environment do
-    Stack.refresh_deployed_revisions
+    Shipit::Stack.refresh_deployed_revisions
   end
 
   desc "Rolls-up output chunks for completed deploys older than an hour"
   task rollup: :environment do
-    Task.due_for_rollup.find_each(&:schedule_rollup_chunks)
+    Shipit::Task.due_for_rollup.find_each(&:schedule_rollup_chunks)
   end
 
   task refresh_users: :environment do
-    User.refresh_shard(Time.now.hour % 24, 24)
+    Shipit::User.refresh_shard(Time.now.hour % 24, 24)
   end
 end

data/test/controllers/api/base_controller_test.rb

@@ -1,20 +1,24 @@
 require 'test_helper'
 
-class Api::BaseControllerTest < ActionController::TestCase
-  test "authentication is required" do
-    get :index
-    assert_response :unauthorized
-    assert_equal({message: 'Bad credentials'}.to_json, response.body)
-  end
+module Shipit
+  module Api
+    class BaseControllerTest < ActionController::TestCase
+      test "authentication is required" do
+        get :index
+        assert_response :unauthorized
+        assert_equal({message: 'Bad credentials'}.to_json, response.body)
+      end
 
-  test "with proper credentials the request is processed" do
-    authenticate!
-    assert_response :ok
-  end
+      test "with proper credentials the request is processed" do
+        authenticate!
+        assert_response :ok
+      end
 
-  test "#index respond with a list of endpoints" do
-    authenticate!
-    get :index, format: :json
-    assert_equal({stacks_url: api_stacks_url}.to_json, response.body)
+      test "#index respond with a list of endpoints" do
+        authenticate!
+        get :index, format: :json
+        assert_equal({stacks_url: api_stacks_url}.to_json, response.body)
+      end
+    end
   end
 end

data/test/controllers/api/deploys_controller_test.rb

@@ -1,57 +1,61 @@
 require 'test_helper'
 
-class Api::DeploysControllerTest < ActionController::TestCase
-  setup do
-    authenticate!
-    @user = users(:walrus)
-    @stack = stacks(:shipit)
-    @commit = commits(:fifth)
-  end
-
-  test "#create triggers a new deploy for the stack" do
-    assert_difference -> { @stack.deploys.count }, 1 do
-      post :create, stack_id: @stack.to_param, sha: @commit.sha
-    end
-    assert_response :accepted
-    assert_json 'status', 'pending'
-  end
-
-  test "#create use the claimed user as author" do
-    request.headers['X-Shipit-User'] = @user.login
-    post :create, stack_id: @stack.to_param, sha: @commit.sha
-    deploy = Deploy.last
-    deploy.user == @user
-  end
-
-  test "#create renders a 422 if the sha isn't found" do
-    post :create, stack_id: @stack.to_param, sha: '123443543545'
-    assert_response :unprocessable_entity
-    assert_json 'errors', 'sha' => ['Unknown revision']
-  end
-
-  test "#create renders a 422 if the sha format is invalid" do
-    post :create, stack_id: @stack.to_param, sha: '1'
-    assert_response :unprocessable_entity
-    assert_json 'errors', 'sha' => ['is too short (minimum is 6 characters)']
-  end
-
-  test "#create refuses to deploy locked stacks" do
-    @stack.update!(lock_reason: 'Something broken')
-
-    assert_no_difference -> { @stack.deploys.count } do
-      post :create, stack_id: @stack.to_param, sha: @commit.sha
-    end
-    assert_response :unprocessable_entity
-    assert_json 'errors.force', ["Can't deploy a locked stack"]
-  end
-
-  test "#create accepts to deploy locked stacks if force mode is enabled" do
-    @stack.update!(lock_reason: 'Something broken')
-
-    assert_difference -> { @stack.deploys.count }, 1 do
-      post :create, stack_id: @stack.to_param, sha: @commit.sha, force: true
+module Shipit
+  module Api
+    class DeploysControllerTest < ActionController::TestCase
+      setup do
+        authenticate!
+        @user = shipit_users(:walrus)
+        @stack = shipit_stacks(:shipit)
+        @commit = shipit_commits(:fifth)
+      end
+
+      test "#create triggers a new deploy for the stack" do
+        assert_difference -> { @stack.deploys.count }, 1 do
+          post :create, stack_id: @stack.to_param, sha: @commit.sha
+        end
+        assert_response :accepted
+        assert_json 'status', 'pending'
+      end
+
+      test "#create use the claimed user as author" do
+        request.headers['X-Shipit-User'] = @user.login
+        post :create, stack_id: @stack.to_param, sha: @commit.sha
+        deploy = Deploy.last
+        deploy.user == @user
+      end
+
+      test "#create renders a 422 if the sha isn't found" do
+        post :create, stack_id: @stack.to_param, sha: '123443543545'
+        assert_response :unprocessable_entity
+        assert_json 'errors', 'sha' => ['Unknown revision']
+      end
+
+      test "#create renders a 422 if the sha format is invalid" do
+        post :create, stack_id: @stack.to_param, sha: '1'
+        assert_response :unprocessable_entity
+        assert_json 'errors', 'sha' => ['is too short (minimum is 6 characters)']
+      end
+
+      test "#create refuses to deploy locked stacks" do
+        @stack.update!(lock_reason: 'Something broken')
+
+        assert_no_difference -> { @stack.deploys.count } do
+          post :create, stack_id: @stack.to_param, sha: @commit.sha
+        end
+        assert_response :unprocessable_entity
+        assert_json 'errors.force', ["Can't deploy a locked stack"]
+      end
+
+      test "#create accepts to deploy locked stacks if force mode is enabled" do
+        @stack.update!(lock_reason: 'Something broken')
+
+        assert_difference -> { @stack.deploys.count }, 1 do
+          post :create, stack_id: @stack.to_param, sha: @commit.sha, force: true
+        end
+        assert_response :accepted
+        assert_json 'status', 'pending'
+      end
     end
-    assert_response :accepted
-    assert_json 'status', 'pending'
   end
 end