evrone-ci-worker 0.2.0.pre0

data/lib/evrone/ci/worker/middlewares/docker_fetch_repo.rb

@@ -0,0 +1,76 @@
+require 'docker'
+require 'evrone/common/spawn'
+require 'net/scp'
+
+module Evrone
+  module CI
+    module Worker
+
+      DockerFetchRepo = Struct.new(:app) do
+
+        include Helper::Logger
+        include Helper::Config
+
+        def call(env)
+          if env.ssh && env.docker_repo_dir
+            code = -1
+
+            code = prepare_ssh_files(env)
+            code = spawn_script(env) if code == 0
+
+            if code == 0
+              app.call env
+            else
+              -1
+            end
+          else
+            app.call env
+          end
+        end
+
+        private
+
+          def prepare_ssh_files(env)
+            dir = config.docker.remote_dir
+            scp = ::Net::SCP.new(env.ssh.connection)
+
+            code = env.ssh.spawn "mkdir -p #{dir}/.ssh && chmod 0700 #{dir}/.ssh", &env.job.method(:add_to_output)
+
+            if code == 0
+              config = "Host *\n"
+              config << "  StrictHostKeyChecking no\n"
+              config << "  UserKnownHostsFile /dev/null\n"
+
+              scp.upload! StringIO.new(config), "#{dir}/.ssh/config"
+              scp.upload! StringIO.new(env.job.message.deploy_key.to_s), "#{dir}/.ssh/id_rsa"
+
+              code = env.ssh.spawn "chmod 0600 #{dir}/.ssh/id_rsa", &env.job.method(:add_to_output)
+            end
+            code
+          end
+
+          def spawn_script(env)
+            scm = create_scm(env)
+
+            env.job.add_command_to_output("git clone #{scm.src} #{scm.path}")
+            env.job.add_command_to_output("git checkout -q #{scm.sha}")
+
+            script = "sh -c '#{scm.make_fetch_command}'"
+            env.ssh.spawn script, &env.job.method(:add_to_output)
+          end
+
+          def create_scm(env)
+            SCM::Git.new(
+              env.job.message.src,
+              env.job.message.sha,
+              env.docker_repo_dir,
+              &env.job.method(:add_to_output)
+            )
+          end
+
+      end
+    end
+  end
+end
+
+

data/lib/evrone/ci/worker/middlewares/docker_script.rb

@@ -0,0 +1,39 @@
+require 'net/scp'
+
+module Evrone
+  module CI
+    module Worker
+
+      DockerScript = Struct.new(:app) do
+
+        include Helper::Logger
+
+        def call(env)
+          if env.ssh && env.docker_repo_dir
+            code = run_script(env)
+            if code == 0
+              app.call env
+            else
+              code
+            end
+          else
+            app.call env
+          end
+        end
+
+        private
+
+          def run_script(env)
+            scp = ::Net::SCP.new(env.ssh.connection)
+            script = [env.docker_repo_dir, ".ci_script.sh"].join("/")
+            puts env.job.message.script.inspect
+            scp.upload! StringIO.new(env.job.message.script), script
+            env.ssh.spawn "env - bash #{script}", chdir: env.docker_repo_dir, &env.job.method(:add_to_output)
+          end
+
+      end
+    end
+  end
+end
+
+

data/lib/evrone/ci/worker/middlewares/docker_start_container.rb

@@ -0,0 +1,81 @@
+require 'docker'
+require 'evrone/common/spawn'
+require 'socket'
+
+module Evrone
+  module CI
+    module Worker
+
+      DockerStartContainer = Struct.new(:app) do
+
+        include Helper::Config
+        include Helper::Logger
+        include Evrone::Common::Spawn
+
+        def call(env)
+          spawn_container(env) do
+            open_ssh_session(env) do
+              app.call env
+            end
+          end
+        end
+
+        private
+
+          def open_ssh_session(env)
+            host = env.container_addr
+            user = config.docker.ssh.user
+            pass = config.docker.ssh.password
+
+            ssh_options = {
+              password:      pass,
+              port:          config.docker.ssh.port || 22,
+              paranoid:      false,
+              forward_agent: false
+            }
+            env.job.add_to_output "open ssh session to #{user}@#{host}:#{ssh_options[:port]}"
+            open_ssh(host, "ci", ssh_options) do |ssh|
+              begin
+                env.ssh = ssh
+                logger.tagged "SSH" do
+                  yield
+                end
+              end
+            end
+          end
+
+          def spawn_container(env)
+            env.container = ::Docker::Container.create create_options
+            env.container.start
+
+            env.job.add_to_output "using #{Socket.gethostname}/#{env.container.id}"
+
+            env.container_addr = config.docker.ssh.host
+            env.container_addr ||= env.container.json['NetworkSettings']['IPAddress']
+            env.docker_repo_dir = [config.docker.remote_dir, env.job.message.name].join('/')
+
+            logger.tagged "DOCKER #{env.container.id}" do
+              begin
+                logger.info "start container"
+                sleep 3
+                yield
+              ensure
+                env.container.stop
+                logger.info "stop container"
+              end
+            end
+          end
+
+          def create_options
+            {
+              'Cmd'       => config.docker.init,
+              'Image'     => config.docker.image,
+            }.merge(config.docker.create_options || {})
+          end
+
+      end
+    end
+  end
+end
+
+

data/lib/evrone/ci/worker/middlewares/local_before_script.rb

@@ -0,0 +1,29 @@
+require 'evrone/ci/common'
+
+module Evrone
+  module CI
+    module Worker
+
+      LocalBeforeScript = Struct.new(:app) do
+
+        include Common::Helper::Shell
+
+        def call(env)
+          script = env.work_dir.join(".ci_before_script.sh")
+          write_file script, env.job.message.before_script, 0700
+
+          code = bash file: script, chdir: env.work_dir, &env.job.method(:add_to_output)
+
+          if code == 0
+            app.call env
+          else
+            -1
+          end
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/evrone/ci/worker/middlewares/local_create_dirs.rb

@@ -0,0 +1,46 @@
+require 'evrone/ci/common'
+
+module Evrone
+  module CI
+    module Worker
+
+      LocalCreateDirs = Struct.new(:app) do
+
+        include Common::Helper::Shell
+        include Worker::Helper::Config
+
+        def call(env)
+          tmp_dir!  env
+          work_dir! env
+          repo_dir! env
+
+          app.call env
+        end
+
+        private
+
+          def tmp_dir!(env)
+            env.tmp_dir = env.path_prefix.join(config.tmp_dir_name)
+                                         .join(env.job.message.name)
+            recreate env.tmp_dir
+          end
+
+          def work_dir!(env)
+            env.work_dir = env.path_prefix.join(config.work_dir_name)
+                                          .join(env.job.message.name)
+
+            recreate env.work_dir
+          end
+
+          def repo_dir!(env)
+            env.repo_dir = env.path_prefix.join(config.repo_dir_name)
+                                          .join(env.job.message.name)
+
+            mkdir env.repo_dir
+          end
+
+      end
+
+    end
+  end
+end

data/lib/evrone/ci/worker/middlewares/local_fetch_repo.rb

@@ -0,0 +1,47 @@
+require 'evrone/ci/common'
+
+module Evrone
+  module CI
+    module Worker
+
+      LocalFetchRepo = Struct.new(:app) do
+
+        include Common::Helper::Shell
+
+        def call(env)
+          scm = create_scm(env)
+
+          if env.repo_dir
+            if scm.fetch == 0 && export(scm, env) == 0
+              app.call env
+            else
+              -1
+            end
+          else
+            app.call env
+          end
+        end
+
+        private
+
+          def export(scm, env)
+            cmd = scm.class.make_export_command env.repo_dir, env.work_dir
+            env.job.add_command_to_output cmd
+            bash cmd, &env.job.method(:add_to_output)
+          end
+
+          def create_scm(env)
+            SCM::Git.new(
+              env.job.message.src,
+              env.job.message.sha,
+              env.repo_dir,
+              deploy_key: env.job.message.deploy_key,
+              &env.job.method(:add_to_output)
+            )
+          end
+
+      end
+
+    end
+  end
+end

data/lib/evrone/ci/worker/middlewares/local_script.rb

@@ -0,0 +1,29 @@
+require 'evrone/ci/common'
+
+module Evrone
+  module CI
+    module Worker
+
+      LocalScript = Struct.new(:app) do
+
+        include Common::Helper::Shell
+
+        def call(env)
+          script = env.work_dir.join(".ci_script.sh")
+          write_file script, env.job.message.script, 0700
+
+          code = bash file: script, chdir: env.work_dir, &env.job.method(:add_to_output)
+
+          if code == 0
+            app.call env
+          else
+            code
+          end
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/evrone/ci/worker/middlewares/log_job.rb

@@ -0,0 +1,25 @@
+module Evrone
+  module CI
+    module Worker
+
+      LogJob = Struct.new(:app) do
+
+        include Helper::Logger
+
+        def call(env)
+          matrix_key = env.job.message.matrix_keys.join(" ")
+
+          logger.tagged("JOB #{env.job.message.id}.#{env.job.message.job_id} #{matrix_key}") do
+            logger.info "starting job"
+            rs = app.call env
+            logger.info "done job"
+            rs
+          end
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/evrone/ci/worker/middlewares/update_job_status.rb

@@ -0,0 +1,64 @@
+require 'evrone/ci/message'
+
+module Evrone
+  module CI
+    module Worker
+
+      UpdateJobStatus = Struct.new(:app) do
+
+        include Helper::Logger
+
+        STARTED  = 2
+        FINISHED = 3
+        BROKEN   = 4
+        FAILED   = 5
+
+        def call(env)
+
+          update_status env.job, STARTED
+          rs = -1
+          begin
+            rs = app.call env
+          rescue Exception => e
+            logger.error("ERROR: #{e.inspect}\n    BACKTRACE:\n#{e.backtrace.map{|i| "    #{i}" }.join("\n")}")
+          end
+
+          case
+          when rs == 0
+            update_status env.job, FINISHED
+          when rs > 0
+            update_status env.job, BROKEN
+          when rs < 0
+            update_status env.job, FAILED
+          end
+
+          rs
+        end
+
+        private
+
+          def update_status(job, status)
+            publish_status create_message(job, status)
+          end
+
+          def create_message(job, status)
+            tm = Time.now
+            Message::JobStatus.new(
+              build_id: job.message.id,
+              job_id:   job.message.job_id,
+              status:   status,
+              tm:       tm.to_i,
+              tm_usec:  tm.usec,
+              matrix:   job.message.matrix_keys
+            )
+          end
+
+          def publish_status(message)
+            logger.info "delivered job status #{message.inspect}"
+            JobStatusConsumer.publish message
+          end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/worker/runner/local.rb

@@ -0,0 +1,52 @@
+module Evrone
+  module CI
+    module Worker
+      class Runner
+        class Local
+
+          include Worker::Helper::Logger
+          include Worker::Helper::Config
+          include Common::Helper::Shell
+
+          attr_reader :build, :path, :build_script
+
+          def initialize(build, path)
+            @build        = build
+            @path         = path
+            @build_script = path.tmp_dir.join(".build.sh").expand_path
+          end
+
+          def perform
+            make_build_script
+            run_build_script
+          end
+
+          private
+
+            def run_build_script
+              bash file: build_script, chdir: path.work_dir, &method(:add_output)
+            end
+
+            def make_build_script
+              content = build.travis.to_queue.to_shell_script script_replaces do |env|
+                prepare_env env
+              end
+              write_file build_script, content, 0755
+              add_output_command build_script
+            end
+
+            def script_replaces
+              { "SHARED_PATH" => path.shared_dir }
+            end
+
+            def prepare_env(env)
+              env['init'].unshift "echo 'copy repo from #{path.repo_dir} to #{path.work_dir}'"
+              env['init'].unshift build.csm.make_export_command path.repo_dir, path.work_dir
+              env
+            end
+        end
+      end
+    end
+  end
+end
+

data/lib/evrone/ci/worker/version.rb

@@ -0,0 +1,7 @@
+module Evrone
+  module CI
+    module Worker
+      VERSION = "0.2.0.pre0"
+    end
+  end
+end

data/lib/evrone/ci/worker.rb

@@ -0,0 +1,84 @@
+require 'rubygems'
+require 'pathname'
+require 'bundler'
+Bundler.require :default
+
+require File.expand_path("../worker/ext/string", __FILE__)
+require File.expand_path("../worker/version",    __FILE__)
+
+module Evrone
+  module CI
+    module Worker
+
+      autoload :JobsConsumer,          File.expand_path("../worker/consumers/jobs_consumer",             __FILE__)
+      autoload :JobLogsConsumer,       File.expand_path("../worker/consumers/job_logs_consumer",         __FILE__)
+      autoload :JobStatusConsumer,     File.expand_path("../worker/consumers/job_status_consumer",       __FILE__)
+      autoload :Configuration,         File.expand_path("../worker/configuration",                       __FILE__)
+      autoload :Job,                   File.expand_path("../worker/job",                                 __FILE__)
+      autoload :Local,                 File.expand_path("../worker/local",                               __FILE__)
+      autoload :Docker,                File.expand_path("../worker/docker",                              __FILE__)
+
+      autoload :LogJob,                File.expand_path("../worker/middlewares/log_job",                 __FILE__)
+      autoload :UpdateJobStatus,       File.expand_path("../worker/middlewares/update_job_status",       __FILE__)
+
+      autoload :LocalCreateDirs,       File.expand_path("../worker/middlewares/local_create_dirs",       __FILE__)
+      autoload :LocalFetchRepo,        File.expand_path("../worker/middlewares/local_fetch_repo",        __FILE__)
+      autoload :LocalBeforeScript,     File.expand_path("../worker/middlewares/local_before_script",     __FILE__)
+      autoload :LocalScript,           File.expand_path("../worker/middlewares/local_script",            __FILE__)
+
+      autoload :DockerStartContainer,  File.expand_path("../worker/middlewares/docker_start_container",  __FILE__)
+      autoload :DockerFetchRepo,       File.expand_path("../worker/middlewares/docker_fetch_repo",       __FILE__)
+      autoload :DockerBeforeScript,    File.expand_path("../worker/middlewares/docker_before_script",    __FILE__)
+      autoload :DockerScript,          File.expand_path("../worker/middlewares/docker_script",           __FILE__)
+
+      module Helper
+        autoload :Logger,              File.expand_path("../worker/helper/logger",                       __FILE__)
+        autoload :Config,              File.expand_path("../worker/helper/config",                       __FILE__)
+      end
+
+      extend self
+
+      @@root   = Pathname.new File.expand_path('../../../..', __FILE__)
+      @@config = Configuration.new
+
+      def logger
+        if ENV['CI_WORKER_SILENT']
+          config.null_logger
+        else
+          config.logger
+        end
+      end
+
+      def configure
+        yield config
+        config
+      end
+
+      def config
+        @@config
+      end
+
+      def root
+        @@root
+      end
+
+      def perform(job, path_prefix)
+        run_class.new(job, path_prefix).perform
+      end
+
+      def run_class
+        self.const_get(config.run.to_s.camelize)
+      end
+
+      def reset_config!
+        @@config = Configuration.new
+      end
+
+    end
+  end
+end
+
+Evrone::CI::Worker.root.join("lib/evrone/ci/worker/initializers").children.each do |e|
+  require e
+end
+

data/spec/lib/worker/configuration_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Evrone::CI::Worker::Configuration do
+  let(:config) { Evrone::CI::Worker.config }
+  subject { config }
+
+  its(:run)             { should eq :docker     }
+  its(:docker)          { should be             }
+  its(:work_dir_name)   { should eq 'work'      }
+  its(:repo_dir_name)   { should eq 'repo'      }
+  its(:timeout)         { should eq 1800        }
+  its(:amqp_url)        { should be_nil         }
+
+  context "docker" do
+    subject { config.docker }
+
+    its(:ssh)           { should be             }
+    its(:remote_dir)    { should eq '/home/ci'  }
+    its(:init)          { should be             }
+    its(:image)         { should eq 'ci'        }
+  end
+
+  context "docker.ssh" do
+    subject { config.docker.ssh }
+
+    its(:user)          { should eq 'ci'        }
+    its(:password)      { should eq 'ci'        }
+  end
+
+  context ".configure" do
+    subject {
+      Evrone::CI::Worker.configure do |c|
+        c.run = "local"
+        c.docker.image = 'image'
+      end
+    }
+    its(:run)           { should eq :local     }
+    its("docker.image") { should eq 'image'    }
+  end
+end

data/spec/lib/worker/docker_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+require 'pathname'
+require 'fileutils'
+
+describe Evrone::CI::Worker::Docker do
+  let(:options) { { } }
+  let(:job)     { create :job, options }
+  let(:path)    { '/tmp/.ci' }
+  let(:local)   { described_class.new job, path }
+  subject { local }
+
+  after { FileUtils.rm_rf path }
+
+  context "perform", run_docker: true do
+    subject { local.perform }
+    it { should eq 0 }
+
+    context "when fail before_script" do
+      let(:options) { { before_script: "/bin/false" } }
+      it { should eq(-1) }
+    end
+
+    context "when fail script" do
+      let(:options) { { script: "/bin/false" } }
+      it { should eq(1) }
+    end
+  end
+
+end