evrone-ci-worker 0.2.0.pre4 → 0.2.0.pre5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68127fd3cbc0497af22109a49d12dd4ce6fc6589
-  data.tar.gz: 615fc7a4536c6dbb205d7f78313efd062e8e6907
+  metadata.gz: 73edf7643aed536ce533c1bf4933c5395952d512
+  data.tar.gz: b7ba035941f4101819506ab44f0dcf9e655d4e8d
 SHA512:
-  metadata.gz: 0f3d0cb41b8f525031ae76b78a6b05c26ff8a63609e1991d8485cd714cf331e580ba7d7c6ea05a319f90c84d4810e011fd9e386bd764fad702f362f6bb8774a2
-  data.tar.gz: dbd57ed0c0c7981b45c00c2f30aed6e71fc9abf5e7ceafb64e9782f98d67dd7a555d8ddb64850899ef0e8bbfcc98f79f1be360476c6b75642afbc6e3b3f2082d
+  metadata.gz: 8c9d3c82ba6750443f38bb71ca029f24128fb04f823af0105bff44093e952fc311be39e347b38992d0b5fcef49fdc9c790bf3c017c76b2527f648aef785eb8ee
+  data.tar.gz: bfeb80657ba8e550dbe0eceb132815eff2080f98442ece836a710a2dc9f5a122f736f8ac06a97f0f52e08fc962c2666ac279ffe1d15b4ef16ca65c5a53f5c049

data/Rakefile

@@ -23,5 +23,5 @@ end
 
 desc "run travis build"
 task :travis do
-  exec "bundle exec rake SPEC_OPTS='--format documentation -t ~run_docker -t ~webdav --order=rand'"
+  exec "bundle exec rake SPEC_OPTS='--format documentation -t ~run_docker --order=rand'"
 end

data/bin/evrone-ci-worker

@@ -6,7 +6,6 @@ cli = Evrone::CI::Worker::CLI.new
 
 if RUBY_PLATFORM =~ /darwin/ # developer
   Evrone::CI::Worker.configure do |c|
-    c.webdav_cache_url = 'http://10.0.2.15:8080'
     c.docker.ssh.port = 2223
     c.docker.ssh.host = 'localhost'
     c.docker.create_options = {

data/docker/bootstrap.sh

@@ -6,12 +6,20 @@ set -e
 echo "deb http://us.archive.ubuntu.com/ubuntu/ precise universe" >> /etc/apt/sources.list
 echo "deb http://us.archive.ubuntu.com/ubuntu/ precise multiverse" >> /etc/apt/sources.list
 
+# fixes locales
+locale-gen en_US.UTF-8
+dpkg-reconfigure -fnoninteractive locales
+update-locale LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US"
+export LANGUAGE=en_US
+export LANG=en_US.UTF-8
+export LC_ALL=en_US.UTF-8
+
 # replace init
  dpkg-divert --local --rename --add /sbin/initctl
  ln -s /bin/true /sbin/initctl
 
 apt-get -qy update
-apt-get install -qy curl git-core build-essential
+apt-get install -qy curl git-core build-essential vim-tiny
 
 # add evrone repo
 mkdir -p /etc/apt/sources.list.d
@@ -21,7 +29,7 @@ apt-get -qy update
 
 # add packages
 apt-get install -qy libmysqlclient-dev libpq-dev imagemagick libmagickwand-dev \
-  libcurl4-openssl-dev nodejs postgresql-contrib-9.1 libxml2-dev libxslt-dev \
+  libcurl4-openssl-dev nodejs libxml2-dev libxslt-dev \
   openjdk-7-jre-headless libsqlite3-dev libgeos-dev ruby
 
 # install runit
@@ -35,24 +43,43 @@ echo "ci:ci" | chpasswd
 apt-get -qy install sudo
 echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 
-# fixes locales
-locale-gen en_US.UTF-8
-dpkg-reconfigure -fnoninteractive locales
-update-locale LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US"
-
 # install ssh
 apt-get install -qy openssh-server
 mkdir -p /var/run/sshd
 sv-gen ssh "/usr/sbin/sshd -D -e"
 
 # install postgresql
-apt-get install -qy postgresql
-echo "host all all all trust" > /etc/postgresql/9.1/main/pg_hba.conf
+apt-get install -qy postgresql-9.1 postgresql-contrib-9.1
+echo "local all all     trust" > /etc/postgresql/9.1/main/pg_hba.conf
+echo "host all all all trust" >> /etc/postgresql/9.1/main/pg_hba.conf
 sv-gen postgresql "/usr/lib/postgresql/9.1/bin/postgres -D /var/lib/postgresql/9.1/main -c config_file=/etc/postgresql/9.1/main/postgresql.conf" postgres
 
 # install mysql
-apt-get install -qy mysql-server
-sv-gen mysqld "/usr/sbin/mysqld" mysql
+apt-get install -qy mysql-server-5.5
+sv-gen mysql "/usr/sbin/mysqld" mysql
+
+# install redis-server
+apt-get install -qy redis-server
+sed -i 's|daemonize yes|daemonize no|g' /etc/redis/redis.conf
+sv-gen redis-server "/usr/bin/redis-server /etc/redis/redis.conf" redis
+
+# install rabbitmq-server
+apt-get install -qy rabbitmq-server
+sv-gen rabbitmq-server "/usr/sbin/rabbitmq-server" root
+
+# install elasticsearch
+ES_PKG=elasticsearch-0.90.5.deb
+curl https://download.elasticsearch.org/elasticsearch/elasticsearch/$ES_PKG -o /tmp/$ES_PKG
+dpkg -i /tmp/$ES_PKG
+rm -f /tmp/$ES_PKG
+sv-gen elasticsearch "/usr/share/elasticsearch/bin/elasticsearch -f -Des.path.data=/var/lib/elasticsearch" elasticsearch
+
+# install phantomjs
+PH_PKG=phantomjs-1.9.2-linux-x86_64.tar.bz2
+curl https://phantomjs.googlecode.com/files/$PH_PKG -o /tmp/$PH_PKG
+tar -jxf /tmp/$PH_PKG -C /usr/local
+rm -f /tmp/$PH_PKG
+ln -sf /usr/local/$(basename $PH_PKG .tar.bz2)/bin/phantomjs /usr/local/bin
 
 # add ruby
 apt-get install -qy rbenv rbenv-2.0.0-p195 rbenv-1.9.3-p448
@@ -64,3 +91,5 @@ apt-get -qy clean autoremove
 
 sv-enable postgresql
 sv-enable ssh
+sv-enable mysql
+sv-enable redis-server

data/evrone-ci-worker.gemspec

@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'evrone-common-amqp', '~> 0.1.1'
   spec.add_runtime_dependency 'docker-api',         '= 1.5.4'
   spec.add_runtime_dependency 'hashr',              '= 0.0.22'
-  spec.add_runtime_dependency 'net-scp',            '= 1.1.2'
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

data/lib/evrone/ci/worker/configuration.rb

@@ -19,15 +19,10 @@ module Evrone
                timeout:      30 * 60,
                logger:       Common::TaggedLogging.new(Logger.new STDOUT),
 
-               work_dir_name:   "work",
-               repo_dir_name:   "repo",
-               tmp_dir_name:    "tmp",
+               tmp_dir_name: "tmp",
 
-               sleep_after_job: 5,
-
-               workers:         1,
-               path_prefix:     nil,
-               webdav_cache_url:nil,
+               workers:          1,
+               path_prefix:      nil,
 
                docker: {
                  remote_dir: "/home/ci",

data/lib/evrone/ci/worker/docker.rb

@@ -14,8 +14,6 @@ module Evrone
           use LogJob
           use UpdateJobStatus
           use DockerStartContainer
-          use DockerFetchRepo
-          use DockerWebdavCache
           use DockerScript
         end
 

data/lib/evrone/ci/worker/job.rb

@@ -12,14 +12,13 @@ module Evrone
 
         def initialize(perform_job_message)
           @output_counter = 0
-          @output         = ""
           @message        = perform_job_message
+          @output         = Common::OutputBuffer.new(&method(:publish_job_log_message))
         end
 
         def add_to_output(str)
           output << str
           logger.debug str.strip if logger.level == 0
-          JobLogsConsumer.publish create_job_log_message(str)
         end
 
         def add_command_to_output(cmd)
@@ -30,14 +29,20 @@ module Evrone
           add_to_output log.split(/\n/).map{|i| " ===> #{i}\n" }.join
         end
 
-        def create_job_log_message(data)
+        def release
+          output.close
+        end
+
+        def publish_job_log_message(str)
           @output_counter += 1
-          Message::JobLog.new(
+          log = Message::JobLog.new(
             build_id: message.id,
             job_id:   message.job_id,
-            tm:       @output_counter,
-            log:      data
+            tm:       output_counter,
+            log:      str
           )
+          JobLogsConsumer.publish log
+          log
         end
 
       end

data/lib/evrone/ci/worker/local.rb

@@ -15,7 +15,6 @@ module Evrone
           use LogJob
           use UpdateJobStatus
           use LocalCreateDirs
-          use LocalFetchRepo
           use LocalScript
         end
 

data/lib/evrone/ci/worker/middlewares/docker_script.rb

@@ -1,5 +1,3 @@
-require 'net/scp'
-
 module Evrone
   module CI
     module Worker
@@ -7,10 +5,14 @@ module Evrone
       DockerScript = Struct.new(:app) do
 
         include Helper::Logger
+        include Helper::Config
+        include Common::Helper::UploadShCommand
 
         def call(env)
           if env.ssh && env.docker_repo_dir
             code = run_script(env)
+            run_after_script(env)
+
             if code == 0
               app.call env
             else
@@ -29,24 +31,46 @@ module Evrone
         private
 
           def run_script(env)
-            scp = ::Net::SCP.new(env.ssh.connection)
-            script = [env.docker_repo_dir, ".ci_build.sh"].join("/")
-            scp.upload! StringIO.new(content(env)), script
-            env.ssh.spawn "env - HOME=$HOME bash #{script}", chdir: env.docker_repo_dir, &env.job.method(:add_to_output)
+            home = config.docker.remote_dir
+            file = [home, ".ci_build.sh"].join("/")
+
+            script = [upload_sh_command(file, script_content(env, home))]
+            script << "env - HOME=$HOME bash #{file}"
+            script = script.join(" && ")
+
+            env.ssh.spawn script, chdir: home, &env.job.method(:add_to_output)
+          end
+
+          def run_after_script(env)
+            home = config.docker.remote_dir
+            file = [home, ".ci_after_build.sh"].join("/")
+
+            script = [upload_sh_command(file, after_script_content(env))]
+            script << "env - HOME=$HOME bash #{file}"
+            script = script.join(" && ")
+
+            env.ssh.spawn script, chdir: home, &env.job.method(:add_to_output)
           end
 
-          def content(env)
+          def script_content(env, home)
             buf = ["set -e"]
-            buf << "echo before_script > #{env.docker_repo_dir}/.ci_state"
+            buf << "echo before_script > #{home}/.ci_state"
             buf << env.job.message.before_script
-            buf << "echo script > #{env.docker_repo_dir}/.ci_state"
+            buf << "echo script > #{home}/.ci_state"
             buf << env.job.message.script
             buf.join("\n")
           end
 
+          def after_script_content(env)
+            buf = ["set -e"]
+            buf << env.job.message.after_script
+            buf.join("\n")
+          end
+
           def read_state(env)
+            home = config.docker.remote_dir
             buf = ""
-            state_file = env.docker_repo_dir.join(".ci_state")
+            state_file = "#{home}/.ci_state"
             env.ssh.spawn "cat #{state_file}" do |out|
               buf << out
             end

data/lib/evrone/ci/worker/middlewares/docker_start_container.rb

@@ -52,9 +52,10 @@ module Evrone
 
             env.job.add_to_output "using #{Socket.gethostname}/#{env.container.id}\n"
 
-            env.container_addr = config.docker.ssh.host
-            env.container_addr ||= env.container.json['NetworkSettings']['IPAddress']
-            env.docker_repo_dir = Pathname.new(config.docker.remote_dir).join(env.job.message.name)
+            env.container_addr      = config.docker.ssh.host
+            env.container_addr    ||= env.container.json['NetworkSettings']['IPAddress']
+            env.container_gateway ||= env.container.json['NetworkSettings']['Gateway']
+            env.docker_repo_dir     = Pathname.new(config.docker.remote_dir).join(env.job.message.name)
 
             logger.tagged "DOCKER #{env.container.id}" do
               begin

data/lib/evrone/ci/worker/middlewares/local_create_dirs.rb

@@ -11,8 +11,6 @@ module Evrone
 
         def call(env)
           tmp_dir!  env
-          work_dir! env
-          repo_dir! env
 
           app.call env
         end
@@ -24,21 +22,6 @@ module Evrone
                                          .join(env.job.message.name)
             recreate env.tmp_dir
           end
-
-          def work_dir!(env)
-            env.work_dir = env.path_prefix.join(config.work_dir_name)
-                                          .join(env.job.message.name)
-
-            recreate env.work_dir
-          end
-
-          def repo_dir!(env)
-            env.repo_dir = env.path_prefix.join(config.repo_dir_name)
-                                          .join(env.job.message.name)
-
-            mkdir env.repo_dir
-          end
-
       end
 
     end

data/lib/evrone/ci/worker/middlewares/local_script.rb

@@ -12,7 +12,7 @@ module Evrone
           script = env.tmp_dir.join("build.sh")
           write_file script, content(env), 0700
 
-          code = bash file: script, chdir: env.work_dir, &env.job.method(:add_to_output)
+          code = bash file: script, &env.job.method(:add_to_output)
 
           if code == 0
             app.call env

data/lib/evrone/ci/worker/middlewares/update_job_status.rb

@@ -20,7 +20,7 @@ module Evrone
           begin
             rs = app.call env
           rescue Exception => e
-            env.job.add_to_output("ERROR: #{e.inspect}\n")
+            env.job.add_to_output("\n\nERROR: #{e.inspect}\n")
             logger.error("ERROR: #{e.inspect}\n    BACKTRACE:\n#{e.backtrace.map{|i| "    #{i}" }.join("\n")}")
           end
 

data/lib/evrone/ci/worker/version.rb

@@ -1,7 +1,7 @@
 module Evrone
   module CI
     module Worker
-      VERSION = "0.2.0.pre4"
+      VERSION = "0.2.0.pre5"
     end
   end
 end

data/lib/evrone/ci/worker.rb

@@ -20,18 +20,16 @@ module Evrone
       autoload :Local,                 File.expand_path("../worker/local",                               __FILE__)
       autoload :Docker,                File.expand_path("../worker/docker",                              __FILE__)
       autoload :CLI,                   File.expand_path("../worker/cli",                                 __FILE__)
+      autoload :OutputBuffer,          File.expand_path("../worker/output_buffer",                       __FILE__)
 
       autoload :LogJob,                File.expand_path("../worker/middlewares/log_job",                 __FILE__)
       autoload :UpdateJobStatus,       File.expand_path("../worker/middlewares/update_job_status",       __FILE__)
 
       autoload :LocalCreateDirs,       File.expand_path("../worker/middlewares/local_create_dirs",       __FILE__)
-      autoload :LocalFetchRepo,        File.expand_path("../worker/middlewares/local_fetch_repo",        __FILE__)
       autoload :LocalScript,           File.expand_path("../worker/middlewares/local_script",            __FILE__)
 
       autoload :DockerStartContainer,  File.expand_path("../worker/middlewares/docker_start_container",  __FILE__)
-      autoload :DockerFetchRepo,       File.expand_path("../worker/middlewares/docker_fetch_repo",       __FILE__)
       autoload :DockerScript,          File.expand_path("../worker/middlewares/docker_script",           __FILE__)
-      autoload :DockerWebdavCache,     File.expand_path("../worker/middlewares/docker_webdav_cache",     __FILE__)
 
       module Helper
         autoload :Logger,              File.expand_path("../worker/helper/logger",                       __FILE__)
@@ -69,7 +67,9 @@ module Evrone
       end
 
       def perform(job, path_prefix)
-        run_class.new(job, path_prefix).perform
+        rs = run_class.new(job, path_prefix).perform
+        job.release
+        rs
       end
 
       def run_class

data/spec/lib/worker/configuration_spec.rb

@@ -6,8 +6,7 @@ describe Evrone::CI::Worker::Configuration do
 
   its(:run)             { should eq :docker     }
   its(:docker)          { should be             }
-  its(:work_dir_name)   { should eq 'work'      }
-  its(:repo_dir_name)   { should eq 'repo'      }
+  its(:tmp_dir_name)    { should eq 'tmp'       }
   its(:timeout)         { should eq 1800        }
   its(:amqp_url)        { should be_nil         }
 

data/spec/lib/worker/job_spec.rb

@@ -8,13 +8,13 @@ describe Evrone::CI::Worker::Job do
 
   context "just created" do
     its(:message)        { should eq message }
-    its(:output)         { should eq '' }
+    its(:output)         { should be_an_instance_of(Evrone::CI::Common::OutputBuffer) }
     its(:output_counter) { should eq 0 }
   end
 
-  context "create_job_log_message" do
+  context "publish_job_log_message" do
     let(:data) { 'log' }
-    subject { job.create_job_log_message data }
+    subject { job.publish_job_log_message data }
 
     it { should be_an_instance_of(Evrone::CI::Message::JobLog) }
     its(:job_id)   { should eq job.message.job_id }
@@ -32,13 +32,17 @@ describe Evrone::CI::Worker::Job do
   context "add_to_output" do
     let(:data)     { 'data' }
     let(:messages) { Evrone::CI::Worker::JobLogsConsumer.messages }
-    subject { job.add_to_output(data) ; job }
+    subject do
+      job.add_to_output(data)
+      job.output.flush
+      job
+    end
 
-    its(:output) { should eq data }
     it "should delivery message" do
       expect {
         subject
       }.to change(messages, :size).by(1)
+      expect(messages.first.log).to eq data
     end
 
     it "should increment output_counter" do
@@ -52,14 +56,17 @@ describe Evrone::CI::Worker::Job do
   context "add_command_to_output" do
     let(:data)     { 'data' }
     let(:messages) { Evrone::CI::Worker::JobLogsConsumer.messages }
-    subject { job.add_command_to_output(data) ; job }
-
-    its(:output) { should eq "$ #{data}\n" }
+    subject do
+      job.add_command_to_output(data)
+      job.output.flush
+      job
+    end
 
     it "should delivery message" do
       expect {
         subject
       }.to change(messages, :size).by(1)
+      expect(messages.first.log).to eq "$ #{data}\n"
     end
 
     it "should increment output_counter" do
@@ -73,14 +80,17 @@ describe Evrone::CI::Worker::Job do
   context "add_trace_to_output" do
     let(:data)     { 'data' }
     let(:messages) { Evrone::CI::Worker::JobLogsConsumer.messages }
-    subject { job.add_trace_to_output(data) ; job }
-
-    its(:output) { should eq " ===> #{data}\n" }
+    subject do
+      job.add_trace_to_output(data)
+      job.output.flush
+      job
+    end
 
     it "should delivery message" do
       expect {
         subject
       }.to change(messages, :size).by(1)
+      expect(messages.first.log).to eq " ===> #{data}\n"
     end
 
     it "should increment output_counter" do

data/spec/lib/worker/middlewares/docker_script_spec.rb

@@ -5,7 +5,11 @@ describe Evrone::CI::Worker::DockerScript, run_docker: true do
   let(:app)           { ->(_) { exit_code } }
   let(:script)        { "echo script" }
   let(:before_script) { "echo before_script" }
-  let(:job)           { create :job, script: script, before_script: before_script }
+  let(:after_script)  { "echo after_script" }
+  let(:job)           { create :job,
+                          script: script,
+                          before_script: before_script,
+                          after_script: after_script }
   let(:env)           { OpenStruct.new job: job }
   let(:mid)           { described_class.new app }
   let(:docker_mid)    { Evrone::CI::Worker::DockerStartContainer.new(mid) }
@@ -18,13 +22,17 @@ describe Evrone::CI::Worker::DockerScript, run_docker: true do
 
   it "should be" do
     expect(subject).to eq 0
-    expect(job.output).to match("script")
+    job.release
+    expect(all_job_log_output).to match("script")
+    expect(all_job_log_output).to match("after_script")
   end
 
   context "when script failed" do
     let(:script) { "false" }
     it "should be" do
       expect(subject).to eq(1)
+      job.release
+      expect(all_job_log_output).to match("after_script")
     end
   end
 
@@ -32,6 +40,8 @@ describe Evrone::CI::Worker::DockerScript, run_docker: true do
     let(:before_script) { "false" }
     it "should be" do
       expect(subject).to eq(-1)
+      job.release
+      expect(all_job_log_output).to match("after_script")
     end
   end
 end

data/spec/lib/worker/middlewares/local_create_dirs_spec.rb

@@ -23,20 +23,10 @@ describe Evrone::CI::Worker::LocalCreateDirs do
       subject
     end
 
-    it "work_dir" do
-      expect(env.work_dir.to_s).to eq '/tmp/.ci/work/evrone/test-repo'
-      expect(File.directory? env.work_dir).to be
-    end
-
     it "tmp_dir" do
       expect(env.tmp_dir.to_s).to eq '/tmp/.ci/tmp/evrone/test-repo'
       expect(File.directory? env.tmp_dir).to be
     end
 
-    it "repo_dir" do
-      expect(env.repo_dir.to_s).to eq '/tmp/.ci/repo/evrone/test-repo'
-      expect(File.directory? env.repo_dir).to be
-    end
-
   end
 end

data/spec/lib/worker/middlewares/local_script_spec.rb

@@ -28,7 +28,8 @@ describe Evrone::CI::Worker::LocalScript do
 
   it "should capture output" do
     subject
-    expect(job.output).to eq "before_script\nscript\n"
+    job.release
+    expect(last_job_log_message.log).to be_include("before_script\nscript\n")
   end
 
   context "when script failed" do

data/spec/support/all_job_log_output.rb

@@ -0,0 +1,3 @@
+def last_job_log_message
+  Evrone::CI::Worker::JobLogsConsumer.messages.last
+end

data/spec/support/create.rb

@@ -6,12 +6,8 @@ def create(who, *args)
 
   case who
 
-  when :local_repo
-    Evrone::CI::Worker.root.join("fixtures/repo").to_s
-
   when :message
-    name = args.shift
-
+    name  = args.shift
     klass = Evrone::CI::Message.const_get name
     klass.test_message options
 
@@ -19,15 +15,5 @@ def create(who, *args)
     message = options[:message] || create(:message, 'PerformJob', options)
     Evrone::CI::Worker::Job.new message
 
-  when :working_dirs
-
-    Evrone::CI::Worker::WorkingDirs.create args.shift, args.shift || '/tmp/.test'
-
-  when :git
-    build = args.shift
-    path  = args.shift
-    Evrone::CI::SCM::Git.new build.src, build.sha, path, deploy_key: build.deploy_key, &build.method(:add_to_output)
-
   end
-
 end

data/spec/support/last_job_logs_message.rb

@@ -0,0 +1,3 @@
+def all_job_log_output
+  Evrone::CI::Worker::JobLogsConsumer.messages.map{|i| i.log }.join("\n")
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: evrone-ci-worker
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre4
+  version: 0.2.0.pre5
 platform: ruby
 authors:
 - Dmitry Galinsky
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-05 00:00:00.000000000 Z
+date: 2013-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: evrone-ci-common
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre4
+        version: 0.2.0.pre5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre4
+        version: 0.2.0.pre5
 - !ruby/object:Gem::Dependency
   name: evrone-ci-message
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre4
+        version: 0.2.0.pre5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre4
+        version: 0.2.0.pre5
 - !ruby/object:Gem::Dependency
   name: evrone-common-amqp
   requirement: !ruby/object:Gem::Requirement
@@ -80,20 +80,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.0.22
-- !ruby/object:Gem::Dependency
-  name: net-scp
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.1.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.1.2
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -183,34 +169,28 @@ files:
 - lib/evrone/ci/worker/initializers/amqp.rb
 - lib/evrone/ci/worker/job.rb
 - lib/evrone/ci/worker/local.rb
-- lib/evrone/ci/worker/middlewares/docker_fetch_repo.rb
 - lib/evrone/ci/worker/middlewares/docker_script.rb
 - lib/evrone/ci/worker/middlewares/docker_start_container.rb
-- lib/evrone/ci/worker/middlewares/docker_webdav_cache.rb
 - lib/evrone/ci/worker/middlewares/local_create_dirs.rb
-- lib/evrone/ci/worker/middlewares/local_fetch_repo.rb
 - lib/evrone/ci/worker/middlewares/local_script.rb
 - lib/evrone/ci/worker/middlewares/log_job.rb
 - lib/evrone/ci/worker/middlewares/update_job_status.rb
-- lib/evrone/ci/worker/runner/docker.rb
-- lib/evrone/ci/worker/runner/local.rb
 - lib/evrone/ci/worker/version.rb
 - spec/lib/worker/configuration_spec.rb
 - spec/lib/worker/docker_spec.rb
 - spec/lib/worker/job_spec.rb
 - spec/lib/worker/local_spec.rb
-- spec/lib/worker/middlewares/docker_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/docker_script_spec.rb
 - spec/lib/worker/middlewares/docker_start_container_spec.rb
-- spec/lib/worker/middlewares/docker_webdav_cache_spec.rb
 - spec/lib/worker/middlewares/local_create_dirs_spec.rb
-- spec/lib/worker/middlewares/local_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/local_script_spec.rb
 - spec/lib/worker/middlewares/log_job_spec.rb
 - spec/lib/worker/middlewares/update_job_status_spec.rb
 - spec/lib/worker_spec.rb
 - spec/spec_helper.rb
+- spec/support/all_job_log_output.rb
 - spec/support/create.rb
+- spec/support/last_job_logs_message.rb
 - spec/support/shared_examples/update_job_status_message_spec.rb
 homepage: ''
 licenses:
@@ -241,16 +221,15 @@ test_files:
 - spec/lib/worker/docker_spec.rb
 - spec/lib/worker/job_spec.rb
 - spec/lib/worker/local_spec.rb
-- spec/lib/worker/middlewares/docker_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/docker_script_spec.rb
 - spec/lib/worker/middlewares/docker_start_container_spec.rb
-- spec/lib/worker/middlewares/docker_webdav_cache_spec.rb
 - spec/lib/worker/middlewares/local_create_dirs_spec.rb
-- spec/lib/worker/middlewares/local_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/local_script_spec.rb
 - spec/lib/worker/middlewares/log_job_spec.rb
 - spec/lib/worker/middlewares/update_job_status_spec.rb
 - spec/lib/worker_spec.rb
 - spec/spec_helper.rb
+- spec/support/all_job_log_output.rb
 - spec/support/create.rb
+- spec/support/last_job_logs_message.rb
 - spec/support/shared_examples/update_job_status_message_spec.rb

data/lib/evrone/ci/worker/middlewares/docker_fetch_repo.rb

@@ -1,73 +0,0 @@
-require 'docker'
-require 'evrone/common/spawn'
-require 'net/scp'
-
-module Evrone
-  module CI
-    module Worker
-
-      DockerFetchRepo = Struct.new(:app) do
-
-        include Helper::Logger
-        include Helper::Config
-
-        def call(env)
-          if env.ssh && env.docker_repo_dir
-            code = -1
-
-            code = prepare_ssh_files(env)
-            code = spawn_script(env) if code == 0
-
-            if code == 0
-              app.call env
-            else
-              code * -1
-            end
-          else
-            app.call env
-          end
-        end
-
-        private
-
-          def prepare_ssh_files(env)
-            dir = config.docker.remote_dir
-            scp = ::Net::SCP.new(env.ssh.connection)
-
-            code = env.ssh.spawn "mkdir -p #{dir}/.ssh && chmod 0700 #{dir}/.ssh", &env.job.method(:add_to_output)
-
-            if code == 0
-              config = "Host *\n"
-              config << "  StrictHostKeyChecking no\n"
-              config << "  UserKnownHostsFile /dev/null\n"
-
-              scp.upload! StringIO.new(config), "#{dir}/.ssh/config"
-              scp.upload! StringIO.new(env.job.message.deploy_key.to_s), "#{dir}/.ssh/id_rsa"
-
-              code = env.ssh.spawn "chmod 0600 #{dir}/.ssh/id_rsa", &env.job.method(:add_to_output)
-            end
-            code
-          end
-
-          def spawn_script(env)
-            scm = create_scm(env)
-
-            script = "sh -c '#{scm.make_fetch_command}'"
-            env.ssh.spawn script, &env.job.method(:add_to_output)
-          end
-
-          def create_scm(env)
-            SCM::Git.new(
-              env.job.message.src,
-              env.job.message.sha,
-              env.docker_repo_dir,
-              &env.job.method(:add_to_output)
-            )
-          end
-
-      end
-    end
-  end
-end
-
-

data/lib/evrone/ci/worker/middlewares/docker_webdav_cache.rb

@@ -1,95 +0,0 @@
-require 'net/scp'
-
-module Evrone
-  module CI
-    module Worker
-
-      DockerWebdavCache = Struct.new(:app) do
-
-        include Helper::Config
-        include Helper::Logger
-
-        CASHER_URL = "https://raw.github.com/travis-ci/casher/master/bin/casher"
-        CASHER_BIN = "$HOME/.casher/bin/casher"
-
-        def call(env)
-          if env.ssh && config.webdav_cache_url
-            assign_url(env) &&
-            prepare(env) &&
-            fetch(env) &&
-            add(env)
-          end
-
-          rs = app.call env
-
-          if env.ssh && env.webdav_cache_url
-            push(env)
-          end
-
-          rs
-        end
-
-        private
-
-          def assign_url(env)
-            release = nil
-
-            env.ssh.spawn "lsb_release -s -c" do |out|
-              release ||= out.gsub(/[^a-z0-9]/, '')
-            end
-            release = 'unknown' unless release
-
-            name = env.job.message.name.dup
-            name << "-#{release}"
-            name << "-#{env.job.message.matrix_keys.join("_")}.tgz"
-            name = name.gsub(/[^a-z0-9_\-.]/, '')
-
-            env.webdav_cache_url = "#{config.webdav_cache_url.dup}/#{name}"
-            true
-          end
-
-          def push(env)
-            cmd = %{
-              #{CASHER_BIN} push #{env.webdav_cache_url}
-            }.compact
-
-            logger.debug "push cache to #{env.webdav_cache_url.inspect}"
-            env.ssh.spawn cmd, &env.job.method(:add_trace_to_output)
-            true
-          end
-
-          def add(env)
-            cmd = %{
-              #{CASHER_BIN} add $HOME/cached
-            }.compact
-            env.ssh.spawn cmd, &env.job.method(:add_trace_to_output)
-            true
-          end
-
-          def fetch(env)
-            logger.debug "fetch cache from #{env.webdav_cache_url.inspect}"
-            cmd = %{
-              #{CASHER_BIN} fetch #{env.webdav_cache_url}
-            }.compact
-            env.ssh.spawn cmd, &env.job.method(:add_trace_to_output)
-            true
-          end
-
-          def prepare(env)
-            cmd = %{
-              export CASHER_DIR=$HOME/.casher &&
-              mkdir -p $CASHER_DIR/bin &&
-              curl #{CASHER_URL} -s -o #{CASHER_BIN} &&
-              chmod +x #{CASHER_BIN} &&
-              echo casher successfuly downloaded
-            }.compact
-            code = env.ssh.spawn cmd, &env.job.method(:add_trace_to_output)
-            code == 0
-          end
-
-      end
-    end
-  end
-end
-
-

data/lib/evrone/ci/worker/middlewares/local_fetch_repo.rb

@@ -1,47 +0,0 @@
-require 'evrone/ci/common'
-
-module Evrone
-  module CI
-    module Worker
-
-      LocalFetchRepo = Struct.new(:app) do
-
-        include Common::Helper::Shell
-
-        def call(env)
-          scm = create_scm(env)
-
-          if env.repo_dir
-            if scm.fetch == 0 && export(scm, env) == 0
-              app.call env
-            else
-              -1
-            end
-          else
-            app.call env
-          end
-        end
-
-        private
-
-          def export(scm, env)
-            cmd = scm.class.make_export_command env.repo_dir, env.work_dir
-            env.job.add_command_to_output cmd
-            bash cmd, &env.job.method(:add_to_output)
-          end
-
-          def create_scm(env)
-            SCM::Git.new(
-              env.job.message.src,
-              env.job.message.sha,
-              env.repo_dir,
-              deploy_key: env.job.message.deploy_key,
-              &env.job.method(:add_to_output)
-            )
-          end
-
-      end
-
-    end
-  end
-end

data/lib/evrone/ci/worker/runner/local.rb

@@ -1,52 +0,0 @@
-module Evrone
-  module CI
-    module Worker
-      class Runner
-        class Local
-
-          include Worker::Helper::Logger
-          include Worker::Helper::Config
-          include Common::Helper::Shell
-
-          attr_reader :build, :path, :build_script
-
-          def initialize(build, path)
-            @build        = build
-            @path         = path
-            @build_script = path.tmp_dir.join(".build.sh").expand_path
-          end
-
-          def perform
-            make_build_script
-            run_build_script
-          end
-
-          private
-
-            def run_build_script
-              bash file: build_script, chdir: path.work_dir, &method(:add_output)
-            end
-
-            def make_build_script
-              content = build.travis.to_queue.to_shell_script script_replaces do |env|
-                prepare_env env
-              end
-              write_file build_script, content, 0755
-              add_output_command build_script
-            end
-
-            def script_replaces
-              { "SHARED_PATH" => path.shared_dir }
-            end
-
-            def prepare_env(env)
-              env['init'].unshift "echo 'copy repo from #{path.repo_dir} to #{path.work_dir}'"
-              env['init'].unshift build.csm.make_export_command path.repo_dir, path.work_dir
-              env
-            end
-        end
-      end
-    end
-  end
-end
-

data/spec/lib/worker/middlewares/docker_fetch_repo_spec.rb

@@ -1,26 +0,0 @@
-require 'spec_helper'
-
-describe Evrone::CI::Worker::DockerFetchRepo, run_docker: true do
-  let(:exit_code)   { 0 }
-  let(:app)         { ->(_) { exit_code } }
-  let(:options)     { {} }
-  let(:job)         { create :job, options }
-  let(:env)         { OpenStruct.new job: job }
-  let(:mid)         { described_class.new app }
-  let(:docker_mid)  { Evrone::CI::Worker::DockerStartContainer.new(mid) }
-
-  subject { docker_mid.call env }
-
-  it "should be" do
-    expect(subject).to eq 0
-  end
-
-  context "when fail to fetch repo" do
-    let(:options) { { src: "/not-exists-repo.git" } }
-    it "should be" do
-      expect(subject).to eq(-128)
-      expect(job.output).to be_include("does not exist")
-    end
-  end
-end
-

data/spec/lib/worker/middlewares/docker_webdav_cache_spec.rb

@@ -1,49 +0,0 @@
-require 'spec_helper'
-
-describe Evrone::CI::Worker::DockerWebdavCache, run_docker: true, webdav: true do
-  let(:exit_code)     { 0 }
-  let(:app)           {
-    ->(env) {
-      file = "$HOME/cached/foo.txt"
-      cmd = %{
-        if [ -f #{file} ] ; then
-          echo "!exists" ;
-          cat #{file} ;
-        else
-          echo "!touch" ;
-          echo 1 > #{file} ;
-        fi
-      }.compact
-      env.ssh.spawn cmd, &job.method(:add_to_output)
-    }
-  }
-  let(:job)           { create :job }
-  let(:env)           { OpenStruct.new job: job }
-  let(:mid)           { described_class.new app }
-  let(:docker_mid)    { Evrone::CI::Worker::DockerStartContainer.new(mid) }
-
-  subject { docker_mid.call env }
-
-  before do
-    Evrone::CI::Worker.config.webdav_cache_url = 'http://192.168.44.1:8080/'
-  end
-
-  context "first run" do
-    it "should be" do
-      expect(subject).to eq 0
-    end
-  end
-
-  context "next run" do
-    before do
-      expect(docker_mid.call env).to eq 0
-      job.output.clear
-    end
-    it "should be" do
-      expect(subject).to eq 0
-      expect(job.output).to match(/\!exists/)
-    end
-
-  end
-end
-

data/spec/lib/worker/middlewares/local_fetch_repo_spec.rb

@@ -1,56 +0,0 @@
-require 'spec_helper'
-require 'pathname'
-require 'fileutils'
-
-describe Evrone::CI::Worker::LocalFetchRepo do
-  let(:exit_code)   { 0 }
-  let(:app)         { ->(_) { exit_code } }
-  let(:job)         { create :job }
-  let(:path_prefix) { Pathname.new("/tmp/.ci") }
-  let(:repo_dir)    { path_prefix.join("repo") }
-  let(:work_dir)    { path_prefix.join("work") }
-  let(:env)         { OpenStruct.new job: job, repo_dir: repo_dir, work_dir: work_dir  }
-  let(:mid)         { described_class.new app }
-
-  subject { mid.call env }
-
-  after do
-    FileUtils.rm_rf path_prefix
-  end
-
-  it { should eq 0 }
-
-  it "should create repo inside repo_dir" do
-    subject
-    expect(File.directory? "/tmp/.ci/repo/.git" ).to be
-  end
-
-  it "should create copy of repo inside work_dir" do
-    subject
-    expect(File.readable? "/tmp/.ci/work/Gemfile").to be
-  end
-
-  context "when fetch failed" do
-    before do
-      any_instance_of(Evrone::CI::SCM::Git) do |git|
-        mock(git).fetch { 1 }
-      end
-    end
-
-    it { should eq(-1) }
-  end
-
-  context "when export failed" do
-    before do
-      any_instance_of(Evrone::CI::SCM::Git) do |git|
-        mock(git).fetch { 0 }
-      end
-      any_instance_of(described_class) do |m|
-        mock(m).export(anything, anything) { 1 }
-      end
-    end
-
-    it { should eq(-1) }
-  end
-end
-