evrone-ci-worker 0.2.0.pre1 → 0.2.0.pre3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58f2ff7f63d54397ceceb915a1beead7603abb5d
-  data.tar.gz: 3bca9c6de4a0b62ce144e5c2e3356bb66ab88135
+  metadata.gz: c5891371d77f52eeae58e9692fcd97e024584903
+  data.tar.gz: 45e7c0f16129b0b79df1e076b11e638d7608cc40
 SHA512:
-  metadata.gz: b23970e35db6cb112cbda92597ee903444a6ef54e282244d283532f8aaf64e736db833175836797f0f3eac39d9a3c87b2ecda31b2010cf0be6d0be9a8c542a2c
-  data.tar.gz: 7f725fc9fbeca95f22d7d1f48e99b6b1debb405576e05f72f3117d8d8fa32615558c96f3d61e04a4e03822edfe3f90021eec96d2b1c1f235d37add55d32f1fa5
+  metadata.gz: a3331e43272c6b1bfe6d8d5e778f5c097a168a393cf65959a0610ad04b94550b96bbac8f03a10ce80fc5555870bd52a41fc2a19a3d69a8c59a07ee834434856a
+  data.tar.gz: 248eaaad99449fffe60e1c545ede8383f4d51cc4ebe89eb7b059baa7ad32c965daf6a5354c2a894f66dbd8c07797d2b22589e16713a58dca459b130d5479e435

data/Rakefile

@@ -6,9 +6,11 @@ require "bundler/gem_tasks"
 
 RSpec::Core::RakeTask.new(:spec)
 
+desc "download test repo and run spec"
 task :default => ["test:create_repo", :spec]
 
 namespace :test do
+  desc "download test repo"
   task :create_repo do
     dir = "fixtures/repo"
     unless File.directory? dir
@@ -19,6 +21,7 @@ namespace :test do
   end
 end
 
+desc "run travis build"
 task :travis do
-  exec "bundle exec rake SPEC_OPTS='--format documentation -t ~run_docker --order=rand'"
+  exec "bundle exec rake SPEC_OPTS='--format documentation -t ~run_docker -t ~webdav --order=rand'"
 end

data/bin/evrone-ci-worker

@@ -4,14 +4,15 @@ require File.expand_path("../../lib/evrone/ci/worker", __FILE__)
 
 cli = Evrone::CI::Worker::CLI.new
 
-=begin
-Evrone::CI::Worker.configure do |c|
-  c.docker.ssh.port = 2223
-  c.docker.ssh.host = 'localhost'
-  c.docker.create_options = {
-    'PortSpecs' => ['2022:22']
-  }
+if RUBY_PLATFORM =~ /darwin/ # developer
+  Evrone::CI::Worker.configure do |c|
+    c.webdav_cache_url = 'http://10.0.2.15:8080'
+    c.docker.ssh.port = 2223
+    c.docker.ssh.host = 'localhost'
+    c.docker.create_options = {
+      'PortSpecs' => ['2022:22']
+    }
+  end
 end
-=end
 
 cli.run

data/docker/Dockerfile

@@ -6,56 +6,12 @@ ENV DEBIAN_FRONTEND noninteractive
 # disable service autostarting
 ENV RUNLEVEL 1
 
-# replace init
-RUN  dpkg-divert --local --rename --add /sbin/initctl
-RUN  ln -s /bin/true /sbin/initctl
-
-RUN apt-get -qy update
-RUN apt-get install -qy curl git-core build-essential
-
-# add evrone repo
-RUN mkdir -p /etc/apt/sources.list.d
-RUN echo "deb  http://download.opensuse.org/repositories/home:/dmexe/xUbuntu_12.04/  ./" > /etc/apt/sources.list.d/Evrone.list
-RUN curl http://download.opensuse.org/repositories/home:/dmexe/xUbuntu_12.04/Release.key | apt-key add -
-RUN apt-get -qy update
-
-# install runit
-RUN chmod 0000 /sbin/start
-RUN apt-get -qy install runit
-RUN chmod 0755 /sbin/start
+# add runit helpers
 ADD ./sv-gen /usr/local/bin/sv-gen
 ADD ./sv-enable /usr/local/bin/sv-enable
 
-# set user password
-RUN useradd -m ci
-RUN echo "ci:ci" | chpasswd
-RUN apt-get -qy install sudo
-RUN echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-
-# fixes locales
-RUN locale-gen en_US.UTF-8
-RUN dpkg-reconfigure -fnoninteractive locales
-RUN update-locale LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US"
-
-# install ssh
-RUN apt-get install -qy openssh-server
-RUN mkdir -p /var/run/sshd
-RUN sv-gen ssh "/usr/sbin/sshd -D -e"
-
-# install postgresql
-RUN apt-get install -qy postgresql
-RUN echo "host all all all trust" > /etc/postgresql/9.1/main/pg_hba.conf
-RUN sv-gen postgresql "/usr/lib/postgresql/9.1/bin/postgres -D /var/lib/postgresql/9.1/main -c config_file=/etc/postgresql/9.1/main/postgresql.conf" postgres
-
-# add ruby
-RUN apt-get install -qy rbenv rbenv-2.0.0-p195 rbenv-1.9.3-p448
-RUN env RBENV_VERSION=2.0.0-p195 rbenv exec gem install bundler --no-ri --no-rdoc
-RUN env RBENV_VERSION=1.9.3-p448 rbenv exec gem install bundler --no-ri --no-rdoc
-RUN rbenv rehash
-
-RUN apt-get -qy clean autoremove
-
-RUN sv-enable postgresql
-RUN sv-enable ssh
+# run bootstrap
+ADD ./bootstrap.sh /tmp/bootstrap.sh
+RUN /tmp/bootstrap.sh && rm /tmp/bootstrap.sh
 
 ENTRYPOINT ["/usr/bin/runsvdir", "-P", "/etc/service"]

data/docker/bootstrap.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -x
+set -e
+
+# add ubuntu repos
+echo "deb http://us.archive.ubuntu.com/ubuntu/ precise universe" >> /etc/apt/sources.list
+echo "deb http://us.archive.ubuntu.com/ubuntu/ precise multiverse" >> /etc/apt/sources.list
+
+# replace init
+ dpkg-divert --local --rename --add /sbin/initctl
+ ln -s /bin/true /sbin/initctl
+
+apt-get -qy update
+apt-get install -qy curl git-core build-essential
+
+# add evrone repo
+mkdir -p /etc/apt/sources.list.d
+echo "deb  http://download.opensuse.org/repositories/home:/dmexe/xUbuntu_12.04/  ./" > /etc/apt/sources.list.d/Evrone.list
+curl http://download.opensuse.org/repositories/home:/dmexe/xUbuntu_12.04/Release.key | apt-key add -
+apt-get -qy update
+
+# add packages
+apt-get install -qy libmysqlclient-dev libpq-dev imagemagick libmagickwand-dev \
+  libcurl4-openssl-dev nodejs postgresql-contrib-9.1 libxml2-dev libxslt-dev \
+  openjdk-7-jre-headless libsqlite3-dev libgeos-dev ruby
+
+# install runit
+chmod 0000 /sbin/start
+apt-get -qy install runit
+chmod 0755 /sbin/start
+
+# set user password
+useradd -m ci
+echo "ci:ci" | chpasswd
+apt-get -qy install sudo
+echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+# fixes locales
+locale-gen en_US.UTF-8
+dpkg-reconfigure -fnoninteractive locales
+update-locale LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US"
+
+# install ssh
+apt-get install -qy openssh-server
+mkdir -p /var/run/sshd
+sv-gen ssh "/usr/sbin/sshd -D -e"
+
+# install postgresql
+apt-get install -qy postgresql
+echo "host all all all trust" > /etc/postgresql/9.1/main/pg_hba.conf
+sv-gen postgresql "/usr/lib/postgresql/9.1/bin/postgres -D /var/lib/postgresql/9.1/main -c config_file=/etc/postgresql/9.1/main/postgresql.conf" postgres
+
+# install mysql
+apt-get install -qy mysql-server
+sv-gen mysqld "/usr/sbin/mysqld" mysql
+
+# add ruby
+apt-get install -qy rbenv rbenv-2.0.0-p195 rbenv-1.9.3-p448
+env RBENV_VERSION=2.0.0-p195 rbenv exec gem install bundler --no-ri --no-rdoc
+env RBENV_VERSION=1.9.3-p448 rbenv exec gem install bundler --no-ri --no-rdoc
+rbenv rehash
+
+apt-get -qy clean autoremove
+
+sv-enable postgresql
+sv-enable ssh

data/docker/sv-gen

@@ -31,7 +31,7 @@ set -e
 exec 2>&1
 cd ${DIR}
 CMD="${CMD}"
-exec chpst -u ${USER} \$CMD 2>&1
+exec sudo -u ${USER} \$CMD 2>&1
 EOF
 chmod +x ${RUN_FILE}
 

data/lib/evrone/ci/worker.rb

@@ -31,6 +31,7 @@ module Evrone
       autoload :DockerStartContainer,  File.expand_path("../worker/middlewares/docker_start_container",  __FILE__)
       autoload :DockerFetchRepo,       File.expand_path("../worker/middlewares/docker_fetch_repo",       __FILE__)
       autoload :DockerScript,          File.expand_path("../worker/middlewares/docker_script",           __FILE__)
+      autoload :DockerWebdavCache,     File.expand_path("../worker/middlewares/docker_webdav_cache",     __FILE__)
 
       module Helper
         autoload :Logger,              File.expand_path("../worker/helper/logger",                       __FILE__)

data/lib/evrone/ci/worker/configuration.rb

@@ -27,6 +27,7 @@ module Evrone
 
                workers:         1,
                path_prefix:     nil,
+               webdav_cache_url:nil,
 
                docker: {
                  remote_dir: "/home/ci",

data/lib/evrone/ci/worker/docker.rb

@@ -15,6 +15,7 @@ module Evrone
           use UpdateJobStatus
           use DockerStartContainer
           use DockerFetchRepo
+          use DockerWebdavCache
           use DockerScript
         end
 

data/lib/evrone/ci/worker/job.rb

@@ -8,11 +8,12 @@ module Evrone
 
         include Helper::Logger
 
-        attr_reader :output, :message
+        attr_reader :output, :message, :output_counter
 
         def initialize(perform_job_message)
-          @output  = ""
-          @message = perform_job_message
+          @output_counter = 0
+          @output         = ""
+          @message        = perform_job_message
         end
 
         def add_to_output(str)
@@ -25,13 +26,16 @@ module Evrone
           add_to_output "$ #{cmd}\n"
         end
 
+        def add_trace_to_output(log)
+          add_to_output log.split(/\n/).map{|i| " ===> #{i}\n" }.join
+        end
+
         def create_job_log_message(data)
-          tm = Time.now
+          @output_counter += 1
           Message::JobLog.new(
             build_id: message.id,
             job_id:   message.job_id,
-            tm:       tm.to_i,
-            tm_usec:  tm.usec,
+            tm:       @output_counter,
             log:      data
           )
         end

data/lib/evrone/ci/worker/middlewares/docker_fetch_repo.rb

@@ -21,7 +21,7 @@ module Evrone
             if code == 0
               app.call env
             else
-              -1
+              code * -1
             end
           else
             app.call env
@@ -52,9 +52,6 @@ module Evrone
           def spawn_script(env)
             scm = create_scm(env)
 
-            env.job.add_command_to_output("git clone #{scm.src} #{scm.path}")
-            env.job.add_command_to_output("git checkout -q #{scm.sha}")
-
             script = "sh -c '#{scm.make_fetch_command}'"
             env.ssh.spawn script, &env.job.method(:add_to_output)
           end

data/lib/evrone/ci/worker/middlewares/docker_start_container.rb

@@ -34,7 +34,8 @@ module Evrone
               paranoid:      false,
               forward_agent: false
             }
-            env.job.add_to_output "open ssh session to #{user}@#{host}:#{ssh_options[:port]}"
+            sleep 2
+            env.job.add_to_output "open ssh session to #{user}@#{host}:#{ssh_options[:port]}\n"
             open_ssh(host, "ci", ssh_options) do |ssh|
               begin
                 env.ssh = ssh
@@ -49,7 +50,7 @@ module Evrone
             env.container = ::Docker::Container.create create_options
             env.container.start
 
-            env.job.add_to_output "using #{Socket.gethostname}/#{env.container.id}"
+            env.job.add_to_output "using #{Socket.gethostname}/#{env.container.id}\n"
 
             env.container_addr = config.docker.ssh.host
             env.container_addr ||= env.container.json['NetworkSettings']['IPAddress']

data/lib/evrone/ci/worker/middlewares/docker_webdav_cache.rb

@@ -0,0 +1,95 @@
+require 'net/scp'
+
+module Evrone
+  module CI
+    module Worker
+
+      DockerWebdavCache = Struct.new(:app) do
+
+        include Helper::Config
+        include Helper::Logger
+
+        CASHER_URL = "https://raw.github.com/travis-ci/casher/master/bin/casher"
+        CASHER_BIN = "$HOME/.casher/bin/casher"
+
+        def call(env)
+          if env.ssh && config.webdav_cache_url
+            assign_url(env) &&
+            prepare(env) &&
+            fetch(env) &&
+            add(env)
+          end
+
+          rs = app.call env
+
+          if env.ssh && env.webdav_cache_url
+            push(env)
+          end
+
+          rs
+        end
+
+        private
+
+          def assign_url(env)
+            release = nil
+
+            env.ssh.spawn "lsb_release -s -c" do |out|
+              release ||= out.gsub(/[^a-z0-9]/, '')
+            end
+            release = 'unknown' unless release
+
+            name = env.job.message.name.dup
+            name << "-#{release}"
+            name << "-#{env.job.message.matrix_keys.join("_")}.tgz"
+            name = name.gsub(/[^a-z0-9_\-.]/, '')
+
+            env.webdav_cache_url = "#{config.webdav_cache_url.dup}/#{name}"
+            true
+          end
+
+          def push(env)
+            cmd = %{
+              #{CASHER_BIN} push #{env.webdav_cache_url}
+            }.compact
+
+            logger.debug "push cache to #{env.webdav_cache_url.inspect}"
+            env.ssh.spawn cmd, &env.job.method(:add_trace_to_output)
+            true
+          end
+
+          def add(env)
+            cmd = %{
+              #{CASHER_BIN} add $HOME/cached
+            }.compact
+            env.ssh.spawn cmd, &env.job.method(:add_trace_to_output)
+            true
+          end
+
+          def fetch(env)
+            logger.debug "fetch cache from #{env.webdav_cache_url.inspect}"
+            cmd = %{
+              #{CASHER_BIN} fetch #{env.webdav_cache_url}
+            }.compact
+            env.ssh.spawn cmd, &env.job.method(:add_trace_to_output)
+            true
+          end
+
+          def prepare(env)
+            cmd = %{
+              export CASHER_DIR=$HOME/.casher &&
+              mkdir -p $CASHER_DIR/bin &&
+              curl #{CASHER_URL} -s -o #{CASHER_BIN} &&
+              chmod +x #{CASHER_BIN} &&
+              echo casher successfuly downloaded
+            }.compact
+            code = env.ssh.spawn cmd, &env.job.method(:add_trace_to_output)
+            code == 0
+          end
+
+      end
+    end
+  end
+end
+
+

data/lib/evrone/ci/worker/middlewares/update_job_status.rb

@@ -20,9 +20,13 @@ module Evrone
           begin
             rs = app.call env
           rescue Exception => e
+            env.job.add_to_output("ERROR: #{e.inspect}\n")
             logger.error("ERROR: #{e.inspect}\n    BACKTRACE:\n#{e.backtrace.map{|i| "    #{i}" }.join("\n")}")
           end
 
+          msg = "\nDone. Your build exited with %s.\n"
+          env.job.add_to_output(msg % rs.abs)
+
           case
           when rs == 0
             update_status env.job, FINISHED

data/lib/evrone/ci/worker/version.rb

@@ -1,7 +1,7 @@
 module Evrone
   module CI
     module Worker
-      VERSION = "0.2.0.pre1"
+      VERSION = "0.2.0.pre3"
     end
   end
 end

data/spec/lib/worker/job_spec.rb

@@ -7,23 +7,26 @@ describe Evrone::CI::Worker::Job do
   subject { job }
 
   context "just created" do
-    its(:message)       { should eq message }
+    its(:message)        { should eq message }
+    its(:output)         { should eq '' }
+    its(:output_counter) { should eq 0 }
   end
 
   context "create_job_log_message" do
-    let(:tm)   { Time.new(2012, 12, 10, 15, 45) }
     let(:data) { 'log' }
     subject { job.create_job_log_message data }
 
-    before do
-      mock(Time).now { tm }
-    end
-
     it { should be_an_instance_of(Evrone::CI::Message::JobLog) }
     its(:job_id)   { should eq job.message.job_id }
     its(:build_id) { should eq job.message.id     }
-    its(:tm)       { should eq tm.to_i }
+    its(:tm)       { should eq 1 }
     its(:log)      { should eq data }
+
+    it "should increment counter" do
+      expect {
+        subject
+      }.to change(job, :output_counter).by(1)
+    end
   end
 
   context "add_to_output" do
@@ -37,6 +40,13 @@ describe Evrone::CI::Worker::Job do
         subject
       }.to change(messages, :size).by(1)
     end
+
+    it "should increment output_counter" do
+      expect {
+        subject
+      }.to change(job, :output_counter).by(1)
+      expect(messages.first.tm).to eq 1
+    end
   end
 
   context "add_command_to_output" do
@@ -45,10 +55,40 @@ describe Evrone::CI::Worker::Job do
     subject { job.add_command_to_output(data) ; job }
 
     its(:output) { should eq "$ #{data}\n" }
+
     it "should delivery message" do
       expect {
         subject
       }.to change(messages, :size).by(1)
     end
+
+    it "should increment output_counter" do
+      expect {
+        subject
+      }.to change(job, :output_counter).by(1)
+      expect(messages.first.tm).to eq 1
+    end
   end
+
+  context "add_trace_to_output" do
+    let(:data)     { 'data' }
+    let(:messages) { Evrone::CI::Worker::JobLogsConsumer.messages }
+    subject { job.add_trace_to_output(data) ; job }
+
+    its(:output) { should eq " ===> #{data}\n" }
+
+    it "should delivery message" do
+      expect {
+        subject
+      }.to change(messages, :size).by(1)
+    end
+
+    it "should increment output_counter" do
+      expect {
+        subject
+      }.to change(job, :output_counter).by(1)
+      expect(messages.first.tm).to eq 1
+    end
+  end
+
 end

data/spec/lib/worker/middlewares/docker_fetch_repo_spec.rb

@@ -18,7 +18,7 @@ describe Evrone::CI::Worker::DockerFetchRepo, run_docker: true do
   context "when fail to fetch repo" do
     let(:options) { { src: "/not-exists-repo.git" } }
     it "should be" do
-      expect(subject).to eq(-1)
+      expect(subject).to eq(-128)
       expect(job.output).to be_include("does not exist")
     end
   end

data/spec/lib/worker/middlewares/docker_webdav_cache_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe Evrone::CI::Worker::DockerWebdavCache, run_docker: true, webdav: true do
+  let(:exit_code)     { 0 }
+  let(:app)           {
+    ->(env) {
+      file = "$HOME/cached/foo.txt"
+      cmd = %{
+        if [ -f #{file} ] ; then
+          echo "!exists" ;
+          cat #{file} ;
+        else
+          echo "!touch" ;
+          echo 1 > #{file} ;
+        fi
+      }.compact
+      env.ssh.spawn cmd, &job.method(:add_to_output)
+    }
+  }
+  let(:job)           { create :job }
+  let(:env)           { OpenStruct.new job: job }
+  let(:mid)           { described_class.new app }
+  let(:docker_mid)    { Evrone::CI::Worker::DockerStartContainer.new(mid) }
+
+  subject { docker_mid.call env }
+
+  before do
+    Evrone::CI::Worker.config.webdav_cache_url = 'http://192.168.44.1:8080/'
+  end
+
+  context "first run" do
+    it "should be" do
+      expect(subject).to eq 0
+    end
+  end
+
+  context "next run" do
+    before do
+      expect(docker_mid.call env).to eq 0
+      job.output.clear
+    end
+    it "should be" do
+      expect(subject).to eq 0
+      expect(job.output).to match(/\!exists/)
+    end
+
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: evrone-ci-worker
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre1
+  version: 0.2.0.pre3
 platform: ruby
 authors:
 - Dmitry Galinsky
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-03 00:00:00.000000000 Z
+date: 2013-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: evrone-ci-common
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre1
+        version: 0.2.0.pre3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre1
+        version: 0.2.0.pre3
 - !ruby/object:Gem::Dependency
   name: evrone-ci-message
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre1
+        version: 0.2.0.pre3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre1
+        version: 0.2.0.pre3
 - !ruby/object:Gem::Dependency
   name: evrone-common-amqp
   requirement: !ruby/object:Gem::Requirement
@@ -166,6 +166,7 @@ files:
 - Rakefile
 - bin/evrone-ci-worker
 - docker/Dockerfile
+- docker/bootstrap.sh
 - docker/sv-enable
 - docker/sv-gen
 - evrone-ci-worker.gemspec
@@ -185,6 +186,7 @@ files:
 - lib/evrone/ci/worker/middlewares/docker_fetch_repo.rb
 - lib/evrone/ci/worker/middlewares/docker_script.rb
 - lib/evrone/ci/worker/middlewares/docker_start_container.rb
+- lib/evrone/ci/worker/middlewares/docker_webdav_cache.rb
 - lib/evrone/ci/worker/middlewares/local_create_dirs.rb
 - lib/evrone/ci/worker/middlewares/local_fetch_repo.rb
 - lib/evrone/ci/worker/middlewares/local_script.rb
@@ -200,6 +202,7 @@ files:
 - spec/lib/worker/middlewares/docker_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/docker_script_spec.rb
 - spec/lib/worker/middlewares/docker_start_container_spec.rb
+- spec/lib/worker/middlewares/docker_webdav_cache_spec.rb
 - spec/lib/worker/middlewares/local_create_dirs_spec.rb
 - spec/lib/worker/middlewares/local_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/local_script_spec.rb
@@ -241,6 +244,7 @@ test_files:
 - spec/lib/worker/middlewares/docker_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/docker_script_spec.rb
 - spec/lib/worker/middlewares/docker_start_container_spec.rb
+- spec/lib/worker/middlewares/docker_webdav_cache_spec.rb
 - spec/lib/worker/middlewares/local_create_dirs_spec.rb
 - spec/lib/worker/middlewares/local_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/local_script_spec.rb