evrone-ci-worker 0.2.0.pre0 → 0.2.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7b9bb1d659ac857b4f5fdb4bf715a9695360997
-  data.tar.gz: 07a0fd1df69fa7374d2ff7b80bddfbd6a685983f
+  metadata.gz: 58f2ff7f63d54397ceceb915a1beead7603abb5d
+  data.tar.gz: 3bca9c6de4a0b62ce144e5c2e3356bb66ab88135
 SHA512:
-  metadata.gz: 30ee6822495f8aebe134334a77477e98e64041b821a89cf4922e7fdee91cb53eae27c2e380ef515f4c1823722f40dd740f18992eaaec7c8345d5bbd41ccee9b3
-  data.tar.gz: bf174a1ebc89b1b1a3e268553cb1c9e185376cc78697086ae361d5573cf5c9f356c27eac5108226ea548e2a0aad57a75ab911fb9869ac0ebe87bf9e8c1a9c74f
+  metadata.gz: b23970e35db6cb112cbda92597ee903444a6ef54e282244d283532f8aaf64e736db833175836797f0f3eac39d9a3c87b2ecda31b2010cf0be6d0be9a8c542a2c
+  data.tar.gz: 7f725fc9fbeca95f22d7d1f48e99b6b1debb405576e05f72f3117d8d8fa32615558c96f3d61e04a4e03822edfe3f90021eec96d2b1c1f235d37add55d32f1fa5

data/bin/evrone-ci-worker

@@ -2,12 +2,9 @@
 
 require File.expand_path("../../lib/evrone/ci/worker", __FILE__)
 
-trap('INT') {
-  Thread.new do
-    Evrone::Common::AMQP.shutdown
-  end.join
-}
+cli = Evrone::CI::Worker::CLI.new
 
+=begin
 Evrone::CI::Worker.configure do |c|
   c.docker.ssh.port = 2223
   c.docker.ssh.host = 'localhost'
@@ -15,8 +12,6 @@ Evrone::CI::Worker.configure do |c|
     'PortSpecs' => ['2022:22']
   }
 end
+=end
 
-Evrone::Common::AMQP::Supervisor::Threaded.build(
-  Evrone::CI::Worker::JobsConsumer => 4,
-).run
-
+cli.run

data/lib/evrone/ci/worker.rb

@@ -1,5 +1,7 @@
 require 'rubygems'
 require 'pathname'
+require 'thread'
+
 require 'bundler'
 Bundler.require :default
 
@@ -17,18 +19,17 @@ module Evrone
       autoload :Job,                   File.expand_path("../worker/job",                                 __FILE__)
       autoload :Local,                 File.expand_path("../worker/local",                               __FILE__)
       autoload :Docker,                File.expand_path("../worker/docker",                              __FILE__)
+      autoload :CLI,                   File.expand_path("../worker/cli",                                 __FILE__)
 
       autoload :LogJob,                File.expand_path("../worker/middlewares/log_job",                 __FILE__)
       autoload :UpdateJobStatus,       File.expand_path("../worker/middlewares/update_job_status",       __FILE__)
 
       autoload :LocalCreateDirs,       File.expand_path("../worker/middlewares/local_create_dirs",       __FILE__)
       autoload :LocalFetchRepo,        File.expand_path("../worker/middlewares/local_fetch_repo",        __FILE__)
-      autoload :LocalBeforeScript,     File.expand_path("../worker/middlewares/local_before_script",     __FILE__)
       autoload :LocalScript,           File.expand_path("../worker/middlewares/local_script",            __FILE__)
 
       autoload :DockerStartContainer,  File.expand_path("../worker/middlewares/docker_start_container",  __FILE__)
       autoload :DockerFetchRepo,       File.expand_path("../worker/middlewares/docker_fetch_repo",       __FILE__)
-      autoload :DockerBeforeScript,    File.expand_path("../worker/middlewares/docker_before_script",    __FILE__)
       autoload :DockerScript,          File.expand_path("../worker/middlewares/docker_script",           __FILE__)
 
       module Helper
@@ -39,7 +40,7 @@ module Evrone
       extend self
 
       @@root   = Pathname.new File.expand_path('../../../..', __FILE__)
-      @@config = Configuration.new
+      @@config_mutex = Mutex.new
 
       def logger
         if ENV['CI_WORKER_SILENT']
@@ -55,7 +56,11 @@ module Evrone
       end
 
       def config
-        @@config
+        @config ||= begin
+          @@config_mutex.synchronize do
+            Configuration.new
+          end
+        end
       end
 
       def root
@@ -71,14 +76,17 @@ module Evrone
       end
 
       def reset_config!
-        @@config = Configuration.new
+        @config = nil
+      end
+
+      def initialize!
+        root.join("lib/evrone/ci/worker/initializers").children.each do |e|
+          require e
+        end
       end
 
     end
   end
 end
 
-Evrone::CI::Worker.root.join("lib/evrone/ci/worker/initializers").children.each do |e|
-  require e
-end
 

data/lib/evrone/ci/worker/cli.rb

@@ -0,0 +1,67 @@
+require 'optparse'
+require 'evrone/common/amqp'
+
+module Evrone
+  module CI
+    module Worker
+      class CLI
+
+        include Helper::Config
+        include Helper::Logger
+
+        def initialize
+          @options = {}
+          parse!
+          Worker.initialize!
+        end
+
+        def run
+          logger.warn "spawn inside #{config.path_prefix}"
+
+          trap('INT') {
+            Thread.new do
+              Evrone::Common::AMQP.shutdown
+            end.join
+          }
+
+          Evrone::Common::AMQP::Supervisor::Threaded.build(
+            Evrone::CI::Worker::JobsConsumer => config.workers,
+          ).run
+        end
+
+        private
+
+          def parse!
+            OptionParser.new do |opts|
+              opts.banner = "Usage: evrone-ci-worker [options]"
+              opts.on("-w", "--workers NUM", "Number of workers, default 1") do |v|
+                @options[:workers] = v.to_i
+              end
+              opts.on("-p", "--path PATH", "Working directory, default current directory") do |v|
+                @options[:path_prefix] = v.to_s
+              end
+              opts.on("-c", "--config FILE", "Path to configuration file") do |v|
+                read_configuration v
+              end
+            end.parse!
+
+            @options.each_pair do |k,v|
+              config.public_send("#{k}=", v)
+            end
+          end
+
+          def read_configuration(file)
+            file = File.expand_path(file)
+            buf  = File.read(file)
+
+            buf.split("\n").each do |line|
+              puts line
+              env, value = line.split("=").map(&:strip)
+              ENV[env] = value
+            end
+          end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/worker/configuration.rb

@@ -25,6 +25,9 @@ module Evrone
 
                sleep_after_job: 5,
 
+               workers:         1,
+               path_prefix:     nil,
+
                docker: {
                  remote_dir: "/home/ci",
                  init:       %w{ /usr/local/bin/runsvdir -P /etc/service },
@@ -48,6 +51,10 @@ module Evrone
           self[:run].to_sym
         end
 
+        def path_prefix
+          self[:path_prefix] || Dir.pwd
+        end
+
         def null_logger
           @@null_logger
         end

data/lib/evrone/ci/worker/docker.rb

@@ -15,7 +15,6 @@ module Evrone
           use UpdateJobStatus
           use DockerStartContainer
           use DockerFetchRepo
-          use DockerBeforeScript
           use DockerScript
         end
 

data/lib/evrone/ci/worker/initializers/amqp.rb

@@ -50,4 +50,5 @@ Evrone::Common::AMQP.configure do |c|
 
   c.content_type = 'application/x-protobuf'
   c.logger = nil
+  c.url    = Evrone::CI::Worker.config.amqp_url
 end

data/lib/evrone/ci/worker/local.rb

@@ -16,7 +16,6 @@ module Evrone
           use UpdateJobStatus
           use LocalCreateDirs
           use LocalFetchRepo
-          use LocalBeforeScript
           use LocalScript
         end
 

data/lib/evrone/ci/worker/middlewares/docker_script.rb

@@ -14,7 +14,12 @@ module Evrone
             if code == 0
               app.call env
             else
-              code
+              case read_state(env)
+              when "script"
+                code
+              else
+                code * -1
+              end
             end
           else
             app.call env
@@ -25,10 +30,27 @@ module Evrone
 
           def run_script(env)
             scp = ::Net::SCP.new(env.ssh.connection)
-            script = [env.docker_repo_dir, ".ci_script.sh"].join("/")
-            puts env.job.message.script.inspect
-            scp.upload! StringIO.new(env.job.message.script), script
-            env.ssh.spawn "env - bash #{script}", chdir: env.docker_repo_dir, &env.job.method(:add_to_output)
+            script = [env.docker_repo_dir, ".ci_build.sh"].join("/")
+            scp.upload! StringIO.new(content(env)), script
+            env.ssh.spawn "env - HOME=$HOME bash #{script}", chdir: env.docker_repo_dir, &env.job.method(:add_to_output)
+          end
+
+          def content(env)
+            buf = ["set -e"]
+            buf << "echo before_script > #{env.docker_repo_dir}/.ci_state"
+            buf << env.job.message.before_script
+            buf << "echo script > #{env.docker_repo_dir}/.ci_state"
+            buf << env.job.message.script
+            buf.join("\n")
+          end
+
+          def read_state(env)
+            buf = ""
+            state_file = env.docker_repo_dir.join(".ci_state")
+            env.ssh.spawn "cat #{state_file}" do |out|
+              buf << out
+            end
+            buf.strip
           end
 
       end

data/lib/evrone/ci/worker/middlewares/docker_start_container.rb

@@ -1,6 +1,7 @@
 require 'docker'
 require 'evrone/common/spawn'
 require 'socket'
+require 'pathname'
 
 module Evrone
   module CI
@@ -52,7 +53,7 @@ module Evrone
 
             env.container_addr = config.docker.ssh.host
             env.container_addr ||= env.container.json['NetworkSettings']['IPAddress']
-            env.docker_repo_dir = [config.docker.remote_dir, env.job.message.name].join('/')
+            env.docker_repo_dir = Pathname.new(config.docker.remote_dir).join(env.job.message.name)
 
             logger.tagged "DOCKER #{env.container.id}" do
               begin

data/lib/evrone/ci/worker/middlewares/local_script.rb

@@ -9,18 +9,42 @@ module Evrone
         include Common::Helper::Shell
 
         def call(env)
-          script = env.work_dir.join(".ci_script.sh")
-          write_file script, env.job.message.script, 0700
+          script = env.tmp_dir.join("build.sh")
+          write_file script, content(env), 0700
 
           code = bash file: script, chdir: env.work_dir, &env.job.method(:add_to_output)
 
           if code == 0
             app.call env
           else
-            code
+            state = read_state(env)
+            case state
+            when "script"
+              code
+            else
+              code * -1
+            end
           end
         end
 
+        private
+
+          def content(env)
+            buf = ["set -e"]
+            buf << "echo before_script > #{env.tmp_dir}/state"
+            buf << env.job.message.before_script
+            buf << "echo script > #{env.tmp_dir}/state"
+            buf << env.job.message.script
+            buf.join("\n")
+          end
+
+          def read_state(env)
+            state_file = env.tmp_dir.join("state")
+            if File.readable? state_file
+              File.read(state_file).strip
+            end
+          end
+
       end
     end
   end

data/lib/evrone/ci/worker/version.rb

@@ -1,7 +1,7 @@
 module Evrone
   module CI
     module Worker
-      VERSION = "0.2.0.pre0"
+      VERSION = "0.2.0.pre1"
     end
   end
 end

data/spec/lib/worker/docker_spec.rb

@@ -16,12 +16,12 @@ describe Evrone::CI::Worker::Docker do
     it { should eq 0 }
 
     context "when fail before_script" do
-      let(:options) { { before_script: "/bin/false" } }
+      let(:options) { { before_script: "false" } }
       it { should eq(-1) }
     end
 
     context "when fail script" do
-      let(:options) { { script: "/bin/false" } }
+      let(:options) { { script: "false" } }
       it { should eq(1) }
     end
   end

data/spec/lib/worker/middlewares/docker_script_spec.rb

@@ -1,18 +1,19 @@
 require 'spec_helper'
 
 describe Evrone::CI::Worker::DockerScript, run_docker: true do
-  let(:exit_code)   { 0 }
-  let(:app)         { ->(_) { exit_code } }
-  let(:script)      { "echo script" }
-  let(:job)         { create :job, script: script }
-  let(:env)         { OpenStruct.new job: job }
-  let(:mid)         { described_class.new app }
-  let(:docker_mid)  { Evrone::CI::Worker::DockerStartContainer.new(mid) }
+  let(:exit_code)     { 0 }
+  let(:app)           { ->(_) { exit_code } }
+  let(:script)        { "echo script" }
+  let(:before_script) { "echo before_script" }
+  let(:job)           { create :job, script: script, before_script: before_script }
+  let(:env)           { OpenStruct.new job: job }
+  let(:mid)           { described_class.new app }
+  let(:docker_mid)    { Evrone::CI::Worker::DockerStartContainer.new(mid) }
 
   subject { docker_mid.call env }
 
   before do
-    stub(env).docker_repo_dir { '/tmp' }
+    stub(env).docker_repo_dir { Pathname.new '/tmp' }
   end
 
   it "should be" do
@@ -20,11 +21,18 @@ describe Evrone::CI::Worker::DockerScript, run_docker: true do
     expect(job.output).to match("script")
   end
 
-  context "when fail to execute" do
-    let(:script) { "/bin/false" }
+  context "when script failed" do
+    let(:script) { "false" }
     it "should be" do
       expect(subject).to eq(1)
     end
   end
+
+  context "when  before_script failed" do
+    let(:before_script) { "false" }
+    it "should be" do
+      expect(subject).to eq(-1)
+    end
+  end
 end
 

data/spec/lib/worker/middlewares/docker_start_container_spec.rb

@@ -14,7 +14,7 @@ describe Evrone::CI::Worker::DockerStartContainer, run_docker: true do
 
     expect(env.ssh).to be
     expect(env.container).to be
-    expect(env.docker_repo_dir).to eq "/home/ci/evrone/test-repo"
+    expect(env.docker_repo_dir.to_s).to eq "/home/ci/evrone/test-repo"
   end
 
 end

data/spec/lib/worker/middlewares/local_script_spec.rb

@@ -3,33 +3,42 @@ require 'pathname'
 require 'fileutils'
 
 describe Evrone::CI::Worker::LocalScript do
-  let(:command)     { "echo script" }
-  let(:app)         { ->(_) { 0 } }
-  let(:job)         { create :job, script: command }
-  let(:work_dir)    { Pathname.new '/tmp/.ci' }
-  let(:env)         { OpenStruct.new job: job, work_dir: work_dir }
-  let(:mid)         { described_class.new app }
+  let(:before_script) { "echo before_script"  }
+  let(:script)        { "echo script" }
+  let(:app)           { ->(_) { 0 } }
+  let(:job)           { create :job, script: script, before_script: before_script }
+  let(:path_prefix)   { Pathname.new "/tmp/.ci" }
+  let(:work_dir)      { path_prefix.join("work") }
+  let(:tmp_dir)       { path_prefix.join("tmp") }
+  let(:env)           { OpenStruct.new job: job, work_dir: work_dir, tmp_dir: tmp_dir }
+  let(:mid)           { described_class.new app }
 
   subject { mid.call env }
 
   after do
-    FileUtils.rm_rf work_dir
+    FileUtils.rm_rf path_prefix
   end
 
   before do
     FileUtils.mkdir_p work_dir
+    FileUtils.mkdir_p tmp_dir
   end
 
   it { should eq 0 }
 
   it "should capture output" do
     subject
-    expect(job.output).to eq "script\n"
+    expect(job.output).to eq "before_script\nscript\n"
   end
 
   context "when script failed" do
-    let(:command) { '/bin/false' }
+    let(:script) { 'false' }
     it { should satisfy { |n| [1,127].include?(n) } }
   end
+
+  context "when before_script failed" do
+    let(:before_script) { 'false' }
+    it { should satisfy { |n| [-1,-127].include?(n) } }
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: evrone-ci-worker
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre0
+  version: 0.2.0.pre1
 platform: ruby
 authors:
 - Dmitry Galinsky
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-30 00:00:00.000000000 Z
+date: 2013-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: evrone-ci-common
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre0
+        version: 0.2.0.pre1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre0
+        version: 0.2.0.pre1
 - !ruby/object:Gem::Dependency
   name: evrone-ci-message
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre0
+        version: 0.2.0.pre1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0.pre0
+        version: 0.2.0.pre1
 - !ruby/object:Gem::Dependency
   name: evrone-common-amqp
   requirement: !ruby/object:Gem::Requirement
@@ -170,6 +170,7 @@ files:
 - docker/sv-gen
 - evrone-ci-worker.gemspec
 - lib/evrone/ci/worker.rb
+- lib/evrone/ci/worker/cli.rb
 - lib/evrone/ci/worker/configuration.rb
 - lib/evrone/ci/worker/consumers/job_logs_consumer.rb
 - lib/evrone/ci/worker/consumers/job_status_consumer.rb
@@ -181,11 +182,9 @@ files:
 - lib/evrone/ci/worker/initializers/amqp.rb
 - lib/evrone/ci/worker/job.rb
 - lib/evrone/ci/worker/local.rb
-- lib/evrone/ci/worker/middlewares/docker_before_script.rb
 - lib/evrone/ci/worker/middlewares/docker_fetch_repo.rb
 - lib/evrone/ci/worker/middlewares/docker_script.rb
 - lib/evrone/ci/worker/middlewares/docker_start_container.rb
-- lib/evrone/ci/worker/middlewares/local_before_script.rb
 - lib/evrone/ci/worker/middlewares/local_create_dirs.rb
 - lib/evrone/ci/worker/middlewares/local_fetch_repo.rb
 - lib/evrone/ci/worker/middlewares/local_script.rb
@@ -198,11 +197,9 @@ files:
 - spec/lib/worker/docker_spec.rb
 - spec/lib/worker/job_spec.rb
 - spec/lib/worker/local_spec.rb
-- spec/lib/worker/middlewares/docker_before_script_spec.rb
 - spec/lib/worker/middlewares/docker_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/docker_script_spec.rb
 - spec/lib/worker/middlewares/docker_start_container_spec.rb
-- spec/lib/worker/middlewares/local_before_script_spec.rb
 - spec/lib/worker/middlewares/local_create_dirs_spec.rb
 - spec/lib/worker/middlewares/local_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/local_script_spec.rb
@@ -241,11 +238,9 @@ test_files:
 - spec/lib/worker/docker_spec.rb
 - spec/lib/worker/job_spec.rb
 - spec/lib/worker/local_spec.rb
-- spec/lib/worker/middlewares/docker_before_script_spec.rb
 - spec/lib/worker/middlewares/docker_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/docker_script_spec.rb
 - spec/lib/worker/middlewares/docker_start_container_spec.rb
-- spec/lib/worker/middlewares/local_before_script_spec.rb
 - spec/lib/worker/middlewares/local_create_dirs_spec.rb
 - spec/lib/worker/middlewares/local_fetch_repo_spec.rb
 - spec/lib/worker/middlewares/local_script_spec.rb

data/lib/evrone/ci/worker/middlewares/docker_before_script.rb

@@ -1,38 +0,0 @@
-require 'net/scp'
-
-module Evrone
-  module CI
-    module Worker
-
-      DockerBeforeScript = Struct.new(:app) do
-
-        include Helper::Logger
-
-        def call(env)
-          if env.ssh && env.docker_repo_dir
-            code = run_before_script(env)
-            if code == 0
-              app.call env
-            else
-              -1
-            end
-          else
-            app.call env
-          end
-        end
-
-        private
-
-          def run_before_script(env)
-            scp = ::Net::SCP.new(env.ssh.connection)
-            script = [env.docker_repo_dir, ".ci_before_script.sh"].join("/")
-            scp.upload! StringIO.new(env.job.message.before_script), script
-            env.ssh.spawn "env - bash #{script}", chdir: env.docker_repo_dir, &env.job.method(:add_to_output)
-          end
-
-      end
-    end
-  end
-end
-
-

data/lib/evrone/ci/worker/middlewares/local_before_script.rb

@@ -1,29 +0,0 @@
-require 'evrone/ci/common'
-
-module Evrone
-  module CI
-    module Worker
-
-      LocalBeforeScript = Struct.new(:app) do
-
-        include Common::Helper::Shell
-
-        def call(env)
-          script = env.work_dir.join(".ci_before_script.sh")
-          write_file script, env.job.message.before_script, 0700
-
-          code = bash file: script, chdir: env.work_dir, &env.job.method(:add_to_output)
-
-          if code == 0
-            app.call env
-          else
-            -1
-          end
-        end
-
-      end
-    end
-  end
-end
-
-

data/spec/lib/worker/middlewares/docker_before_script_spec.rb

@@ -1,30 +0,0 @@
-require 'spec_helper'
-
-describe Evrone::CI::Worker::DockerBeforeScript, run_docker: true do
-  let(:exit_code)   { 0 }
-  let(:app)         { ->(_) { exit_code } }
-  let(:script)      { "echo before_script" }
-  let(:job)         { create :job, before_script: script }
-  let(:env)         { OpenStruct.new job: job }
-  let(:mid)         { described_class.new app }
-  let(:docker_mid)  { Evrone::CI::Worker::DockerStartContainer.new(mid) }
-
-  subject { docker_mid.call env }
-
-  before do
-    stub(env).docker_repo_dir { '/tmp' }
-  end
-
-  it "should be" do
-    expect(subject).to eq 0
-    expect(job.output).to match("before_script")
-  end
-
-  context "when fail to execute" do
-    let(:script) { "/bin/false" }
-    it "should be" do
-      expect(subject).to eq(-1)
-    end
-  end
-end
-

data/spec/lib/worker/middlewares/local_before_script_spec.rb

@@ -1,35 +0,0 @@
-require 'spec_helper'
-require 'pathname'
-require 'fileutils'
-
-describe Evrone::CI::Worker::LocalBeforeScript do
-  let(:command)     { "echo before_script" }
-  let(:app)         { ->(_) { 0 } }
-  let(:job)         { create :job, before_script: command }
-  let(:work_dir)    { Pathname.new '/tmp/.ci' }
-  let(:env)         { OpenStruct.new job: job, work_dir: work_dir }
-  let(:mid)         { described_class.new app }
-
-  subject { mid.call env }
-
-  after do
-    FileUtils.rm_rf work_dir
-  end
-
-  before do
-    FileUtils.mkdir_p work_dir
-  end
-
-  it { should eq 0 }
-
-  it "should capture output" do
-    subject
-    expect(job.output).to eq "before_script\n"
-  end
-
-  context "when script failed" do
-    let(:command) { '/bin/false' }
-    it { should eq(-1) }
-  end
-end
-