evrone-ci-worker 0.2.0.pre0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b7b9bb1d659ac857b4f5fdb4bf715a9695360997
+  data.tar.gz: 07a0fd1df69fa7374d2ff7b80bddfbd6a685983f
+SHA512:
+  metadata.gz: 30ee6822495f8aebe134334a77477e98e64041b821a89cf4922e7fdee91cb53eae27c2e380ef515f4c1823722f40dd740f18992eaaec7c8345d5bbd41ccee9b3
+  data.tar.gz: bf174a1ebc89b1b1a3e268553cb1c9e185376cc78697086ae361d5573cf5c9f356c27eac5108226ea548e2a0aad57a75ab911fb9869ac0ebe87bf9e8c1a9c74f

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+fixtures/

data/.rspec

@@ -0,0 +1,3 @@
+-f d
+--color
+--order=rand

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in evrone-ci-job.gemspec
+gemspec
+gem 'evrone-ci-common', path: File.expand_path("../../common", __FILE__)
+gem 'evrone-ci-message', path: File.expand_path("../../message", __FILE__)

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Dmitry Galinsky
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Evrone::Ci::Job
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'evrone-ci-job'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install evrone-ci-job
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rubygems'
+require 'bundler'
+Bundler.require
+require 'rspec/core/rake_task'
+require "bundler/gem_tasks"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => ["test:create_repo", :spec]
+
+namespace :test do
+  task :create_repo do
+    dir = "fixtures/repo"
+    unless File.directory? dir
+      cmd = "git clone https://github.com/evrone/ci-worker-test-repo.git fixtures/repo"
+      puts cmd
+      system cmd
+    end
+  end
+end
+
+task :travis do
+  exec "bundle exec rake SPEC_OPTS='--format documentation -t ~run_docker --order=rand'"
+end

data/bin/evrone-ci-worker

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/evrone/ci/worker", __FILE__)
+
+trap('INT') {
+  Thread.new do
+    Evrone::Common::AMQP.shutdown
+  end.join
+}
+
+Evrone::CI::Worker.configure do |c|
+  c.docker.ssh.port = 2223
+  c.docker.ssh.host = 'localhost'
+  c.docker.create_options = {
+    'PortSpecs' => ['2022:22']
+  }
+end
+
+Evrone::Common::AMQP::Supervisor::Threaded.build(
+  Evrone::CI::Worker::JobsConsumer => 4,
+).run
+

data/docker/Dockerfile

@@ -0,0 +1,61 @@
+FROM ubuntu:12.04
+
+# disable questions
+ENV DEBIAN_FRONTEND noninteractive
+
+# disable service autostarting
+ENV RUNLEVEL 1
+
+# replace init
+RUN  dpkg-divert --local --rename --add /sbin/initctl
+RUN  ln -s /bin/true /sbin/initctl
+
+RUN apt-get -qy update
+RUN apt-get install -qy curl git-core build-essential
+
+# add evrone repo
+RUN mkdir -p /etc/apt/sources.list.d
+RUN echo "deb  http://download.opensuse.org/repositories/home:/dmexe/xUbuntu_12.04/  ./" > /etc/apt/sources.list.d/Evrone.list
+RUN curl http://download.opensuse.org/repositories/home:/dmexe/xUbuntu_12.04/Release.key | apt-key add -
+RUN apt-get -qy update
+
+# install runit
+RUN chmod 0000 /sbin/start
+RUN apt-get -qy install runit
+RUN chmod 0755 /sbin/start
+ADD ./sv-gen /usr/local/bin/sv-gen
+ADD ./sv-enable /usr/local/bin/sv-enable
+
+# set user password
+RUN useradd -m ci
+RUN echo "ci:ci" | chpasswd
+RUN apt-get -qy install sudo
+RUN echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+# fixes locales
+RUN locale-gen en_US.UTF-8
+RUN dpkg-reconfigure -fnoninteractive locales
+RUN update-locale LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US"
+
+# install ssh
+RUN apt-get install -qy openssh-server
+RUN mkdir -p /var/run/sshd
+RUN sv-gen ssh "/usr/sbin/sshd -D -e"
+
+# install postgresql
+RUN apt-get install -qy postgresql
+RUN echo "host all all all trust" > /etc/postgresql/9.1/main/pg_hba.conf
+RUN sv-gen postgresql "/usr/lib/postgresql/9.1/bin/postgres -D /var/lib/postgresql/9.1/main -c config_file=/etc/postgresql/9.1/main/postgresql.conf" postgres
+
+# add ruby
+RUN apt-get install -qy rbenv rbenv-2.0.0-p195 rbenv-1.9.3-p448
+RUN env RBENV_VERSION=2.0.0-p195 rbenv exec gem install bundler --no-ri --no-rdoc
+RUN env RBENV_VERSION=1.9.3-p448 rbenv exec gem install bundler --no-ri --no-rdoc
+RUN rbenv rehash
+
+RUN apt-get -qy clean autoremove
+
+RUN sv-enable postgresql
+RUN sv-enable ssh
+
+ENTRYPOINT ["/usr/bin/runsvdir", "-P", "/etc/service"]

data/docker/sv-enable

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+NAME=${1}
+USAGE="Usage: sv-enable <service>"
+SRC=/etc/sv/${NAME}
+DST=/etc/service/${NAME}
+
+if [ "x${1}" = "x" ] ; then
+  echo $USAGE
+  exit 1
+fi
+
+if [ ! -d ${SRC} ] ; then
+  echo "Service ${NAME} does not exists"
+  exit 1
+fi
+
+mkdir -p /etc/service
+
+if [ ! -L $DST ] ; then
+  echo " ---> enable ${NAME}"
+  rm -rf ${DST}
+  ln -s ${SRC} ${DST}
+fi

data/docker/sv-gen

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+
+NAME=${1}
+CMD=${2}
+USER=${3:-root}
+DIR=${4:-/}
+RUN_FILE="/etc/sv/${NAME}/run"
+LOG_FILE="/etc/sv/${NAME}/log/run"
+
+USAGE="Usage: sv-gen <name> <command> [user] [directory]"
+
+if [ "x${1}" = "x" ] ; then
+  echo $USAGE
+  exit 1
+fi
+
+if [ "x${2}" = "x" ] ; then
+  echo $USAGE
+  exit 1
+fi
+
+echo " ---> create /etc/sv/${NAME}"
+mkdir -p /etc/sv/${NAME}
+
+echo " ---> create ${RUN_FILE}"
+cat > $RUN_FILE << EOF
+#!/bin/sh
+set -e
+exec 2>&1
+cd ${DIR}
+CMD="${CMD}"
+exec chpst -u ${USER} \$CMD 2>&1
+EOF
+chmod +x ${RUN_FILE}
+
+echo " ---> create /etc/sv/${NAME}/log"
+mkdir -p /etc/sv/${NAME}/log
+
+echo " ---> create ${LOG_FILE}"
+cat > $LOG_FILE << EOF
+#!/bin/sh
+set -e
+LOG=/var/log/${NAME}
+
+test -d "\$LOG" || mkdir -p m2750 "\$LOG"
+exec svlogd "\$LOG"
+EOF
+chmod +x ${LOG_FILE}

data/evrone-ci-worker.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require File.expand_path('../lib/evrone/ci/worker/version.rb', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = "evrone-ci-worker"
+  spec.version       = Evrone::CI::Worker::VERSION
+  spec.authors       = ["Dmitry Galinsky"]
+  spec.email         = ["dima.exe@gmail.com"]
+  spec.description   = %q{ ci worker }
+  spec.summary       = %q{ ci worker }
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'evrone-ci-common',   "= #{Evrone::CI::Worker::VERSION}"
+  spec.add_runtime_dependency 'evrone-ci-message',  "= #{Evrone::CI::Worker::VERSION}"
+  spec.add_runtime_dependency 'evrone-common-amqp', '~> 0.1.1'
+  spec.add_runtime_dependency 'docker-api',         '= 1.5.4'
+  spec.add_runtime_dependency 'hashr',              '= 0.0.22'
+  spec.add_runtime_dependency 'net-scp',            '= 1.1.2'
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rr"
+end

data/lib/evrone/ci/worker/configuration.rb

@@ -0,0 +1,58 @@
+require 'hashr'
+require 'logger'
+require 'evrone/ci/common/tagged_logging'
+
+module Evrone
+  module CI
+    module Worker
+      class Configuration < ::Hashr
+
+        extend Hashr::EnvDefaults
+
+        @@null_logger = Logger.new("/dev/null")
+
+        self.env_namespace = 'ci_worker'
+        self.raise_missing_keys = true
+
+        define amqp_url:     nil,
+               run:          "docker",
+               timeout:      30 * 60,
+               logger:       Common::TaggedLogging.new(Logger.new STDOUT),
+
+               work_dir_name:   "work",
+               repo_dir_name:   "repo",
+               tmp_dir_name:    "tmp",
+
+               sleep_after_job: 5,
+
+               docker: {
+                 remote_dir: "/home/ci",
+                 init:       %w{ /usr/local/bin/runsvdir -P /etc/service },
+                 image:      "ci",
+
+                 ssh: {
+                   host:     nil, # used in tests
+                   port:     nil, # used in tests
+                   user:     "ci",
+                   password: "ci"
+                 },
+
+                 create_options: nil # used in tests
+               }
+
+        def timeout
+          self[:timeout].to_i
+        end
+
+        def run
+          self[:run].to_sym
+        end
+
+        def null_logger
+          @@null_logger
+        end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/worker/consumers/job_logs_consumer.rb

@@ -0,0 +1,15 @@
+require 'evrone/common/amqp'
+
+module Evrone
+  module CI
+    module Worker
+      class JobLogsConsumer
+
+        include Evrone::Common::AMQP::Consumer
+
+        exchange 'ci.jobs.log'
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/worker/consumers/job_status_consumer.rb

@@ -0,0 +1,16 @@
+require 'evrone/common/amqp'
+
+module Evrone
+  module CI
+    module Worker
+      class JobStatusConsumer
+
+        include Evrone::Common::AMQP::Consumer
+
+        exchange 'ci.jobs.status'
+
+      end
+    end
+  end
+end
+

data/lib/evrone/ci/worker/consumers/jobs_consumer.rb

@@ -0,0 +1,27 @@
+require 'evrone/common/amqp'
+
+module Evrone
+  module CI
+    module Worker
+      class JobsConsumer
+
+        include Evrone::Common::AMQP::Consumer
+
+        exchange 'ci.jobs'
+        queue    'ci.worker.jobs.generic'
+        ack      true
+
+        model Message::PerformJob
+
+        def perform(message)
+          job         = Job.new message
+          number      = Thread.current[:consumer_id] || 0
+          path_prefix = "/tmp/.test/job.#{number}"
+          Worker.perform(job, path_prefix)
+          ack!
+        end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/worker/docker.rb

@@ -0,0 +1,35 @@
+require 'evrone/ci/common'
+
+module Evrone
+  module CI
+    module Worker
+
+      class Docker
+
+        include Common::Helper::Middlewares
+
+        attr_reader :job
+
+        middlewares do
+          use LogJob
+          use UpdateJobStatus
+          use DockerStartContainer
+          use DockerFetchRepo
+          use DockerBeforeScript
+          use DockerScript
+        end
+
+        def initialize(job, _)
+          @job         = job
+        end
+
+        def perform
+          env = OpenStruct.new job: job
+          run_middlewares(env){ |_| 0 }
+        end
+
+      end
+
+    end
+  end
+end

data/lib/evrone/ci/worker/ext/string.rb

@@ -0,0 +1,10 @@
+class String
+  def compact
+    gsub(/\n/, ' ').gsub(/ +/, ' ').strip
+  end
+
+  def camelize
+    split("_").each {|s| s.capitalize! }.join("")
+  end
+end
+

data/lib/evrone/ci/worker/helper/config.rb

@@ -0,0 +1,14 @@
+module Evrone
+  module CI
+    module Worker
+      module Helper::Config
+
+        def config
+          Worker.config
+        end
+
+      end
+    end
+  end
+end
+

data/lib/evrone/ci/worker/helper/logger.rb

@@ -0,0 +1,14 @@
+module Evrone
+  module CI
+    module Worker
+      module Helper::Logger
+
+        def logger
+          Worker.logger
+        end
+
+      end
+    end
+  end
+end
+

data/lib/evrone/ci/worker/initializers/amqp.rb

@@ -0,0 +1,53 @@
+require 'evrone/common/amqp'
+
+module Evrone::CI::Worker::AMQP
+
+  Base = Struct.new("Subscribing", :app) do
+    include Evrone::CI::Worker::Helper::Logger
+
+    def consumer_name
+      Thread.current[:consumer_name]
+    end
+
+    def consumer_id
+      Thread.current[:consumer_id]
+    end
+
+    def consumer_tag
+      consumer_id ? "#{consumer_name.split('::').last} #{consumer_id}" : consumer_name
+    end
+  end
+
+  class Subscribing < Base
+    def call(env)
+      logger.tagged(consumer_tag) do
+        logger.warn "subsribing #{env[:exchange].name}"
+        rs = app.call env
+        logger.warn "shutdown"
+        rs
+      end
+    end
+  end
+
+  class Recieving < Base
+    def call(env)
+      logger.warn "payload recieved #{env[:payload].inspect[0...60]}..."
+      rs = app.call env
+      logger.warn "commit message"
+      rs
+    end
+  end
+end
+
+Evrone::Common::AMQP.configure do |c|
+  c.subscribing do
+    use Evrone::CI::Worker::AMQP::Subscribing
+  end
+
+  c.recieving do
+    use Evrone::CI::Worker::AMQP::Recieving
+  end
+
+  c.content_type = 'application/x-protobuf'
+  c.logger = nil
+end

data/lib/evrone/ci/worker/job.rb

@@ -0,0 +1,42 @@
+require 'evrone/ci/message'
+require 'evrone/ci/common'
+
+module Evrone
+  module CI
+    module Worker
+      class Job
+
+        include Helper::Logger
+
+        attr_reader :output, :message
+
+        def initialize(perform_job_message)
+          @output  = ""
+          @message = perform_job_message
+        end
+
+        def add_to_output(str)
+          output << str
+          logger.debug str.strip if logger.level == 0
+          JobLogsConsumer.publish create_job_log_message(str)
+        end
+
+        def add_command_to_output(cmd)
+          add_to_output "$ #{cmd}\n"
+        end
+
+        def create_job_log_message(data)
+          tm = Time.now
+          Message::JobLog.new(
+            build_id: message.id,
+            job_id:   message.job_id,
+            tm:       tm.to_i,
+            tm_usec:  tm.usec,
+            log:      data
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/worker/local.rb

@@ -0,0 +1,37 @@
+require 'evrone/ci/common'
+require 'pathname'
+
+module Evrone
+  module CI
+    module Worker
+
+      class Local
+
+        include Common::Helper::Middlewares
+
+        attr_reader :job, :path_prefix
+
+        middlewares do
+          use LogJob
+          use UpdateJobStatus
+          use LocalCreateDirs
+          use LocalFetchRepo
+          use LocalBeforeScript
+          use LocalScript
+        end
+
+        def initialize(job, path_prefix)
+          @job         = job
+          @path_prefix = Pathname.new(path_prefix).expand_path
+        end
+
+        def perform
+          env = OpenStruct.new job: job, path_prefix: path_prefix
+          run_middlewares(env){ |_| 0 }
+        end
+
+      end
+
+    end
+  end
+end

data/lib/evrone/ci/worker/middlewares/docker_before_script.rb

@@ -0,0 +1,38 @@
+require 'net/scp'
+
+module Evrone
+  module CI
+    module Worker
+
+      DockerBeforeScript = Struct.new(:app) do
+
+        include Helper::Logger
+
+        def call(env)
+          if env.ssh && env.docker_repo_dir
+            code = run_before_script(env)
+            if code == 0
+              app.call env
+            else
+              -1
+            end
+          else
+            app.call env
+          end
+        end
+
+        private
+
+          def run_before_script(env)
+            scp = ::Net::SCP.new(env.ssh.connection)
+            script = [env.docker_repo_dir, ".ci_before_script.sh"].join("/")
+            scp.upload! StringIO.new(env.job.message.before_script), script
+            env.ssh.spawn "env - bash #{script}", chdir: env.docker_repo_dir, &env.job.method(:add_to_output)
+          end
+
+      end
+    end
+  end
+end
+
+