evrone-ci-router 0.2.0.pre0 → 0.2.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bfbf13a7842e030d4d20aa825df0c5aede860177
-  data.tar.gz: da99904e2ddeacee3c87c9b9c56a63e6981dd860
+  metadata.gz: edee6c03fb2fd412081abfcc5d0a1728f18a277b
+  data.tar.gz: 8de6b06ce57d13adff8d12ead02ac3a0560436d7
 SHA512:
-  metadata.gz: d645737756afe9953e85d2054fe436dff8b3dd8b5e7aa209d9f023e2f732c114809db6ded81b28a5f708c7f71405791b69cbdccf3574934e3ee9ff599be9f5e5
-  data.tar.gz: 9cc5c15fb61e8ffb93809e24c6a979f5218bfca721fceb676456c2b1e6a44831759b38b5ea2359a75753cc773af41ec776ead1df21e9130822376bde7245e3a4
+  metadata.gz: 41e077af700321ee78461ced593a229000544e47372fc88dc2c18fa46318de8c1747204460fb990c60297b06edc7f8db43397bc70065cde1cf22a7db2fb23d0a
+  data.tar.gz: 553fe1631f488f868241ff00ca844a20ae049fe054712d47bc66d91854b14e086454c0f2b14a6c02d93afd5ac52081d30cdcff0feb3f9dea1b12a2973deb6546

data/bin/evrone-ci-router

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/evrone/ci/router", __FILE__)
+
+Evrone::CI::Router::CLI.new.run

data/fixtures/jobs_publisher

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/evrone/ci/router", __FILE__)
+
+require 'evrone/ci/message/testing'
+
+include Evrone::CI
+Router::CLI.new
+
+num = ENV['NUM'] || 1
+num.to_i.times do
+  Router::BuildsConsumer.publish Message::PerformBuild.test_message
+end

data/lib/evrone/ci/router.rb

@@ -1,9 +1,11 @@
 require 'rubygems'
+require 'pathname'
+require 'fileutils'
+require 'thread'
+
 require 'bundler'
 Bundler.require :default
 
-require 'pathname'
-
 require File.expand_path("../..", __FILE__) + "/ci/router/ext/string.rb"
 require File.expand_path("../..", __FILE__) + "/ci/router/ext/array.rb"
 
@@ -14,8 +16,9 @@ module Evrone
       autoload :Configuration,        File.expand_path("../router/configuration",                    __FILE__)
       autoload :Build,                File.expand_path("../router/build",                            __FILE__)
       autoload :BuildMatrix,          File.expand_path("../router/build_matrix",                     __FILE__)
-      autoload :Queue,                File.expand_path("../router/queue",                            __FILE__)
+      autoload :ScriptBuilder,        File.expand_path("../router/script_builder",                   __FILE__)
       autoload :Travis,               File.expand_path("../router/travis",                           __FILE__)
+      autoload :CLI,                  File.expand_path("../router/cli",                              __FILE__)
 
       autoload :BuildLogsConsumer,    File.expand_path("../router/consumers/build_logs_consumer",    __FILE__)
       autoload :BuildStatusConsumer,  File.expand_path("../router/consumers/build_status_consumer",  __FILE__)
@@ -28,22 +31,8 @@ module Evrone
         autoload :TraceShCommand,     File.expand_path("../router/helper/trace_sh_command",          __FILE__)
       end
 
-      module Middleware
-        module Travis
-          autoload :Env,              File.expand_path("../router/middleware/travis/env",            __FILE__)
-          autoload :Ruby,             File.expand_path("../router/middleware/travis/ruby",           __FILE__)
-          autoload :Script,           File.expand_path("../router/middleware/travis/script",         __FILE__)
-        end
-        autoload :CreateBuildMatrix,  File.expand_path("../router/middleware/create_build_matrix",   __FILE__)
-        autoload :FetchSource,        File.expand_path("../router/middleware/fetch_source",          __FILE__)
-        autoload :FetchCommitInfo,    File.expand_path("../router/middleware/fetch_commit_info",     __FILE__)
-        autoload :LogBuild,           File.expand_path("../router/middleware/log_build",             __FILE__)
-        autoload :UpdateBuildStatus,  File.expand_path("../router/middleware/update_build_status",   __FILE__)
-        autoload :CreateDirs,         File.expand_path("../router/middleware/create_dirs",           __FILE__)
-      end
-
-      @@root   = Pathname.new File.expand_path('../../../..', __FILE__)
-      @@config = Configuration.new
+      @@root = Pathname.new File.expand_path('../../../..', __FILE__)
+      @@config_mutex = Mutex.new
 
       class << self
         def root
@@ -60,41 +49,127 @@ module Evrone
         end
 
         def config
-          @@config
+          @config ||= begin
+            @@config_mutex.synchronize do
+              Configuration.new
+            end
+          end
         end
 
         def reset_config!
-          @@config = Configuration.new
+          @config = nil
         end
-      end
 
-      include Common::Helper::Middlewares
-
-      middlewares do
-        use Middleware::LogBuild
-        use Middleware::UpdateBuildStatus
-        use Middleware::CreateDirs
-        use Middleware::FetchSource
-        use Middleware::FetchCommitInfo
-        use Middleware::CreateBuildMatrix
+        def initialize!
+          root.join("lib/evrone/ci/router/initializers").children.each do |e|
+            require e
+          end
+        end
       end
 
-      attr_reader :build, :path_prefix
+      include Helper::Logger
+      include Helper::Config
+
+      attr_reader :build, :path_prefix, :repo_dir, :travis
 
       def initialize(build, path_prefix)
         @build        = build
         @path_prefix  = Pathname.new(path_prefix).expand_path
+        @repo_dir     = @path_prefix.join(config.repo_dir_name)
+                                    .join(build.message.name)
+        @travis       = nil
       end
 
       def perform
-        env = OpenStruct.new build: build, path_prefix: path_prefix
-        run_middlewares(env) { |_| 0 }
+        log_build do
+          update_build_status do
+            create_repo_dir    &&
+            fetch_repo         &&
+            assign_commit_info &&
+            load_travis        &&
+            create_and_delivery_build_matrix
+          end
+        end
+      end
+
+      def create_and_delivery_build_matrix
+        matrix = BuildMatrix.new travis
+        build.matrix     = matrix.keys
+        build.jobs_count = matrix.travises.size
+
+        matrix.travises.each_with_index do |travis, idx|
+          number = idx + 1
+          message = build.to_perform_job_message travis, number
+          logger.info "delivery job #{message.id}.#{number} #{travis.to_matrix_s}"
+          JobsConsumer.publish message
+        end
+
+        true
       end
 
+      def load_travis
+        @travis = Travis.from_file repo_dir.join(".travis.yml")
+      end
+
+      def create_repo_dir
+        FileUtils.mkdir_p(repo_dir) unless repo_dir.directory?
+        true
+      end
+
+      def fetch_repo
+        scm.fetch == 0
+      end
+
+      def assign_commit_info
+        build.commit_info = scm.commit_info
+        true
+      end
+
+      def log_build
+        logger.tagged("BUILD #{build.message.id}") do
+          logger.info "starting build"
+          rs = yield
+          logger.info "done build"
+          rs
+        end
+      end
+
+      def update_build_status
+        publish_build_status_message Build::STARTED
+        rs = false
+        begin
+          rs = yield
+        rescue Exception => e
+          logger.error("ERROR: #{e.inspect}\n    BACKTRACE:\n#{e.backtrace.map{|i| "    #{i}" }.join("\n")}")
+        end
+
+        if rs
+          publish_build_status_message Build::FINISHED
+        else
+          publish_build_status_message Build::FAILED
+        end
+        rs
+      end
+
+      private
+
+        def scm
+          @scm ||= SCM::Git.new(
+            build.message.src,
+            build.message.sha,
+            repo_dir,
+            deploy_key: build.message.deploy_key,
+            &build.method(:add_to_output)
+          )
+        end
+
+        def publish_build_status_message(status)
+          message = build.to_build_status_message(status)
+          logger.info "delivered build status #{message.inspect}"
+          BuildStatusConsumer.publish message
+        end
+
     end
   end
 end
 
-Evrone::CI::Router.root.join("lib/evrone/ci/router/initializers").children.each do |e|
-  require e
-end

data/lib/evrone/ci/router/build.rb

@@ -6,6 +6,10 @@ module Evrone
     class Router
       class Build
 
+        STARTED  = 2
+        FINISHED = 3
+        FAILED   = 5
+
         include Router::Helper::Logger
         include Router::Helper::Config
 
@@ -18,7 +22,7 @@ module Evrone
         end
 
         def to_perform_job_message(travis, job_id)
-          queue   = travis.to_queue
+          script_builder   = travis.to_script_builder
           job_message = Message::PerformJob.new(
             id:               message.id,
             name:             message.name,
@@ -27,13 +31,35 @@ module Evrone
             pull_request_id:  message.pull_request_id,
             deploy_key:       message.deploy_key,
             job_id:           job_id,
-            before_script:    queue.to_before_script,
-            script:           queue.to_script,
+            before_script:    script_builder.to_before_script,
+            script:           script_builder.to_script,
             matrix_keys:      travis.matrix_keys,
           )
           job_message
         end
 
+        def to_build_status_message(status)
+          tm = Time.now
+          attributes = {
+            build_id:   message.id,
+            status:     status,
+            tm:         tm.to_i,
+            tm_usec:    tm.usec,
+            matrix:     matrix || [],
+            jobs_count: jobs_count || 0,
+          }
+
+          if commit_info
+            attributes.merge!(
+              commit_author:       commit_info.author,
+              commit_author_email: commit_info.email,
+              commit_sha:          commit_info.sha,
+              commit_message:      commit_info.message
+            )
+          end
+          Message::BuildStatus.new attributes
+        end
+
         def add_to_output(str)
           output << str
           logger.debug str.strip if logger.level == 0

data/lib/evrone/ci/router/build_matrix.rb

@@ -4,6 +4,7 @@ module Evrone
       class BuildMatrix
 
         KEYS = (Travis::LANGS + %w{ env }).freeze
+        NOT_MATRIX_KEYS = %w{ script before_script }
 
         attr_reader :travis
 
@@ -17,7 +18,12 @@ module Evrone
 
         def travises
           attributes_for_new_travises.map do |attrs|
-            Travis.new attrs
+            Travis.new attrs.merge(
+              NOT_MATRIX_KEYS.inject({}) do |a,v|
+                a[v] = travis.public_send(v)
+                a
+              end
+            )
           end
         end
 

data/lib/evrone/ci/router/cli.rb

@@ -0,0 +1,67 @@
+require 'optparse'
+require 'evrone/common/amqp'
+
+module Evrone
+  module CI
+    class Router
+      class CLI
+
+        include Helper::Config
+        include Helper::Logger
+
+        def initialize
+          @options = {}
+          parse!
+          Router.initialize!
+        end
+
+        def run
+          logger.warn "spawn inside #{config.path_prefix}"
+
+          trap('INT') {
+            Thread.new do
+              Evrone::Common::AMQP.shutdown
+            end.join
+          }
+
+          Evrone::Common::AMQP::Supervisor::Threaded.build(
+            Evrone::CI::Router::BuildsConsumer => config.workers,
+          ).run
+        end
+
+        private
+
+          def parse!
+            OptionParser.new do |opts|
+              opts.banner = "Usage: evrone-ci-router [options]"
+              opts.on("-w", "--workers NUM", "Number of workers, default 1") do |v|
+                @options[:workers] = v.to_i
+              end
+              opts.on("-p", "--path PATH", "Working directory, default current directory") do |v|
+                @options[:path_prefix] = v.to_s
+              end
+              opts.on("-c", "--config FILE", "Path to configuration file") do |v|
+                read_configuration v
+              end
+            end.parse!
+
+            @options.each_pair do |k,v|
+              config.public_send("#{k}=", v)
+            end
+          end
+
+          def read_configuration(file)
+            file = File.expand_path(file)
+            buf  = File.read(file)
+
+            buf.split("\n").each do |line|
+              puts line
+              env, value = line.split("=").map(&:strip)
+              ENV[env] = value
+            end
+          end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/router/configuration.rb

@@ -15,12 +15,18 @@ module Evrone
         define amqp_url:      nil,
                timeout:       30 * 60,
                logger:        Common::TaggedLogging.new(Logger.new STDOUT),
-               repo_dir_name: "repo"
+               repo_dir_name: "repo",
+               workers:       1,
+               path_prefix:   nil
 
         def timeout
           self[:timeout].to_i
         end
 
+        def path_prefix
+          self[:path_prefix] || Dir.pwd
+        end
+
       end
     end
   end

data/lib/evrone/ci/router/consumers/builds_consumer.rb

@@ -16,7 +16,7 @@ module Evrone
         def perform(message)
           build       = Build.new message
           number      = Thread.current[:consumer_id] || 0
-          path_prefix = "/tmp/.test/build.#{number}"
+          path_prefix = "#{Router.config.path_prefix}/build.#{number}"
 
           Router.new(build, path_prefix).perform
           ack!

data/lib/evrone/ci/router/initializers/amqp.rb

@@ -59,5 +59,6 @@ Evrone::Common::AMQP.configure do |c|
   end
 
   c.content_type = 'application/x-protobuf'
-  c.logger = nil
+  c.logger       = nil
+  c.url          = Evrone::CI::Router.config.amqp_url
 end

data/lib/evrone/ci/router/{queue.rb → script_builder.rb}

@@ -4,14 +4,18 @@ require 'evrone/ci/common'
 module Evrone
   module CI
     class Router
-      class Queue
+      class ScriptBuilder
+
+        autoload :Env,    File.expand_path("../script_builder/env",     __FILE__)
+        autoload :Ruby,   File.expand_path("../script_builder/ruby",    __FILE__)
+        autoload :Script, File.expand_path("../script_builder/script",  __FILE__)
 
         include Common::Helper::Middlewares
 
         middlewares do
-          use Middleware::Travis::Env
-          use Middleware::Travis::Ruby
-          use Middleware::Travis::Script
+          use ScriptBuilder::Env
+          use ScriptBuilder::Ruby
+          use ScriptBuilder::Script
         end
 
         attr_reader :travis
@@ -21,7 +25,7 @@ module Evrone
         end
 
         def to_before_script
-          a = ["set -e"]
+          a = []
           a += env.init
           a += env.before_install
           a += env.install
@@ -30,7 +34,7 @@ module Evrone
         end
 
         def to_script
-          a = ["set -e"]
+          a = []
           a << env.script
           a.join("\n")
         end