evrone-ci-router 0.2.0.pre0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bfbf13a7842e030d4d20aa825df0c5aede860177
+  data.tar.gz: da99904e2ddeacee3c87c9b9c56a63e6981dd860
+SHA512:
+  metadata.gz: d645737756afe9953e85d2054fe436dff8b3dd8b5e7aa209d9f023e2f732c114809db6ded81b28a5f708c7f71405791b69cbdccf3574934e3ee9ff599be9f5e5
+  data.tar.gz: 9cc5c15fb61e8ffb93809e24c6a979f5218bfca721fceb676456c2b1e6a44831759b38b5ea2359a75753cc773af41ec776ead1df21e9130822376bde7245e3a4

data/.gitignore

@@ -0,0 +1,7 @@
+fixtures/repo
+.test
+.vagrant
+.bundle
+pkg/
+fixtures/repo
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+-f d
+--color
+--order=rand

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in evrone-ci-job.gemspec
+gemspec
+gem 'evrone-ci-common', path: File.expand_path("../../common", __FILE__)
+gem 'evrone-ci-message', path: File.expand_path("../../message", __FILE__)
+

data/README.md

@@ -0,0 +1 @@
+

data/Rakefile

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'bundler'
+Bundler.require
+require 'rspec/core/rake_task'
+require "bundler/gem_tasks"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => ["test:create_repo", :spec]
+
+namespace :test do
+  task :create_repo do
+    dir = "fixtures/repo"
+    unless File.directory? dir
+      cmd = "git clone https://github.com/evrone/ci-worker-test-repo.git fixtures/repo"
+      puts cmd
+      system cmd
+    end
+  end
+end
+
+task :travis => :default

data/bin/cli

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+bundle exec pry -r$(pwd)/lib/evrone/ci/worker.rb

data/bin/git_ssh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec /usr/bin/ssh -A -o StrictHostKeyChecking=no "$@"

data/bin/jobs_publisher

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/evrone/ci/router", __FILE__)
+
+require File.expand_path("../../spec/support/fixture", __FILE__)
+require File.expand_path("../../spec/support/create", __FILE__)
+
+require 'evrone/ci/message/testing'
+
+include Evrone::CI
+
+Router::BuildsConsumer.publish Message::PerformBuild.test_message

data/bin/workers

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/evrone/ci/router", __FILE__)
+
+trap('INT') {
+  Thread.new do
+    Evrone::Common::AMQP.shutdown
+  end.join
+}
+
+Evrone::Common::AMQP::Supervisor::Threaded.build(
+  Evrone::CI::Router::BuildsConsumer => 1,
+).run

data/evrone-ci-router.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require File.expand_path('../lib/evrone/ci/router/version.rb', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = "evrone-ci-router"
+  spec.version       = Evrone::CI::Router::VERSION
+  spec.authors       = ["Dmitry Galinsky"]
+  spec.email         = ["dima.exe@gmail.com"]
+  spec.description   = %q{ ci router }
+  spec.summary       = %q{ ci router }
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'evrone-ci-common',   "= #{Evrone::CI::Router::VERSION}"
+  spec.add_runtime_dependency 'evrone-ci-message',  "= #{Evrone::CI::Router::VERSION}"
+  spec.add_runtime_dependency 'evrone-common-amqp', '~> 0.1.1'
+  spec.add_runtime_dependency 'hashr',              '= 0.0.22'
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rr"
+end

data/fixtures/travis.yml

@@ -0,0 +1,5 @@
+rvm:
+  - 2.0.0
+before_script:
+  - "echo before_script"
+script: "RAILS_ENV=test ls -1 && echo DONE!"

data/lib/evrone/ci/router.rb

@@ -0,0 +1,100 @@
+require 'rubygems'
+require 'bundler'
+Bundler.require :default
+
+require 'pathname'
+
+require File.expand_path("../..", __FILE__) + "/ci/router/ext/string.rb"
+require File.expand_path("../..", __FILE__) + "/ci/router/ext/array.rb"
+
+module Evrone
+  module CI
+    class Router
+
+      autoload :Configuration,        File.expand_path("../router/configuration",                    __FILE__)
+      autoload :Build,                File.expand_path("../router/build",                            __FILE__)
+      autoload :BuildMatrix,          File.expand_path("../router/build_matrix",                     __FILE__)
+      autoload :Queue,                File.expand_path("../router/queue",                            __FILE__)
+      autoload :Travis,               File.expand_path("../router/travis",                           __FILE__)
+
+      autoload :BuildLogsConsumer,    File.expand_path("../router/consumers/build_logs_consumer",    __FILE__)
+      autoload :BuildStatusConsumer,  File.expand_path("../router/consumers/build_status_consumer",  __FILE__)
+      autoload :BuildsConsumer,       File.expand_path("../router/consumers/builds_consumer",        __FILE__)
+      autoload :JobsConsumer,         File.expand_path("../router/consumers/jobs_consumer",          __FILE__)
+
+      module Helper
+        autoload :Config,             File.expand_path("../router/helper/config",                    __FILE__)
+        autoload :Logger,             File.expand_path("../router/helper/logger",                    __FILE__)
+        autoload :TraceShCommand,     File.expand_path("../router/helper/trace_sh_command",          __FILE__)
+      end
+
+      module Middleware
+        module Travis
+          autoload :Env,              File.expand_path("../router/middleware/travis/env",            __FILE__)
+          autoload :Ruby,             File.expand_path("../router/middleware/travis/ruby",           __FILE__)
+          autoload :Script,           File.expand_path("../router/middleware/travis/script",         __FILE__)
+        end
+        autoload :CreateBuildMatrix,  File.expand_path("../router/middleware/create_build_matrix",   __FILE__)
+        autoload :FetchSource,        File.expand_path("../router/middleware/fetch_source",          __FILE__)
+        autoload :FetchCommitInfo,    File.expand_path("../router/middleware/fetch_commit_info",     __FILE__)
+        autoload :LogBuild,           File.expand_path("../router/middleware/log_build",             __FILE__)
+        autoload :UpdateBuildStatus,  File.expand_path("../router/middleware/update_build_status",   __FILE__)
+        autoload :CreateDirs,         File.expand_path("../router/middleware/create_dirs",           __FILE__)
+      end
+
+      @@root   = Pathname.new File.expand_path('../../../..', __FILE__)
+      @@config = Configuration.new
+
+      class << self
+        def root
+          @@root
+        end
+
+        def logger
+          config.logger
+        end
+
+        def configure
+          yield config
+          config
+        end
+
+        def config
+          @@config
+        end
+
+        def reset_config!
+          @@config = Configuration.new
+        end
+      end
+
+      include Common::Helper::Middlewares
+
+      middlewares do
+        use Middleware::LogBuild
+        use Middleware::UpdateBuildStatus
+        use Middleware::CreateDirs
+        use Middleware::FetchSource
+        use Middleware::FetchCommitInfo
+        use Middleware::CreateBuildMatrix
+      end
+
+      attr_reader :build, :path_prefix
+
+      def initialize(build, path_prefix)
+        @build        = build
+        @path_prefix  = Pathname.new(path_prefix).expand_path
+      end
+
+      def perform
+        env = OpenStruct.new build: build, path_prefix: path_prefix
+        run_middlewares(env) { |_| 0 }
+      end
+
+    end
+  end
+end
+
+Evrone::CI::Router.root.join("lib/evrone/ci/router/initializers").children.each do |e|
+  require e
+end

data/lib/evrone/ci/router/build.rb

@@ -0,0 +1,60 @@
+require 'evrone/ci/message'
+require 'evrone/ci/common'
+
+module Evrone
+  module CI
+    class Router
+      class Build
+
+        include Router::Helper::Logger
+        include Router::Helper::Config
+
+        attr_reader :message, :output
+        attr_accessor :matrix, :jobs_count, :commit_info
+
+        def initialize(perform_build_message)
+          @output  = ''
+          @message = perform_build_message
+        end
+
+        def to_perform_job_message(travis, job_id)
+          queue   = travis.to_queue
+          job_message = Message::PerformJob.new(
+            id:               message.id,
+            name:             message.name,
+            src:              message.src,
+            sha:              message.sha,
+            pull_request_id:  message.pull_request_id,
+            deploy_key:       message.deploy_key,
+            job_id:           job_id,
+            before_script:    queue.to_before_script,
+            script:           queue.to_script,
+            matrix_keys:      travis.matrix_keys,
+          )
+          job_message
+        end
+
+        def add_to_output(str)
+          output << str
+          logger.debug str.strip if logger.level == 0
+          BuildLogsConsumer.publish create_build_log_message(str)
+        end
+
+        def add_command_to_output(cmd)
+          add_to_output "$ #{cmd}\n"
+        end
+
+        def create_build_log_message(data)
+          tm = Time.now
+          Message::BuildLog.new(
+            build_id:  message.id,
+            tm:        tm.to_i,
+            tm_usec:   tm.usec,
+            log:       data
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/router/build_matrix.rb

@@ -0,0 +1,89 @@
+module Evrone
+  module CI
+    class Router
+      class BuildMatrix
+
+        KEYS = (Travis::LANGS + %w{ env }).freeze
+
+        attr_reader :travis
+
+        def initialize(travis)
+          @travis = travis
+        end
+
+        def keys
+          extract_pair_of_key_and_values.map(&:first).sort
+        end
+
+        def travises
+          attributes_for_new_travises.map do |attrs|
+            Travis.new attrs
+          end
+        end
+
+        def attributes_for_new_travises
+          permutate_and_build_values.inject([]) do |ac, values|
+            ac << values.inject({}) do |a,val|
+              a[val.key] = val.value
+              a
+            end
+            ac
+          end
+        end
+
+        def permutate_and_build_values
+          values = extract_pair_of_key_and_values.map do |key, vals|
+            vals.map{|it| Value.new(key, it) }
+          end
+          if matrix_values?(values)
+            array_permutations(values).map do |it|
+              if it.is_a?(Array)
+                it.flatten
+              else
+                [it]
+              end
+            end
+          else
+            values
+          end.sort_by(&:to_s)
+        end
+
+        def extract_pair_of_key_and_values
+          KEYS.map.inject([]) do |a, k|
+            if (val = travis[k]) && !val.empty?
+              a << [k, val]
+            end
+            a
+          end
+        end
+
+        private
+
+          def matrix_values?(values)
+            !values.all?{|i| i.size == 1 }
+          end
+
+          def array_permutations array, index=0
+            # index is 0 by default : start at the beginning, more elegant.
+            return array[-1] if index == array.size - 1 # Return last element if at end.
+
+            result = []
+
+            array[index].each do |element| # For each array
+              array_permutations(array, index + 1).each do |x| # Permute permute permute
+                result << [element, x]
+              end
+            end
+            result
+          end
+
+          Value = Struct.new(:key, :value) do
+            def to_s
+              [key, value].join(":")
+            end
+          end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/router/configuration.rb

@@ -0,0 +1,27 @@
+require 'hashr'
+require 'logger'
+require 'evrone/ci/common/tagged_logging'
+
+module Evrone
+  module CI
+    class Router
+      class Configuration < ::Hashr
+
+        extend Hashr::EnvDefaults
+
+        self.env_namespace = 'ci_router'
+        self.raise_missing_keys = true
+
+        define amqp_url:      nil,
+               timeout:       30 * 60,
+               logger:        Common::TaggedLogging.new(Logger.new STDOUT),
+               repo_dir_name: "repo"
+
+        def timeout
+          self[:timeout].to_i
+        end
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/router/consumers/build_logs_consumer.rb

@@ -0,0 +1,15 @@
+require 'evrone/common/amqp'
+
+module Evrone
+  module CI
+    class Router
+      class BuildLogsConsumer
+
+        include Evrone::Common::AMQP::Consumer
+
+        exchange 'ci.builds.log'
+
+      end
+    end
+  end
+end

data/lib/evrone/ci/router/consumers/build_status_consumer.rb

@@ -0,0 +1,15 @@
+require 'evrone/common/amqp'
+
+module Evrone
+  module CI
+    class Router
+      class BuildStatusConsumer
+
+        include Evrone::Common::AMQP::Consumer
+
+        exchange 'ci.builds.status'
+
+      end
+    end
+  end
+end