vx-router 0.2.0.pre28

data/lib/vx/router/build.rb

@@ -0,0 +1,80 @@
+require 'vx/message'
+require 'vx/common'
+
+module Vx
+  class Router
+    class Build
+
+      INITIALIED = 0
+      STARTED    = 2
+      FINISHED   = 3
+      FAILED     = 5
+
+      include Router::Helper::Logger
+      include Router::Helper::Config
+
+      attr_reader :message, :output, :output_counter
+      attr_accessor :jobs_count
+
+      def initialize(perform_build_message)
+        @output_counter = 0
+        @output         = Common::OutputBuffer.new(&method(:publish_build_log_message))
+        @message        = perform_build_message
+      end
+
+      def release
+        output.close
+      end
+
+      def to_perform_job_message(configuration, job_id)
+        script_builder   = ScriptBuilder.new self, configuration
+        job_message = Message::PerformJob.new(
+          id:               message.id,
+          name:             message.name,
+          job_id:           job_id,
+          before_script:    script_builder.to_before_script,
+          script:           script_builder.to_script,
+          after_script:     script_builder.to_after_script,
+          matrix_keys:      configuration.matrix_keys,
+        )
+        job_message
+      end
+
+      def to_build_status_message(status)
+        tm = Time.now
+        attributes = {
+          build_id:   message.id,
+          status:     status,
+          tm:         tm.to_i,
+          jobs_count: jobs_count || 0,
+        }
+        Message::BuildStatus.new attributes
+      end
+
+      def add_to_output(str)
+        output << str
+        logger.debug str.strip if logger.level == 0
+      end
+
+      def add_command_to_output(cmd)
+        add_to_output "$ #{cmd}\n"
+      end
+
+      def publish_build_log_message(str)
+        @output_counter += 1
+        log = Message::BuildLog.new(
+          build_id:  message.id,
+          tm:        output_counter,
+          log:       str
+        )
+        BuildLogsConsumer.publish log
+        log
+      end
+
+      def scm_class
+        Vx::SCM::Git
+      end
+
+    end
+  end
+end

data/lib/vx/router/cli.rb

@@ -0,0 +1,53 @@
+require 'optparse'
+require 'vx/common/amqp_setup'
+require 'vx/common/env_file'
+
+module Vx
+  class Router
+    class CLI
+
+      include Helper::Config
+      include Helper::Logger
+      include Common::EnvFile
+
+      def initialize
+        @options = {}
+        parse!
+        Router.initialize!
+      end
+
+      def run
+        trap('INT') {
+          Thread.new do
+            Vx::Common::AMQP.shutdown
+          end.join
+        }
+
+        Vx::Common::AMQP::Supervisor::Threaded.build(
+          Vx::Router::BuildsConsumer => config.workers,
+        ).run
+      end
+
+      private
+
+        def parse!
+          OptionParser.new do |opts|
+            opts.banner = "Usage: vx-router [options]"
+            opts.on("-w", "--workers NUM", "Number of workers, default 1") do |v|
+              @options[:workers] = v.to_i
+            end
+            opts.on("-c", "--config FILE", "Path to configuration file, default /etc/vexor/ci") do |v|
+              @options[:config] = v
+            end
+          end.parse!
+
+          read_env_file @options.delete(:config)
+
+          @options.each_pair do |k,v|
+            config.public_send("#{k}=", v)
+          end
+        end
+
+    end
+  end
+end

data/lib/vx/router/configuration.rb

@@ -0,0 +1,27 @@
+require 'hashr'
+require 'logger'
+require 'vx/common/tagged_logging'
+
+module Vx
+  class Router
+    class Configuration < ::Hashr
+
+      extend Hashr::EnvDefaults
+
+      self.env_namespace = 'ci_router'
+      self.raise_missing_keys = true
+
+      define amqp_url:          nil,
+             timeout:           30 * 60,
+             logger:            Common::TaggedLogging.new(Logger.new STDOUT),
+             workers:           1,
+             webdav_cache_url:  nil
+
+
+      def timeout
+        self[:timeout].to_i
+      end
+
+    end
+  end
+end

data/lib/vx/router/consumers/build_logs_consumer.rb

@@ -0,0 +1,13 @@
+require 'vx/common/amqp'
+
+module Vx
+  class Router
+    class BuildLogsConsumer
+
+      include Vx::Common::AMQP::Consumer
+
+      exchange 'vx.builds.log'
+
+    end
+  end
+end

data/lib/vx/router/consumers/build_status_consumer.rb

@@ -0,0 +1,13 @@
+require 'vx/common/amqp'
+
+module Vx
+  class Router
+    class BuildStatusConsumer
+
+      include Vx::Common::AMQP::Consumer
+
+      exchange 'vx.builds.status'
+
+    end
+  end
+end

data/lib/vx/router/consumers/builds_consumer.rb

@@ -0,0 +1,27 @@
+require 'vx/common/amqp'
+require 'vx/message'
+
+module Vx
+  class Router
+    class BuildsConsumer
+
+      include Vx::Common::AMQP::Consumer
+      include Helper::Logger
+
+      exchange 'vx.builds'
+      queue    'vx.worker.builds'
+      ack      true
+
+      model Message::PerformBuild
+
+      def perform(message)
+        logger.tagged self.class.consumer_id do
+          build = Build.new message
+          Router.new(build).perform
+        end
+        ack!
+      end
+
+    end
+  end
+end

data/lib/vx/router/consumers/job_status_consumer.rb

@@ -0,0 +1,13 @@
+require 'vx/common/amqp'
+
+module Vx
+  class Router
+    class JobStatusConsumer
+
+      include Vx::Common::AMQP::Consumer
+
+      exchange 'vx.jobs.status'
+
+    end
+  end
+end

data/lib/vx/router/consumers/jobs_consumer.rb

@@ -0,0 +1,13 @@
+require 'vx/common/amqp'
+
+module Vx
+  class Router
+    class JobsConsumer
+
+      include Vx::Common::AMQP::Consumer
+
+      exchange 'vx.jobs'
+
+    end
+  end
+end

data/lib/vx/router/ext/array.rb

@@ -0,0 +1,5 @@
+class Array
+  def extract_options!
+    self.last.is_a?(Hash) ? self.pop : {}
+  end
+end

data/lib/vx/router/ext/string.rb

@@ -0,0 +1,9 @@
+class String
+  def compact
+    gsub(/\n/, ' ').gsub(/ +/, ' ').strip
+  end
+
+  def camelize
+    split("_").each {|s| s.capitalize! }.join("")
+  end
+end

data/lib/vx/router/helper/config.rb

@@ -0,0 +1,11 @@
+module Vx
+  class Router
+    module Helper::Config
+
+      def config
+        Router.config
+      end
+
+    end
+  end
+end

data/lib/vx/router/helper/logger.rb

@@ -0,0 +1,11 @@
+module Vx
+  class Router
+    module Helper::Logger
+
+      def logger
+        Router.logger
+      end
+
+    end
+  end
+end

data/lib/vx/router/helper/trace_sh_command.rb

@@ -0,0 +1,12 @@
+module Vx
+  class Router
+
+    module Helper::TraceShCommand
+      def trace_sh_command(cmd, options = {})
+        trace = options[:trace] || cmd
+        "echo #{Shellwords.escape "$ #{trace}"}\n#{cmd}"
+      end
+    end
+
+  end
+end

data/lib/vx/router/initializers/amqp.rb

@@ -0,0 +1,3 @@
+require 'vx/common/amqp'
+
+Vx::Common::AMQP.setup(Vx::Router.logger, url: Vx::Router.config.amqp_url)

data/lib/vx/router/script_builder.rb

@@ -0,0 +1,98 @@
+require 'shellwords'
+require 'vx/common'
+
+module Vx
+  class Router
+    class ScriptBuilder
+
+      autoload :Env,          File.expand_path("../script_builder/env",           __FILE__)
+      autoload :Ruby,         File.expand_path("../script_builder/ruby",          __FILE__)
+      autoload :Script,       File.expand_path("../script_builder/script",        __FILE__)
+      autoload :Prepare,      File.expand_path("../script_builder/prepare",       __FILE__)
+      autoload :Databases,    File.expand_path("../script_builder/databases",     __FILE__)
+      autoload :WebdavCache,  File.expand_path("../script_builder/webdav_cache",  __FILE__)
+
+      include Common::Helper::Middlewares
+
+      middlewares do
+        use ScriptBuilder::WebdavCache
+        use ScriptBuilder::Env
+        use ScriptBuilder::Prepare
+        use ScriptBuilder::Databases
+        use ScriptBuilder::Ruby
+        use ScriptBuilder::Script
+      end
+
+      attr_reader :configuration, :build
+
+      def initialize(build, configuration)
+        @configuration = configuration
+        @build         = build
+      end
+
+      def to_before_script
+        a = []
+        a << "\n# init"
+        a += env.init
+
+        a << "\n# before install"
+        a += env.before_install
+
+        a << "\n# announce"
+        a += env.announce
+
+        a << "\n# install"
+        a += env.install
+
+        a << "\n# before script"
+        a += env.before_script
+        a.join("\n")
+      end
+
+      def to_after_script
+        a = []
+        a << "\n# after script"
+        a += env.after_script
+        a.join("\n")
+      end
+
+      def to_script
+        a = []
+        a << "\n# script"
+        a += env.script
+        a.join("\n")
+      end
+
+      private
+
+        def env
+          @env ||= run_middlewares(default_env) {|_| _ }
+        end
+
+        def default_env
+          OpenStruct.new(
+            # initialization, repo does not exists
+            init:            [],
+
+            # before instalation, using for system setup
+            before_install:  [],
+
+            # instalation, using for application setup
+            install:         [],
+
+            # announce software and services version
+            announce:        [],
+
+            before_script:   [],
+            script:          [],
+            after_script:    [],
+
+            configuration:   configuration,
+            build:           build,
+            cache_key:       []
+          )
+        end
+
+    end
+  end
+end

data/lib/vx/router/script_builder/databases.rb

@@ -0,0 +1,28 @@
+module Vx
+  class Router
+    class ScriptBuilder
+
+      Databases = Struct.new(:app) do
+
+        def call(env)
+          psql = []
+          psql << %{sudo -u postgres psql -q -h localhost -c "create role ci with login superuser password 'ci'" || true}
+          psql << %{sudo -u postgres psql -q -h localhost -c "create database ci" || true}
+
+          mysql = []
+
+          mysql << %{echo "CREATE USER 'ci'@'localhost';" | mysql -u root }
+          mysql << %{echo "GRANT ALL PRIVILEGES ON *.* TO 'ci'@'localhost';" | mysql -u root }
+
+          env.init.tap do |i|
+            psql.each {|c| i << c }
+            mysql.each {|c| i << c }
+          end
+
+          app.call(env)
+        end
+
+      end
+    end
+  end
+end

data/lib/vx/router/script_builder/env.rb

@@ -0,0 +1,21 @@
+module Vx
+  class Router
+    class ScriptBuilder
+
+      Env = Struct.new(:app) do
+
+        include Helper::TraceShCommand
+
+        def call(env)
+          env.init << 'export LC_ALL=en_US.UTF8'
+          env.configuration.global_env.each do |e|
+            env.init << trace_sh_command("export #{e}")
+          end
+          env.announce << trace_sh_command("env")
+          app.call(env)
+        end
+
+      end
+    end
+  end
+end