radagast 0.0.1.pre.dev → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f007912a5b7b16f956098e6deb659ea4867c7716
-  data.tar.gz: 51dd34ce29364443a50a5a40d36bb5adcb7fa9ec
+  metadata.gz: 560f233b948a123e0fb646dceeb36fcb04d59e70
+  data.tar.gz: 708db7870f7d5d310bc9075bd5a410d2e3892b1c
 SHA512:
-  metadata.gz: 0fa4bcc392cc02f97951f8987ca09b036c8eb906eaee8a5e86f77d4fb44e8c97379427838c3692d9578f241c20db36defe0cccf60117d2d290d40e0d19816559
-  data.tar.gz: 7dba764106400d4fbf9ab603763a69298af36f9348f405dd50420ae29c8d8b4a07c51998710795e07d9b65f1355f1a7ce1b42bd23cddcee588f387cc1c16363f
+  metadata.gz: 625289d4021c881f27e268fe41f202e02bddb279bb9cbdf7426444440382bc28d7f14964074c3d7811120b1e1bcfeb86ed46951d41fe378af41876017c9197b2
+  data.tar.gz: bcdd6fabb52f4dee4e4fa2aeae653ca6059554011cc41f410ced512b9d4ebb141012e98117b378b3b9be92eea7e2717bf270f3c1def0403f85857bfd8e2c70ae

data/bin/rabbit-restart.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+docker stop rabbit-radagast
+docker rm rabbit-radagast
+
+docker run -d \
+  --hostname rabbit-radagast \
+  --name rabbit-radagast \
+  -p 5672:5672 \
+  -p 15672:15672 \
+  -e RABBITMQ_ERLANG_COOKIE='radagast' \
+  rabbitmq:3
+
+sleep 3
+docker exec rabbit-radagast rabbitmq-plugins enable rabbitmq_management
+docker restart rabbit-radagast

data/bin/rabbit-stop.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+docker stop rabbit-radagast
+docker rm rabbit-radagast

data/bin/radagast

@@ -1,3 +1,13 @@
 #!/usr/bin/env ruby
 
-puts "Nothing here yet"
+require_relative '../lib/radagast/config'
+require_relative '../lib/radagast/worker'
+
+config = Radagast::Config.parse_argv
+worker = Radagast::Worker.new config
+
+begin
+  worker.start
+rescue SystemExit, Interrupt
+  worker.finish
+end

data/lib/radagast/config.rb

@@ -0,0 +1,22 @@
+require 'optparse'
+require 'logger'
+
+module Radagast
+  Config = Struct.new(:key, :rabbit, :log_level) do
+    def initialize
+      self.key = 'default'
+      self.rabbit = 'amqp://guest:guest@127.0.0.1:5672'
+      self.log_level = Logger::UNKNOWN
+    end
+
+    def self.parse_argv(argv = ARGV)
+      config = new
+      OptionParser.new do |opt|
+        opt.on('--key KEY') { |o| config.key = o }
+        opt.on('--rabbit RABBIT') { |o| config.rabbit = o }
+        opt.on('--log_level LOG_LEVEL') { |o| config.log_level = const_get(o) }
+      end.parse!(argv)
+      config
+    end
+  end
+end

data/lib/radagast/manager.rb

@@ -0,0 +1,58 @@
+require_relative 'config'
+require_relative 'rabbit_helper'
+require_relative 'result'
+
+module Radagast
+  # Dispatches tasks and aggregates the results
+  class Manager < RabbitHelper
+    def initialize(config)
+      super(
+        queue_name: "results-#{config.key}",
+        routing_key: "tasks-#{config.key}",
+        config: config
+      )
+      @callbacks = {}
+      @processed_cnt = 0
+      @published_cnt = 0
+      @all_results = []
+    end
+
+    def task(cmd, &blk)
+      logger.info 'Publishing task'
+      @published_cnt += 1
+      @callbacks[cmd] = blk
+      publish(cmd: cmd)
+    end
+
+    def process_callback(data)
+      logger.info "Result #{@processed_cnt}/#{@published_cnt} is here: #{data}"
+      result = Result.new
+      result.exit_code = data['exitstatus']
+      result.stderr = data['stderr'].strip
+      result.stdout = data['stdout'].strip
+      result.meta = { cmd: data['cmd'] }
+      @callbacks[data['cmd']].call result
+    end
+
+    def finish
+      logger.info 'Finishing manager'
+      @t.join
+      yield @all_results if block_given?
+    end
+
+    def start
+      @t = Thread.new do
+        logger.info 'Manager subscribe to results queue'
+        subscribe do |data|
+          @processed_cnt += 1
+          process_callback data
+          @all_results << data
+          if @processed_cnt == @published_cnt
+            logger.info "All #{@processed_cnt} messages have been processed"
+            cleanup
+          end
+        end
+      end
+    end
+  end
+end

data/lib/radagast/rabbit_helper.rb

@@ -0,0 +1,38 @@
+require 'bunny'
+require 'json'
+require 'logger'
+
+module Radagast
+  # Wraps RabbitMQ internals to provide simple API to Worker and Manager
+  class RabbitHelper
+    attr_reader :logger
+
+    def initialize(queue_name:, routing_key:, config:)
+      @rabbit = Bunny.new config.rabbit
+      @rabbit.start
+      @routing_key = routing_key
+      @channel = @rabbit.create_channel
+      @exchange = @channel.default_exchange
+      @queue = @channel.queue(queue_name, auto_delete: true)
+      @logger = Logger.new(STDOUT)
+      @logger.level = config.log_level
+    end
+
+    def cleanup
+      @rabbit.close
+    end
+
+    def publish(data)
+      logger.info "publishing #{data}"
+      @exchange.publish(JSON.generate(data), routing_key: @routing_key)
+    end
+
+    def subscribe
+      @queue.subscribe(block: true) do |_delivery_info, _metadata, payload|
+        data = JSON.parse(payload)
+        logger.info "processing #{data}"
+        yield data
+      end
+    end
+  end
+end

data/lib/radagast/result.rb

@@ -0,0 +1,3 @@
+module Radagast
+  Result = Struct.new(:exit_code, :stdout, :stderr, :meta)
+end

data/lib/radagast/worker.rb

@@ -0,0 +1,38 @@
+require 'json'
+require 'open3'
+
+require_relative 'config'
+require_relative 'rabbit_helper'
+
+module Radagast
+  # Fetches the task, processes it and returns the result
+  class Worker < RabbitHelper
+    def initialize(config)
+      super(
+        queue_name: "tasks-#{config.key}",
+        routing_key: "results-#{config.key}",
+        config: config
+      )
+    end
+
+    def start
+      logger.info 'Worker setup, subscribing to tasks queue'
+      subscribe do |data|
+        stdout, stderr, status = Open3.capture3(data['cmd'])
+        response = {
+          cmd: data['cmd'],
+          meta: data['meta'],
+          stdout: stdout,
+          stderr: stderr,
+          exitstatus: status.exitstatus
+        }
+        publish(response)
+      end
+    end
+
+    def finish
+      logger.info 'Finishing worker'
+      cleanup
+    end
+  end
+end

data/lib/radagast.rb

@@ -0,0 +1,4 @@
+require_relative 'radagast/config'
+require_relative 'radagast/manager'
+require_relative 'radagast/worker'
+require_relative 'radagast/result'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: radagast
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre.dev
+  version: 0.0.1
 platform: ruby
 authors:
 - Bartek Wilczek
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-16 00:00:00.000000000 Z
+date: 2017-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -24,15 +24,38 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.6'
-description: Use docker swarm and rabbitmq to distribute tasks. Designed for faster
-  test execution.
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+description: |-
+  Use docker swarm and rabbitmq to distribute tasks. \
+      Designed for faster test execution.
 email: bwilczek@gmail.com
 executables:
 - radagast
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/rabbit-restart.sh
+- bin/rabbit-stop.sh
 - bin/radagast
+- lib/radagast.rb
+- lib/radagast/config.rb
+- lib/radagast/manager.rb
+- lib/radagast/rabbit_helper.rb
+- lib/radagast/result.rb
+- lib/radagast/worker.rb
 homepage: https://github.com/bwilczek/radagast
 licenses:
 - MIT
@@ -48,9 +71,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.8