abricot 0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZGMzNGZkOWVmOWM5Y2U0MWZhYjYxNjliNWVmMmI0YmMxNjhlNDg0Ng==
+  data.tar.gz: !binary |-
+    ZDYyYWI1ZjRmNDkwNzI4NDdlM2IyNzliMmQ2MTI5YTA3M2Y5NTM3Mg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZTVlZTU2MTg1MTZjN2E1MzU2YjYxZDA1Y2YxNTUyZDdiMDZlYjNjMzkwZTkw
+    MTBkNjFiY2E2NTIyNTcwNzE1MzZlMzJiZTJmNjY1ZTg0NGRhNGRkMGZmZThm
+    ZmRiZWUxZmI5Y2ZmZWIzZTQ4ZmU1NjljMDBlYzQyZDgxMjdkYzY=
+  data.tar.gz: !binary |-
+    ZjBhMmY5NmQ3MDk1OTJlNGM5OTUzYTA0YjNkNmE3NTlkNGM3MWE2ZWI5NzRh
+    ZDQ5NWQ2YmM3YzAwY2I3OGM1MjYyYjhjODQ5Nzc1YzA0N2VlMGZhZWQ2N2Rm
+    ZTRhZDZjOTY0NGM2Yzg5ZDZlNDRjN2RmM2UxZTllYzE5ZDRjOGM=

data/README.md

@@ -0,0 +1,44 @@
+Abricot
+=======
+
+Fast cloud command dispatcher tool with Redis pub/sub.
+
+Abricot was built to run benchmarks on a large amount of machines.
+
+How to use
+-----------
+
+On each slave:
+
+```
+$ abricot listen
+```
+
+On the master:
+
+```
+$ abricot exec echo hello
+```
+
+### Specifying the redis server
+
+Both the slaves and master accept the `--redis` argument (default is localhost).
+Example:
+
+```
+$ abricot listen --redis redis://redis-server:port/db
+```
+
+### Running a job
+
+To run a job, you may pass several arguments:
+
+* `-c CMD`: Run your command through bash
+* `-f FILE`: Run a script file, which will be uploaded. You may use arbitrary
+  scripts with `#!...` in the header.
+* `-n NUM_WORKERS`: Run the job on exactly `NUM_WORKERS`.
+
+License
+--------
+
+Abricot is released under LGPLv3

data/bin/abricot

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'abricot'
+  require 'abricot/cli'
+  Abricot::CLI.start(ARGV)
+rescue => e
+  $stderr.puts "#{e.class}: #{e.message}"
+  $stderr.puts '-' * 80
+  $stderr.puts e.backtrace.join("\n")
+  exit 1
+end

data/lib/abricot.rb

@@ -0,0 +1,5 @@
+require 'redis'
+require 'json'
+
+module Abricot
+end

data/lib/abricot/cli.rb

@@ -0,0 +1,21 @@
+require 'thor'
+
+class Abricot::CLI < Thor
+  desc "worker", "Start listening for orders"
+  option :redis
+  def listen
+    require 'abricot/worker'
+    Abricot::Worker.new(options).listen
+  end
+
+  desc "exec ARGS...", "Run a command on slaves"
+  option :redis,       :type => :string
+  option :cmd,         :type => :boolean, :aliases => :c
+  option :file,        :type => :string,  :aliases => :f
+  option :num_workers, :type => :numeric, :aliases => :n
+  option :id,          :type => :string
+  def exec(*args)
+    require 'abricot/master'
+    Abricot::Master.new(options).exec(args, options)
+  end
+end

data/lib/abricot/master.rb

@@ -0,0 +1,117 @@
+require 'ruby-progressbar'
+
+class Abricot::Master
+  class JobFailure < RuntimeError; end
+  class NotEnoughSlaves < RuntimeError; end
+
+  attr_accessor :redis, :redis_sub
+
+  def initialize(options={})
+    @redis = Redis.new(:url => options[:redis])
+    @redis_sub = Redis.new(:url => options[:redis])
+  end
+
+  def num_workers_available
+    redis.pubsub('numsub', 'abricot:slave_control').last.to_i
+  end
+
+  def exec(args, options={})
+    options = options.dup
+    options['id'] ||= (0...10).map { (65 + rand(26)).chr }.join
+
+    trap(:INT) { puts; send_kill(options['id']) }
+    _exec(args, options)
+  end
+
+  def _exec(args, options={})
+    script = File.read(options['file']) if options['file']
+    script ||= args.join(" ") if options['cmd']
+
+    if script
+      lines = script.lines.to_a
+      unless lines.first =~ /^#!/
+        lines = ['#!/bin/bash'] + lines
+        script = lines.join("\n")
+      end
+      payload = {:type => 'script', :script => script}
+    else
+      payload = {:type => 'exec', :args => args}
+    end
+
+    num_workers = options['num_workers']
+    if num_workers
+      if num_workers_available < num_workers
+        raise NotEnoughSlaves.new("found #{num_workers_available} slaves, but wanted #{num_workers}")
+      end
+    else
+      num_workers = num_workers_available
+    end
+
+    id = options['id']
+    payload[:id] = id
+    payload[:num_workers] = num_workers
+
+    num_worker_start = 0
+    num_worker_done = 0
+
+    format = '%t |%b>%i| %c/%C'
+    start_pb = ProgressBar.create(:format => format,  :title => 'start', :total => num_workers)
+    done_pb = nil
+
+    status = nil
+
+    redis_sub.subscribe("abricot:job:#{id}:progress") do |on|
+      on.subscribe do
+        redis.set("abricot:job:#{id}:num_workers", 0)
+        redis.expire("abricot:job:#{id}:num_workers", 600)
+
+        redis.publish('abricot:slave_control', payload.to_json)
+      end
+
+      on.message do |channel, message|
+        msg = JSON.parse(message)
+        case msg['type']
+        when 'start' then
+          num_worker_start += 1
+          start_pb.progress = num_worker_start if start_pb
+          if num_worker_start == num_workers
+            start_pb.finish
+            start_pb = nil
+            done_pb = ProgressBar.create(:format => format, :title => 'done ', :total => num_workers)
+          end
+        when 'done' then
+          if status != :fail
+            if msg['status'] != 0
+              start_pb = done_pb = nil
+              STDERR.puts
+              STDERR.puts "-" * 80
+              STDERR.puts "JOB FAILURE:"
+              STDERR.puts msg['output']
+              STDERR.puts "-" * 80
+              redis_sub.unsubscribe("abricot:job:#{id}:progress")
+              status = :fail
+            else
+              num_worker_done += 1
+              done_pb.progress = num_worker_done if done_pb
+              if num_worker_done == num_workers
+                done_pb.finish if done_pb
+                redis_sub.unsubscribe("abricot:job:#{id}:progress")
+                status = :success
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def send_kill(id)
+    Thread.new { redis.publish('abricot:slave_control', {'type' => 'kill', 'id' => id.to_s}.to_json) }.join
+    exit
+  end
+
+  def send_kill_all
+    Thread.new { redis.publish('abricot:slave_control', {'type' => 'killall'}.to_json) }.join
+    exit
+  end
+end

data/lib/abricot/worker.rb

@@ -0,0 +1,124 @@
+require 'tempfile'
+
+class Abricot::Worker
+  attr_accessor :redis, :redis_sub
+  attr_accessor :runner_threads
+
+
+  def initialize(options={})
+    @redis = Redis.new(:url => options[:redis])
+    @redis_sub = Redis.new(:url => options[:redis])
+    @runner_threads = {}
+  end
+
+  def listen
+    trap(:INT) { puts; exit }
+
+    redis_sub.subscribe('abricot:slave_control') do |on|
+      on.message do |channel, message|
+        msg = JSON.parse(message)
+        id = msg['id']
+        case msg['type']
+        when 'killall' then kill_all_jobs
+        when 'kill'    then kill_job(id)
+        else
+          if redis.incr("abricot:job:#{id}:num_workers") <= msg['num_workers']
+            kill_job(id)
+            @runner_threads[id] = Thread.new { run(msg) }
+          end
+        end
+      end
+    end
+  end
+
+  def kill_all_jobs
+    runner_threads.keys.each { |k| kill_job(k) }
+  end
+
+  def kill_job(id)
+    if thread = @runner_threads.delete(id.to_s)
+      thread.join unless thread == Thread.current
+    end
+  end
+
+  def run(options)
+    id = options['id'].to_s
+
+    STDERR.puts "-" * 80
+    STDERR.puts "Running job: #{options}"
+    STDERR.puts "-" * 80
+
+    redis.publish("abricot:job:#{id}:progress", {'type' => 'start'}.to_json)
+
+    output, status = case options['type']
+    when 'exec' then exec_and_capture(id, *options['args'])
+    when 'script' then
+      file = Tempfile.new('abricot-')
+      begin
+        file.write(options['script'])
+        file.chmod(0755)
+        file.close
+        exec_and_capture(id, file.path)
+      ensure
+        file.delete
+      end
+    else raise "Unknown type"
+    end
+
+    return unless status
+
+    STDERR.puts output
+    STDERR.puts "exited with #{status}"
+    STDERR.puts "-" * 80
+    STDERR.puts ""
+
+    payload = {'type' => 'done'}
+    payload['status'] = status
+    payload['output'] = output if status != 0
+    redis.publish("abricot:job:#{id}:progress", payload.to_json)
+
+    kill_job(id)
+  rescue Exception => e
+    STDERR.puts e
+  end
+
+  def exec_and_capture(job_id, *args)
+    args = args.map(&:to_s)
+    IO.popen('-') do |io|
+      unless io
+        trap("SIGINT", "IGNORE")
+        trap("SIGTERM", "IGNORE")
+        $stderr.reopen($stdout)
+        begin
+          exec(*args)
+        rescue Exception => e
+          STDERR.puts "#{e} while running #{args}"
+        end
+        exit! 1
+      end
+
+      status = nil
+
+      output = []
+      loop do
+        unless @runner_threads[job_id]
+          STDERR.puts "WARNING: Killing Running Job!"
+          Process.kill('KILL', io.pid)
+          break
+        end
+
+        if IO.select([io], [], [], 0.1)
+          buffer = io.read
+          break if buffer.empty?
+          output << buffer
+        end
+      end
+
+      _, status = Process.waitpid2(io.pid)
+      [output.join, status.exitstatus]
+    end
+  end
+end
+
+# pubsub numsub <= num_worker
+# command en cours -> non

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: abricot
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Nicolas Viennot
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  prerelease: false
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.7
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.7
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  prerelease: false
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  prerelease: false
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  prerelease: false
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+description: Fast cloud command dispatcher tool with Redis pub/sub
+email:
+- nicolas@viennot.biz
+executables:
+- abricot
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/abricot.rb
+- lib/abricot/cli.rb
+- lib/abricot/master.rb
+- lib/abricot/worker.rb
+- bin/abricot
+- README.md
+homepage: https://github.com/nviennot/abricot
+licenses:
+- LGPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.7
+signing_key: 
+specification_version: 4
+summary: Fast cloud command dispatcher tool with Redis pub/sub
+test_files: []
+has_rdoc: false