RubyGems - right_chimp - Versions diffs - 1.0 - Mend

right_chimp 1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (38) hide show

checksums.yaml +7 -0
data/CHANGES +88 -0
data/Gemfile +4 -0
data/Gemfile.lock +49 -0
data/INSTALL +25 -0
data/LICENSE +20 -0
data/README +80 -0
data/Rakefile +18 -0
data/bin/chimp +12 -0
data/bin/chimpd +12 -0
data/chimp.gemspec +27 -0
data/lib/right_chimp/Chimp.rb +1181 -0
data/lib/right_chimp/IDManager.rb +18 -0
data/lib/right_chimp/Log.rb +34 -0
data/lib/right_chimp/daemon/ChimpDaemon.rb +416 -0
data/lib/right_chimp/daemon/ChimpDaemonClient.rb +74 -0
data/lib/right_chimp/exec/ExecArray.rb +43 -0
data/lib/right_chimp/exec/ExecCallback.rb +15 -0
data/lib/right_chimp/exec/ExecNoop.rb +9 -0
data/lib/right_chimp/exec/ExecReport.rb +57 -0
data/lib/right_chimp/exec/ExecRightScript.rb +35 -0
data/lib/right_chimp/exec/ExecSSH.rb +45 -0
data/lib/right_chimp/exec/Executor.rb +180 -0
data/lib/right_chimp/queue/ChimpQueue.rb +187 -0
data/lib/right_chimp/queue/ExecutionGroup.rb +277 -0
data/lib/right_chimp/queue/QueueWorker.rb +42 -0
data/lib/right_chimp/templates/all_jobs.erb +214 -0
data/lib/right_chimp/version.rb +3 -0
data/lib/right_chimp.rb +32 -0
data/spec/spec_chimp.rb +20 -0
data/spec/spec_chimp_commandline.rb +55 -0
data/spec/spec_chimpd.rb +33 -0
data/spec/spec_chimpd_client.rb +44 -0
data/spec/spec_execution_group.rb +135 -0
data/spec/spec_queue.rb +37 -0
data/spec/spec_queue_worker.rb +30 -0
data/spec/spec_selection.rb +33 -0
metadata +159 -0

data/lib/right_chimp/templates/all_jobs.erb ADDED Viewed

@@ -0,0 +1,214 @@
+<html><head>
+<title>chimpd</title>
+<meta http-equiv="refresh" content="5">
+<style type="text/css">
+  body {
+    color: #088A29;
+    background-color: black;
+    font-family: monospace;
+    font-size: 10px;
+    margin: 0;
+  }
+  h3 {
+    font-weight: bold;
+    border-bottom: 2px solid #088A29;
+  }
+  ol, ul {
+    list-style-type: none;
+  }
+  ol {
+    padding-left: 10px;
+  }
+  ul {
+    padding-bottom: 5px;
+    border-bottom: solid #088A29 2px;
+  }
+  div#stats {
+    background-color: black; /* #002A5C */
+    color: #088A29;
+    float: left;
+    border-right: solid 2px #088A29;
+    position: fixed;
+    height: 100%;
+    padding: 5px;
+    padding-right: 15px;
+  }
+  div#jobs {
+    position: absolute;
+    left: 375px;
+    top: 10px;
+    width: 500px;
+    padding-left: 20px;
+    color: #088A29;
+  }
+  div#groups {
+    color: #088A29;
+    background-color: black;
+  }
+  div#groups a {
+    color: #088A29;
+    background-color: black;
+  }
+  .highlight {
+    background-color: #088A29;
+    color: black;
+  }
+  td {
+    padding-right: 10px;
+  }
+  a {
+    text-decoration: underline;
+    color: #088A29;
+    font-weight: bold;
+  }
+  a:hover, ol a:hover {
+    text-decoration: underline;
+  }
+</style>
+</head><body>
+<div id="stats">
+<h1>chimpd <%=VERSION%> console</h1>
+<p style="font-size: 90%; padding-left: 10px">FEATURING CUTTING EDGE META REFRESH<br/> TECHNOLOGY
+SINCE 2012</p>
+<h3>Queue Status</h3>
+<table>
+<tr>
+  <td>Jobs running:</td>
+  <td><%= count_jobs_running %>
+</tr><tr>
+  <td>Jobs waiting:</td>
+  <td><%= count_jobs_queued %>
+</tr><tr>
+  <td>Jobs failed:</td>
+  <td><%= count_jobs_failed %>
+</tr><tr>
+  <td>Jobs completed:</td>
+  <td><%= count_jobs_done %>
+</tr>
+</table>
+<h3>Job Groups</h3>
+<div id="groups" width="90%" style="border: solid 1px #088A29; overflow-y: auto; height: 150px; background: black; color: #088A29;">
+  <ol style="margin: 0; padding: 3px;">
+  <% queue.group.values.each do |g| %>
+    <li>
+      [<%= sprintf( "%03d", g.get_jobs_by_status(:running).size) %>/<%= sprintf( "%03d", g.get_job_ids.size) %> <%= g.short_name  %>] <% if group_name == g.group_id %><span class="highlight"><%=g.group_id%></span><% else %><a href="/display/group/<%=g.group_id%>"><%= g.group_id %></a><% end %>
+    </li>
+  <% end %>
+  </ol>
+</div>
+<h3>Filter Results</h3>
+<p>[
+<% if job_filter == 'running' or job_filter == nil %>running jobs <% else %><a href="/display/running">running jobs</a><% end %> |
+<% if job_filter == 'all' %>all jobs <% else %><a href="/display/all">all jobs</a> <% end %> |
+<% if job_filter == 'done' %>complete jobs <% else %><a href="/display/done">complete jobs</a><% end %> |
+<% if job_filter == 'error' %>failed jobs <% else %><a href="/display/error">failed jobs</a><% end %>
+ ]
+</p>
+<h3>Commands</h3>
+<ol>
+  <li><a href="/job/running/cancel">cancel running jobs</a></li>
+  <li><a href="/job/error/retry">retry failed jobs </a></li>
+  <li><a href="/job/all/report">download execution report</a></li>
+</ol>
+</div>
+<div id="jobs">
+<% if group_name == nil %>
+  <h1>Job Information</h1>
+<% else %>
+  <h1>Job Group: <%=CGI::escapeHTML(group_name)%></h1>
+<% end %>
+<%
+#
+# Print out all the jobs
+#
+jobs.sort! do |a,b|
+  a.job_id <=> b.job_id
+end
+jobs.each do |j|
+  next unless j
+  status      = j.status
+  id          = j.job_id
+  server_name = CGI::escapeHTML(j.server['nickname']) if j.server
+  if j.exec
+    action      = j.exec.to_s
+    action      = j.exec['right_script']['name'] if j.exec['right_script']
+  else
+    action      = "unknown"
+  end
+  if job_filter and not (job_filter == 'all' or job_filter == 'group' )
+    next unless status == job_filter.to_sym
+  end
+  execution_time_seconds = j.get_total_exec_time
+  h = (execution_time_seconds/3600).to_i
+  m = (execution_time_seconds/60 - h*60).to_i
+  s = (execution_time_seconds - (m*60 + h*3600))
+  time = sprintf("%02d:%02d:%02d", h, m, s)
+%>
+<h2 style="float: left; color: #088A29">#<%= id %></h2>
+<%
+  color = "#088A29"
+  color = "brightgreen" if status == Executor::STATUS_RUNNING
+  color = "orange"      if status == Executor::STATUS_RETRYING
+  color = "red"         if status == Executor::STATUS_ERROR
+%>
+  <ul style="color: <%= color %>">
+    <li>Server: <b><%= server_name %></b></li>
+    <li>Group: <b><%= j.group.group_id %></b></li>
+    <li>Action: <%= action %></li>
+    <li>Status: <%= status %></li>
+    <li>Time: <%= time %></li>
+    <% if status != Executor::STATUS_RUNNING %>
+      <li>Results: <span style="color:#777"><%= j.results %></span></li>
+    <% end %>
+    <li> <span class="action">
+    <% if status == Executor::STATUS_ERROR %>
+      <a href="/job/<%=id%>/ack">ack</a> |
+      <a href="/job/<%=id%>/retry">retry</a>
+    <% elsif status == Executor::STATUS_RUNNING %>
+      <a href="/job/<%=id%>/cancel">cancel</a>
+    <% else %>
+      <a href="/job/<%=id%>/retry">requeue</a>
+    <% end %>
+    </li>
+  </ul>
+<% end %>
+</div>
+</body></html>

data/lib/right_chimp/version.rb ADDED Viewed

@@ -0,0 +1,3 @@
+module Chimp
+  VERSION = "1.0"
+end

data/lib/right_chimp.rb ADDED Viewed

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'bundler'
+require 'getoptlong'
+require 'thread'
+require 'webrick'
+require 'singleton'
+require 'base64'
+require 'rake'
+require 'rest_connection'
+require 'rest-client'
+require 'progressbar'
+require 'json'
+module Chimp
+  require 'right_chimp/version'
+  require 'right_chimp/Chimp'
+  require 'right_chimp/Log'
+  require 'right_chimp/IDManager'
+  require 'right_chimp/daemon/ChimpDaemon'
+  require 'right_chimp/daemon/ChimpDaemonClient'
+  require 'right_chimp/queue/ChimpQueue'
+  require 'right_chimp/queue/QueueWorker'
+  require 'right_chimp/queue/ExecutionGroup'
+  require 'right_chimp/exec/Executor'
+  require 'right_chimp/exec/ExecArray'
+  require 'right_chimp/exec/ExecRightScript'
+  require 'right_chimp/exec/ExecSSH'
+  require 'right_chimp/exec/ExecReport'
+  require 'right_chimp/exec/ExecNoop'
+end

data/spec/spec_chimp.rb ADDED Viewed

@@ -0,0 +1,20 @@
+#
+# Test chimpd client
+#
+$LOAD_PATH << File.dirname(__FILE__) + "/.."
+require 'lib/right_chimp.rb'
+require 'rspec'
+require 'pp'
+include Chimp
+describe Chimp::ExecRightScript do
+  it "can select servers with a tag query" do
+    c = Chimp::Chimp.new
+    c.tags = ["info:deployment=moo:localring91"]
+    c.script = "SYS DNSMadeEasy Register Addresses"
+    c.dry_run = false
+    c.run
+  end
+end

data/spec/spec_chimp_commandline.rb ADDED Viewed

@@ -0,0 +1,55 @@
+#
+# Test chimpd client
+#
+$LOAD_PATH << File.dirname(__FILE__) + "/.."
+require 'lib/right_chimp.rb'
+require 'rspec'
+require 'pp'
+include Chimp
+chimp_command = "CHIMP_TEST=true bundle exec bin/chimp --noprompt"
+test_tag      = "--tag='info:deployment=moo:localring91'"
+test_script   = "--script='SYS DNSMadeEasy Register Addresses'"
+describe Chimp::Chimp do
+  #
+  # Selection options
+  #
+  #it "should not prompt when there is no action" do
+  #  data =`CHIMP_TEST=true bundle exec bin/chimp #{test_tag}`
+  #  $?.should == 0
+  #  data.match("No actions to perform.").should != nil
+  #end
+  #
+  # RightScript execution
+  #
+  it "should run a rightscript with a tag query" do
+    system("#{chimp_command} #{test_tag} #{test_script}")
+    $?.should == 0
+  end
+  it "should run scripts on an array with a concurrency of 1 and a delay of 5" do
+    system("#{chimp_command} --array='Core91' --concurrency=1 --delay=5 --script='SYS MAIL postfix configuration'")
+    $?.should == 0
+  end
+  it "should run scripts on an array with a concurrency of 4 and a delay of 0" do
+    system("#{chimp_command} --array='Core94' --array='Core91' --concurrency=4 --delay=0 --script='SYS MAIL postfix configuration'")
+    $?.should == 0
+  end
+  #
+  # chimpd submission
+  #
+  #it "should run a rightscript with a tag query via chimpd" do
+  #  system('CHIMP_TEST=true bundle exec bin/chimpd')
+  #  system('CHIMP_TEST=true bundle exec bin/chimp --chimpd --dry-run --noprompt --tag="info:deployment=moo:localring91" --script="SYS DNSMadeEasy Register Addresses"')
+  #  $?.should == 0
+  #  system('CHIMP_TEST=true bundle exec bin/chimpd --quit')
+  #end
+end

data/spec/spec_chimpd.rb ADDED Viewed

@@ -0,0 +1,33 @@
+#
+# Test chimpd
+#
+$LOAD_PATH << File.dirname(__FILE__) + "/.."
+require 'lib/right_chimp.rb'
+require 'rspec'
+require 'pp'
+include Chimp
+uri =
+describe Chimp::ChimpDaemon do
+  before :all do
+    @c = ChimpDaemon.instance
+    @c.spawn_queue_runner
+    @c.spawn_webserver
+  end
+  #
+  # .spawn_queue_runner
+  #
+  it "should have 50 threads" do
+    ChimpQueue.instance.max_threads.should == 50
+  end
+  #
+  # .quit
+  #
+  after :all do
+    @c.quit
+  end
+end

data/spec/spec_chimpd_client.rb ADDED Viewed

@@ -0,0 +1,44 @@
+#
+# Test chimpd client
+#
+$LOAD_PATH << File.dirname(__FILE__) + "/.."
+require 'lib/right_chimp.rb'
+require 'rspec'
+require 'pp'
+include Chimp
+host = "localhost"
+port = "9055"
+describe Chimp::ChimpDaemonClient do
+  before :all do
+    @c = ChimpDaemon.new
+    @c.verbose = false
+    @c.concurrency = 3
+    @c.spawn_queue_runner
+    @c.spawn_webserver
+  end
+  after :all do
+    @c.quit if @c
+  end
+  #it "can quit a chimpd" do
+  #  response_code = ChimpDaemonClient.quit(host, port)
+  #  response_code.should == 200
+  #end
+  #
+  # .submit
+  #
+  it "can submit work to chimpd" do
+    c = Chimp::Chimp.new
+    c.tags = ["service:auditor=true"]
+    c.script = "SYS DNSMadeEasy Register Addresses"
+    c.dry_run = false
+    ChimpDaemonClient.submit(host, port, c)
+  end
+end

data/spec/spec_execution_group.rb ADDED Viewed

@@ -0,0 +1,135 @@
+#
+# Test QueueWorker
+#
+$LOAD_PATH << File.dirname(__FILE__) + "/.."
+require 'lib/right_chimp.rb'
+require 'rspec'
+include Chimp
+describe SerialExecutionGroup do
+  before :each do
+    @eg = SerialExecutionGroup.new(:test)
+  end
+  #
+  # .ready?
+  #
+  it "should be ready when it has work in its queue" do
+    @eg.ready?.should == false
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.ready?.should == true
+  end
+  it "should not be ready while a job is executing" do
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.push(ExecNoop.new(:job_id => 1))
+    @eg.get_job(0).status = Executor::STATUS_RUNNING
+    @eg.ready?.should == false
+  end
+  #
+  # .done?
+  #
+  it "should not be done when it has work in its queue" do
+    @eg.done?.should == true
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.done?.should == false
+  end
+  #
+  # .size
+  #
+  it "should be able to report the correct number of items in its queue" do
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.push(ExecNoop.new(:job_id => 1))
+    @eg.push(ExecNoop.new(:job_id => 2))
+    @eg.push(ExecNoop.new(:job_id => 3))
+    @eg.push(ExecNoop.new(:job_id => 4))
+    @eg.size.should == 5
+    @eg.shift
+    @eg.size.should == 4
+    @eg.shift
+    @eg.size.should == 3
+  end
+  #
+  # .sort!
+  #
+  it "should be able to sort its queue by server name" do
+    @eg.push(ExecNoop.new(:job_id => 1, :server => { "nickname" => "BBB", "name" => "BBB"}))
+    @eg.push(ExecNoop.new(:job_id => 1, :server => { "nickname" => "CCC", "name" => "CCC"}))
+    @eg.push(ExecNoop.new(:job_id => 0, :server => { "nickname" => "AAA", "name" => "AAA"}))
+    @eg.sort!
+  end
+  #
+  # .running?
+  #
+  it "should tell us whether it is running" do
+    @eg.running?.should == false
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.running?.should == true
+  end
+  #
+  # .requeue(id)
+  #
+  it "should requeue jobs correctly" do
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.reset!
+    @eg.size.should == 0
+    @eg.requeue(0)
+    @eg.size.should == 1
+  end
+  #
+  # .requeue_failed_jobs
+  #
+  it "should requeue all failed jobs" do
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.reset!
+    @eg.size.should == 0
+    @eg.get_job(0).status = Executor::STATUS_ERROR
+    @eg.requeue_failed_jobs!
+    @eg.size.should == 1
+  end
+  #
+  # .cancel(id)
+  #
+  it "should cancel a job" do
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.size.should == 1
+    @eg.cancel(0)
+    @eg.size.should == 0
+  end
+end
+describe ParallelExecutionGroup do
+  before :each do
+    @eg = ParallelExecutionGroup.new(:test)
+  end
+  #
+  # .ready?
+  #
+  it "should be ready when it has work in its queue" do
+    @eg.ready?.should == false
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.push(ExecNoop.new(:job_id => 1))
+    @eg.ready?.should == true
+  end
+  it "should be ready while a job is executing" do
+    @eg.push(ExecNoop.new(:job_id => 0))
+    @eg.push(ExecNoop.new(:job_id => 1))
+    @eg.get_job(0).status = Executor::STATUS_RUNNING
+    @eg.ready?.should == true
+  end
+end

data/spec/spec_queue.rb ADDED Viewed

@@ -0,0 +1,37 @@
+#
+# Test ChimpQueue
+#
+require 'lib/right_chimp.rb'
+require 'rspec'
+include Chimp
+describe ChimpQueue do
+  before :all do
+    @queue = ChimpQueue.instance
+    @queue.max_threads = 3
+    @queue.start
+  end
+  before :each do
+    @queue.reset!
+  end
+  it "should accept work" do
+    @queue.push(:default, ExecNoop.new(:job_id => 0))
+    @queue.group[:default].get_job(0).status.should == Executor::STATUS_NONE
+  end
+  it "should distribute work" do
+    @queue.push(:default, ExecNoop.new(:job_id => 0))
+    @queue.shift.status.should == Executor::STATUS_NONE
+  end
+  it "should process the queue" do
+    @queue.push(:default, ExecNoop.new(:job_id => 0))
+    @queue.wait_until_done(:default) { }
+    @queue.group[:default].get_job(0).status.should == Executor::STATUS_DONE
+  end
+end

data/spec/spec_queue_worker.rb ADDED Viewed

@@ -0,0 +1,30 @@
+#
+# Test QueueWorker
+#
+$LOAD_PATH << File.dirname(__FILE__) + "/.."
+require 'lib/right_chimp.rb'
+require 'rspec'
+include Chimp
+describe QueueWorker do
+  it "should instantiate" do
+    q = QueueWorker.new
+    q.delay = 10
+    q.retry_count = 10
+    q.never_exit = false
+    ChimpQueue.instance.group = {}
+    q.delay.should == 10
+    q.retry_count.should == 10
+    q.never_exit.should == false
+  end
+  #it "should run" do
+  #  q = QueueWorker.new
+  #  q.run
+  #end
+end

data/spec/spec_selection.rb ADDED Viewed

@@ -0,0 +1,33 @@
+#
+# Test Chimp
+#
+$LOAD_PATH << File.dirname(__FILE__) + "/.."
+require 'lib/right_chimp.rb'
+require 'rspec'
+describe Chimp::Chimp do
+  before :each do
+    @c = Chimp::Chimp.new
+    @c.quiet = true
+    @c.prompt = false
+    @c.progress = false
+    @c.interactive = false
+  end
+  it "performs a tag query" do
+    @c.tags = ['service:dataservice=true']
+    @c.run
+    @c.servers.size.should > 0
+  end
+  it "performs a deployment query" do
+    @c.deployment_names = ['moo:globalring']
+    @c.run
+    @c.servers.size.should > 0
+  end
+  it "performs an array query" do
+    @c.array_names = ['Chimp Test Array']
+    @c.run
+  end
+end