right_chimp 2.1.22.2 → 2.1.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c22b012ce9b5f4ec956a9e8bc3d8b1fe0660d0b0
-  data.tar.gz: 1c51f41cc9575e6134d9a51519634e70ae149963
+  metadata.gz: 3b8353c389aaf578df02f964a87778706144628b
+  data.tar.gz: 42bae4bf2b3e4246cf5ee31fea1709e42a3be2d7
 SHA512:
-  metadata.gz: 7983b6254f8dac499342b10bfc9edd60149b6b53f8e8e81aac24c0af86999e72294274c7e9c52d89d028b6238c505e6fbfb55ab82a5d4033df3be4d94149386c
-  data.tar.gz: e6c4ca7bc4ae4336fad48b4a4f4d5805bf099edcf2049fdb85fedbfd55995331b5b269bc34c5f192fee8e85fd562b8ddc2c83a9dc5cd78eb509738ae5ea12a97
+  metadata.gz: a60272b66e310894fdbbc393b5010dd12eeb9380c65aeeea84c5d152390edbf8a7f7a5330aea4188c2e8a4dd747b4502656278cac1ba4e29d71d22f6c3956559
+  data.tar.gz: 8edc1310533df69e7a192019c77aca13a799f529bc5b07de5eb0f7ea3af9d02a7b46cbb8efc796ab4b3f8fcf380a54f73252eacf1b4a7d704da1f61e3420ee47

data/CHANGES

@@ -1,190 +1,199 @@
-Version 0.2.0
+Version 2.1.24
 --------------
-* Initial non-alpha release
+* Feature: We now ensure jobs are added and tracked into the processing status before sending a 200 back to
+the client. As a result chimp wait until will never skip a group because of slow API performance.
 
-Version 0.2.2
--------------
-* Feature: you can now specify a --script via its URI
-* Feature: initial release of chimpd
+Version 2.1.23
+--------------
+* Feature: better chimp wait until done messaging.
 
-Version 0.2.3
--------------
-* Bugfix: chimpd job id management was broken in some places
+Version 2.1.22.2
+--------------
+* Bugfix: don't consume CPU in 'pause' mode
 
-Version 0.3.0
--------------
-* Feature: New --dry-run option
-* Feature: Array instance execution now includes error checking (requires API 0.1)
-* Change:  The work queue is now sorted by server name initially
-* Change:  Refactored chimp and chimpd scripts into Real Ruby Objects
+Version 2.1.22.1
+--------------
+* Bugfix: don't show retry/abort/ignore prompt in 'pause' mode
 
-Version 0.4.0
--------------
-* Feature: "chimpd --exit" can be used to stop a chimpd server
-* Feature: chimp will now display a list of scripts to choose from
+Version 2.1.22
+--------------
+* Bugfix: use 'pause' behaviour if there are no contents in stdin
 
-Version 0.4.1
--------------
-* Feature: specify --script via numerical script id instead of full href
-* Bugfix:  fixed --script href bug
-* Bugfix:  fix chimpd submission from chimp exit code
+Version 2.1.21
+--------------
+* Bugfix: Pull the correct path whenever loading up the erb template in webui.
 
-Version 0.4.2
+Version 2.1.20
+--------------
+* Bugfix: We now correctly report the correct number of tasks waiting, in the json interface
+
+Version 2.1.19.1
+--------------
+Reverted bad release
+
+Version 2.1.16
+--------------
+* Bugfix: chimp --wait-until-done now honors the noprompt flag.
+
+Version 2.1.15
+---------------
+* Feature: We now are able to query a subset of job_uuids and total stats format.
+
+Version 2.1.13
+--------------
+* Feature: Web UI now shows job_uuid
+* Feature: Changed error reporting format
+* Feature: One can now associate a "note" to a job by using --job-notes="foo"
+
+Version 2.1.4
 -------------
-* Bugfix:  fixed chimpd submission
-* Feature: Improved ability to ignore runaway tasks in chimpd
+* Bugfix: Fixed UI issues casued by long job names
 
-Version 0.4.3
+Version 2.1.3
 -------------
-* Feature: If ServerArrays cannot be found, print a warning instead of failing
+* Feature: chimpd-wait-until now dsiplays the group name it is waiting on every 120secs
 
-Version 0.4.5
+Version 2.1
+-----------
+* 'Feature': Updated to run with Ruby 2.2.2, with minor version increment accoridngly.
+
+Version 2.0
+-----------
+* Feature: Deprecated all Rightscale API1.0 calls in favor of Rightscale API1.5 and API1.6 calls.
+
+Version 1.1.3
 -------------
-* Feature: add support for setting inputs when executing RightScripts (--input)
-* Feature: add ignore-errors command line flag
-* Feature: now using the USER environment variable for ssh jobs
+* Bugfix: Fix issue where queued/held jobs move into a waiting state and do nothing
 
-Version 0.4.6
+Version 1.1.2
 -------------
-* Bugfix: add --ssh-user flag to replace buggy USER env. variable feature
+* Bugfix: Fix issue where ssh commands were non-functional
+* Bugfix: Fix queueing of 'held' jobs to pull no more than one job from the queue
 
-Version 0.4.8
+Version 1.1.1
 -------------
-* Bugfix: chimpd --verbose
+* Bugfix: Enable timeouts on server arrays
 
-Version 0.4.9
+Version 1.1.0
 -------------
-* Feature: log actions to file
-* Feature: retry/cancel all button added to chimpd
+* Feature: Add a job state called 'holding' where jobs can be held until the user queues them
 
-Version 0.5.0
+Version 1.0.9
 -------------
-* Feature: rewrite of queue system to support task grouping
-* Feature: detailed execution time reporting
-* Feature: new chimpd web interface skin
-* Feature: retry failed and cancel running
-* Feature: improved logging in chimpd
+* Feature: Improved support for running chimp commands on any chimp daemon port
+* Feature: Added inverted hot dog stand theme
 
-Version 0.5.1
+Version 1.0.5
 -------------
-* Feature: move run_script_on_instances from API 0.1 to API 1.0 (rest_connection 06df205d7f)
+* Feature: improved error handling and logging when submitting to chimpd
 
-Version 0.5.2
+Version 1.0.4
 -------------
-* Feature: --noverify flag disables interactive verification of errors
+* Feature: chimp output when using chimpd is better behaved
+* Feature: chimpd client now log errors better, retries on failure
+* Feature: chimp --chimpd-wait-until-done now returns immediately during a dry run
+* Feature: chimp --chimpd-wait-until-done now blocks until a group has completed > 1 job
 
-Version 0.5.3
+Version 1.0.3
 -------------
-* Bugfix: non-operational array instances are no longer operated upon
+* Bugfix: fix race condition in chimpd serial groups
 
-Version 0.5.4
+Version 1.0.2
 -------------
-* Bugfix:  reduce log output during reporting
-* Feature: The --report flag now sets quiet mode and concurrency=5 by default
+* Feature: Add support for specifying inputs on the command line when working with arrays
 
-Version 0.5.5
+Version 1.0.1
 -------------
-* Bugfix: --limit flag wasn't sorting servers before executing
+* Add missing ERB template for chimpd
 
 Version 1.0
 -----------
 * Initial open source release
 
-Version 1.0.1
+Version 0.5.5
 -------------
-* Add missing ERB template for chimpd
+* Bugfix: --limit flag wasn't sorting servers before executing
 
-Version 1.0.2
+Version 0.5.4
 -------------
-* Feature: Add support for specifying inputs on the command line when working with arrays
+* Feature: The --report flag now sets quiet mode and concurrency=5 by default
+* Bugfix:  reduce log output during reporting
 
-Version 1.0.3
+Version 0.5.3
 -------------
-* Bugfix: fix race condition in chimpd serial groups
+* Bugfix: non-operational array instances are no longer operated upon
 
-Version 1.0.4
+Version 0.5.2
 -------------
-* Feature: chimp --chimpd-wait-until-done now blocks until a group has completed > 1 job
-* Feature: chimp --chimpd-wait-until-done now returns immediately during a dry run
-* Feature: chimpd client now log errors better, retries on failure
-* Feature: chimp output when using chimpd is better behaved
+* Feature: --noverify flag disables interactive verification of errors
 
-Version 1.0.5
+Version 0.5.1
 -------------
-* Feature: improved error handling and logging when submitting to chimpd
+* Feature: move run_script_on_instances from API 0.1 to API 1.0 (rest_connection 06df205d7f)
 
-Version 1.0.9
+Version 0.5.0
 -------------
-* Feature: Improved support for running chimp commands on any chimp daemon port
-* Feature: Added inverted hot dog stand theme
+* Feature: improved logging in chimpd
+* Feature: retry failed and cancel running
+* Feature: new chimpd web interface skin
+* Feature: detailed execution time reporting
+* Feature: rewrite of queue system to support task grouping
 
-Version 1.1.0
+Version 0.4.9
 -------------
-* Feature: Add a job state called 'holding' where jobs can be held until the user queues them
+* Feature: retry/cancel all button added to chimpd
+* Feature: log actions to file
 
-Version 1.1.1
+Version 0.4.8
 -------------
-* Bugfix: Enable timeouts on server arrays
+* Bugfix: chimpd --verbose
 
-Version 1.1.2
+Version 0.4.6
 -------------
-* Bugfix: Fix queueing of 'held' jobs to pull no more than one job from the queue
-* Bugfix: Fix issue where ssh commands were non-functional
+* Bugfix: add --ssh-user flag to replace buggy USER env. variable feature
 
-Version 1.1.3
+Version 0.4.5
 -------------
-* Bugfix: Fix issue where queued/held jobs move into a waiting state and do nothing
-
-Version 2.0
------------
-* Feature: Deprecated all Rightscale API1.0 calls in favor of Rightscale API1.5 and API1.6 calls.
-
-Version 2.1
------------
-* 'Feature': Updated to run with Ruby 2.2.2, with minor version increment accoridngly.
+* Feature: now using the USER environment variable for ssh jobs
+* Feature: add ignore-errors command line flag
+* Feature: add support for setting inputs when executing RightScripts (--input)
 
-Version 2.1.3
+Version 0.4.3
 -------------
-* Feature: chimpd-wait-until now dsiplays the group name it is waiting on every 120secs
+* Feature: If ServerArrays cannot be found, print a warning instead of failing
 
-Version 2.1.4
+Version 0.4.2
 -------------
-* Bugfix: Fixed UI issues casued by long job names
-
-Version 2.1.13
---------------
-* Feature: Changed error reporting format
-* Feature: Web UI now shows job_uuid
-* Feature: One can now associate a "note" to a job by using --job-notes="foo"
-
-Version 2.1.15
----------------
-* Feature: We now are able to query a subset of job_uuids and total stats format.
-
-Version 2.1.16
---------------
-* Bugfix: chimp --wait-until-done now honors the noprompt flag.
+* Feature: Improved ability to ignore runaway tasks in chimpd
+* Bugfix:  fixed chimpd submission
 
-Version 2.1.19.1
---------------
-Reverted bad release
+Version 0.4.1
+-------------
+* Bugfix:  fix chimpd submission from chimp exit code
+* Bugfix:  fixed --script href bug
+* Feature: specify --script via numerical script id instead of full href
 
-Version 2.1.20
---------------
-* Bugfix: We now correctly report the correct number of tasks waiting, in the json interface
+Version 0.4.0
+-------------
+* Feature: chimp will now display a list of scripts to choose from
+* Feature: "chimpd --exit" can be used to stop a chimpd server
 
-Version 2.1.21
---------------
-* Bugfix: Pull the correct path whenever loading up the erb template in webui.
+Version 0.3.0
+-------------
+* Change:  Refactored chimp and chimpd scripts into Real Ruby Objects
+* Change:  The work queue is now sorted by server name initially
+* Feature: Array instance execution now includes error checking (requires API 0.1)
+* Feature: New --dry-run option
 
-Version 2.1.22
---------------
-* Bugfix: use 'pause' behaviour if there are no contents in stdin
+Version 0.2.3
+-------------
+* Bugfix: chimpd job id management was broken in some places
 
-Version 2.1.22.1
---------------
-* Bugfix: don't show retry/abort/ignore prompt in 'pause' mode
+Version 0.2.2
+-------------
+* Feature: initial release of chimpd
+* Feature: you can now specify a --script via its URI
 
-Version 2.1.22.2
---------------
-* Bugfix: don't consume CPU in 'pause' mode
+Version 0.2.0
+-------------
+* Initial non-alpha release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    right_chimp (2.1.22.2)
+    right_chimp (2.1.24)
       highline (~> 1.7.2)
       nokogiri (~> 1.6.7.1)
       progressbar (~> 0.11.0)
@@ -16,7 +16,7 @@ GEM
     coderay (1.1.1)
     debug_inspector (0.0.2)
     diff-lcs (1.1.3)
-    domain_name (0.5.20160615)
+    domain_name (0.5.20160826)
       unf (>= 0.0.5, < 1.0.0)
     highline (1.7.8)
     http-cookie (1.0.2)

data/lib/right_chimp/Chimp.rb

@@ -1103,36 +1103,40 @@ module Chimp
     # This is used by chimpd, when processing a task.
     #
     def process
+
       Chimp.set_failure(false)
-      Chimp.set_job_uuid(self.job_uuid)
+      Chimp.set_job_uuid(job_uuid)
 
-      Log.debug "[#{Chimp.get_job_uuid}] Processing task"
+      Log.debug "[#{job_uuid}] Processing task"
       # Add to our "processing" counter
 
-      Log.debug "[#{Chimp.get_job_uuid}] Trying to get array_info" unless Chimp.failure
+      Log.debug "[#{job_uuid}] Trying to get array_info" unless Chimp.failure
       get_array_info unless Chimp.failure
 
-      Log.debug "[#{Chimp.get_job_uuid}] Trying to get server_info" unless Chimp.failure
+      Log.debug "[#{job_uuid}] Trying to get server_info" unless Chimp.failure
       get_server_info unless Chimp.failure
 
-      Log.debug "[#{Chimp.get_job_uuid}] Trying to get template_info" unless Chimp.failure
+      Log.debug "[#{job_uuid}] Trying to get template_info" unless Chimp.failure
       get_template_info unless Chimp.failure
 
-      Log.debug "[#{Chimp.get_job_uuid}] Trying to get executable_info" unless Chimp.failure
+      Log.debug "[#{job_uuid}] Trying to get executable_info" unless Chimp.failure
       get_executable_info unless Chimp.failure
 
       # All elements of task have been processed
       ChimpDaemon.instance.semaphore.synchronize do
+        # remove from the processing queue
+        Log.debug 'Removing job: ' + job_uuid + ' from the processing queue for group: ' + group.to_s
+        ChimpDaemon.instance.queue.processing[group].delete(job_uuid)
         ChimpDaemon.instance.proc_counter -= 1
       end
 
       if Chimp.failure
 
-        Log.error "##################################################"
-        Log.error "["+self.job_uuid+"] API CALL FAILED FOR:"
-        Log.error "["+self.job_uuid+"] chimp #{@cli_args} "
-        Log.error "["+self.job_uuid+"] Run manually!"
-        Log.error "##################################################"
+        Log.error '##################################################'
+        Log.error '[' + job_uuid + '] API CALL FAILED FOR:'
+        Log.error '[' + job_uuid + "] chimp #{@cli_args} "
+        Log.error '[' + job_uuid + '] Run manually!'
+        Log.error '##################################################'
         return []
       else
         if @servers.first.nil? or @executable.nil?
@@ -1183,17 +1187,24 @@ module Chimp
               all = ChimpDaemonClient.retrieve_group_info(@chimpd_host, @chimpd_port, @group, :all)
             rescue RestClient::ResourceNotFound
               sleep 5
-              $stdout.print "\nINFO: Waiting on group #{@group}"
+              $stdout.print "\nINFO: Waiting on group #{@group} to populate"
               retry
             end
 
             ChimpQueue.instance.create_group(@group)
             ChimpQueue[@group].set_jobs(all)
 
-            break if ChimpQueue[@group].done?
+            if ChimpQueue[@group].done?
+              Log.debug 'Group ' + @group.to_s + ' is completed'
+              jobs = ChimpQueue[@group].size
+              $stdout.print "\nINFO: Group #{@group} has completed (#{jobs} jobs)"
+              break
+            else
+              Log.debug 'Group ' + @group.to_s + ' is not done.'
+            end
 
             if sleeping_counter % 240 == 0
-              $stdout.print "\nWaiting for group #{@group}" unless sleeping_counter == 0
+              $stdout.print "\n(Still) Waiting for group #{@group}" unless sleeping_counter == 0
             end
             $stdout.print "."
             $stdout.flush
@@ -1205,11 +1216,11 @@ module Chimp
           # If verify_results returns false, then ask chimpd to requeue all failed jobs.
           #
           case verify_results(@group)
-          when "continue"
+          when 'continue'
             break
-          when "retry"
+          when 'retry'
             ChimpDaemonClient.retry_group(@chimpd_host, @chimpd_port, @group)
-          when "pause"
+          when 'pause'
             @paused = true
             #stuck in this loop until action is taken
           end

data/lib/right_chimp/daemon/ChimpDaemon.rb

@@ -250,11 +250,9 @@ module Chimp
       (1..n).each do |n|
         @threads ||=[]
         @threads << Thread.new {
-
           while true
             begin
 
-
               queued_request = @chimp_queue.pop
               group = queued_request.group
               queued_request.interactive = false
@@ -343,14 +341,38 @@ module Chimp
       # /group/<name>/<status>
       #
       def do_GET(req, resp)
-        jobs = {}
+        jobs = []
+        Log.debug 'get group info'
 
         group_name = req.request_uri.path.split('/')[-2]
         filter     = req.request_uri.path.split('/')[-1]
+        # Quickly check processing jobs just in case
+        # Instance the entire queue
+        q = ChimpQueue.instance
+        g2 = q.processing[group_name.to_sym]
+
+        if g2
+          Log.debug 'Found processing job(s): ' + g2.inspect
+        else
+          Log.debug 'not found any processing jobs for that group: ' + g2.inspect
+        end
+
         g = ChimpQueue[group_name.to_sym]
-        raise WEBrick::HTTPStatus::NotFound, "Group not found" unless g
-        jobs = g.get_jobs_by_status(filter)
+        raise WEBrick::HTTPStatus::NotFound, 'Group not found' unless g || g2
+        jobs = g.get_jobs_by_status(filter) if g
+
+        # If there are processing jobs, add them as dummy executions.
+        if g2
+          Log.debug 'Group: ' + group_name + ' is processing:'
+          g2.each do |job|
+            Log.debug 'Job: ' + job
+            j = ExecRightScript.new(group: group_name, job_uuid: job)
+            jobs.push j
+          end
+        end
+
         resp.body = jobs.to_yaml
+
         raise WEBrick::HTTPStatus::OK
       end
 
@@ -407,19 +429,28 @@ module Chimp
         # Ask chimpd to process a Chimp object directly
         #
         if verb == 'process' or verb == 'add'
+          # comment the next line to GET STUCK IN PROCESSING forever
           ChimpDaemon.instance.chimp_queue.push payload
           ChimpDaemon.instance.semaphore.synchronize do
-            ChimpDaemon.instance.proc_counter +=1
+            # While we are at it, we will store these processing jobs to prevent issues in the event
+            # of a very slow API response.
+            Log.debug 'Adding job: ' + job_uuid + ' to the processing queue for group: ' + group.to_s
+            q.processing[payload.group] = [] if q.processing[payload.group].nil?
+            q.processing[payload.group].push(payload.job_uuid)
+
+            ChimpDaemon.instance.proc_counter += 1
           end
-          Log.debug "Tasks in the processing queue: #{ChimpDaemon.instance.proc_counter.to_s}"
-          id = 0
+
+          Log.debug 'Tasks in the processing queue:' + ChimpDaemon.instance.proc_counter.to_s
+          Log.debug 'Pocessing:'
+          Log.debug q.processing.inspect
         elsif verb == 'update'
-          puts "UPDATE"
+          puts 'UPDATE'
           q.get_job(job_id).status = payload.status
         end
 
         resp.body = {
-          'job_uuid' => job_uuid ,
+          'job_uuid' => job_uuid,
           'id' => job_id
         }.to_yaml
 
@@ -429,7 +460,7 @@ module Chimp
       def do_GET(req, resp)
         id          = self.get_id(req)
         verb        = self.get_verb(req)
-        job_results = "OK"
+        job_results = 'OK'
         queue       = ChimpQueue.instance
 
         #

data/lib/right_chimp/daemon/ChimpDaemonClient.rb

@@ -23,10 +23,10 @@ module Chimp
           id = YAML::load(response.body)['id']
           #ID changes upon execution, not upon submission.
           job_uuid = YAML::load(response.body)['job_uuid']
-          puts "["+job_uuid+"]"
+          puts '[' + job_uuid + ']'
           return true
         else
-          $stderr.puts "["+job_uuid+"] WARNING: error submitting to chimpd! response code: #{response.code}"
+          $stderr.puts '[' + job_uuid + "] WARNING: error submitting to chimpd! response code: #{response.code}"
           return false
         end
 

data/lib/right_chimp/queue/ChimpQueue.rb

@@ -6,7 +6,7 @@ module Chimp
   class ChimpQueue
     include Singleton
 
-    attr_accessor :delay, :retry_count, :max_threads, :group
+    attr_accessor :delay, :retry_count, :max_threads, :group, :processing
 
     def initialize
       @delay = 0
@@ -15,6 +15,7 @@ module Chimp
       @workers_never_exit = true
       @threads = []
       @semaphore = Mutex.new
+      @processing = {}
       self.reset!
     end
 

data/lib/right_chimp/queue/ExecutionGroup.rb

@@ -172,8 +172,8 @@ module Chimp
       return (
         get_jobs_by_status(Executor::STATUS_NONE).size == 0 &&
         get_jobs_by_status(Executor::STATUS_RUNNING).size == 0 &&
-         get_jobs_by_status(Executor::STATUS_DONE).size > 0
-        )
+        get_jobs_by_status(Executor::STATUS_DONE).size > 0
+      )
     end
 
     #
@@ -181,9 +181,9 @@ module Chimp
     #
     def running?
       total_jobs_running = get_jobs_by_status(Executor::STATUS_NONE).size +
-          get_jobs_by_status(Executor::STATUS_RUNNING).size +
-          get_jobs_by_status(Executor::STATUS_RETRYING).size
-      return(total_jobs_running > 0)
+                           get_jobs_by_status(Executor::STATUS_RUNNING).size +
+                           get_jobs_by_status(Executor::STATUS_RETRYING).size
+      (total_jobs_running > 0)
     end
 
     #

data/lib/right_chimp/version.rb

@@ -1,3 +1,3 @@
 module Chimp
-  VERSION = "2.1.22.2"
+  VERSION = "2.1.24"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: right_chimp
 version: !ruby/object:Gem::Version
-  version: 2.1.22.2
+  version: 2.1.24
 platform: ruby
 authors:
 - RightScale Operations
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-08 00:00:00.000000000 Z
+date: 2016-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake