right_chimp 2.1.24 → 2.1.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b8353c389aaf578df02f964a87778706144628b
-  data.tar.gz: 42bae4bf2b3e4246cf5ee31fea1709e42a3be2d7
+  metadata.gz: 6c548668e38fd58794b473964f7ba204a7e75892
+  data.tar.gz: 195e3977c81913af9ffd816f607901fa83150557
 SHA512:
-  metadata.gz: a60272b66e310894fdbbc393b5010dd12eeb9380c65aeeea84c5d152390edbf8a7f7a5330aea4188c2e8a4dd747b4502656278cac1ba4e29d71d22f6c3956559
-  data.tar.gz: 8edc1310533df69e7a192019c77aca13a799f529bc5b07de5eb0f7ea3af9d02a7b46cbb8efc796ab4b3f8fcf380a54f73252eacf1b4a7d704da1f61e3420ee47
+  metadata.gz: bec51a0eea3e14118a9a72f6df1218339ffc0885c127e34fe0c6574c8e422c8857bd3e3a8b440fe47fa12bdde8810323c42baf19cf55c0b5dc706cecfcead239
+  data.tar.gz: 0445c453033f4371afc70d7e2a8ada37bb7e51e3de8e358577719d275f57daa3556dde1d898348f851dbabc2672a91b3f570bf5a132a36cbb6546705bc4ea77a

data/CHANGES

@@ -1,3 +1,6 @@
+Version 2.1.25
+--------------
+* Bugfix: Fixed some corner case race conditions that affected the proper tracking of processing tasks. 
 Version 2.1.24
 --------------
 * Feature: We now ensure jobs are added and tracked into the processing status before sending a 200 back to

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    right_chimp (2.1.24)
+    right_chimp (2.1.25)
       highline (~> 1.7.2)
       nokogiri (~> 1.6.7.1)
       progressbar (~> 0.11.0)
@@ -68,4 +68,4 @@ DEPENDENCIES
   rspec (~> 2.6.0)
 
 BUNDLED WITH
-   1.12.5
+   1.13.1

data/lib/right_chimp/Chimp.rb

@@ -1103,7 +1103,6 @@ module Chimp
     # This is used by chimpd, when processing a task.
     #
     def process
-
       Chimp.set_failure(false)
       Chimp.set_job_uuid(job_uuid)
 
@@ -1123,12 +1122,6 @@ module Chimp
       get_executable_info unless Chimp.failure
 
       # All elements of task have been processed
-      ChimpDaemon.instance.semaphore.synchronize do
-        # remove from the processing queue
-        Log.debug 'Removing job: ' + job_uuid + ' from the processing queue for group: ' + group.to_s
-        ChimpDaemon.instance.queue.processing[group].delete(job_uuid)
-        ChimpDaemon.instance.proc_counter -= 1
-      end
 
       if Chimp.failure
 
@@ -1139,11 +1132,19 @@ module Chimp
         Log.error '##################################################'
         return []
       else
-        if @servers.first.nil? or @executable.nil?
+        if @servers.first.nil? || @executable.nil?
           Log.warn "[#{Chimp.get_job_uuid}] Nothing to do for \"chimp #{@cli_args}\"."
+          # decrease our counter
+          ChimpDaemon.instance.queue.processing[@group].delete(job_uuid.to_sym)
+          ChimpDaemon.instance.proc_counter -= 1
           return []
         else
-          Log.debug "[#{Chimp.get_job_uuid}] Generating job..."
+          Log.debug "[#{Chimp.get_job_uuid}] Generating job(s)..."
+          Log.debug 'Increasing processing counter (' + ChimpDaemon.instance.proc_counter.to_s + ") + " +  @servers.size.to_s + ' for group ' + @group.to_s
+
+          ChimpDaemon.instance.queue.processing[@group][job_uuid.to_sym] += @servers.size
+          ChimpDaemon.instance.proc_counter += @servers.size - 1
+
           return generate_jobs(@servers, @server_template, @executable)
         end
       end
@@ -1174,7 +1175,6 @@ module Chimp
       begin
         while !@dry_run
           local_queue = ChimpQueue.instance
-
           #
           # load up remote chimpd jobs into the local queue
           # this makes all the standard queue control methods available to us

data/lib/right_chimp/daemon/ChimpDaemon.rb

@@ -89,9 +89,9 @@ module Chimp
             when '--help', '-h'
               help
             when '--exit', '-x'
-            	uri = "http://localhost:#{@port}/admin"
-							response = RestClient.post uri, { 'shutdown' => true }.to_yaml
-							exit 0
+              uri = "http://localhost:#{@port}/admin"
+              response = RestClient.post uri, { 'shutdown' => true }.to_yaml
+              exit 0
           end
         end
       rescue GetoptLong::InvalidOption => ex
@@ -341,6 +341,7 @@ module Chimp
       # /group/<name>/<status>
       #
       def do_GET(req, resp)
+
         jobs = []
         Log.debug 'get group info'
 
@@ -362,10 +363,11 @@ module Chimp
         jobs = g.get_jobs_by_status(filter) if g
 
         # If there are processing jobs, add them as dummy executions.
-        if g2
+        if g2 && !g2.empty?
+
           Log.debug 'Group: ' + group_name + ' is processing:'
           g2.each do |job|
-            Log.debug 'Job: ' + job
+            Log.debug 'Job: ' + job.to_s
             j = ExecRightScript.new(group: group_name, job_uuid: job)
             jobs.push j
           end
@@ -429,20 +431,26 @@ module Chimp
         # Ask chimpd to process a Chimp object directly
         #
         if verb == 'process' or verb == 'add'
-          # comment the next line to GET STUCK IN PROCESSING forever
-          ChimpDaemon.instance.chimp_queue.push payload
           ChimpDaemon.instance.semaphore.synchronize do
             # While we are at it, we will store these processing jobs to prevent issues in the event
             # of a very slow API response.
             Log.debug 'Adding job: ' + job_uuid + ' to the processing queue for group: ' + group.to_s
-            q.processing[payload.group] = [] if q.processing[payload.group].nil?
-            q.processing[payload.group].push(payload.job_uuid)
+
+            q.processing[payload.group] = {} if q.processing[payload.group].nil?
+            q.processing[payload.group][payload.job_uuid.to_sym] = 0
 
             ChimpDaemon.instance.proc_counter += 1
           end
+          # comment the next line to GET STUCK IN PROCESSING forever
+          ChimpDaemon.instance.chimp_queue.push payload
 
-          Log.debug 'Tasks in the processing queue:' + ChimpDaemon.instance.proc_counter.to_s
-          Log.debug 'Pocessing:'
+          # Proper count of processing Tasks
+          counter = 0
+          q.processing.each{|k,v|
+            counter += v.size
+          }
+          Log.debug 'Processing:'
+          Log.debug 'Tasks in the processing queue:' + counter.to_s
           Log.debug q.processing.inspect
         elsif verb == 'update'
           puts 'UPDATE'
@@ -571,7 +579,7 @@ module Chimp
           stats << "waiting: #{queue.get_jobs_by_status(:none).size} / "
           stats << "failed: #{queue.get_jobs_by_status(:error).size} / "
           stats << "done: #{queue.get_jobs_by_status(:done).size} / "
-          stats << "processing: #{ChimpDaemon.instance.proc_counter.to_i} / "
+          stats << "processing: #{ChimpDaemon.instance.proc_counter} / "
           stats << "\n"
 
           resp.body = stats
@@ -587,7 +595,7 @@ module Chimp
                         "waiting" => queue.get_jobs_by_status(:none).size,
                         "failed" => queue.get_jobs_by_status(:error).size,
                         "done" => queue.get_jobs_by_status(:done).size,
-                        "processing" => ChimpDaemon.instance.proc_counter.to_i,
+                        "processing" => ChimpDaemon.instance.proc_counter,
                         "holding" => queue.get_jobs_by_status(:holding).size
                       }
 

data/lib/right_chimp/queue/QueueWorker.rb

@@ -22,8 +22,27 @@ module Chimp
 
         begin
           if work_item != nil
+            job_uuid = work_item.job_uuid
+            group = work_item.group.group_id
+
             work_item.retry_count = @retry_count
             work_item.owner = Thread.current.object_id
+            ChimpDaemon.instance.semaphore.synchronize do
+              # remove from the processing queue
+              Log.debug 'Decreasing processing counter (' + (ChimpDaemon.instance.proc_counter-1).to_s + ') for [' + job_uuid + '] group: ' + group.to_s
+              ChimpDaemon.instance.queue.processing[group][job_uuid.to_sym] -= 1
+              Log.debug ChimpDaemon.instance.queue.processing[group].inspect
+              if ChimpDaemon.instance.queue.processing[group][job_uuid.to_sym] == 0
+                Log.debug 'Completed processing task ' + job_uuid
+                ChimpDaemon.instance.queue.processing[group].delete(job_uuid.to_sym)
+                Log.debug ChimpDaemon.instance.queue.processing.inspect
+              else
+                Log.debug 'Still counting down for ' + job_uuid
+              end
+
+              ChimpDaemon.instance.proc_counter -= 1
+            end
+
             work_item.run
             sleep @delay
           else

data/lib/right_chimp/version.rb

@@ -1,3 +1,3 @@
 module Chimp
-  VERSION = "2.1.24"
+  VERSION = "2.1.25"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: right_chimp
 version: !ruby/object:Gem::Version
-  version: 2.1.24
+  version: 2.1.25
 platform: ruby
 authors:
 - RightScale Operations
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-13 00:00:00.000000000 Z
+date: 2016-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake