carrousel 0.0.5 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 26861d8c73faf25d8721f6410233aec083306da7
+  data.tar.gz: f546da79e7ea3bff9b54f95b45b365e7864c5f22
+SHA512:
+  metadata.gz: f526dd00c4ebdeb7715c5dcbd3c72a3c77e72f92ce8204329f4e715da29cd2f9f6e1ead2ff760609373f6b7e7badc6ad2178b25e00c22d072df9259a6e77587e
+  data.tar.gz: e1c031d2467987ac87dcad3e8d833851044fa67b680c67807d20dbd6651e0bc348f769b5bc4c8517bc1f5f72cfec80d627d0d1ab7ab41ff3a2d7b4f6d477cd12

data/README.md

@@ -1,6 +1,7 @@
 # Carrousel
 
 [![Gem Version](https://badge.fury.io/rb/carrousel.png)](http://badge.fury.io/rb/carrousel)
+[![carrousel API Documentation](https://www.omniref.com/ruby/gems/carrousel.png)](https://www.omniref.com/ruby/gems/carrousel)
 
 The Carrousel gem is a command line utility for running a single command on
 multiple targets. Carrousel tracks which commands have succeeded or failed

data/bin/carrousel

@@ -11,8 +11,8 @@ OptionParser.new do |op|
     @opts[:command] = c
   end
 
-  op.on('-d', '--delay DELAY', 'Sleep for DELAY seconds between commands') do |d|
-    @opts[:delay] = d.to_i
+  op.on('-j', '--jobs N', 'Allow N jobs at one time') do |n|
+    @opts[:maxjobs] = n.to_i
   end
 
   op.on('-l', '--listfile FILE', 'Load list of arguments from FILE') do |l|
@@ -35,11 +35,12 @@ end.parse!
 
 @opts = {
   :command => nil,
-  :delay => 0,
+  :delay => 1,
   :listfile => nil,
   :statusfile => nil,
   :verbose => false,
-  :debug => false
+  :debug => false,
+  :maxjobs => 1,
 }.merge(@opts)
 
 Carrousel::Runner.new(ARGV, @opts).run

data/carrousel.gemspec

@@ -23,6 +23,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency 'proc-wait3'
+
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"

data/lib/carrousel/runner.rb

@@ -3,6 +3,8 @@
 
 require 'digest'
 require 'yaml'
+require 'yaml/store'
+require 'proc/wait3'
 
 ##
 # "Enter the Carrousel. This is the time of renewal."
@@ -12,22 +14,31 @@ module Carrousel
 
   class Runner
 
+    CONTINUE_SIGNAL = 25
+
     def initialize(args, opts = {})
-      @args = args
-      @opts = opts
-      @incomplete = []
-      @complete = []
+      @args      = args
+      @opts      = opts
+
+      incomplete = []
+      complete   = []
+
+      warn @opts.inspect if @opts[:debug]
 
       unless @opts[:listfile].nil?
         lines = File.readlines(@opts[:listfile]).map(&:strip)
-        @incomplete.concat(lines)
+        incomplete.concat(lines)
       end
-      @incomplete.concat(@args)
+      incomplete.concat(@args)
 
-      @opts[:statusfile] ||= generate_status_filename
-      open_status_file
+      warn "incomplete after cli parse: #{incomplete}" if @opts[:debug]
+      warn "complete after cli parse: #{complete}"   if @opts[:debug]
 
-      p self if @opts[:debug]
+      @opts[:statusfile] ||= generate_status_filename(incomplete.sort.join + Time.now.to_s)
+      open_status_file(incomplete, complete)
+
+      warn "incomplete after statusfile parse: #{incomplete}" if @opts[:debug]
+      warn "complete after statusfile parse: #{complete}"     if @opts[:debug]
 
       raise ArgumentError.new("Command option is required") if @opts[:command].nil?
     end # def initialize
@@ -38,62 +49,125 @@ module Carrousel
       # succeeds, we move the item to the completed list. If we are interrupted
       # in the middle of processing, we ensure that the item is saved in the
       # normal list, and we ensure that we write out the completed list.
-      until @incomplete.empty?
-        begin
-          command = [@opts[:command], @incomplete.first].join(' ')
-          warn "Executing command: #{command}" if @opts[:verbose]
-          resp = system(command)
-          warn "System response: #{resp}" if @opts[:verbose]
-          if resp
-            @complete << @incomplete.delete(@incomplete.first)
-          else
-            @incomplete.rotate!
+
+      begin
+
+        until @store.transaction(true) { @store[:incomplete].empty? && @store[:processing].empty? }
+
+          until @store.transaction(true) { @store[:incomplete].empty? }
+
+            if @store.transaction(true) { @store[:pids].size < @opts[:maxjobs] }
+              target = nil
+              @store.transaction do
+                target = @store[:incomplete].delete_at(0)
+                @store[:processing].push(target)
+              end
+
+              warn "creating new job for target: #{target}" if @opts[:debug]
+              pid = Process.fork { create_new_job(target) }
+              warn "Forked job: #{pid}" if @opts[:debug]
+
+              @store.transaction do
+                @store[:pids].push(pid)
+                warn "Num jobs: #{@store[:pids].size} Current jobs: #{@store[:pids]}" if @opts[:debug]
+              end
+
+              warn "Detaching #{pid}" if @opts[:debug]
+              Process.detach(pid) # We don't plan to monitor these  
+
+              warn "Sending continue signal to #{pid}" if @opts[:debug]
+              Process.kill(CONTINUE_SIGNAL, pid)
+            end
+
           end
-        ensure
-          save_status_file
         end
 
-        if @opts[:delay] > 0
-          warn "Sleeping for #{@opts[:delay]} seconds" if @opts[:verbose]
-          sleep @opts[:delay]
+      ensure
+        save_status_file
+      end
+
+    end
+
+    private
+    def create_new_job(target)
+      warn "<#{target}> Job created. Pausing #{Process.pid}" if @opts[:debug]
+      Process.pause(CONTINUE_SIGNAL)
+      warn "<#{target}> Resuming job #{Process.pid}" if @opts[:debug]
+
+      command = [@opts[:command], target].join(' ')
+      warn "<#{target}> Executing command: #{command}" if @opts[:verbose]
+      resp = system(command)
+      warn "<#{target}> System response: #{resp}" if @opts[:verbose]
+
+      @store.transaction do
+        @store[:processing].delete(target)
+
+        if resp
+          @store[:complete] << target
+        else
+          @store[:incomplete] << target
         end
-      end # until @incomplete.empty?
+
+        @store[:pids].delete(Process.pid)
+
+        warn "Removing pid from queue: #{Process.pid}" if @opts[:debug]
+        warn "Num jobs: #{@store[:pids].size} Current jobs: #{@store[:pids]}" if @opts[:debug]
+      end
+
     end # def run
 
     private
-    def generate_status_filename
-      key = Digest::SHA256.hexdigest(@incomplete.sort.join).slice(0...7)
+    def generate_status_filename(string)
+      key = Digest::SHA256.hexdigest(string).slice(0...7)
       warn "status file key: #{key}" if @opts[:debug]
       name = self.class.name.gsub('::', '_').downcase
       File.expand_path(".#{name}_status_#{key}", Dir.pwd)
     end # def generate_status_filename
 
     private
-    def open_status_file
-      if File.exists?(@opts[:statusfile])
-        dbs = YAML.load(File.read(@opts[:statusfile]))
-        warn "opened status file:\n#{dbs}" if @opts[:debug]
-        if dbs
-          @opts[:command] ||= dbs[:command]
-          @complete.concat(dbs[:complete])
-          @incomplete.concat(dbs[:incomplete])
+    def open_status_file(incomplete, complete)
+      resume = File.exists?(@opts[:statusfile])
+      @store = YAML::Store.new @opts[:statusfile]
+      warn "opened status file: #{@store.path}" if @opts[:debug]
+
+      @store.transaction do
+        @store[:processing] = []
+        @store[:pids] = []
+      end
+
+      warn "resuming: #{resume}" if @opts[:debug]
+
+      if resume
+        @store.transaction(true) do # read-only transaction
+          @opts[:command] ||= @store[:command]
+          @store[:incomplete].concat(incomplete)
+          @store[:complete].concat(complete)
+        end
+      else
+        @store.transaction do
+          @store[:command]    = @opts[:command]
+          @store[:incomplete] = incomplete
+          @store[:complete]   = complete
         end
       end
+
     end # def open_status_file
 
     private
     def save_status_file
-      warn "Saving status file: #{@opts[:statusfile]}" if @opts[:verbose]
-      File.open(@opts[:statusfile], 'w') do |f|
-        ydb = {
-          :command => @opts[:command],
-          :complete => @complete,
-          :incomplete => @incomplete
-        }.to_yaml
-        f.puts(ydb)
-        warn "Saved status file:\n#{ydb}" if @opts[:debug]
+
+      @store.transaction do
+        @store[:pids].each do |process|
+          warn "Killing #{process}" if @opts[:debug]
+          Process.kill('KILL', process)
+        end
+
+        @store[:incomplete].concat(@store[:processing])
+        @store.delete(:processing)
+        @store.delete(:pids)
       end
-      true
+
+      warn "Saved status file: #{@store.path}" if @opts[:debug]
     end # def save_status_file
 
   end # class Runner

data/lib/carrousel/version.rb

@@ -1,3 +1,3 @@
 module Carrousel
-  VERSION = "0.0.5"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,65 +1,72 @@
 --- !ruby/object:Gem::Specification
 name: carrousel
 version: !ruby/object:Gem::Version
-  version: 0.0.5
-  prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - Raj Sahae
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-27 00:00:00.000000000 Z
+date: 2015-05-05 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: proc-wait3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ! "Carrousel is a robust utility designed to take a list\n  of generic
+description: "Carrousel is a robust utility designed to take a list\n  of generic
   items, and given some command, perform that command on each item\n  in that list.
   Depending on the commands return value, Carrousel will track \n  which items have
   been completed successfully, and retry items as necessary.\n  It will save your
@@ -72,8 +79,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -88,27 +95,26 @@ files:
 homepage: https://github.com/rajsahae/carrousel
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.4.6
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Robust list based action tracking utility.
 test_files:
 - test/minitest_helper.rb