carrousel 0.0.5 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +1 -0
- data/bin/carrousel +5 -4
- data/carrousel.gemspec +2 -0
- data/lib/carrousel/runner.rb +119 -45
- data/lib/carrousel/version.rb +1 -1
- metadata +30 -24
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 26861d8c73faf25d8721f6410233aec083306da7
|
4
|
+
data.tar.gz: f546da79e7ea3bff9b54f95b45b365e7864c5f22
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f526dd00c4ebdeb7715c5dcbd3c72a3c77e72f92ce8204329f4e715da29cd2f9f6e1ead2ff760609373f6b7e7badc6ad2178b25e00c22d072df9259a6e77587e
|
7
|
+
data.tar.gz: e1c031d2467987ac87dcad3e8d833851044fa67b680c67807d20dbd6651e0bc348f769b5bc4c8517bc1f5f72cfec80d627d0d1ab7ab41ff3a2d7b4f6d477cd12
|
data/README.md
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# Carrousel
|
2
2
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/carrousel.png)](http://badge.fury.io/rb/carrousel)
|
4
|
+
[![carrousel API Documentation](https://www.omniref.com/ruby/gems/carrousel.png)](https://www.omniref.com/ruby/gems/carrousel)
|
4
5
|
|
5
6
|
The Carrousel gem is a command line utility for running a single command on
|
6
7
|
multiple targets. Carrousel tracks which commands have succeeded or failed
|
data/bin/carrousel
CHANGED
@@ -11,8 +11,8 @@ OptionParser.new do |op|
|
|
11
11
|
@opts[:command] = c
|
12
12
|
end
|
13
13
|
|
14
|
-
op.on('-
|
15
|
-
@opts[:
|
14
|
+
op.on('-j', '--jobs N', 'Allow N jobs at one time') do |n|
|
15
|
+
@opts[:maxjobs] = n.to_i
|
16
16
|
end
|
17
17
|
|
18
18
|
op.on('-l', '--listfile FILE', 'Load list of arguments from FILE') do |l|
|
@@ -35,11 +35,12 @@ end.parse!
|
|
35
35
|
|
36
36
|
@opts = {
|
37
37
|
:command => nil,
|
38
|
-
:delay =>
|
38
|
+
:delay => 1,
|
39
39
|
:listfile => nil,
|
40
40
|
:statusfile => nil,
|
41
41
|
:verbose => false,
|
42
|
-
:debug => false
|
42
|
+
:debug => false,
|
43
|
+
:maxjobs => 1,
|
43
44
|
}.merge(@opts)
|
44
45
|
|
45
46
|
Carrousel::Runner.new(ARGV, @opts).run
|
data/carrousel.gemspec
CHANGED
@@ -23,6 +23,8 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
24
24
|
spec.require_paths = ["lib"]
|
25
25
|
|
26
|
+
spec.add_runtime_dependency 'proc-wait3'
|
27
|
+
|
26
28
|
spec.add_development_dependency "bundler", "~> 1.3"
|
27
29
|
spec.add_development_dependency "rake"
|
28
30
|
spec.add_development_dependency "minitest"
|
data/lib/carrousel/runner.rb
CHANGED
@@ -3,6 +3,8 @@
|
|
3
3
|
|
4
4
|
require 'digest'
|
5
5
|
require 'yaml'
|
6
|
+
require 'yaml/store'
|
7
|
+
require 'proc/wait3'
|
6
8
|
|
7
9
|
##
|
8
10
|
# "Enter the Carrousel. This is the time of renewal."
|
@@ -12,22 +14,31 @@ module Carrousel
|
|
12
14
|
|
13
15
|
class Runner
|
14
16
|
|
17
|
+
CONTINUE_SIGNAL = 25
|
18
|
+
|
15
19
|
def initialize(args, opts = {})
|
16
|
-
@args
|
17
|
-
@opts
|
18
|
-
|
19
|
-
|
20
|
+
@args = args
|
21
|
+
@opts = opts
|
22
|
+
|
23
|
+
incomplete = []
|
24
|
+
complete = []
|
25
|
+
|
26
|
+
warn @opts.inspect if @opts[:debug]
|
20
27
|
|
21
28
|
unless @opts[:listfile].nil?
|
22
29
|
lines = File.readlines(@opts[:listfile]).map(&:strip)
|
23
|
-
|
30
|
+
incomplete.concat(lines)
|
24
31
|
end
|
25
|
-
|
32
|
+
incomplete.concat(@args)
|
26
33
|
|
27
|
-
@opts[:
|
28
|
-
|
34
|
+
warn "incomplete after cli parse: #{incomplete}" if @opts[:debug]
|
35
|
+
warn "complete after cli parse: #{complete}" if @opts[:debug]
|
29
36
|
|
30
|
-
|
37
|
+
@opts[:statusfile] ||= generate_status_filename(incomplete.sort.join + Time.now.to_s)
|
38
|
+
open_status_file(incomplete, complete)
|
39
|
+
|
40
|
+
warn "incomplete after statusfile parse: #{incomplete}" if @opts[:debug]
|
41
|
+
warn "complete after statusfile parse: #{complete}" if @opts[:debug]
|
31
42
|
|
32
43
|
raise ArgumentError.new("Command option is required") if @opts[:command].nil?
|
33
44
|
end # def initialize
|
@@ -38,62 +49,125 @@ module Carrousel
|
|
38
49
|
# succeeds, we move the item to the completed list. If we are interrupted
|
39
50
|
# in the middle of processing, we ensure that the item is saved in the
|
40
51
|
# normal list, and we ensure that we write out the completed list.
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
@
|
49
|
-
|
50
|
-
|
52
|
+
|
53
|
+
begin
|
54
|
+
|
55
|
+
until @store.transaction(true) { @store[:incomplete].empty? && @store[:processing].empty? }
|
56
|
+
|
57
|
+
until @store.transaction(true) { @store[:incomplete].empty? }
|
58
|
+
|
59
|
+
if @store.transaction(true) { @store[:pids].size < @opts[:maxjobs] }
|
60
|
+
target = nil
|
61
|
+
@store.transaction do
|
62
|
+
target = @store[:incomplete].delete_at(0)
|
63
|
+
@store[:processing].push(target)
|
64
|
+
end
|
65
|
+
|
66
|
+
warn "creating new job for target: #{target}" if @opts[:debug]
|
67
|
+
pid = Process.fork { create_new_job(target) }
|
68
|
+
warn "Forked job: #{pid}" if @opts[:debug]
|
69
|
+
|
70
|
+
@store.transaction do
|
71
|
+
@store[:pids].push(pid)
|
72
|
+
warn "Num jobs: #{@store[:pids].size} Current jobs: #{@store[:pids]}" if @opts[:debug]
|
73
|
+
end
|
74
|
+
|
75
|
+
warn "Detaching #{pid}" if @opts[:debug]
|
76
|
+
Process.detach(pid) # We don't plan to monitor these
|
77
|
+
|
78
|
+
warn "Sending continue signal to #{pid}" if @opts[:debug]
|
79
|
+
Process.kill(CONTINUE_SIGNAL, pid)
|
80
|
+
end
|
81
|
+
|
51
82
|
end
|
52
|
-
ensure
|
53
|
-
save_status_file
|
54
83
|
end
|
55
84
|
|
56
|
-
|
57
|
-
|
58
|
-
|
85
|
+
ensure
|
86
|
+
save_status_file
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
def create_new_job(target)
|
93
|
+
warn "<#{target}> Job created. Pausing #{Process.pid}" if @opts[:debug]
|
94
|
+
Process.pause(CONTINUE_SIGNAL)
|
95
|
+
warn "<#{target}> Resuming job #{Process.pid}" if @opts[:debug]
|
96
|
+
|
97
|
+
command = [@opts[:command], target].join(' ')
|
98
|
+
warn "<#{target}> Executing command: #{command}" if @opts[:verbose]
|
99
|
+
resp = system(command)
|
100
|
+
warn "<#{target}> System response: #{resp}" if @opts[:verbose]
|
101
|
+
|
102
|
+
@store.transaction do
|
103
|
+
@store[:processing].delete(target)
|
104
|
+
|
105
|
+
if resp
|
106
|
+
@store[:complete] << target
|
107
|
+
else
|
108
|
+
@store[:incomplete] << target
|
59
109
|
end
|
60
|
-
|
110
|
+
|
111
|
+
@store[:pids].delete(Process.pid)
|
112
|
+
|
113
|
+
warn "Removing pid from queue: #{Process.pid}" if @opts[:debug]
|
114
|
+
warn "Num jobs: #{@store[:pids].size} Current jobs: #{@store[:pids]}" if @opts[:debug]
|
115
|
+
end
|
116
|
+
|
61
117
|
end # def run
|
62
118
|
|
63
119
|
private
|
64
|
-
def generate_status_filename
|
65
|
-
key = Digest::SHA256.hexdigest(
|
120
|
+
def generate_status_filename(string)
|
121
|
+
key = Digest::SHA256.hexdigest(string).slice(0...7)
|
66
122
|
warn "status file key: #{key}" if @opts[:debug]
|
67
123
|
name = self.class.name.gsub('::', '_').downcase
|
68
124
|
File.expand_path(".#{name}_status_#{key}", Dir.pwd)
|
69
125
|
end # def generate_status_filename
|
70
126
|
|
71
127
|
private
|
72
|
-
def open_status_file
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
128
|
+
def open_status_file(incomplete, complete)
|
129
|
+
resume = File.exists?(@opts[:statusfile])
|
130
|
+
@store = YAML::Store.new @opts[:statusfile]
|
131
|
+
warn "opened status file: #{@store.path}" if @opts[:debug]
|
132
|
+
|
133
|
+
@store.transaction do
|
134
|
+
@store[:processing] = []
|
135
|
+
@store[:pids] = []
|
136
|
+
end
|
137
|
+
|
138
|
+
warn "resuming: #{resume}" if @opts[:debug]
|
139
|
+
|
140
|
+
if resume
|
141
|
+
@store.transaction(true) do # read-only transaction
|
142
|
+
@opts[:command] ||= @store[:command]
|
143
|
+
@store[:incomplete].concat(incomplete)
|
144
|
+
@store[:complete].concat(complete)
|
145
|
+
end
|
146
|
+
else
|
147
|
+
@store.transaction do
|
148
|
+
@store[:command] = @opts[:command]
|
149
|
+
@store[:incomplete] = incomplete
|
150
|
+
@store[:complete] = complete
|
80
151
|
end
|
81
152
|
end
|
153
|
+
|
82
154
|
end # def open_status_file
|
83
155
|
|
84
156
|
private
|
85
157
|
def save_status_file
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
158
|
+
|
159
|
+
@store.transaction do
|
160
|
+
@store[:pids].each do |process|
|
161
|
+
warn "Killing #{process}" if @opts[:debug]
|
162
|
+
Process.kill('KILL', process)
|
163
|
+
end
|
164
|
+
|
165
|
+
@store[:incomplete].concat(@store[:processing])
|
166
|
+
@store.delete(:processing)
|
167
|
+
@store.delete(:pids)
|
95
168
|
end
|
96
|
-
|
169
|
+
|
170
|
+
warn "Saved status file: #{@store.path}" if @opts[:debug]
|
97
171
|
end # def save_status_file
|
98
172
|
|
99
173
|
end # class Runner
|
data/lib/carrousel/version.rb
CHANGED
metadata
CHANGED
@@ -1,65 +1,72 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: carrousel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
5
|
-
prerelease:
|
4
|
+
version: 0.1.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Raj Sahae
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2015-05-05 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: proc-wait3
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
14
27
|
- !ruby/object:Gem::Dependency
|
15
28
|
name: bundler
|
16
29
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
30
|
requirements:
|
19
|
-
- - ~>
|
31
|
+
- - "~>"
|
20
32
|
- !ruby/object:Gem::Version
|
21
33
|
version: '1.3'
|
22
34
|
type: :development
|
23
35
|
prerelease: false
|
24
36
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
37
|
requirements:
|
27
|
-
- - ~>
|
38
|
+
- - "~>"
|
28
39
|
- !ruby/object:Gem::Version
|
29
40
|
version: '1.3'
|
30
41
|
- !ruby/object:Gem::Dependency
|
31
42
|
name: rake
|
32
43
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
44
|
requirements:
|
35
|
-
- -
|
45
|
+
- - ">="
|
36
46
|
- !ruby/object:Gem::Version
|
37
47
|
version: '0'
|
38
48
|
type: :development
|
39
49
|
prerelease: false
|
40
50
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
51
|
requirements:
|
43
|
-
- -
|
52
|
+
- - ">="
|
44
53
|
- !ruby/object:Gem::Version
|
45
54
|
version: '0'
|
46
55
|
- !ruby/object:Gem::Dependency
|
47
56
|
name: minitest
|
48
57
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
58
|
requirements:
|
51
|
-
- -
|
59
|
+
- - ">="
|
52
60
|
- !ruby/object:Gem::Version
|
53
61
|
version: '0'
|
54
62
|
type: :development
|
55
63
|
prerelease: false
|
56
64
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
65
|
requirements:
|
59
|
-
- -
|
66
|
+
- - ">="
|
60
67
|
- !ruby/object:Gem::Version
|
61
68
|
version: '0'
|
62
|
-
description:
|
69
|
+
description: "Carrousel is a robust utility designed to take a list\n of generic
|
63
70
|
items, and given some command, perform that command on each item\n in that list.
|
64
71
|
Depending on the commands return value, Carrousel will track \n which items have
|
65
72
|
been completed successfully, and retry items as necessary.\n It will save your
|
@@ -72,8 +79,8 @@ executables:
|
|
72
79
|
extensions: []
|
73
80
|
extra_rdoc_files: []
|
74
81
|
files:
|
75
|
-
- .gitignore
|
76
|
-
- .travis.yml
|
82
|
+
- ".gitignore"
|
83
|
+
- ".travis.yml"
|
77
84
|
- Gemfile
|
78
85
|
- LICENSE.txt
|
79
86
|
- README.md
|
@@ -88,27 +95,26 @@ files:
|
|
88
95
|
homepage: https://github.com/rajsahae/carrousel
|
89
96
|
licenses:
|
90
97
|
- MIT
|
98
|
+
metadata: {}
|
91
99
|
post_install_message:
|
92
100
|
rdoc_options: []
|
93
101
|
require_paths:
|
94
102
|
- lib
|
95
103
|
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
-
none: false
|
97
104
|
requirements:
|
98
|
-
- -
|
105
|
+
- - ">="
|
99
106
|
- !ruby/object:Gem::Version
|
100
107
|
version: '0'
|
101
108
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
102
|
-
none: false
|
103
109
|
requirements:
|
104
|
-
- -
|
110
|
+
- - ">="
|
105
111
|
- !ruby/object:Gem::Version
|
106
112
|
version: '0'
|
107
113
|
requirements: []
|
108
114
|
rubyforge_project:
|
109
|
-
rubygems_version:
|
115
|
+
rubygems_version: 2.4.6
|
110
116
|
signing_key:
|
111
|
-
specification_version:
|
117
|
+
specification_version: 4
|
112
118
|
summary: Robust list based action tracking utility.
|
113
119
|
test_files:
|
114
120
|
- test/minitest_helper.rb
|