scout-gear 6.0.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +465 -432
  3. data/VERSION +1 -1
  4. data/bin/scout +5 -1
  5. data/lib/rbbt-scout.rb +5 -0
  6. data/lib/scout/concurrent_stream.rb +6 -2
  7. data/lib/scout/config.rb +168 -0
  8. data/lib/scout/exceptions.rb +9 -0
  9. data/lib/scout/indiferent_hash/options.rb +1 -0
  10. data/lib/scout/indiferent_hash.rb +4 -2
  11. data/lib/scout/log/color.rb +31 -2
  12. data/lib/scout/log/progress/report.rb +1 -0
  13. data/lib/scout/log/progress/util.rb +3 -1
  14. data/lib/scout/log/progress.rb +7 -3
  15. data/lib/scout/log.rb +8 -3
  16. data/lib/scout/misc/digest.rb +1 -3
  17. data/lib/scout/misc/monitor.rb +3 -0
  18. data/lib/scout/misc/system.rb +15 -0
  19. data/lib/scout/misc.rb +1 -0
  20. data/lib/scout/named_array.rb +68 -0
  21. data/lib/scout/open/stream.rb +58 -26
  22. data/lib/scout/path/find.rb +27 -3
  23. data/lib/scout/path/util.rb +7 -4
  24. data/lib/scout/persist/serialize.rb +7 -14
  25. data/lib/scout/persist.rb +21 -1
  26. data/lib/scout/resource/produce.rb +7 -94
  27. data/lib/scout/resource/software.rb +176 -0
  28. data/lib/scout/tsv/dumper.rb +107 -0
  29. data/lib/scout/tsv/index.rb +49 -0
  30. data/lib/scout/tsv/parser.rb +317 -0
  31. data/lib/scout/tsv/path.rb +13 -0
  32. data/lib/scout/tsv/persist/adapter.rb +348 -0
  33. data/lib/scout/tsv/persist/tokyocabinet.rb +113 -0
  34. data/lib/scout/tsv/persist.rb +15 -0
  35. data/lib/scout/tsv/traverse.rb +48 -0
  36. data/lib/scout/tsv/util.rb +24 -0
  37. data/lib/scout/tsv.rb +27 -0
  38. data/lib/scout/work_queue/worker.rb +16 -11
  39. data/lib/scout/work_queue.rb +63 -21
  40. data/lib/scout/workflow/definition.rb +93 -4
  41. data/lib/scout/workflow/step/config.rb +18 -0
  42. data/lib/scout/workflow/step/dependencies.rb +40 -0
  43. data/lib/scout/workflow/step/file.rb +15 -0
  44. data/lib/scout/workflow/step/info.rb +33 -6
  45. data/lib/scout/workflow/step/provenance.rb +148 -0
  46. data/lib/scout/workflow/step.rb +70 -20
  47. data/lib/scout/workflow/task.rb +5 -4
  48. data/lib/scout/workflow/usage.rb +1 -1
  49. data/lib/scout/workflow.rb +11 -3
  50. data/lib/scout-gear.rb +1 -0
  51. data/lib/scout.rb +1 -0
  52. data/scout-gear.gemspec +38 -3
  53. data/scout_commands/find +1 -1
  54. data/scout_commands/workflow/task +16 -10
  55. data/share/software/install_helpers +523 -0
  56. data/test/scout/log/test_progress.rb +0 -2
  57. data/test/scout/misc/test_system.rb +21 -0
  58. data/test/scout/open/test_stream.rb +160 -1
  59. data/test/scout/path/test_find.rb +14 -7
  60. data/test/scout/resource/test_software.rb +24 -0
  61. data/test/scout/test_config.rb +66 -0
  62. data/test/scout/test_meta_extension.rb +10 -0
  63. data/test/scout/test_named_array.rb +19 -0
  64. data/test/scout/test_persist.rb +35 -0
  65. data/test/scout/test_semaphore.rb +1 -1
  66. data/test/scout/test_tmpfile.rb +2 -2
  67. data/test/scout/test_tsv.rb +74 -0
  68. data/test/scout/test_work_queue.rb +63 -8
  69. data/test/scout/tsv/persist/test_adapter.rb +34 -0
  70. data/test/scout/tsv/persist/test_tokyocabinet.rb +92 -0
  71. data/test/scout/tsv/test_dumper.rb +44 -0
  72. data/test/scout/tsv/test_index.rb +64 -0
  73. data/test/scout/tsv/test_parser.rb +173 -0
  74. data/test/scout/tsv/test_persist.rb +36 -0
  75. data/test/scout/tsv/test_traverse.rb +9 -0
  76. data/test/scout/tsv/test_util.rb +0 -0
  77. data/test/scout/work_queue/test_worker.rb +49 -1
  78. data/test/scout/workflow/step/test_dependencies.rb +25 -0
  79. data/test/scout/workflow/step/test_info.rb +15 -17
  80. data/test/scout/workflow/step/test_load.rb +16 -18
  81. data/test/scout/workflow/step/test_provenance.rb +25 -0
  82. data/test/scout/workflow/test_step.rb +206 -10
  83. data/test/scout/workflow/test_task.rb +0 -3
  84. data/test/test_helper.rb +6 -0
  85. metadata +37 -2
data/VERSION CHANGED
@@ -1 +1 @@
1
- 6.0.0
1
+ 7.2.0
data/bin/scout CHANGED
@@ -37,6 +37,11 @@ if dev_dir
37
37
  $LOAD_PATH.unshift f
38
38
  end
39
39
  end
40
+ ['rbbt-*/lib'].each do |pattern|
41
+ Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
42
+ $LOAD_PATH.unshift f
43
+ end
44
+ end
40
45
  end
41
46
 
42
47
  Log.nocolor = true if ARGV.include? "--nocolor"
@@ -112,7 +117,6 @@ def commands(prev)
112
117
  end
113
118
 
114
119
  def scout_usage(prev = nil)
115
- puts
116
120
  puts SOPT.doc
117
121
 
118
122
  if prev
data/lib/rbbt-scout.rb ADDED
@@ -0,0 +1,5 @@
1
+ $LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
2
+ module Rbbt
3
+ extend Resource
4
+ self.pkgdir = 'rbbt'
5
+ end
@@ -9,11 +9,11 @@ module AbortedStream
9
9
  end
10
10
 
11
11
  module ConcurrentStream
12
- attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
12
+ attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
13
13
 
14
14
  def self.setup(stream, options = {}, &block)
15
15
 
16
- threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
16
+ threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair, :next
17
17
  stream.extend ConcurrentStream unless ConcurrentStream === stream
18
18
 
19
19
  stream.threads ||= []
@@ -24,6 +24,7 @@ module ConcurrentStream
24
24
  stream.no_fail = no_fail unless no_fail.nil?
25
25
  stream.std_err = ""
26
26
 
27
+ stream.next = next_stream unless next_stream.nil?
27
28
  stream.pair = pair unless pair.nil?
28
29
 
29
30
  callback = block if block_given?
@@ -230,6 +231,9 @@ module ConcurrentStream
230
231
  def read(*args)
231
232
  begin
232
233
  super(*args)
234
+ rescue
235
+ raise stream_exception if stream_exception
236
+ raise $!
233
237
  ensure
234
238
  begin
235
239
  close unless closed?
@@ -0,0 +1,168 @@
1
+ require_relative 'path'
2
+ require_relative 'resource'
3
+ require_relative 'resource/scout'
4
+
5
+ module Scout::Config
6
+
7
+ CACHE = IndiferentHash.setup({})
8
+
9
+ GOT_KEYS=[]
10
+
11
+ def self.add_entry(key, value, tokens)
12
+ tokens = [tokens] unless Array === tokens
13
+ tokens << "key:#{key}" unless tokens.include?("key:#{key}")
14
+ CACHE[key.to_s] ||= []
15
+ CACHE[key.to_s] << [tokens, value]
16
+ end
17
+
18
+ def self.load_file(file)
19
+ Log.debug "Loading config file: #{ file }"
20
+ TSV.traverse file, :type => :array do |line|
21
+ next if line =~ /^#/
22
+ key, value, *tokens = line.strip.split(/\s/)
23
+
24
+ self.add_entry(key, value, tokens) if key
25
+ end
26
+ end
27
+
28
+ def self.load_config
29
+ Scout.etc.config.find_all.reverse.each do |file|
30
+ self.load_file(file)
31
+ end
32
+ end
33
+
34
+ def self.set(values, *tokens)
35
+ if not Hash === values
36
+ values = {values => tokens.shift}
37
+ end
38
+
39
+ values.each do |key,value|
40
+ add_entry key, value, tokens
41
+ end
42
+ end
43
+
44
+ def self.token_priority(token)
45
+ token, _sep, priority = token.to_s.partition("::")
46
+
47
+ if priority.nil? || priority.empty?
48
+ type, _sep, rest = token.partition(":")
49
+ priority = case type
50
+ when "workflow"
51
+ 4
52
+ when "task"
53
+ 3
54
+ when "file"
55
+ 2
56
+ when "line"
57
+ 1
58
+ when "key"
59
+ 20
60
+ else
61
+ 10
62
+ end
63
+ else
64
+ priority = priority.to_i
65
+ end
66
+
67
+ [token, priority]
68
+ end
69
+
70
+ def self.match(entries, give_token)
71
+ priorities = {}
72
+ entries.each do |tokens, value|
73
+ best_prio = nil
74
+ tokens = [tokens] unless Array === tokens
75
+ tokens.each do |tok|
76
+ tok, prio = token_priority tok
77
+ next unless tok == give_token
78
+
79
+ best_prio = prio if best_prio.nil? or best_prio > prio
80
+ next if prio > best_prio
81
+
82
+ priorities[prio] ||= []
83
+ priorities[prio].unshift value
84
+ end
85
+ end if entries
86
+ priorities
87
+ end
88
+
89
+ # For equal priorities the matching prioritizes tokens ealier in the list
90
+ def self.get(key, *tokens)
91
+ options = tokens.pop if Hash === tokens.last
92
+ default = options.nil? ? nil : options[:default]
93
+
94
+ tokens = ["key:" + key] if tokens.empty?
95
+
96
+ tokens = tokens.flatten
97
+ file, _sep, line = caller.reject{|l|
98
+ l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
99
+ l =~ /rbbt\/resource\/path\.rb/ or
100
+ l =~ /rbbt\/util\/misc\.rb/ or
101
+ l =~ /accessor\.rb/ or
102
+ l =~ /progress-monitor\.rb/
103
+ }.first.partition(":")
104
+
105
+ File.expand_path(file)
106
+
107
+ tokens << ("file:" << file)
108
+ tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
109
+
110
+ entries = CACHE[key.to_s]
111
+ priorities = {}
112
+ tokens.each do |token|
113
+ token_prio = match entries, token.to_s
114
+ token_prio.each do |prio, values|
115
+ priorities[prio] ||= []
116
+ priorities[prio].concat(values)
117
+ end
118
+ end
119
+
120
+ value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
121
+ value = false if value == 'false'
122
+
123
+ Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
124
+ GOT_KEYS << [key, value, tokens]
125
+
126
+ if String === value && m = value.match(/^env:(.*)/)
127
+ variable = m.captures.first
128
+ ENV[variable]
129
+ elsif value == 'nil'
130
+ nil
131
+ else
132
+ value
133
+ end
134
+ end
135
+
136
+ def self.with_config
137
+ saved_config = {}
138
+ CACHE.each do |k,v|
139
+ saved_config[k] = v.dup
140
+ end
141
+ saved_got_keys = GOT_KEYS.dup
142
+ begin
143
+ yield
144
+ ensure
145
+ CACHE.replace(saved_config)
146
+ GOT_KEYS.replace(saved_got_keys)
147
+ end
148
+ end
149
+
150
+ def self.process_config(config)
151
+ if Misc.is_filename?(config) && File.exist?(config)
152
+ Scout::Config.load_file(config)
153
+ elsif Scout.etc.config_profile[config].exists?
154
+ Scout::Config.load_file(Scout.etc.config_profile[config].find)
155
+ else
156
+ key, value, *tokens = config.split(/\s/)
157
+ tokens = tokens.collect do |tok|
158
+ tok, _sep, prio = tok.partition("::")
159
+ prio = "0" if prio.nil? or prio.empty?
160
+ [tok, prio] * "::"
161
+ end
162
+ Scout::Config.set({key => value}, *tokens)
163
+ end
164
+ end
165
+
166
+
167
+ self.load_config
168
+ end
@@ -84,6 +84,14 @@ class DoneProcessing < Exception
84
84
  end
85
85
  end
86
86
 
87
+ class WorkerException < ScoutException
88
+ attr_accessor :worker_exception, :pid
89
+ def initialize(worker_exception, pid)
90
+ @worker_exception = worker_exception
91
+ @pid = pid
92
+ end
93
+ end
94
+
87
95
 
88
96
  #class OpenGzipError < StandardError; end
89
97
  #
@@ -129,3 +137,4 @@ end
129
137
  #
130
138
  #
131
139
  #
140
+ class ResourceNotFound < ScoutException; end
@@ -29,6 +29,7 @@ module IndiferentHash
29
29
 
30
30
  def self.pull_keys(hash, prefix)
31
31
  new = {}
32
+ prefix = prefix.to_s
32
33
  hash.keys.each do |key|
33
34
  if key.to_s =~ /#{ prefix }_(.*)/
34
35
  case
@@ -58,9 +58,11 @@ module IndiferentHash
58
58
  def delete(key)
59
59
  case key
60
60
  when Symbol, Module
61
- super(key) || super(key.to_s)
61
+ v = super(key)
62
+ v.nil? ? super(key.to_s) : v
62
63
  when String
63
- super(key) || super(key.to_sym)
64
+ v = super(key)
65
+ v.nil? ? super(key.to_sym) : v
64
66
  else
65
67
  super(key)
66
68
  end
@@ -143,13 +143,14 @@ module Log
143
143
  CONCEPT_COLORS = IndiferentHash.setup({
144
144
  :title => magenta,
145
145
  :path => blue,
146
- :input => blue,
146
+ :input => cyan,
147
147
  :value => green,
148
148
  :integer => green,
149
149
  :negative => red,
150
150
  :float => green,
151
151
  :waiting => yellow,
152
- :started => blue,
152
+ :started => cyan,
153
+ :start => cyan,
153
154
  :done => green,
154
155
  :error => red,
155
156
  })
@@ -166,10 +167,38 @@ module Log
166
167
  def self.color(color, str = nil, reset = false)
167
168
  return str.dup || "" if nocolor
168
169
 
170
+ if (color == :integer || color == :float) && Numeric === str
171
+ color = if str < 0
172
+ :red
173
+ elsif str > 1
174
+ :cyan
175
+ else
176
+ :green
177
+ end
178
+ end
179
+
180
+ if color == :status
181
+ color = case str.to_sym
182
+ when :done
183
+ :green
184
+ when :error, :aborted
185
+ :red
186
+ when :waiting, :queued
187
+ :yellow
188
+ when :started, :streamming
189
+ :cyan
190
+ when :start
191
+ :title
192
+ else
193
+ :cyan
194
+ end
195
+ end
196
+
169
197
  color = SEVERITY_COLOR[color] if Integer === color
170
198
  color = CONCEPT_COLORS[color] if CONCEPT_COLORS.include?(color)
171
199
  color = Term::ANSIColor.send(color) if Symbol === color and Term::ANSIColor.respond_to?(color)
172
200
 
201
+ str = str.to_s unless str.nil?
173
202
  return str if Symbol === color
174
203
  color_str = reset ? Term::ANSIColor.reset : ""
175
204
  color_str << color
@@ -2,6 +2,7 @@ require 'yaml'
2
2
  module Log
3
3
  class ProgressBar
4
4
  def print(io, str)
5
+ return if self.severity && self.severity < Log.severity
5
6
  return if Log.no_bar
6
7
  STDERR.print str
7
8
  Log.logfile.puts str unless Log.logfile.nil?
@@ -28,6 +28,8 @@ module Log
28
28
  end
29
29
 
30
30
  def self.new_bar(max, options = {})
31
+ options, max = max, nil if Hash === max
32
+ max = options[:max] if max.nil?
31
33
  cleanup_bars
32
34
  BAR_MUTEX.synchronize do
33
35
  Log::LAST.replace "new_bar" if Log::LAST == "progress"
@@ -79,7 +81,7 @@ module Log
79
81
  Log::ProgressBar.remove_bar self, error
80
82
  end
81
83
 
82
- def self.with_bar(max, options = {})
84
+ def self.with_bar(max = nil, options = {})
83
85
  bar = new_bar(max, options)
84
86
  begin
85
87
  error = false
@@ -15,18 +15,22 @@ module Log
15
15
 
16
16
  class << self
17
17
  attr_accessor :default_file
18
+ attr_accessor :default_severity
18
19
  end
19
20
 
20
21
  attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback, :severity
21
22
 
22
23
  def initialize(max = nil, options = {})
23
- depth, desc, file, bytes, frequency, process, callback =
24
- IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback,
25
- :depth => 0, :frequency => 2
24
+ depth, desc, file, bytes, frequency, process, callback, severity =
25
+ IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback, :severity,
26
+ :depth => 0, :frequency => 2, :severity => Log::ProgressBar.default_severity
27
+
28
+ max = nil if TrueClass === max
26
29
 
27
30
  @max = max
28
31
  @ticks = 0
29
32
  @frequency = frequency
33
+ @severity = severity
30
34
  @last_time = nil
31
35
  @last_count = nil
32
36
  @last_percent = nil
data/lib/scout/log.rb CHANGED
@@ -186,11 +186,12 @@ module Log
186
186
 
187
187
  def self.exception(e)
188
188
  stack = caller
189
+ backtrace = e.backtrace || []
189
190
  if ENV["RBBT_ORIGINAL_STACK"] == 'true'
190
191
  error([e.class.to_s, e.message].compact * ": " )
191
- error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
192
+ error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
192
193
  else
193
- error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
194
+ error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace.reverse)*"\n")
194
195
  error([e.class.to_s, e.message].compact * ": " )
195
196
  end
196
197
  end
@@ -206,7 +207,11 @@ module Log
206
207
  line = line.sub('`',"'")
207
208
  color = :green if line =~ /workflow/
208
209
  color = :blue if line =~ /scout-/
209
- Log.color color, line
210
+ if color
211
+ Log.color color, line
212
+ else
213
+ line
214
+ end
210
215
  end unless stack.nil?
211
216
  end
212
217
 
@@ -41,9 +41,7 @@ module Misc
41
41
 
42
42
  def self.digest(obj)
43
43
  str = Misc.digest_str(obj)
44
- hash = Digest::MD5.hexdigest(str)
45
- Log.debug "Digest #{hash} - #{str}"
46
- hash
44
+ Digest::MD5.hexdigest(str)
47
45
  end
48
46
 
49
47
  def self.file_md5(file)
@@ -1,4 +1,7 @@
1
1
  module Misc
2
+ def self.pid_alive?(pid)
3
+ !! Process.kill(0, pid) rescue false
4
+ end
2
5
  def self.benchmark(repeats = 1, message = nil)
3
6
  require 'benchmark'
4
7
  res = nil
@@ -0,0 +1,15 @@
1
+ module Misc
2
+ def self.env_add(var, value, sep = ":", prepend = true)
3
+ if ENV[var].nil?
4
+ ENV[var] = value
5
+ elsif ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
6
+ return
7
+ else
8
+ if prepend
9
+ ENV[var] = value + sep + ENV[var]
10
+ else
11
+ ENV[var] += sep + value
12
+ end
13
+ end
14
+ end
15
+ end
data/lib/scout/misc.rb CHANGED
@@ -3,6 +3,7 @@ require_relative 'misc/insist'
3
3
  require_relative 'misc/digest'
4
4
  require_relative 'misc/filesystem'
5
5
  require_relative 'misc/monitor'
6
+ require_relative 'misc/system'
6
7
 
7
8
  module Misc
8
9
  end
@@ -0,0 +1,68 @@
1
+ require_relative 'meta_extension'
2
+ module NamedArray
3
+ extend MetaExtension
4
+ extension_attr :fields
5
+
6
+ def self.identify_name(names, selected)
7
+ res = (Array === selected ? selected : [selected]).collect do |field|
8
+ case field
9
+ when nil
10
+ 0
11
+ when Integer
12
+ field
13
+ when Symbol
14
+ field == :key ? field : identify_name(names, field.to_s)
15
+ when (names.nil? and String)
16
+ if field =~ /^\d+$/
17
+ identify_field(key_field, fields, field.to_i)
18
+ else
19
+ raise "No name information available and specified name not numeric: #{ field }"
20
+ end
21
+ when Symbol
22
+ names.index{|f| f.to_s == field.to_s }
23
+ when String
24
+ pos = names.index{|f| f.to_s == field }
25
+ next pos if pos
26
+ if field =~ /^\d+$/
27
+ next identify_names(names, field.to_i)
28
+ end
29
+ pos = names.index{|name| name.start_with?(field) }
30
+ next pos if pos
31
+ nil
32
+ else
33
+ raise "Field '#{ Log.fingerprint field }' was not understood. Options: (#{ Log.fingerprint names })"
34
+ end
35
+ end
36
+
37
+ Array === selected ? res : res.first
38
+ end
39
+
40
+ def positions(fields)
41
+ if Array == fields
42
+ fields.collect{|field|
43
+ NamedArray.identify_name(@fields, field)
44
+ }
45
+ else
46
+ NamedArray.identify_name(@fields, fields)
47
+ end
48
+ end
49
+
50
+ def [](key)
51
+ pos = NamedArray.identify_name(@fields, key)
52
+ super(pos)
53
+ end
54
+
55
+ def concat(other)
56
+ super(other)
57
+ self.fields.concat(other.fields) if NamedArray === other
58
+ self
59
+ end
60
+
61
+ def to_hash
62
+ hash = {}
63
+ self.fields.zip(self) do |field,value|
64
+ hash[field] = value
65
+ end
66
+ IndiferentHash.setup hash
67
+ end
68
+ end