scout-gear 9.1.0 → 10.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (116) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +3 -3
  3. data/Rakefile +1 -2
  4. data/VERSION +1 -1
  5. data/lib/scout/offsite/exceptions.rb +9 -0
  6. data/lib/scout/offsite/ssh.rb +1 -0
  7. data/lib/scout/semaphore.rb +2 -0
  8. data/lib/scout/tsv/open.rb +1 -1
  9. data/lib/scout/tsv/parser.rb +1 -1
  10. data/lib/scout/tsv/persist/adapter.rb +2 -2
  11. data/lib/scout/tsv/persist.rb +1 -0
  12. data/lib/scout/tsv.rb +1 -1
  13. data/lib/scout/work_queue/exceptions.rb +18 -0
  14. data/lib/scout/work_queue.rb +1 -0
  15. data/lib/scout/workflow/definition.rb +1 -1
  16. data/lib/scout/workflow/step/config.rb +1 -1
  17. data/lib/scout/workflow/step.rb +2 -2
  18. data/lib/scout/workflow/task/inputs.rb +1 -1
  19. data/lib/scout/workflow/task.rb +2 -2
  20. data/lib/scout/workflow.rb +2 -2
  21. data/lib/scout-gear.rb +1 -10
  22. data/scout-gear.gemspec +8 -98
  23. data/test/scout/workflow/task/test_inputs.rb +0 -4
  24. metadata +7 -110
  25. data/lib/scout/cmd.rb +0 -347
  26. data/lib/scout/concurrent_stream.rb +0 -284
  27. data/lib/scout/config.rb +0 -168
  28. data/lib/scout/exceptions.rb +0 -151
  29. data/lib/scout/indiferent_hash/case_insensitive.rb +0 -30
  30. data/lib/scout/indiferent_hash/options.rb +0 -115
  31. data/lib/scout/indiferent_hash.rb +0 -96
  32. data/lib/scout/log/color.rb +0 -224
  33. data/lib/scout/log/color_class.rb +0 -269
  34. data/lib/scout/log/fingerprint.rb +0 -69
  35. data/lib/scout/log/progress/report.rb +0 -244
  36. data/lib/scout/log/progress/util.rb +0 -173
  37. data/lib/scout/log/progress.rb +0 -106
  38. data/lib/scout/log/trap.rb +0 -107
  39. data/lib/scout/log.rb +0 -441
  40. data/lib/scout/meta_extension.rb +0 -100
  41. data/lib/scout/misc/digest.rb +0 -63
  42. data/lib/scout/misc/filesystem.rb +0 -25
  43. data/lib/scout/misc/format.rb +0 -255
  44. data/lib/scout/misc/helper.rb +0 -31
  45. data/lib/scout/misc/insist.rb +0 -56
  46. data/lib/scout/misc/monitor.rb +0 -66
  47. data/lib/scout/misc/system.rb +0 -73
  48. data/lib/scout/misc.rb +0 -10
  49. data/lib/scout/named_array.rb +0 -138
  50. data/lib/scout/open/lock/lockfile.rb +0 -587
  51. data/lib/scout/open/lock.rb +0 -68
  52. data/lib/scout/open/remote.rb +0 -135
  53. data/lib/scout/open/stream.rb +0 -491
  54. data/lib/scout/open/util.rb +0 -244
  55. data/lib/scout/open.rb +0 -170
  56. data/lib/scout/path/find.rb +0 -204
  57. data/lib/scout/path/tmpfile.rb +0 -8
  58. data/lib/scout/path/util.rb +0 -127
  59. data/lib/scout/path.rb +0 -51
  60. data/lib/scout/persist/open.rb +0 -17
  61. data/lib/scout/persist/path.rb +0 -15
  62. data/lib/scout/persist/serialize.rb +0 -157
  63. data/lib/scout/persist.rb +0 -104
  64. data/lib/scout/resource/open.rb +0 -8
  65. data/lib/scout/resource/path.rb +0 -80
  66. data/lib/scout/resource/produce/rake.rb +0 -69
  67. data/lib/scout/resource/produce.rb +0 -151
  68. data/lib/scout/resource/scout.rb +0 -3
  69. data/lib/scout/resource/software.rb +0 -178
  70. data/lib/scout/resource/util.rb +0 -59
  71. data/lib/scout/resource.rb +0 -41
  72. data/lib/scout/simple_opt/accessor.rb +0 -54
  73. data/lib/scout/simple_opt/doc.rb +0 -126
  74. data/lib/scout/simple_opt/get.rb +0 -57
  75. data/lib/scout/simple_opt/parse.rb +0 -67
  76. data/lib/scout/simple_opt/setup.rb +0 -26
  77. data/lib/scout/simple_opt.rb +0 -5
  78. data/lib/scout/tmpfile.rb +0 -129
  79. data/test/scout/indiferent_hash/test_case_insensitive.rb +0 -16
  80. data/test/scout/indiferent_hash/test_options.rb +0 -46
  81. data/test/scout/log/test_progress.rb +0 -108
  82. data/test/scout/misc/test_digest.rb +0 -30
  83. data/test/scout/misc/test_filesystem.rb +0 -30
  84. data/test/scout/misc/test_insist.rb +0 -13
  85. data/test/scout/misc/test_system.rb +0 -21
  86. data/test/scout/open/test_lock.rb +0 -52
  87. data/test/scout/open/test_remote.rb +0 -25
  88. data/test/scout/open/test_stream.rb +0 -676
  89. data/test/scout/open/test_util.rb +0 -73
  90. data/test/scout/path/test_find.rb +0 -119
  91. data/test/scout/path/test_util.rb +0 -22
  92. data/test/scout/persist/test_open.rb +0 -37
  93. data/test/scout/persist/test_path.rb +0 -37
  94. data/test/scout/persist/test_serialize.rb +0 -114
  95. data/test/scout/resource/test_path.rb +0 -46
  96. data/test/scout/resource/test_produce.rb +0 -92
  97. data/test/scout/resource/test_software.rb +0 -24
  98. data/test/scout/resource/test_util.rb +0 -36
  99. data/test/scout/simple_opt/test_doc.rb +0 -16
  100. data/test/scout/simple_opt/test_get.rb +0 -11
  101. data/test/scout/simple_opt/test_parse.rb +0 -10
  102. data/test/scout/simple_opt/test_setup.rb +0 -77
  103. data/test/scout/test_cmd.rb +0 -85
  104. data/test/scout/test_concurrent_stream.rb +0 -29
  105. data/test/scout/test_config.rb +0 -66
  106. data/test/scout/test_indiferent_hash.rb +0 -26
  107. data/test/scout/test_log.rb +0 -32
  108. data/test/scout/test_meta_extension.rb +0 -80
  109. data/test/scout/test_misc.rb +0 -6
  110. data/test/scout/test_named_array.rb +0 -43
  111. data/test/scout/test_open.rb +0 -146
  112. data/test/scout/test_path.rb +0 -54
  113. data/test/scout/test_persist.rb +0 -186
  114. data/test/scout/test_resource.rb +0 -26
  115. data/test/scout/test_tmpfile.rb +0 -53
  116. /data/test/scout/{log/test_color.rb → test_offsite.rb} +0 -0
data/lib/scout/config.rb DELETED
@@ -1,168 +0,0 @@
1
- require_relative 'path'
2
- require_relative 'resource'
3
- require_relative 'resource/scout'
4
-
5
- module Scout::Config
6
-
7
- CACHE ||= IndiferentHash.setup({})
8
-
9
- GOT_KEYS=[]
10
-
11
- def self.add_entry(key, value, tokens)
12
- tokens = [tokens] unless Array === tokens
13
- tokens << "key:#{key}" unless tokens.include?("key:#{key}")
14
- CACHE[key.to_s] ||= []
15
- CACHE[key.to_s] << [tokens, value]
16
- end
17
-
18
- def self.load_file(file)
19
- Log.debug "Loading config file: #{ file }"
20
- TSV.traverse file, :type => :array do |line|
21
- next if line =~ /^#/
22
- key, value, *tokens = line.strip.split(/\s/)
23
-
24
- self.add_entry(key, value, tokens) if key
25
- end
26
- end
27
-
28
- def self.load_config
29
- Scout.etc.config.find_all.reverse.each do |file|
30
- self.load_file(file)
31
- end
32
- end
33
-
34
- def self.set(values, *tokens)
35
- if not Hash === values
36
- values = {values => tokens.shift}
37
- end
38
-
39
- values.each do |key,value|
40
- add_entry key, value, tokens
41
- end
42
- end
43
-
44
- def self.token_priority(token)
45
- token, _sep, priority = token.to_s.partition("::")
46
-
47
- if priority.nil? || priority.empty?
48
- type, _sep, rest = token.partition(":")
49
- priority = case type
50
- when "workflow"
51
- 4
52
- when "task"
53
- 3
54
- when "file"
55
- 2
56
- when "line"
57
- 1
58
- when "key"
59
- 20
60
- else
61
- 10
62
- end
63
- else
64
- priority = priority.to_i
65
- end
66
-
67
- [token, priority]
68
- end
69
-
70
- def self.match(entries, give_token)
71
- priorities = {}
72
- entries.each do |tokens, value|
73
- best_prio = nil
74
- tokens = [tokens] unless Array === tokens
75
- tokens.each do |tok|
76
- tok, prio = token_priority tok
77
- next unless tok == give_token
78
-
79
- best_prio = prio if best_prio.nil? or best_prio > prio
80
- next if prio > best_prio
81
-
82
- priorities[prio] ||= []
83
- priorities[prio].unshift value
84
- end
85
- end if entries
86
- priorities
87
- end
88
-
89
- # For equal priorities the matching prioritizes tokens ealier in the list
90
- def self.get(key, *tokens)
91
- options = tokens.pop if Hash === tokens.last
92
- default = options.nil? ? nil : options[:default]
93
-
94
- tokens = ["key:" + key] if tokens.empty?
95
-
96
- tokens = tokens.flatten
97
- file, _sep, line = caller.reject{|l|
98
- l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
99
- l =~ /rbbt\/resource\/path\.rb/ or
100
- l =~ /rbbt\/util\/misc\.rb/ or
101
- l =~ /accessor\.rb/ or
102
- l =~ /progress-monitor\.rb/
103
- }.first.partition(":")
104
-
105
- File.expand_path(file)
106
-
107
- tokens << ("file:" << file)
108
- tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
109
-
110
- entries = CACHE[key.to_s]
111
- priorities = {}
112
- tokens.each do |token|
113
- token_prio = match entries, token.to_s
114
- token_prio.each do |prio, values|
115
- priorities[prio] ||= []
116
- priorities[prio].concat(values)
117
- end
118
- end
119
-
120
- value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
121
- value = false if value == 'false'
122
-
123
- Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
124
- GOT_KEYS << [key, value, tokens]
125
-
126
- if String === value && m = value.match(/^env:(.*)/)
127
- variable = m.captures.first
128
- ENV[variable]
129
- elsif value == 'nil'
130
- nil
131
- else
132
- value
133
- end
134
- end
135
-
136
- def self.with_config
137
- saved_config = {}
138
- CACHE.each do |k,v|
139
- saved_config[k] = v.dup
140
- end
141
- saved_got_keys = GOT_KEYS.dup
142
- begin
143
- yield
144
- ensure
145
- CACHE.replace(saved_config)
146
- GOT_KEYS.replace(saved_got_keys)
147
- end
148
- end
149
-
150
- def self.process_config(config)
151
- if Path.is_filename?(config) && File.exist?(config)
152
- Scout::Config.load_file(config)
153
- elsif Scout.etc.config_profile[config].exists?
154
- Scout::Config.load_file(Scout.etc.config_profile[config].find)
155
- else
156
- key, value, *tokens = config.split(/\s/)
157
- tokens = tokens.collect do |tok|
158
- tok, _sep, prio = tok.partition("::")
159
- prio = "0" if prio.nil? or prio.empty?
160
- [tok, prio] * "::"
161
- end
162
- Scout::Config.set({key => value}, *tokens)
163
- end
164
- end
165
-
166
-
167
- self.load_config
168
- end
@@ -1,151 +0,0 @@
1
- class ScoutDeprecated < StandardError; end
2
- class ScoutException < StandardError; end
3
-
4
- class FieldNotFoundError < StandardError;end
5
-
6
- class TryAgain < StandardError; end
7
- class StopInsist < Exception
8
- attr_accessor :exception
9
- def initialize(exception)
10
- @exception = exception
11
- end
12
- end
13
-
14
- class Aborted < StandardError; end
15
-
16
- class ParameterException < ScoutException; end
17
- class MissingParameterException < ParameterException
18
- def initialize(parameter)
19
- super("Missing parameter '#{parameter}'")
20
- end
21
- end
22
- class ProcessFailed < StandardError;
23
- attr_accessor :pid, :msg
24
- def initialize(pid = Process.pid, msg = nil)
25
- @pid = pid
26
- @msg = msg
27
- if @pid
28
- if @msg
29
- message = "Process #{@pid} failed - #{@msg}"
30
- else
31
- message = "Process #{@pid} failed"
32
- end
33
- else
34
- message = "Failed to run #{@msg}"
35
- end
36
- super(message)
37
- end
38
- end
39
-
40
- class ConcurrentStreamProcessFailed < ProcessFailed
41
- attr_accessor :concurrent_stream
42
- def initialize(pid = Process.pid, msg = nil, concurrent_stream = nil)
43
- super(pid, msg)
44
- @concurrent_stream = concurrent_stream
45
- end
46
- end
47
-
48
- class OpenURLError < StandardError; end
49
-
50
- class DontClose < Exception
51
- attr_accessor :payload
52
- def initialize(payload = nil)
53
- @payload = payload
54
- end
55
- end
56
-
57
-
58
- class KeepLocked < Exception
59
- attr_accessor :payload
60
- def initialize(payload)
61
- @payload = payload
62
- end
63
- end
64
-
65
- class KeepBar < Exception
66
- attr_accessor :payload
67
- def initialize(payload)
68
- @payload = payload
69
- end
70
- end
71
-
72
- class LockInterrupted < TryAgain; end
73
-
74
- class ClosedStream < StandardError; end
75
-
76
- class DoneProcessing < Exception
77
- attr_accessor :pid
78
- def initialize(pid = Process.pid)
79
- @pid = pid
80
- end
81
-
82
- def message
83
- "Done processing pid #{pid}"
84
- end
85
- end
86
-
87
- class WorkerException < ScoutException
88
- attr_accessor :worker_exception, :pid
89
- def initialize(worker_exception, pid)
90
- @worker_exception = worker_exception
91
- @pid = pid
92
- end
93
- end
94
-
95
- class SemaphoreInterrupted < TryAgain; end
96
-
97
- #class OpenGzipError < StandardError; end
98
- #
99
- #
100
- #class TryThis < StandardError
101
- # attr_accessor :payload
102
- # def initialize(payload = nil)
103
- # @payload = payload
104
- # end
105
- #end
106
- #
107
- #class SemaphoreInterrupted < TryAgain; end
108
- #
109
- #class RemoteServerError < StandardError; end
110
- #
111
- #class DependencyError < Aborted
112
- # def initialize(msg)
113
- # if defined? Step and Step === msg
114
- # step = msg
115
- # new_msg = [step.path, step.messages.last] * ": "
116
- # super(new_msg)
117
- # else
118
- # super(msg)
119
- # end
120
- # end
121
- #end
122
- #
123
- #class DependencyScoutException < ScoutException
124
- # def initialize(msg)
125
- # if defined? Step and Step === msg
126
- # step = msg
127
- #
128
- # new_msg = nil
129
- # new_msg = [step.path, step.messages.last] * ": "
130
- #
131
- # super(new_msg)
132
- # else
133
- # super(msg)
134
- # end
135
- # end
136
- #end
137
- #
138
- #
139
- #
140
- #
141
- class ResourceNotFound < ScoutException; end
142
-
143
- class SSHProcessFailed < StandardError
144
- attr_accessor :host, :cmd
145
- def initialize(host, cmd)
146
- @host = host
147
- @cmd = cmd
148
- message = "SSH server #{host} failed cmd '#{cmd}'"
149
- super(message)
150
- end
151
- end
@@ -1,30 +0,0 @@
1
- module CaseInsensitiveHash
2
-
3
- def self.setup(hash)
4
- hash.extend CaseInsensitiveHash
5
- end
6
-
7
- def downcase_keys
8
- @downcase_keys ||= begin
9
- down = {}
10
- keys.collect{|key|
11
- down[key.to_s.downcase] = key
12
- }
13
- down
14
- end
15
- end
16
-
17
- def [](key, *rest)
18
- value = super(key, *rest)
19
- return value unless value.nil?
20
- key_downcase = key.to_s.downcase
21
- super(downcase_keys[key_downcase])
22
- end
23
-
24
- def values_at(*keys)
25
- keys.collect do |key|
26
- self[key]
27
- end
28
- end
29
-
30
- end
@@ -1,115 +0,0 @@
1
- module IndiferentHash
2
- def self.add_defaults(options, defaults = {})
3
- options = string2hash options if String === options
4
- IndiferentHash.setup(options)
5
-
6
- defaults = string2hash defaults if String === defaults
7
-
8
- defaults.each do |key, value|
9
- next if options.include?(key)
10
-
11
- options[key] = value
12
- end
13
-
14
- options
15
- end
16
-
17
- def self.process_options(hash, *keys)
18
- IndiferentHash.setup(hash)
19
-
20
- defaults = keys.pop if Hash === keys.last
21
- hash = IndiferentHash.add_defaults hash, defaults if defaults
22
-
23
- if keys.length == 1
24
- hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s)
25
- else
26
- keys.collect do |key| hash.include?(key.to_sym) ? hash.delete(key.to_sym) : hash.delete(key.to_s) end
27
- end
28
- end
29
-
30
- def self.pull_keys(hash, prefix)
31
- new = {}
32
- prefix = prefix.to_s
33
- hash.keys.each do |key|
34
- if key.to_s =~ /#{ prefix }_(.*)/
35
- case
36
- when String === key
37
- new[$1] = hash.delete key
38
- when Symbol === key
39
- new[$1.to_sym] = hash.delete key
40
- end
41
- else
42
- if key.to_s == prefix.to_s
43
- new[key] = hash.delete key
44
- end
45
- end
46
- end
47
-
48
- IndiferentHash.setup(new)
49
- end
50
-
51
- def self.zip2hash(list1, list2)
52
- hash = {}
53
- list1.each_with_index do |e,i|
54
- hash[e] = list2[i]
55
- end
56
- IndiferentHash.setup(hash)
57
- end
58
-
59
- def self.positional2hash(keys, *values)
60
- if Hash === values.last
61
- extra = values.pop
62
- inputs = IndiferentHash.zip2hash(keys, values)
63
- inputs.delete_if{|k,v| v.nil? or (String === v and v.empty?)}
64
- inputs = IndiferentHash.add_defaults inputs, extra
65
- inputs.delete_if{|k,v| not keys.include?(k) and not (Symbol === k ? keys.include?(k.to_s) : keys.include?(k.to_sym))}
66
- inputs
67
- else
68
- IndiferentHash.zip2hash(keys, values)
69
- end
70
- end
71
-
72
- def self.array2hash(array, default = nil)
73
- hash = {}
74
- array.each do |key, value|
75
- value = default.dup if value.nil? and not default.nil?
76
- hash[key] = value
77
- end
78
- IndiferentHash.setup(hash)
79
- end
80
-
81
- def self.process_to_hash(list)
82
- result = yield list
83
- zip2hash(list, result)
84
- end
85
-
86
- def self.hash2string(hash)
87
- hash.sort_by{|k,v| k.to_s}.collect{|k,v|
88
- next unless %w(Symbol String Float Fixnum Integer Numeric TrueClass FalseClass Module Class Object).include? v.class.to_s
89
- [ Symbol === k ? ":" << k.to_s : k.to_s.chomp,
90
- Symbol === v ? ":" << v.to_s : v.to_s.chomp] * "="
91
- }.compact * "#"
92
- end
93
-
94
- def self.string2hash(string)
95
- options = {}
96
-
97
- string.split('#').each do |str|
98
- key, _, value = str.partition "="
99
-
100
- key = key[1..-1].to_sym if key[0] == ":"
101
-
102
- options[key] = true and next if value.empty?
103
- options[key] = value[1..-1].to_sym and next if value[0] == ":"
104
- options[key] = Regexp.new(/#{value[1..-2]}/) and next if value[0] == "/" and value[-1] == "/"
105
- options[key] = value[1..-2] and next if value =~ /^['"].*['"]$/
106
- options[key] = value.to_i and next if value =~ /^\d+$/
107
- options[key] = value.to_f and next if value =~ /^\d*\.\d+$/
108
- options[key] = true and next if value == "true"
109
- options[key] = false and next if value == "false"
110
- options[key] = value
111
- end
112
-
113
- IndiferentHash.setup(options)
114
- end
115
- end
@@ -1,96 +0,0 @@
1
- require_relative 'indiferent_hash/options'
2
- require_relative 'indiferent_hash/case_insensitive'
3
-
4
- module IndiferentHash
5
-
6
- def self.setup(hash)
7
- hash.extend IndiferentHash
8
- hash
9
- end
10
-
11
- def merge(other)
12
- new = self.dup
13
- IndiferentHash.setup(new)
14
- other.each do |k,value|
15
- new[k] = value
16
- end
17
- new
18
- end
19
-
20
- def []=(key,value)
21
- delete(key)
22
- super(key,value)
23
- end
24
-
25
- def _default?
26
- @_default ||= self.default or self.default_proc
27
- end
28
-
29
- def [](key)
30
- res = super(key)
31
- return res unless res.nil? or (_default? and not keys.include? key)
32
-
33
- case key
34
- when Symbol, Module
35
- super(key.to_s)
36
- when String
37
- super(key.to_sym)
38
- else
39
- res
40
- end
41
- end
42
-
43
- def values_at(*key_list)
44
- key_list.inject([]){|acc,key| acc << self[key]}
45
- end
46
-
47
- def include?(key)
48
- case key
49
- when Symbol, Module
50
- super(key) || super(key.to_s)
51
- when String
52
- super(key) || super(key.to_sym)
53
- else
54
- super(key)
55
- end
56
- end
57
-
58
- def delete(key)
59
- case key
60
- when Symbol, Module
61
- v = super(key)
62
- v.nil? ? super(key.to_s) : v
63
- when String
64
- v = super(key)
65
- v.nil? ? super(key.to_sym) : v
66
- else
67
- super(key)
68
- end
69
- end
70
-
71
- def clean_version
72
- clean = {}
73
- each do |k,v|
74
- clean[k.to_s] = v unless clean.include? k.to_s
75
- end
76
- clean
77
- end
78
-
79
- def slice(*list)
80
- ext_list = []
81
- list.each do |e|
82
- case e
83
- when Symbol
84
- ext_list << e
85
- ext_list << e.to_s
86
- when String
87
- ext_list << e
88
- ext_list << e.to_sym
89
- else
90
- ext_list << e
91
- end
92
- end
93
- IndiferentHash.setup(super(*ext_list))
94
- end
95
- end
96
-