scout-gear 1.2.0 → 5.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +4 -0
  3. data/.vimproject +663 -4
  4. data/Rakefile +1 -0
  5. data/VERSION +1 -1
  6. data/bin/scout +235 -0
  7. data/lib/scout/cmd.rb +344 -0
  8. data/lib/scout/concurrent_stream.rb +259 -0
  9. data/lib/scout/exceptions.rb +15 -8
  10. data/lib/scout/indiferent_hash/options.rb +8 -26
  11. data/lib/scout/indiferent_hash.rb +0 -30
  12. data/lib/scout/log/color.rb +2 -2
  13. data/lib/scout/log/fingerprint.rb +11 -1
  14. data/lib/scout/log/progress/report.rb +0 -1
  15. data/lib/scout/log/progress/util.rb +1 -1
  16. data/lib/scout/log/progress.rb +4 -4
  17. data/lib/scout/log.rb +10 -2
  18. data/lib/scout/meta_extension.rb +15 -1
  19. data/lib/scout/misc/digest.rb +56 -0
  20. data/lib/scout/misc/filesystem.rb +26 -0
  21. data/lib/scout/misc/format.rb +226 -0
  22. data/lib/scout/misc/insist.rb +56 -0
  23. data/lib/scout/misc.rb +5 -11
  24. data/lib/scout/open/lock.rb +61 -0
  25. data/lib/scout/open/remote.rb +120 -0
  26. data/lib/scout/open/stream.rb +372 -0
  27. data/lib/scout/open/util.rb +225 -0
  28. data/lib/scout/open.rb +169 -0
  29. data/lib/scout/path/find.rb +78 -26
  30. data/lib/scout/path/tmpfile.rb +8 -0
  31. data/lib/scout/path/util.rb +17 -5
  32. data/lib/scout/path.rb +13 -31
  33. data/lib/scout/persist/open.rb +17 -0
  34. data/lib/scout/persist/path.rb +15 -0
  35. data/lib/scout/persist/serialize.rb +140 -0
  36. data/lib/scout/persist.rb +54 -0
  37. data/lib/scout/resource/path.rb +15 -0
  38. data/lib/scout/resource/produce/rake.rb +69 -0
  39. data/lib/scout/resource/produce.rb +246 -0
  40. data/lib/scout/resource/scout.rb +3 -0
  41. data/lib/scout/resource.rb +37 -0
  42. data/lib/scout/simple_opt/accessor.rb +54 -0
  43. data/lib/scout/simple_opt/doc.rb +102 -0
  44. data/lib/scout/simple_opt/get.rb +57 -0
  45. data/lib/scout/simple_opt/parse.rb +67 -0
  46. data/lib/scout/simple_opt/setup.rb +26 -0
  47. data/lib/scout/simple_opt.rb +5 -0
  48. data/lib/scout/tmpfile.rb +39 -1
  49. data/lib/scout/workflow/definition.rb +72 -0
  50. data/lib/scout/workflow/documentation.rb +77 -0
  51. data/lib/scout/workflow/step/info.rb +77 -0
  52. data/lib/scout/workflow/step.rb +96 -0
  53. data/lib/scout/workflow/task/inputs.rb +112 -0
  54. data/lib/scout/workflow/task.rb +141 -0
  55. data/lib/scout/workflow/usage.rb +294 -0
  56. data/lib/scout/workflow/util.rb +11 -0
  57. data/lib/scout/workflow.rb +39 -0
  58. data/lib/scout-gear.rb +5 -0
  59. data/lib/scout.rb +1 -0
  60. data/lib/workflow-scout.rb +2 -0
  61. data/scout-gear.gemspec +78 -5
  62. data/scout_commands/alias +48 -0
  63. data/scout_commands/find +83 -0
  64. data/scout_commands/glob +0 -0
  65. data/scout_commands/rbbt +23 -0
  66. data/scout_commands/workflow/task +707 -0
  67. data/test/scout/indiferent_hash/test_case_insensitive.rb +16 -0
  68. data/test/scout/indiferent_hash/test_options.rb +11 -1
  69. data/test/scout/misc/test_digest.rb +30 -0
  70. data/test/scout/misc/test_filesystem.rb +30 -0
  71. data/test/scout/misc/test_insist.rb +13 -0
  72. data/test/scout/open/test_lock.rb +52 -0
  73. data/test/scout/open/test_remote.rb +25 -0
  74. data/test/scout/open/test_stream.rb +515 -0
  75. data/test/scout/open/test_util.rb +73 -0
  76. data/test/scout/path/test_find.rb +37 -1
  77. data/test/scout/persist/test_open.rb +37 -0
  78. data/test/scout/persist/test_path.rb +37 -0
  79. data/test/scout/persist/test_serialize.rb +114 -0
  80. data/test/scout/resource/test_path.rb +40 -0
  81. data/test/scout/resource/test_produce.rb +62 -0
  82. data/test/scout/simple_opt/test_get.rb +11 -0
  83. data/test/scout/simple_opt/test_parse.rb +10 -0
  84. data/test/scout/simple_opt/test_setup.rb +77 -0
  85. data/test/scout/test_cmd.rb +85 -0
  86. data/test/scout/test_concurrent_stream.rb +29 -0
  87. data/test/scout/test_misc.rb +0 -7
  88. data/test/scout/test_open.rb +146 -0
  89. data/test/scout/test_path.rb +3 -1
  90. data/test/scout/test_persist.rb +83 -0
  91. data/test/scout/test_resource.rb +26 -0
  92. data/test/scout/test_workflow.rb +87 -0
  93. data/test/scout/workflow/step/test_info.rb +28 -0
  94. data/test/scout/workflow/task/test_inputs.rb +182 -0
  95. data/test/scout/workflow/test_step.rb +36 -0
  96. data/test/scout/workflow/test_task.rb +178 -0
  97. data/test/scout/workflow/test_usage.rb +26 -0
  98. data/test/scout/workflow/test_util.rb +17 -0
  99. data/test/test_helper.rb +17 -0
  100. data/test/test_scout-gear.rb +0 -0
  101. metadata +76 -3
@@ -0,0 +1,226 @@
1
+ module Misc
2
+ COLOR_LIST = %w(#BC80BD #CCEBC5 #FFED6F #8DD3C7 #FFFFB3 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9)
3
+
4
+ def self.colors_for(list)
5
+ unused = COLOR_LIST.dup
6
+
7
+ used = {}
8
+ colors = list.collect do |elem|
9
+ if used.include? elem
10
+ used[elem]
11
+ else
12
+ color = unused.shift
13
+ used[elem]=color
14
+ color
15
+ end
16
+ end
17
+
18
+ [colors, used]
19
+ end
20
+
21
+ def self.format_seconds(time, extended = false)
22
+ seconds = time.to_i
23
+ str = [seconds/3600, seconds/60 % 60, seconds % 60].map{|t| "%02i" % t }.join(':')
24
+ str << ".%02i" % ((time - seconds) * 100) if extended
25
+ str
26
+ end
27
+
28
+ def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
29
+ i = 0
30
+ size = size + offset + indent
31
+ re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
32
+ text.split(re).collect do |paragraph|
33
+ i += 1
34
+ str = if i % 2 == 1
35
+ words = paragraph.gsub(/\s+/, "\s").split(" ")
36
+ lines = []
37
+ line = " "*offset
38
+ word = words.shift
39
+ while word
40
+ word = word[0..size-indent-offset-4] + '...' if word.length >= size - indent - offset
41
+ while word and Log.uncolor(line).length + Log.uncolor(word).length <= size - indent
42
+ line << word << " "
43
+ word = words.shift
44
+ end
45
+ offset = 0
46
+ lines << ((" " * indent) << line[0..-2])
47
+ line = ""
48
+ end
49
+ (lines * "\n")
50
+ else
51
+ paragraph
52
+ end
53
+ offset = 0
54
+ str
55
+ end*""
56
+ end
57
+
58
+ def self.format_definition_list_item(dt, dd, size = 80, indent = 20, color = :yellow)
59
+ dd = "" if dd.nil?
60
+ dt = Log.color color, dt if color
61
+ dt = dt.to_s unless dd.empty?
62
+ len = Log.uncolor(dt).length
63
+
64
+ if indent < 0
65
+ text = format_paragraph(dd, size, indent.abs-1, 0)
66
+ text = dt << "\n" << text
67
+ else
68
+ offset = len - indent
69
+ offset = 0 if offset < 0
70
+ text = format_paragraph(dd, size, indent.abs+1, offset)
71
+ text[0..len-1] = dt
72
+ end
73
+ text
74
+ end
75
+
76
+ def self.format_definition_list(defs, size = 80, indent = 20, color = :yellow, sep = "\n\n")
77
+ entries = []
78
+ defs.each do |dt,dd|
79
+ text = format_definition_list_item(dt,dd,size,indent,color)
80
+ entries << text
81
+ end
82
+ entries * sep
83
+ end
84
+
85
+ def self.camel_case(string)
86
+ return string if string !~ /_/ && string =~ /[A-Z]+.*/
87
+ string.split(/_|(\d+)/).map{|e|
88
+ (e =~ /^[A-Z]{2,}$/ ? e : e.capitalize)
89
+ }.join
90
+ end
91
+
92
+ def self.camel_case_lower(string)
93
+ string.split('_').inject([]){ |buffer,e|
94
+ buffer.push(buffer.empty? ? e.downcase : (e =~ /^[A-Z]{2,}$/ ? e : e.capitalize))
95
+ }.join
96
+ end
97
+
98
+ def self.snake_case(string)
99
+ return nil if string.nil?
100
+ string = string.to_s if Symbol === string
101
+ string.
102
+ gsub(/([A-Z]{2,})([A-Z][a-z])/,'\1_\2').
103
+ gsub(/([a-z])([A-Z])/,'\1_\2').
104
+ gsub(/\s/,'_').gsub(/[^\w]/, '').
105
+ split("_").collect{|p| p.match(/[A-Z]{2,}/) ? p : p.downcase } * "_"
106
+ end
107
+
108
+ # source: https://gist.github.com/ekdevdes/2450285
109
+ # author: Ethan Kramer (https://github.com/ekdevdes)
110
+ def self.humanize(value, options = {})
111
+ if options.empty?
112
+ options[:format] = :sentence
113
+ end
114
+
115
+ values = value.to_s.split('_')
116
+ values.each_index do |index|
117
+ # lower case each item in array
118
+ # Miguel Vazquez edit: Except for acronyms
119
+ values[index].downcase! unless values[index].match(/[a-zA-Z][A-Z]/)
120
+ end
121
+ if options[:format] == :allcaps
122
+ values.each do |value|
123
+ value.capitalize!
124
+ end
125
+
126
+ if options.empty?
127
+ options[:seperator] = " "
128
+ end
129
+
130
+ return values.join " "
131
+ end
132
+
133
+ if options[:format] == :class
134
+ values.each do |value|
135
+ value.capitalize!
136
+ end
137
+
138
+ return values.join ""
139
+ end
140
+
141
+ if options[:format] == :sentence
142
+ values[0].capitalize! unless values[0].match(/[a-zA-Z][A-Z]/)
143
+
144
+ return values.join " "
145
+ end
146
+
147
+ if options[:format] == :nocaps
148
+ return values.join " "
149
+ end
150
+ end
151
+
152
+ def self.fixascii(string)
153
+ if string.respond_to?(:encode)
154
+ self.fixutf8(string).encode("ASCII-8BIT")
155
+ else
156
+ string
157
+ end
158
+ end
159
+
160
+ def self.to_utf8(string)
161
+ string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
162
+ end
163
+
164
+ def self.fixutf8(string)
165
+ return nil if string.nil?
166
+ return string if string.respond_to?(:encoding) && string.encoding.to_s == "UTF-8" && (string.respond_to?(:valid_encoding?) && string.valid_encoding?) ||
167
+ (string.respond_to?(:valid_encoding) && string.valid_encoding)
168
+
169
+ if string.respond_to?(:encode)
170
+ string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
171
+ else
172
+ require 'iconv'
173
+ @@ic ||= Iconv.new('UTF-8//IGNORE', 'UTF-8')
174
+ @@ic.iconv(string)
175
+ end
176
+ end
177
+
178
+ def self.humanize_list(list)
179
+ return "" if list.empty?
180
+ if list.length == 1
181
+ list.first
182
+ else
183
+ list[0..-2].collect{|e| e.to_s} * ", " << " and " << list[-1].to_s
184
+ end
185
+ end
186
+
187
+ def self.parse_sql_values(txt)
188
+ io = StringIO.new txt.strip
189
+
190
+ values = []
191
+ fields = []
192
+ current = nil
193
+ quoted = false
194
+ while c = io.getc
195
+ if quoted
196
+ if c == "'"
197
+ quoted = false
198
+ else
199
+ current << c
200
+ end
201
+ else
202
+ case c
203
+ when "("
204
+ current = ""
205
+ when ")"
206
+ fields << current
207
+ values << fields
208
+ fields = []
209
+ current = nil
210
+ when ','
211
+ if not current.nil?
212
+ fields << current
213
+ current = ""
214
+ end
215
+ when "'"
216
+ quoted = true
217
+ when ";"
218
+ break
219
+ else
220
+ current << c
221
+ end
222
+ end
223
+ end
224
+ values
225
+ end
226
+ end
@@ -0,0 +1,56 @@
1
+ module Misc
2
+ def self.insist(times = 4, sleep = nil, msg = nil)
3
+ sleep_array = nil
4
+
5
+ try = 0
6
+ begin
7
+ begin
8
+ yield
9
+ rescue Exception
10
+ if Array === times
11
+ sleep_array = times
12
+ times = sleep_array.length
13
+ sleep = sleep_array.shift
14
+ end
15
+
16
+ if sleep.nil?
17
+ sleep_array = ([0] + [0.001, 0.01, 0.1, 0.5] * (times / 3)).sort[0..times-1]
18
+ sleep = sleep_array.shift
19
+ end
20
+ raise $!
21
+ end
22
+ rescue TryAgain
23
+ sleep sleep
24
+ retry
25
+ rescue StopInsist
26
+ raise $!.exception
27
+ rescue Aborted, Interrupt
28
+ if msg
29
+ Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
30
+ else
31
+ Log.warn("Not Insisting after Aborted: #{$!.message}")
32
+ end
33
+ raise $!
34
+ rescue Exception
35
+ Log.exception $! if ENV["RBBT_LOG_INSIST"] == 'true'
36
+ if msg
37
+ Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
38
+ elsif FalseClass === msg
39
+ nil
40
+ else
41
+ Log.warn("Insisting after exception: #{$!.class} #{$!.message}")
42
+ end
43
+
44
+ if sleep and try > 0
45
+ sleep sleep
46
+ sleep = sleep_array.shift || sleep if sleep_array
47
+ else
48
+ Thread.pass
49
+ end
50
+
51
+ try += 1
52
+ retry if try < times
53
+ raise $!
54
+ end
55
+ end
56
+ end
data/lib/scout/misc.rb CHANGED
@@ -1,13 +1,7 @@
1
- module Misc
2
- def self.in_dir(dir)
3
- old_pwd = FileUtils.pwd
4
- begin
5
- FileUtils.mkdir_p dir unless File.exist?(dir)
6
- FileUtils.cd dir
7
- yield
8
- ensure
9
- FileUtils.cd old_pwd
10
- end
11
- end
1
+ require_relative 'misc/format'
2
+ require_relative 'misc/insist'
3
+ require_relative 'misc/digest'
4
+ require_relative 'misc/filesystem'
12
5
 
6
+ module Misc
13
7
  end
@@ -0,0 +1,61 @@
1
+ require_relative '../path'
2
+ require_relative '../log'
3
+ require_relative '../exceptions'
4
+ require 'lockfile'
5
+
6
+ module Open
7
+ def self.lock(file, unlock = true, options = {})
8
+ unlock, options = true, unlock if Hash === unlock
9
+ return yield if file.nil? and not Lockfile === options[:lock]
10
+
11
+ if Lockfile === file
12
+ lockfile = file
13
+ else
14
+ file = file.find if Path === file
15
+ FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exist? File.dirname(File.expand_path(file))
16
+
17
+ case options[:lock]
18
+ when Lockfile
19
+ lockfile = options[:lock]
20
+ when FalseClass
21
+ lockfile = nil
22
+ unlock = false
23
+ when Path, String
24
+ lock_path = options[:lock].find
25
+ lockfile = Lockfile.new(lock_path, options)
26
+ else
27
+ lock_path = File.expand_path(file + '.lock')
28
+ lockfile = Lockfile.new(lock_path, options)
29
+ end
30
+ end
31
+
32
+ begin
33
+ lockfile.lock unless lockfile.nil? || lockfile.locked?
34
+ rescue Aborted, Interrupt
35
+ raise LockInterrupted
36
+ end
37
+
38
+ res = nil
39
+
40
+ begin
41
+ res = yield lockfile
42
+ rescue KeepLocked
43
+ unlock = false
44
+ res = $!.payload
45
+ ensure
46
+ if unlock
47
+ begin
48
+ if lockfile.locked?
49
+ lockfile.unlock
50
+ else
51
+ end
52
+ rescue Exception
53
+ Log.warn "Exception unlocking: #{lockfile.path}"
54
+ Log.exception $!
55
+ end
56
+ end
57
+ end
58
+
59
+ res
60
+ end
61
+ end
@@ -0,0 +1,120 @@
1
+ require_relative '../misc'
2
+ require_relative '../path'
3
+ require_relative '../cmd'
4
+
5
+ module Open
6
+ class << self
7
+ attr_accessor :remote_cache_dir
8
+
9
+ def remote_cache_dir
10
+ @remote_cache_dir ||= Path.setup("var/cache/open-remote/").find
11
+ end
12
+ end
13
+
14
+ def self.remote?(file)
15
+ !! (file =~ /^(?:https?|ftp|ssh):\/\//)
16
+ end
17
+
18
+ def self.ssh?(file)
19
+ !! (file =~ /^ssh:\/\//)
20
+ end
21
+
22
+ def self.ssh(file, options = {})
23
+ m = file.match(/ssh:\/\/([^:]+):(.*)/)
24
+ server = m[1]
25
+ file = m[2]
26
+ CMD.cmd("ssh '#{server}' cat '#{file}'", :pipe => true, :autojoin => true)
27
+ end
28
+
29
+ def self.wget(url, options = {})
30
+ if ! (options[:force] || options[:nocache]) && cache_file = in_cache(url, options)
31
+ return file_open(cache_file)
32
+ end
33
+
34
+ Log.low "WGET:\n -URL: #{ url }\n -OPTIONS: #{options.inspect}"
35
+ options = IndiferentHash.add_defaults options, "--user-agent=" => 'rbbt', :pipe => true, :autojoin => true
36
+
37
+ wait(options[:nice], options[:nice_key]) if options[:nice]
38
+ options.delete(:nice)
39
+ options.delete(:nice_key)
40
+
41
+ pipe = options.delete(:pipe)
42
+ quiet = options.delete(:quiet)
43
+ post = options.delete(:post)
44
+ cookies = options.delete(:cookies)
45
+ nocache = options.delete(:nocache)
46
+
47
+ options["--quiet"] = quiet if options["--quiet"].nil?
48
+ options["--post-data="] ||= post if post
49
+
50
+ if cookies
51
+ options["--save-cookies"] = cookies
52
+ options["--load-cookies"] = cookies
53
+ options["--keep-session-cookies"] = true
54
+ end
55
+
56
+ stderr = case
57
+ when options['stderr']
58
+ options['stderr']
59
+ when options['--quiet']
60
+ false
61
+ else
62
+ nil
63
+ end
64
+
65
+ begin
66
+ wget_options = options.dup
67
+ wget_options = wget_options.merge( '-O' => '-') unless options.include?('--output-document')
68
+ wget_options[:pipe] = pipe unless pipe.nil?
69
+ wget_options[:stderr] = stderr unless stderr.nil?
70
+
71
+ io = CMD.cmd("wget '#{ url }'", wget_options)
72
+ if nocache && nocache.to_s != 'update'
73
+ io
74
+ else
75
+ add_cache(url, io, options)
76
+ open_cache(url, options)
77
+ end
78
+ rescue
79
+ STDERR.puts $!.backtrace.inspect
80
+ raise OpenURLError, "Error reading remote url: #{ url }.\n#{$!.message}"
81
+ end
82
+ end
83
+
84
+ def self.digest_url(url, options = {})
85
+ params = [url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]).split("\n").sort * "\n" : "")]
86
+ Misc.digest([url, params])
87
+ end
88
+
89
+ def self.cache_file(url, options)
90
+ File.join(self.remote_cache_dir, digest_url(url, options))
91
+ end
92
+
93
+ def self.in_cache(url, options = {})
94
+ filename = cache_file(url, options)
95
+ if File.exist? filename
96
+ return filename
97
+ else
98
+ nil
99
+ end
100
+ end
101
+
102
+ def self.remove_from_cache(url, options = {})
103
+ filename = cache_file(url, options)
104
+ if File.exist? filename
105
+ FileUtils.rm filename
106
+ else
107
+ nil
108
+ end
109
+ end
110
+
111
+ def self.add_cache(url, data, options = {})
112
+ filename = cache_file(url, options)
113
+ Open.sensible_write(filename, data, :force => true)
114
+ end
115
+
116
+ def self.open_cache(url, options = {})
117
+ filename = cache_file(url, options)
118
+ Open.open(filename)
119
+ end
120
+ end