scout-gear 2.0.0 → 5.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +65 -2
  3. data/Rakefile +2 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +233 -24
  6. data/lib/scout/cmd.rb +344 -0
  7. data/lib/scout/concurrent_stream.rb +259 -0
  8. data/lib/scout/exceptions.rb +15 -8
  9. data/lib/scout/indiferent_hash/options.rb +8 -26
  10. data/lib/scout/log/color.rb +2 -2
  11. data/lib/scout/log/fingerprint.rb +11 -1
  12. data/lib/scout/log/progress/report.rb +0 -1
  13. data/lib/scout/log/progress/util.rb +1 -1
  14. data/lib/scout/log/progress.rb +4 -4
  15. data/lib/scout/log.rb +10 -2
  16. data/lib/scout/meta_extension.rb +19 -3
  17. data/lib/scout/misc/digest.rb +56 -0
  18. data/lib/scout/misc/filesystem.rb +26 -0
  19. data/lib/scout/misc/format.rb +17 -6
  20. data/lib/scout/misc/insist.rb +56 -0
  21. data/lib/scout/misc/monitor.rb +23 -0
  22. data/lib/scout/misc.rb +5 -11
  23. data/lib/scout/open/lock.rb +61 -0
  24. data/lib/scout/open/remote.rb +120 -0
  25. data/lib/scout/open/stream.rb +373 -0
  26. data/lib/scout/open/util.rb +225 -0
  27. data/lib/scout/open.rb +169 -0
  28. data/lib/scout/path/find.rb +68 -21
  29. data/lib/scout/path/tmpfile.rb +8 -0
  30. data/lib/scout/path/util.rb +14 -1
  31. data/lib/scout/path.rb +6 -30
  32. data/lib/scout/persist/open.rb +17 -0
  33. data/lib/scout/persist/path.rb +15 -0
  34. data/lib/scout/persist/serialize.rb +151 -0
  35. data/lib/scout/persist.rb +54 -0
  36. data/lib/scout/resource/path.rb +20 -0
  37. data/lib/scout/resource/produce/rake.rb +69 -0
  38. data/lib/scout/resource/produce.rb +246 -0
  39. data/lib/scout/resource/scout.rb +3 -0
  40. data/lib/scout/resource/util.rb +48 -0
  41. data/lib/scout/resource.rb +39 -0
  42. data/lib/scout/simple_opt/accessor.rb +1 -1
  43. data/lib/scout/simple_opt/doc.rb +29 -23
  44. data/lib/scout/simple_opt/parse.rb +4 -3
  45. data/lib/scout/tmpfile.rb +39 -1
  46. data/lib/scout/workflow/definition.rb +78 -0
  47. data/lib/scout/workflow/documentation.rb +83 -0
  48. data/lib/scout/workflow/step/info.rb +77 -0
  49. data/lib/scout/workflow/step/load.rb +18 -0
  50. data/lib/scout/workflow/step.rb +132 -0
  51. data/lib/scout/workflow/task/inputs.rb +114 -0
  52. data/lib/scout/workflow/task.rb +155 -0
  53. data/lib/scout/workflow/usage.rb +314 -0
  54. data/lib/scout/workflow/util.rb +11 -0
  55. data/lib/scout/workflow.rb +40 -0
  56. data/lib/scout-gear.rb +4 -0
  57. data/lib/scout.rb +1 -0
  58. data/lib/workflow-scout.rb +2 -0
  59. data/scout-gear.gemspec +77 -5
  60. data/scout_commands/alias +48 -0
  61. data/scout_commands/find +83 -0
  62. data/scout_commands/glob +0 -0
  63. data/scout_commands/rbbt +23 -0
  64. data/scout_commands/workflow/info +29 -0
  65. data/scout_commands/workflow/list +27 -0
  66. data/scout_commands/workflow/task +58 -0
  67. data/scout_commands/workflow/task_old +706 -0
  68. data/test/scout/indiferent_hash/test_options.rb +11 -1
  69. data/test/scout/misc/test_digest.rb +30 -0
  70. data/test/scout/misc/test_filesystem.rb +30 -0
  71. data/test/scout/misc/test_insist.rb +13 -0
  72. data/test/scout/open/test_lock.rb +52 -0
  73. data/test/scout/open/test_remote.rb +25 -0
  74. data/test/scout/open/test_stream.rb +515 -0
  75. data/test/scout/open/test_util.rb +73 -0
  76. data/test/scout/path/test_find.rb +28 -0
  77. data/test/scout/persist/test_open.rb +37 -0
  78. data/test/scout/persist/test_path.rb +37 -0
  79. data/test/scout/persist/test_serialize.rb +114 -0
  80. data/test/scout/resource/test_path.rb +40 -0
  81. data/test/scout/resource/test_produce.rb +62 -0
  82. data/test/scout/resource/test_util.rb +27 -0
  83. data/test/scout/simple_opt/test_doc.rb +16 -0
  84. data/test/scout/test_cmd.rb +85 -0
  85. data/test/scout/test_concurrent_stream.rb +29 -0
  86. data/test/scout/test_meta_extension.rb +9 -0
  87. data/test/scout/test_misc.rb +0 -7
  88. data/test/scout/test_open.rb +146 -0
  89. data/test/scout/test_path.rb +3 -1
  90. data/test/scout/test_persist.rb +83 -0
  91. data/test/scout/test_resource.rb +26 -0
  92. data/test/scout/test_workflow.rb +87 -0
  93. data/test/scout/workflow/step/test_info.rb +30 -0
  94. data/test/scout/workflow/step/test_load.rb +65 -0
  95. data/test/scout/workflow/task/test_inputs.rb +182 -0
  96. data/test/scout/workflow/test_definition.rb +0 -0
  97. data/test/scout/workflow/test_documentation.rb +30 -0
  98. data/test/scout/workflow/test_step.rb +36 -0
  99. data/test/scout/workflow/test_task.rb +179 -0
  100. data/test/scout/workflow/test_usage.rb +35 -0
  101. data/test/scout/workflow/test_util.rb +17 -0
  102. data/test/test_helper.rb +17 -0
  103. data/test/test_scout-gear.rb +0 -0
  104. metadata +75 -3
@@ -63,7 +63,7 @@ module Colorize
63
63
  if percent
64
64
  array = array.collect{|v| n = v.to_f; n = n > 100 ? 100 : n; n < 0.001 ? 0.001 : n}
65
65
  else
66
- array = array.collect{|v| n = v.to_f; }
66
+ array = array.collect{|v| v.to_f }
67
67
  end
68
68
  max = array.max
69
69
  min = array.min
@@ -103,7 +103,7 @@ module Colorize
103
103
 
104
104
  value_color = Hash[*array.uniq.zip(all_colors).flatten]
105
105
 
106
- value_color.values_at *array
106
+ value_color.values_at(*array)
107
107
  end
108
108
 
109
109
  def self.tsv(tsv, options = {})
@@ -19,8 +19,12 @@ module Log
19
19
  else
20
20
  "'" << obj << "'"
21
21
  end
22
+ when ConcurrentStream
23
+ name = obj.inspect + " " + obj.object_id.to_s
24
+ name += " #{obj.filename}" if obj.filename
25
+ name
22
26
  when IO
23
- (obj.respond_to?(:filename) and obj.filename ) ? "<IO:" + (obj.filename || obj.inspect + rand(100000)) + ">" : obj.inspect
27
+ (obj.respond_to?(:filename) and obj.filename ) ? "<IO:" + (obj.filename || obj.inspect + rand(100000)) + ">" : obj.inspect + " " + obj.object_id.to_s
24
28
  when File
25
29
  "<File:" + obj.path + ">"
26
30
  when Array
@@ -52,6 +56,12 @@ module Log
52
56
  else
53
57
  "%.6f" % obj
54
58
  end
59
+ when Thread
60
+ if obj["name"]
61
+ obj["name"]
62
+ else
63
+ obj.inspect
64
+ end
55
65
  else
56
66
  obj.to_s
57
67
  end
@@ -169,7 +169,6 @@ module Log
169
169
  bar = bars.sort_by{|b| b.depth }.first
170
170
  print(io, Log.color(:magenta ,bar.report_msg) << "\n")
171
171
  else
172
- length = Log::ProgressBar.cleanup_bars
173
172
  print(io, Log.color(:magenta, "···Progress\n"))
174
173
  bars.sort_by{|b| b.depth }.reverse.each do |bar|
175
174
  if SILENCED.include? bar
@@ -91,7 +91,7 @@ module Log
91
91
  error = true
92
92
  raise $!
93
93
  ensure
94
- remove_bar(bar, error) if bar
94
+ remove_bar(bar, error) if bar && ! keep
95
95
  end
96
96
  end
97
97
  end
@@ -17,12 +17,12 @@ module Log
17
17
  attr_accessor :default_file
18
18
  end
19
19
 
20
- attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback
20
+ attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback, :severity
21
21
 
22
22
  def initialize(max = nil, options = {})
23
- depth, num_reports, desc, io, severity, file, bytes, frequency, process, callback =
24
- IndiferentHash.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency, :process, :callback,
25
- :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
23
+ depth, desc, file, bytes, frequency, process, callback =
24
+ IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback,
25
+ :depth => 0, :frequency => 2
26
26
 
27
27
  @max = max
28
28
  @ticks = 0
data/lib/scout/log.rb CHANGED
@@ -14,8 +14,8 @@ module Log
14
14
  end
15
15
 
16
16
  class << self
17
- attr_accessor :logfile, :severity
18
- attr_writer :tty_size
17
+ attr_accessor :severity
18
+ attr_writer :tty_size, :logfile
19
19
  end
20
20
 
21
21
 
@@ -360,3 +360,11 @@ def eef(obj=nil, file = $stdout)
360
360
  Log.log_obj_fingerprint(obj, :error, file)
361
361
  end
362
362
 
363
+ def sss(level, &block)
364
+ if block_given?
365
+ Log.with_severity level, &block
366
+ else
367
+ Log.severity = level
368
+ end
369
+ end
370
+
@@ -11,15 +11,23 @@ module MetaExtension
11
11
  end
12
12
  end
13
13
 
14
- meta.define_method(:setup) do |obj,*rest|
14
+ meta.define_method(:setup) do |*args,&block|
15
+ if block_given?
16
+ obj, rest = block, args
17
+ else
18
+ obj, *rest = args
19
+ end
20
+ obj = block if obj.nil?
15
21
  obj.extend base
16
22
  attrs = self.class_variable_get("@@extension_attrs")
17
23
 
18
24
  return if attrs.nil? || attrs.empty?
19
25
 
20
26
  if rest.length == 1 && Hash === (rlast = rest.last) &&
21
- ! (rlkey = rlast.keys.first).nil? &&
22
- attrs.include?(rlkey.to_sym)
27
+ ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
28
+ (! attrs.length != 1 ))
29
+
30
+
23
31
 
24
32
  pairs = rlast
25
33
  else
@@ -33,6 +41,14 @@ module MetaExtension
33
41
  obj
34
42
  end
35
43
 
44
+ base.define_method(:extension_attr_hash) do
45
+ attr_hash = {}
46
+ meta.class_variable_get("@@extension_attrs").each do |name|
47
+ attr_hash[name] = self.instance_variable_get("@#{name}")
48
+ end
49
+ attr_hash
50
+ end
51
+
36
52
  base.define_method(:annotate) do |other|
37
53
  attr_values = meta.class_variable_get("@@extension_attrs").collect do |a|
38
54
  self.instance_variable_get("@#{a}")
@@ -0,0 +1,56 @@
1
+ module Misc
2
+ def self.digest_str(obj)
3
+ if obj.respond_to?(:digest_str)
4
+ obj.digest_str
5
+ else
6
+ case obj
7
+ when String
8
+ #'\'' << obj << '\''
9
+ if Path === obj || ! Open.exists?(obj)
10
+ '\'' << obj << '\''
11
+ else
12
+ Misc.file_md5(obj)
13
+ end
14
+ when Integer, Symbol
15
+ obj.to_s
16
+ when Array
17
+ '[' << obj.inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
18
+ when Hash
19
+ '{' << obj.inject(""){|acc,p| s = Misc.digest_str(p.first) << "=" << Misc.digest_str(p.last); acc.empty? ? s : acc << ', ' << s } << '}'
20
+ when Integer
21
+ obj.to_s
22
+ when Float
23
+ if obj % 1 == 0
24
+ obj.to_i
25
+ elsif obj.abs > 10
26
+ "%.1f" % obj
27
+ elsif obj.abs > 1
28
+ "%.3f" % obj
29
+ else
30
+ "%.6f" % obj
31
+ end
32
+ when TrueClass
33
+ "true"
34
+ when FalseClass
35
+ "false"
36
+ else
37
+ obj.inspect
38
+ end
39
+ end
40
+ end
41
+
42
+ def self.digest(obj)
43
+ str = Misc.digest_str(obj)
44
+ hash = Digest::MD5.hexdigest(str)
45
+ Log.debug "Digest #{hash} - #{str}"
46
+ hash
47
+ end
48
+
49
+ def self.file_md5(file)
50
+ file = file.find if Path === file
51
+ #md5file = file + '.md5'
52
+ Persist.persist("MD5:#{file}", :string) do
53
+ Digest::MD5.file(file).hexdigest
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,26 @@
1
+ module Misc
2
+ def self.in_dir(dir)
3
+ old_pwd = FileUtils.pwd
4
+ begin
5
+ FileUtils.mkdir_p dir unless File.exist?(dir)
6
+ FileUtils.cd dir
7
+ yield
8
+ ensure
9
+ FileUtils.cd old_pwd
10
+ end
11
+ end
12
+
13
+ def self.path_relative_to(basedir, path)
14
+ path = File.expand_path(path) unless path.slice(0,1) == "/"
15
+ basedir = File.expand_path(basedir) unless basedir.slice(0,1) == "/"
16
+
17
+ basedir += "/" unless basedir.slice(-2,-1) == "/"
18
+
19
+ if path.index(basedir) == 0
20
+ return path[basedir.length..-1]
21
+ else
22
+ return nil
23
+ end
24
+ end
25
+
26
+ end
@@ -25,7 +25,14 @@ module Misc
25
25
  str
26
26
  end
27
27
 
28
- def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
28
+
29
+ MAX_WIDTH = 100
30
+ def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
31
+ size ||= Log.tty_size || MAX_WIDTH
32
+ size = MAX_WIDTH if size > MAX_WIDTH
33
+ indent ||= 0
34
+ offset ||= 0
35
+
29
36
  i = 0
30
37
  size = size + offset + indent
31
38
  re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
@@ -55,7 +62,10 @@ module Misc
55
62
  end*""
56
63
  end
57
64
 
58
- def self.format_definition_list_item(dt, dd, size = 80, indent = 20, color = :yellow)
65
+ def self.format_definition_list_item(dt, dd, indent = nil, size = nil, color = :yellow)
66
+ size ||= Log.tty_size || MAX_WIDTH
67
+ size = MAX_WIDTH if size > MAX_WIDTH
68
+ indent ||= size / 3
59
69
  dd = "" if dd.nil?
60
70
  dt = Log.color color, dt if color
61
71
  dt = dt.to_s unless dd.empty?
@@ -73,10 +83,12 @@ module Misc
73
83
  text
74
84
  end
75
85
 
76
- def self.format_definition_list(defs, size = 80, indent = 20, color = :yellow, sep = "\n\n")
86
+ def self.format_definition_list(defs, indent = nil, size = nil, color = :yellow, sep = "\n\n")
87
+ size ||= Log.tty_size || MAX_WIDTH
88
+ indent ||= 30
77
89
  entries = []
78
90
  defs.each do |dt,dd|
79
- text = format_definition_list_item(dt,dd,size,indent,color)
91
+ text = format_definition_list_item(dt,dd,indent, size,color)
80
92
  entries << text
81
93
  end
82
94
  entries * sep
@@ -101,7 +113,7 @@ module Misc
101
113
  string.
102
114
  gsub(/([A-Z]{2,})([A-Z][a-z])/,'\1_\2').
103
115
  gsub(/([a-z])([A-Z])/,'\1_\2').
104
- gsub(/\s/,'_').gsub(/[^\w_]/, '').
116
+ gsub(/\s/,'_').gsub(/[^\w]/, '').
105
117
  split("_").collect{|p| p.match(/[A-Z]{2,}/) ? p : p.downcase } * "_"
106
118
  end
107
119
 
@@ -223,5 +235,4 @@ module Misc
223
235
  end
224
236
  values
225
237
  end
226
-
227
238
  end
@@ -0,0 +1,56 @@
1
+ module Misc
2
+ def self.insist(times = 4, sleep = nil, msg = nil)
3
+ sleep_array = nil
4
+
5
+ try = 0
6
+ begin
7
+ begin
8
+ yield
9
+ rescue Exception
10
+ if Array === times
11
+ sleep_array = times
12
+ times = sleep_array.length
13
+ sleep = sleep_array.shift
14
+ end
15
+
16
+ if sleep.nil?
17
+ sleep_array = ([0] + [0.001, 0.01, 0.1, 0.5] * (times / 3)).sort[0..times-1]
18
+ sleep = sleep_array.shift
19
+ end
20
+ raise $!
21
+ end
22
+ rescue TryAgain
23
+ sleep sleep
24
+ retry
25
+ rescue StopInsist
26
+ raise $!.exception
27
+ rescue Aborted, Interrupt
28
+ if msg
29
+ Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
30
+ else
31
+ Log.warn("Not Insisting after Aborted: #{$!.message}")
32
+ end
33
+ raise $!
34
+ rescue Exception
35
+ Log.exception $! if ENV["RBBT_LOG_INSIST"] == 'true'
36
+ if msg
37
+ Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
38
+ elsif FalseClass === msg
39
+ nil
40
+ else
41
+ Log.warn("Insisting after exception: #{$!.class} #{$!.message}")
42
+ end
43
+
44
+ if sleep and try > 0
45
+ sleep sleep
46
+ sleep = sleep_array.shift || sleep if sleep_array
47
+ else
48
+ Thread.pass
49
+ end
50
+
51
+ try += 1
52
+ retry if try < times
53
+ raise $!
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,23 @@
1
+ module Misc
2
+ def self.benchmark(repeats = 1, message = nil)
3
+ require 'benchmark'
4
+ res = nil
5
+ begin
6
+ measure = Benchmark.measure do
7
+ repeats.times do
8
+ res = yield
9
+ end
10
+ end
11
+ if message
12
+ puts "#{message }: #{ repeats } repeats"
13
+ else
14
+ puts "Benchmark for #{ repeats } repeats"
15
+ end
16
+ puts measure
17
+ rescue Exception
18
+ puts "Benchmark aborted"
19
+ raise $!
20
+ end
21
+ res
22
+ end
23
+ end
data/lib/scout/misc.rb CHANGED
@@ -1,14 +1,8 @@
1
1
  require_relative 'misc/format'
2
- module Misc
3
- def self.in_dir(dir)
4
- old_pwd = FileUtils.pwd
5
- begin
6
- FileUtils.mkdir_p dir unless File.exist?(dir)
7
- FileUtils.cd dir
8
- yield
9
- ensure
10
- FileUtils.cd old_pwd
11
- end
12
- end
2
+ require_relative 'misc/insist'
3
+ require_relative 'misc/digest'
4
+ require_relative 'misc/filesystem'
5
+ require_relative 'misc/monitor'
13
6
 
7
+ module Misc
14
8
  end
@@ -0,0 +1,61 @@
1
+ require_relative '../path'
2
+ require_relative '../log'
3
+ require_relative '../exceptions'
4
+ require 'lockfile'
5
+
6
+ module Open
7
+ def self.lock(file, unlock = true, options = {})
8
+ unlock, options = true, unlock if Hash === unlock
9
+ return yield if file.nil? and not Lockfile === options[:lock]
10
+
11
+ if Lockfile === file
12
+ lockfile = file
13
+ else
14
+ file = file.find if Path === file
15
+ FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exist? File.dirname(File.expand_path(file))
16
+
17
+ case options[:lock]
18
+ when Lockfile
19
+ lockfile = options[:lock]
20
+ when FalseClass
21
+ lockfile = nil
22
+ unlock = false
23
+ when Path, String
24
+ lock_path = options[:lock].find
25
+ lockfile = Lockfile.new(lock_path, options)
26
+ else
27
+ lock_path = File.expand_path(file + '.lock')
28
+ lockfile = Lockfile.new(lock_path, options)
29
+ end
30
+ end
31
+
32
+ begin
33
+ lockfile.lock unless lockfile.nil? || lockfile.locked?
34
+ rescue Aborted, Interrupt
35
+ raise LockInterrupted
36
+ end
37
+
38
+ res = nil
39
+
40
+ begin
41
+ res = yield lockfile
42
+ rescue KeepLocked
43
+ unlock = false
44
+ res = $!.payload
45
+ ensure
46
+ if unlock
47
+ begin
48
+ if lockfile.locked?
49
+ lockfile.unlock
50
+ else
51
+ end
52
+ rescue Exception
53
+ Log.warn "Exception unlocking: #{lockfile.path}"
54
+ Log.exception $!
55
+ end
56
+ end
57
+ end
58
+
59
+ res
60
+ end
61
+ end
@@ -0,0 +1,120 @@
1
+ require_relative '../misc'
2
+ require_relative '../path'
3
+ require_relative '../cmd'
4
+
5
+ module Open
6
+ class << self
7
+ attr_accessor :remote_cache_dir
8
+
9
+ def remote_cache_dir
10
+ @remote_cache_dir ||= Path.setup("var/cache/open-remote/").find
11
+ end
12
+ end
13
+
14
+ def self.remote?(file)
15
+ !! (file =~ /^(?:https?|ftp|ssh):\/\//)
16
+ end
17
+
18
+ def self.ssh?(file)
19
+ !! (file =~ /^ssh:\/\//)
20
+ end
21
+
22
+ def self.ssh(file, options = {})
23
+ m = file.match(/ssh:\/\/([^:]+):(.*)/)
24
+ server = m[1]
25
+ file = m[2]
26
+ CMD.cmd("ssh '#{server}' cat '#{file}'", :pipe => true, :autojoin => true)
27
+ end
28
+
29
+ def self.wget(url, options = {})
30
+ if ! (options[:force] || options[:nocache]) && cache_file = in_cache(url, options)
31
+ return file_open(cache_file)
32
+ end
33
+
34
+ Log.low "WGET:\n -URL: #{ url }\n -OPTIONS: #{options.inspect}"
35
+ options = IndiferentHash.add_defaults options, "--user-agent=" => 'rbbt', :pipe => true, :autojoin => true
36
+
37
+ wait(options[:nice], options[:nice_key]) if options[:nice]
38
+ options.delete(:nice)
39
+ options.delete(:nice_key)
40
+
41
+ pipe = options.delete(:pipe)
42
+ quiet = options.delete(:quiet)
43
+ post = options.delete(:post)
44
+ cookies = options.delete(:cookies)
45
+ nocache = options.delete(:nocache)
46
+
47
+ options["--quiet"] = quiet if options["--quiet"].nil?
48
+ options["--post-data="] ||= post if post
49
+
50
+ if cookies
51
+ options["--save-cookies"] = cookies
52
+ options["--load-cookies"] = cookies
53
+ options["--keep-session-cookies"] = true
54
+ end
55
+
56
+ stderr = case
57
+ when options['stderr']
58
+ options['stderr']
59
+ when options['--quiet']
60
+ false
61
+ else
62
+ nil
63
+ end
64
+
65
+ begin
66
+ wget_options = options.dup
67
+ wget_options = wget_options.merge( '-O' => '-') unless options.include?('--output-document')
68
+ wget_options[:pipe] = pipe unless pipe.nil?
69
+ wget_options[:stderr] = stderr unless stderr.nil?
70
+
71
+ io = CMD.cmd("wget '#{ url }'", wget_options)
72
+ if nocache && nocache.to_s != 'update'
73
+ io
74
+ else
75
+ add_cache(url, io, options)
76
+ open_cache(url, options)
77
+ end
78
+ rescue
79
+ STDERR.puts $!.backtrace.inspect
80
+ raise OpenURLError, "Error reading remote url: #{ url }.\n#{$!.message}"
81
+ end
82
+ end
83
+
84
+ def self.digest_url(url, options = {})
85
+ params = [url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]).split("\n").sort * "\n" : "")]
86
+ Misc.digest([url, params])
87
+ end
88
+
89
+ def self.cache_file(url, options)
90
+ File.join(self.remote_cache_dir, digest_url(url, options))
91
+ end
92
+
93
+ def self.in_cache(url, options = {})
94
+ filename = cache_file(url, options)
95
+ if File.exist? filename
96
+ return filename
97
+ else
98
+ nil
99
+ end
100
+ end
101
+
102
+ def self.remove_from_cache(url, options = {})
103
+ filename = cache_file(url, options)
104
+ if File.exist? filename
105
+ FileUtils.rm filename
106
+ else
107
+ nil
108
+ end
109
+ end
110
+
111
+ def self.add_cache(url, data, options = {})
112
+ filename = cache_file(url, options)
113
+ Open.sensible_write(filename, data, :force => true)
114
+ end
115
+
116
+ def self.open_cache(url, options = {})
117
+ filename = cache_file(url, options)
118
+ Open.open(filename)
119
+ end
120
+ end