scout-gear 7.1.0 → 7.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +65 -2
  3. data/VERSION +1 -1
  4. data/bin/scout +5 -1
  5. data/lib/rbbt-scout.rb +5 -0
  6. data/lib/scout/concurrent_stream.rb +13 -8
  7. data/lib/scout/config.rb +168 -0
  8. data/lib/scout/exceptions.rb +5 -3
  9. data/lib/scout/indiferent_hash/options.rb +1 -0
  10. data/lib/scout/indiferent_hash.rb +4 -2
  11. data/lib/scout/log/color.rb +3 -2
  12. data/lib/scout/log/progress/report.rb +1 -0
  13. data/lib/scout/log/progress/util.rb +66 -1
  14. data/lib/scout/log/progress.rb +5 -3
  15. data/lib/scout/log.rb +3 -2
  16. data/lib/scout/misc/helper.rb +31 -0
  17. data/lib/scout/misc/monitor.rb +4 -1
  18. data/lib/scout/misc/system.rb +15 -0
  19. data/lib/scout/misc.rb +2 -0
  20. data/lib/scout/named_array.rb +68 -0
  21. data/lib/scout/open/stream.rb +58 -33
  22. data/lib/scout/path/find.rb +27 -3
  23. data/lib/scout/path/util.rb +7 -4
  24. data/lib/scout/persist/serialize.rb +7 -14
  25. data/lib/scout/persist.rb +46 -12
  26. data/lib/scout/resource/produce.rb +7 -94
  27. data/lib/scout/resource/software.rb +176 -0
  28. data/lib/scout/semaphore.rb +8 -1
  29. data/lib/scout/tsv/dumper.rb +112 -0
  30. data/lib/scout/tsv/index.rb +161 -0
  31. data/lib/scout/tsv/open.rb +128 -0
  32. data/lib/scout/tsv/parser.rb +230 -30
  33. data/lib/scout/tsv/path.rb +13 -0
  34. data/lib/scout/tsv/persist/adapter.rb +367 -0
  35. data/lib/scout/tsv/persist/fix_width_table.rb +324 -0
  36. data/lib/scout/tsv/persist/serialize.rb +117 -0
  37. data/lib/scout/tsv/persist/tokyocabinet.rb +113 -0
  38. data/lib/scout/tsv/persist.rb +13 -0
  39. data/lib/scout/tsv/traverse.rb +143 -0
  40. data/lib/scout/tsv/util/filter.rb +303 -0
  41. data/lib/scout/tsv/util/process.rb +73 -0
  42. data/lib/scout/tsv/util/select.rb +220 -0
  43. data/lib/scout/tsv/util.rb +82 -0
  44. data/lib/scout/tsv.rb +16 -3
  45. data/lib/scout/work_queue/worker.rb +4 -4
  46. data/lib/scout/work_queue.rb +22 -7
  47. data/lib/scout/workflow/definition.rb +101 -4
  48. data/lib/scout/workflow/step/config.rb +18 -0
  49. data/lib/scout/workflow/step/dependencies.rb +40 -0
  50. data/lib/scout/workflow/step/file.rb +15 -0
  51. data/lib/scout/workflow/step/info.rb +35 -4
  52. data/lib/scout/workflow/step/progress.rb +14 -0
  53. data/lib/scout/workflow/step/provenance.rb +148 -0
  54. data/lib/scout/workflow/step.rb +71 -17
  55. data/lib/scout/workflow/task.rb +10 -5
  56. data/lib/scout/workflow/usage.rb +3 -1
  57. data/lib/scout/workflow.rb +11 -3
  58. data/lib/scout-gear.rb +1 -0
  59. data/lib/scout.rb +1 -0
  60. data/scout-gear.gemspec +64 -10
  61. data/scout_commands/find +1 -1
  62. data/scout_commands/workflow/task +16 -9
  63. data/scout_commands/workflow/task_old +2 -2
  64. data/share/software/install_helpers +523 -0
  65. data/test/scout/log/test_progress.rb +0 -2
  66. data/test/scout/misc/test_system.rb +21 -0
  67. data/test/scout/open/test_stream.rb +160 -1
  68. data/test/scout/path/test_find.rb +14 -7
  69. data/test/scout/resource/test_software.rb +24 -0
  70. data/test/scout/test_config.rb +66 -0
  71. data/test/scout/test_meta_extension.rb +10 -0
  72. data/test/scout/test_named_array.rb +19 -0
  73. data/test/scout/test_persist.rb +96 -0
  74. data/test/scout/test_tmpfile.rb +1 -1
  75. data/test/scout/test_tsv.rb +50 -1
  76. data/test/scout/test_work_queue.rb +41 -13
  77. data/test/scout/tsv/persist/test_adapter.rb +44 -0
  78. data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
  79. data/test/scout/tsv/persist/test_tokyocabinet.rb +92 -0
  80. data/test/scout/tsv/test_dumper.rb +44 -0
  81. data/test/scout/tsv/test_index.rb +156 -0
  82. data/test/scout/tsv/test_open.rb +9 -0
  83. data/test/scout/tsv/test_parser.rb +114 -3
  84. data/test/scout/tsv/test_persist.rb +43 -0
  85. data/test/scout/tsv/test_traverse.rb +116 -0
  86. data/test/scout/tsv/test_util.rb +23 -0
  87. data/test/scout/tsv/util/test_filter.rb +188 -0
  88. data/test/scout/tsv/util/test_process.rb +47 -0
  89. data/test/scout/tsv/util/test_select.rb +44 -0
  90. data/test/scout/work_queue/test_worker.rb +66 -9
  91. data/test/scout/workflow/step/test_dependencies.rb +25 -0
  92. data/test/scout/workflow/step/test_info.rb +15 -17
  93. data/test/scout/workflow/step/test_load.rb +19 -21
  94. data/test/scout/workflow/step/test_provenance.rb +25 -0
  95. data/test/scout/workflow/test_step.rb +206 -10
  96. data/test/scout/workflow/test_task.rb +0 -3
  97. data/test/test_helper.rb +9 -1
  98. metadata +50 -6
@@ -0,0 +1,68 @@
1
+ require_relative 'meta_extension'
2
+ module NamedArray
3
+ extend MetaExtension
4
+ extension_attr :fields
5
+
6
+ def self.identify_name(names, selected)
7
+ res = (Array === selected ? selected : [selected]).collect do |field|
8
+ case field
9
+ when nil
10
+ 0
11
+ when Integer
12
+ field
13
+ when Symbol
14
+ field == :key ? field : identify_name(names, field.to_s)
15
+ when (names.nil? and String)
16
+ if field =~ /^\d+$/
17
+ identify_field(key_field, fields, field.to_i)
18
+ else
19
+ raise "No name information available and specified name not numeric: #{ field }"
20
+ end
21
+ when Symbol
22
+ names.index{|f| f.to_s == field.to_s }
23
+ when String
24
+ pos = names.index{|f| f.to_s == field }
25
+ next pos if pos
26
+ if field =~ /^\d+$/
27
+ next identify_names(names, field.to_i)
28
+ end
29
+ pos = names.index{|name| name.start_with?(field) }
30
+ next pos if pos
31
+ nil
32
+ else
33
+ raise "Field '#{ Log.fingerprint field }' was not understood. Options: (#{ Log.fingerprint names })"
34
+ end
35
+ end
36
+
37
+ Array === selected ? res : res.first
38
+ end
39
+
40
+ def positions(fields)
41
+ if Array == fields
42
+ fields.collect{|field|
43
+ NamedArray.identify_name(@fields, field)
44
+ }
45
+ else
46
+ NamedArray.identify_name(@fields, fields)
47
+ end
48
+ end
49
+
50
+ def [](key)
51
+ pos = NamedArray.identify_name(@fields, key)
52
+ super(pos)
53
+ end
54
+
55
+ def concat(other)
56
+ super(other)
57
+ self.fields.concat(other.fields) if NamedArray === other
58
+ self
59
+ end
60
+
61
+ def to_hash
62
+ hash = {}
63
+ self.fields.zip(self) do |field,value|
64
+ hash[field] = value
65
+ end
66
+ IndiferentHash.setup hash
67
+ end
68
+ end
@@ -1,5 +1,5 @@
1
1
  module Open
2
- BLOCK_SIZE = 1024
2
+ BLOCK_SIZE = 1024 * 8
3
3
 
4
4
  class << self
5
5
  attr_accessor :sensible_write_lock_dir
@@ -52,15 +52,11 @@ module Open
52
52
  into_path, into = into, File.open(into, 'w')
53
53
  end
54
54
 
55
- into.sync = true if IO === into
56
55
  into_close = false unless into.respond_to? :close
57
- io.sync = true
58
56
 
59
- begin
60
- while c = io.readpartial(BLOCK_SIZE)
61
- into << c if into
62
- end
63
- rescue EOFError
57
+ while c = io.read(BLOCK_SIZE)
58
+ into << c if into
59
+ break if io.closed?
64
60
  end
65
61
 
66
62
  io.join if io.respond_to? :join
@@ -77,7 +73,7 @@ module Open
77
73
  FileUtils.rm into_path if into_path and File.exist?(into_path)
78
74
  rescue Exception
79
75
  Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
80
- exception = io.stream_exception || $!
76
+ exception = (io.respond_to?(:stream_exception) && io.stream_exception) ? io.stream_exception : $!
81
77
  io.abort exception if io.respond_to? :abort
82
78
  into.close if into.respond_to?(:closed?) && ! into.closed?
83
79
  into_path = into if into_path.nil? && String === into
@@ -110,7 +106,7 @@ module Open
110
106
  Log.warn "Path exists in sensible_write, not forcing update: #{ path }"
111
107
  Open.consume_stream content
112
108
  else
113
- FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory? File.dirname(tmp_path)
109
+ FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?(File.dirname(tmp_path))
114
110
  FileUtils.rm_f tmp_path if File.exist? tmp_path
115
111
  begin
116
112
 
@@ -121,13 +117,11 @@ module Open
121
117
  File.open(tmp_path, 'wb') do |f| f.write content end
122
118
  when (IO === content or StringIO === content or File === content)
123
119
  Open.write(tmp_path) do |f|
124
- f.sync = true
125
- begin
126
- while block = content.readpartial(BLOCK_SIZE)
127
- f.write block
128
- end
129
- rescue EOFError
130
- end
120
+ #f.sync = true
121
+ while block = content.read(BLOCK_SIZE)
122
+ f.write block
123
+ break if content.closed?
124
+ end
131
125
  end
132
126
  else
133
127
  File.open(tmp_path, 'wb') do |f| end
@@ -292,22 +286,20 @@ module Open
292
286
  Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
293
287
 
294
288
  skip = [false] * num
295
- begin
296
- while block = stream.readpartial(BLOCK_SIZE)
297
-
298
- in_pipes.each_with_index do |sin,i|
299
- begin
300
- sin.write block
301
- rescue IOError
302
- Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
303
- skip[i] = true
304
- rescue
305
- Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
306
- raise $!
307
- end unless skip[i]
308
- end
289
+ while block = stream.read(BLOCK_SIZE)
290
+
291
+ in_pipes.each_with_index do |sin,i|
292
+ begin
293
+ sin.write block
294
+ rescue IOError
295
+ Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
296
+ skip[i] = true
297
+ rescue
298
+ Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
299
+ raise $!
300
+ end unless skip[i]
309
301
  end
310
- rescue IOError
302
+ break if stream.closed?
311
303
  end
312
304
 
313
305
  stream.join if stream.respond_to? :join
@@ -345,7 +337,6 @@ module Open
345
337
  end
346
338
  end
347
339
 
348
-
349
340
  out_pipes.each do |sout|
350
341
  ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :pair => stream
351
342
  end
@@ -401,4 +392,38 @@ module Open
401
392
  str
402
393
  end
403
394
 
395
+ def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
396
+ Open.open_pipe do |sin|
397
+ line = stream.gets
398
+ while line =~ /^#{header_hash}/ do
399
+ sin.puts line
400
+ line = stream.gets
401
+ end
402
+
403
+ line_stream = Open.open_pipe do |line_stream_in|
404
+ line_stream_in.puts line
405
+ begin
406
+ Open.consume_stream(stream, false, line_stream_in)
407
+ rescue
408
+ raise $!
409
+ end
410
+ end
411
+
412
+ sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
413
+
414
+ begin
415
+ Open.consume_stream(sorted, false, sin)
416
+ rescue
417
+ Log.exception $!
418
+ begin
419
+ sorted.raise($!) if sorted.respond_to? :raise
420
+ stream.raise($!) if stream.respond_to? :raise
421
+ ensure
422
+ raise $!
423
+ end
424
+ end
425
+ end
426
+ end
427
+
428
+
404
429
  end
@@ -9,7 +9,8 @@ module Path
9
9
  break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
10
10
  file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
11
11
  file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
12
- file =~ /(?:scout|rbbt)\/persist.rb/
12
+ file =~ /(?:scout|rbbt)\/persist.rb/ or
13
+ file =~ /modules\/rbbt-util/
13
14
  end
14
15
  file = file.sub(/\.rb[^\w].*/,'.rb')
15
16
  end
@@ -136,15 +137,38 @@ module Path
136
137
  found
137
138
  end
138
139
 
140
+ def self.exists_file_or_alternatives(file)
141
+ return file if File.exist?(file) or File.directory?(file)
142
+ %w(gz bgz zip).each do |extension|
143
+ alt_file = file + '.' + extension
144
+ return alt_file if File.exist?(alt_file) or File.directory?(alt_file)
145
+ end
146
+ nil
147
+ end
148
+
139
149
  def find(where = nil)
140
- return self if located?
150
+ if located?
151
+ if File.exist?(self)
152
+ return self if located?
153
+ else
154
+ found = Path.exists_file_or_alternatives(self)
155
+ if found
156
+ return self.annotate(found)
157
+ else
158
+ return self if located?
159
+ end
160
+ end
161
+ end
162
+
141
163
  return find_all if where == 'all' || where == :all
164
+
142
165
  return follow(where) if where
143
166
 
144
167
  map_order.each do |map_name|
145
168
  found = follow(map_name, false)
146
169
 
147
- return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
170
+ found = Path.exists_file_or_alternatives(found)
171
+ return annotate_found_where(found, map_name) if found
148
172
  end
149
173
 
150
174
  return follow(:default)
@@ -1,4 +1,9 @@
1
1
  module Path
2
+ def no_method_missing
3
+ class << self
4
+ undef_method :method_missing
5
+ end
6
+ end
2
7
 
3
8
  def self.is_filename?(string, need_to_exists = true)
4
9
  return false if string.nil?
@@ -56,9 +61,7 @@ module Path
56
61
  end.flatten.uniq
57
62
  end
58
63
 
59
- def no_method_missing
60
- class << self
61
- undef_method :method_missing
62
- end
64
+ def set_extension(extension)
65
+ self.annotate(self + ".#{extension}")
63
66
  end
64
67
  end
@@ -93,22 +93,15 @@ module Persist
93
93
 
94
94
  Log.debug "Save #{Log.fingerprint type} on #{file}"
95
95
  if save_drivers[type]
96
- Open.write(file, save_drivers[type].call(content))
97
- return
98
- end
99
-
100
- if IO === content || StringIO === content
101
- main, copy = Open.tee_stream_thread content
102
- t = Thread.new do
103
- Thread.current["name"] = "file saver: " + file
104
- Open.sensible_write(file, main)
96
+ if save_drivers[type].arity == 1
97
+ return Open.sensible_write(file, save_drivers[type].call(content))
98
+ else
99
+ return save_drivers[type].call(file, content)
105
100
  end
106
- ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
107
- else
108
- serialized = serialize(content, type)
109
- Open.sensible_write(file, serialized, :force => true)
110
- content
111
101
  end
102
+ serialized = serialize(content, type)
103
+ Open.sensible_write(file, serialized, :force => true)
104
+ return nil
112
105
  end
113
106
 
114
107
  def self.load(file, type = :serializer)
data/lib/scout/persist.rb CHANGED
@@ -14,7 +14,7 @@ module Persist
14
14
 
15
15
  attr_writer :lock_dir
16
16
  def lock_dir
17
- @lock_dir ||= Path.setup("var/cache/persist_locks")
17
+ @lock_dir ||= Path.setup("tmp/persist_locks").find
18
18
  end
19
19
  end
20
20
 
@@ -24,26 +24,60 @@ module Persist
24
24
  TmpFile.tmp_for_file(name, options, other_options)
25
25
  end
26
26
 
27
+ MEMORY_CACHE = {}
28
+ CONNECTIONS = {}
27
29
  def self.persist(name, type = :serializer, options = {}, &block)
28
30
  persist_options = IndiferentHash.pull_keys options, :persist
31
+ return yield if FalseClass === persist_options[:persist]
29
32
  file = persist_options[:path] || options[:path] || persistence_path(name, options)
30
33
 
34
+ lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
35
+
31
36
  update = options[:update] || persist_options[:update]
32
37
  update = Open.mtime(update) if Path === update
33
38
  update = Open.mtime(file) >= update ? false : true if Time === update
34
39
 
35
- if Open.exist?(file) && ! update
36
- Persist.load(file, type)
37
- else
38
- res = yield
39
- begin
40
- Open.rm(file)
41
- res = Persist.save(res, file, type)
42
- rescue
43
- raise $! unless options[:canfail]
44
- Log.debug "Could not persist #{type} on #{file}"
40
+ if type == :memory
41
+ repo = options[:memory] || options[:repo] || MEMORY_CACHE
42
+ repo[file] ||= yield
43
+ return repo[file]
44
+ end
45
+
46
+ Open.lock lockfile do |lock|
47
+ if Open.exist?(file) && ! update
48
+ Persist.load(file, type)
49
+ else
50
+ return yield(file) if block.arity == 1
51
+ res = yield
52
+ begin
53
+ Open.rm(file)
54
+
55
+ if IO === res || StringIO === res
56
+ tee_copies = options[:tee_copies] || 1
57
+ main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
58
+ main.lock = lock
59
+ t = Thread.new do
60
+ Thread.current.report_on_exception = false
61
+ Thread.current["name"] = "file saver: " + file
62
+ Open.sensible_write(file, main)
63
+ end
64
+ Thread.pass until t["name"]
65
+ copies.each_with_index do |copy,i|
66
+ next_stream = copies[i+1] if copies.length > i
67
+ ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
68
+ end
69
+ res = copies.first
70
+ raise KeepLocked.new(res)
71
+ else
72
+ pres = Persist.save(res, file, type)
73
+ res = pres unless pres.nil?
74
+ end
75
+ rescue
76
+ raise $! unless options[:canfail]
77
+ Log.debug "Could not persist #{type} on #{file}"
78
+ end
79
+ res
45
80
  end
46
- res
47
81
  end
48
82
  end
49
83
 
@@ -39,7 +39,7 @@ module Resource
39
39
  else
40
40
  ScoutRake.run(rakefile, rake_dir, task)
41
41
  end
42
- rescue Rake::TaskNotFound
42
+ rescue ScoutRake::TaskNotFound
43
43
  if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
44
44
  raise $!
45
45
  end
@@ -51,9 +51,9 @@ module Resource
51
51
 
52
52
  def produce(path, force = false)
53
53
  case
54
- when @resources.include?(path)
54
+ when (@resources && @resources.include?(path))
55
55
  type, content = @resources[path]
56
- when (Path === path && @resources.include?(path.original))
56
+ when (Path === path && @resources && @resources.include?(path.original))
57
57
  type, content = @resources[path.original]
58
58
  when has_rake(path)
59
59
  type = :rake
@@ -126,97 +126,10 @@ module Resource
126
126
  when :rake
127
127
  run_rake(path, content, rake_dir)
128
128
  when :install
129
- Log.debug "Installing software: #{path}"
130
-
131
- $set_software_env = false unless File.exist? path
132
-
133
- software_dir = path.resource.root.software.find :user
134
- helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find(:lib, caller_lib_dir(__FILE__)))
135
- #helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find)
136
-
137
- preamble = <<-EOF
138
- #!/bin/bash
139
-
140
- RBBT_SOFTWARE_DIR="#{software_dir}"
141
-
142
- INSTALL_HELPER_FILE="#{helper_file}"
143
- source "$INSTALL_HELPER_FILE"
144
- EOF
145
-
146
- content = content.call if Proc === content
147
-
148
- content = if content =~ /git:|\.git$/
149
- {:git => content}
150
- else
151
- {:src => content}
152
- end if String === content and Open.remote?(content)
153
-
154
- script_text = case content
155
- when nil
156
- raise "No way to install #{path}"
157
- when Path
158
- Open.read(content)
159
- when String
160
- if Path.is_filename?(content) and Open.exists?(content)
161
- Open.read(content)
162
- else
163
- content
164
- end
165
- when Hash
166
- name = content[:name] || File.basename(path)
167
- git = content[:git]
168
- src = content[:src]
169
- url = content[:url]
170
- jar = content[:jar]
171
- extra = content[:extra]
172
- commands = content[:commands]
173
- if git
174
- <<-EOF
175
-
176
- name='#{name}'
177
- url='#{git}'
178
-
179
- install_git "$name" "$url" #{extra}
180
-
181
- #{commands}
182
- EOF
183
- elsif src
184
- <<-EOF
185
-
186
- name='#{name}'
187
- url='#{src}'
188
-
189
- install_src "$name" "$url" #{extra}
190
-
191
- #{commands}
192
- EOF
193
- elsif jar
194
- <<-EOF
195
-
196
- name='#{name}'
197
- url='#{jar}'
198
-
199
- install_jar "$name" "$url" #{extra}
200
-
201
- #{commands}
202
- EOF
203
- else
204
- <<-EOF
205
-
206
- name='#{name}'
207
- url='#{url}'
208
-
209
- #{commands}
210
- EOF
211
- end
212
- end
213
-
214
- script = preamble + "\n" + script_text
215
- Log.debug "Installing software with script:\n" << script
216
- CMD.cmd_log('bash', :in => script)
217
-
218
- set_software_env(software_dir) unless $set_software_env
219
- $set_software_env = true
129
+ software_dir = self.root.software
130
+ name = File.basename(path)
131
+ Resource.install(content, name, software_dir)
132
+ set_software_env(software_dir)
220
133
  else
221
134
  raise "Could not produce #{ resource }. (#{ type }, #{ content })"
222
135
  end
@@ -0,0 +1,176 @@
1
+ module Resource
2
+
3
+ def self.install_helpers
4
+ File.expand_path(Scout.share.software.install_helpers.find(:lib))
5
+ end
6
+
7
+ def self.install(content, name, software_dir = Path.setup('software'), &block)
8
+ software_dir ||= Path.setup('software')
9
+ software_dir = software_dir.find if Path === software_dir
10
+
11
+ content = block if block_given?
12
+
13
+ preamble = <<-EOF
14
+ #!/bin/bash
15
+
16
+ SOFTWARE_DIR="#{software_dir}"
17
+
18
+ INSTALL_HELPER_FILE="#{install_helpers}"
19
+ source "$INSTALL_HELPER_FILE"
20
+ EOF
21
+
22
+ content = content.call if Proc === content
23
+
24
+ name = content[:name] if Hash === content && content.include?(:name)
25
+ content =
26
+ if content =~ /git:|\.git$/
27
+ {:git => content}
28
+ else
29
+ {:src => content}
30
+ end if String === content and Open.remote?(content)
31
+
32
+ script_text =
33
+ case content
34
+ when nil
35
+ raise "No way to install #{name}"
36
+ when Path
37
+ Open.read(content)
38
+ when String
39
+ if Path.is_filename?(content) and Open.exists?(content)
40
+ Open.read(content)
41
+ else
42
+ content
43
+ end
44
+ when Hash
45
+ name = content[:name] || name
46
+ git = content[:git]
47
+ src = content[:src]
48
+ url = content[:url]
49
+ jar = content[:jar]
50
+ extra = content[:extra]
51
+ commands = content[:commands]
52
+ if git
53
+ <<-EOF
54
+
55
+ name='#{name}'
56
+ url='#{git}'
57
+
58
+ install_git "$name" "$url" #{extra}
59
+
60
+ #{commands}
61
+ EOF
62
+ elsif src
63
+ <<-EOF
64
+
65
+ name='#{name}'
66
+ url='#{src}'
67
+
68
+ install_src "$name" "$url" #{extra}
69
+
70
+ #{commands}
71
+ EOF
72
+ elsif jar
73
+ <<-EOF
74
+
75
+ name='#{name}'
76
+ url='#{jar}'
77
+
78
+ install_jar "$name" "$url" #{extra}
79
+
80
+ #{commands}
81
+ EOF
82
+ else
83
+ <<-EOF
84
+
85
+ name='#{name}'
86
+ url='#{url}'
87
+
88
+ #{commands}
89
+ EOF
90
+ end
91
+ end
92
+
93
+ script = preamble + "\n" + script_text
94
+ Log.debug "Installing software #{name} into #{software_dir} with script:\n" << script
95
+ CMD.cmd_log('bash', :in => script)
96
+ Resource.set_software_env(software_dir)
97
+ end
98
+
99
+ def self.set_software_env(software_dir = Path.setup('software'))
100
+ software_dir.opt.find_all.collect{|d| d.annotate(File.dirname(d)) }.reverse.each do |software_dir|
101
+ next unless software_dir.exists?
102
+ Log.medium "Preparing software env at #{software_dir}"
103
+
104
+ software_dir = File.expand_path(software_dir)
105
+ opt_dir = File.join(software_dir, 'opt')
106
+ bin_dir = File.join(opt_dir, 'bin')
107
+
108
+ Misc.env_add 'PATH', bin_dir
109
+
110
+ FileUtils.mkdir_p opt_dir unless File.exist? opt_dir
111
+
112
+ %w(.ld-paths .c-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
113
+ filename = File.join(opt_dir, file)
114
+ begin
115
+ FileUtils.touch filename unless File.exist? filename
116
+ rescue
117
+ Log.warn("Could not touch #{ filename }")
118
+ end
119
+ end
120
+
121
+ Open.read(File.join opt_dir, '.c-paths').split(/\n/).each do |line|
122
+ dir = line.chomp
123
+ dir = File.join(opt_dir, dir) unless dir[0] == "/"
124
+ Misc.env_add('CPLUS_INCLUDE_PATH',dir)
125
+ Misc.env_add('C_INCLUDE_PATH',dir)
126
+ end if File.exist? File.join(opt_dir, '.c-paths')
127
+
128
+ Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
129
+ dir = line.chomp
130
+ dir = File.join(opt_dir, dir) unless dir[0] == "/"
131
+ Misc.env_add('LIBRARY_PATH',dir)
132
+ Misc.env_add('LD_LIBRARY_PATH',dir)
133
+ Misc.env_add('LD_RUN_PATH',dir)
134
+ end if File.exist? File.join(opt_dir, '.ld-paths')
135
+
136
+ Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
137
+ dir = line.chomp
138
+ dir = File.join(opt_dir, dir) unless dir[0] == "/"
139
+ Misc.env_add('PKG_CONFIG_PATH',dir)
140
+ end if File.exist? File.join(opt_dir, '.pkgconfig-paths')
141
+
142
+ Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
143
+ dir = line.chomp
144
+ dir = File.join(opt_dir, dir) unless dir[0] == "/"
145
+ Misc.env_add('ACLOCAL_FLAGS', "-I #{dir}", ' ')
146
+ end if File.exist? File.join(opt_dir, '.aclocal-paths')
147
+
148
+ Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
149
+ dir = line.chomp
150
+ dir = File.join(opt_dir, dir) unless dir[0] == "/"
151
+ Misc.env_add('CLASSPATH', "#{dir}")
152
+ end if File.exist? File.join(opt_dir, '.java-classpaths')
153
+
154
+ Dir.glob(File.join opt_dir, 'jars', '*.jar').each do |file|
155
+ Misc.env_add('CLASSPATH', "#{file}")
156
+ end
157
+
158
+ if File.exist?(File.join(opt_dir, '.post_install')) and File.directory?(File.join(opt_dir, '.post_install'))
159
+ Dir.glob(File.join(opt_dir, '.post_install','*')).each do |file|
160
+
161
+ # Load exports
162
+ Open.read(file).split("\n").each do |line|
163
+ next unless line =~ /^\s*export\s+([^=]+)=(.*)/
164
+ var = $1.strip
165
+ value = $2.strip
166
+ value.sub!(/^['"]/,'')
167
+ value.sub!(/['"]$/,'')
168
+ value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
169
+ Log.debug "Set variable export from .post_install: #{Misc.fingerprint [var,value]*"="}"
170
+ ENV[var] = value
171
+ end
172
+ end
173
+ end
174
+ end
175
+ end
176
+ end