scout-gear 7.1.0 → 7.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +65 -2
- data/VERSION +1 -1
- data/bin/scout +5 -1
- data/lib/rbbt-scout.rb +5 -0
- data/lib/scout/concurrent_stream.rb +13 -8
- data/lib/scout/config.rb +168 -0
- data/lib/scout/exceptions.rb +5 -3
- data/lib/scout/indiferent_hash/options.rb +1 -0
- data/lib/scout/indiferent_hash.rb +4 -2
- data/lib/scout/log/color.rb +3 -2
- data/lib/scout/log/progress/report.rb +1 -0
- data/lib/scout/log/progress/util.rb +66 -1
- data/lib/scout/log/progress.rb +5 -3
- data/lib/scout/log.rb +3 -2
- data/lib/scout/misc/helper.rb +31 -0
- data/lib/scout/misc/monitor.rb +4 -1
- data/lib/scout/misc/system.rb +15 -0
- data/lib/scout/misc.rb +2 -0
- data/lib/scout/named_array.rb +68 -0
- data/lib/scout/open/stream.rb +58 -33
- data/lib/scout/path/find.rb +27 -3
- data/lib/scout/path/util.rb +7 -4
- data/lib/scout/persist/serialize.rb +7 -14
- data/lib/scout/persist.rb +46 -12
- data/lib/scout/resource/produce.rb +7 -94
- data/lib/scout/resource/software.rb +176 -0
- data/lib/scout/semaphore.rb +8 -1
- data/lib/scout/tsv/dumper.rb +112 -0
- data/lib/scout/tsv/index.rb +161 -0
- data/lib/scout/tsv/open.rb +128 -0
- data/lib/scout/tsv/parser.rb +230 -30
- data/lib/scout/tsv/path.rb +13 -0
- data/lib/scout/tsv/persist/adapter.rb +367 -0
- data/lib/scout/tsv/persist/fix_width_table.rb +324 -0
- data/lib/scout/tsv/persist/serialize.rb +117 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +113 -0
- data/lib/scout/tsv/persist.rb +13 -0
- data/lib/scout/tsv/traverse.rb +143 -0
- data/lib/scout/tsv/util/filter.rb +303 -0
- data/lib/scout/tsv/util/process.rb +73 -0
- data/lib/scout/tsv/util/select.rb +220 -0
- data/lib/scout/tsv/util.rb +82 -0
- data/lib/scout/tsv.rb +16 -3
- data/lib/scout/work_queue/worker.rb +4 -4
- data/lib/scout/work_queue.rb +22 -7
- data/lib/scout/workflow/definition.rb +101 -4
- data/lib/scout/workflow/step/config.rb +18 -0
- data/lib/scout/workflow/step/dependencies.rb +40 -0
- data/lib/scout/workflow/step/file.rb +15 -0
- data/lib/scout/workflow/step/info.rb +35 -4
- data/lib/scout/workflow/step/progress.rb +14 -0
- data/lib/scout/workflow/step/provenance.rb +148 -0
- data/lib/scout/workflow/step.rb +71 -17
- data/lib/scout/workflow/task.rb +10 -5
- data/lib/scout/workflow/usage.rb +3 -1
- data/lib/scout/workflow.rb +11 -3
- data/lib/scout-gear.rb +1 -0
- data/lib/scout.rb +1 -0
- data/scout-gear.gemspec +64 -10
- data/scout_commands/find +1 -1
- data/scout_commands/workflow/task +16 -9
- data/scout_commands/workflow/task_old +2 -2
- data/share/software/install_helpers +523 -0
- data/test/scout/log/test_progress.rb +0 -2
- data/test/scout/misc/test_system.rb +21 -0
- data/test/scout/open/test_stream.rb +160 -1
- data/test/scout/path/test_find.rb +14 -7
- data/test/scout/resource/test_software.rb +24 -0
- data/test/scout/test_config.rb +66 -0
- data/test/scout/test_meta_extension.rb +10 -0
- data/test/scout/test_named_array.rb +19 -0
- data/test/scout/test_persist.rb +96 -0
- data/test/scout/test_tmpfile.rb +1 -1
- data/test/scout/test_tsv.rb +50 -1
- data/test/scout/test_work_queue.rb +41 -13
- data/test/scout/tsv/persist/test_adapter.rb +44 -0
- data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
- data/test/scout/tsv/persist/test_tokyocabinet.rb +92 -0
- data/test/scout/tsv/test_dumper.rb +44 -0
- data/test/scout/tsv/test_index.rb +156 -0
- data/test/scout/tsv/test_open.rb +9 -0
- data/test/scout/tsv/test_parser.rb +114 -3
- data/test/scout/tsv/test_persist.rb +43 -0
- data/test/scout/tsv/test_traverse.rb +116 -0
- data/test/scout/tsv/test_util.rb +23 -0
- data/test/scout/tsv/util/test_filter.rb +188 -0
- data/test/scout/tsv/util/test_process.rb +47 -0
- data/test/scout/tsv/util/test_select.rb +44 -0
- data/test/scout/work_queue/test_worker.rb +66 -9
- data/test/scout/workflow/step/test_dependencies.rb +25 -0
- data/test/scout/workflow/step/test_info.rb +15 -17
- data/test/scout/workflow/step/test_load.rb +19 -21
- data/test/scout/workflow/step/test_provenance.rb +25 -0
- data/test/scout/workflow/test_step.rb +206 -10
- data/test/scout/workflow/test_task.rb +0 -3
- data/test/test_helper.rb +9 -1
- metadata +50 -6
@@ -0,0 +1,68 @@
|
|
1
|
+
require_relative 'meta_extension'
|
2
|
+
module NamedArray
|
3
|
+
extend MetaExtension
|
4
|
+
extension_attr :fields
|
5
|
+
|
6
|
+
def self.identify_name(names, selected)
|
7
|
+
res = (Array === selected ? selected : [selected]).collect do |field|
|
8
|
+
case field
|
9
|
+
when nil
|
10
|
+
0
|
11
|
+
when Integer
|
12
|
+
field
|
13
|
+
when Symbol
|
14
|
+
field == :key ? field : identify_name(names, field.to_s)
|
15
|
+
when (names.nil? and String)
|
16
|
+
if field =~ /^\d+$/
|
17
|
+
identify_field(key_field, fields, field.to_i)
|
18
|
+
else
|
19
|
+
raise "No name information available and specified name not numeric: #{ field }"
|
20
|
+
end
|
21
|
+
when Symbol
|
22
|
+
names.index{|f| f.to_s == field.to_s }
|
23
|
+
when String
|
24
|
+
pos = names.index{|f| f.to_s == field }
|
25
|
+
next pos if pos
|
26
|
+
if field =~ /^\d+$/
|
27
|
+
next identify_names(names, field.to_i)
|
28
|
+
end
|
29
|
+
pos = names.index{|name| name.start_with?(field) }
|
30
|
+
next pos if pos
|
31
|
+
nil
|
32
|
+
else
|
33
|
+
raise "Field '#{ Log.fingerprint field }' was not understood. Options: (#{ Log.fingerprint names })"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
Array === selected ? res : res.first
|
38
|
+
end
|
39
|
+
|
40
|
+
def positions(fields)
|
41
|
+
if Array == fields
|
42
|
+
fields.collect{|field|
|
43
|
+
NamedArray.identify_name(@fields, field)
|
44
|
+
}
|
45
|
+
else
|
46
|
+
NamedArray.identify_name(@fields, fields)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def [](key)
|
51
|
+
pos = NamedArray.identify_name(@fields, key)
|
52
|
+
super(pos)
|
53
|
+
end
|
54
|
+
|
55
|
+
def concat(other)
|
56
|
+
super(other)
|
57
|
+
self.fields.concat(other.fields) if NamedArray === other
|
58
|
+
self
|
59
|
+
end
|
60
|
+
|
61
|
+
def to_hash
|
62
|
+
hash = {}
|
63
|
+
self.fields.zip(self) do |field,value|
|
64
|
+
hash[field] = value
|
65
|
+
end
|
66
|
+
IndiferentHash.setup hash
|
67
|
+
end
|
68
|
+
end
|
data/lib/scout/open/stream.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Open
|
2
|
-
BLOCK_SIZE = 1024
|
2
|
+
BLOCK_SIZE = 1024 * 8
|
3
3
|
|
4
4
|
class << self
|
5
5
|
attr_accessor :sensible_write_lock_dir
|
@@ -52,15 +52,11 @@ module Open
|
|
52
52
|
into_path, into = into, File.open(into, 'w')
|
53
53
|
end
|
54
54
|
|
55
|
-
into.sync = true if IO === into
|
56
55
|
into_close = false unless into.respond_to? :close
|
57
|
-
io.sync = true
|
58
56
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
end
|
63
|
-
rescue EOFError
|
57
|
+
while c = io.read(BLOCK_SIZE)
|
58
|
+
into << c if into
|
59
|
+
break if io.closed?
|
64
60
|
end
|
65
61
|
|
66
62
|
io.join if io.respond_to? :join
|
@@ -77,7 +73,7 @@ module Open
|
|
77
73
|
FileUtils.rm into_path if into_path and File.exist?(into_path)
|
78
74
|
rescue Exception
|
79
75
|
Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
|
80
|
-
exception = io.stream_exception
|
76
|
+
exception = (io.respond_to?(:stream_exception) && io.stream_exception) ? io.stream_exception : $!
|
81
77
|
io.abort exception if io.respond_to? :abort
|
82
78
|
into.close if into.respond_to?(:closed?) && ! into.closed?
|
83
79
|
into_path = into if into_path.nil? && String === into
|
@@ -110,7 +106,7 @@ module Open
|
|
110
106
|
Log.warn "Path exists in sensible_write, not forcing update: #{ path }"
|
111
107
|
Open.consume_stream content
|
112
108
|
else
|
113
|
-
FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?
|
109
|
+
FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?(File.dirname(tmp_path))
|
114
110
|
FileUtils.rm_f tmp_path if File.exist? tmp_path
|
115
111
|
begin
|
116
112
|
|
@@ -121,13 +117,11 @@ module Open
|
|
121
117
|
File.open(tmp_path, 'wb') do |f| f.write content end
|
122
118
|
when (IO === content or StringIO === content or File === content)
|
123
119
|
Open.write(tmp_path) do |f|
|
124
|
-
f.sync = true
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
rescue EOFError
|
130
|
-
end
|
120
|
+
#f.sync = true
|
121
|
+
while block = content.read(BLOCK_SIZE)
|
122
|
+
f.write block
|
123
|
+
break if content.closed?
|
124
|
+
end
|
131
125
|
end
|
132
126
|
else
|
133
127
|
File.open(tmp_path, 'wb') do |f| end
|
@@ -292,22 +286,20 @@ module Open
|
|
292
286
|
Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
|
293
287
|
|
294
288
|
skip = [false] * num
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
end unless skip[i]
|
308
|
-
end
|
289
|
+
while block = stream.read(BLOCK_SIZE)
|
290
|
+
|
291
|
+
in_pipes.each_with_index do |sin,i|
|
292
|
+
begin
|
293
|
+
sin.write block
|
294
|
+
rescue IOError
|
295
|
+
Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
|
296
|
+
skip[i] = true
|
297
|
+
rescue
|
298
|
+
Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
|
299
|
+
raise $!
|
300
|
+
end unless skip[i]
|
309
301
|
end
|
310
|
-
|
302
|
+
break if stream.closed?
|
311
303
|
end
|
312
304
|
|
313
305
|
stream.join if stream.respond_to? :join
|
@@ -345,7 +337,6 @@ module Open
|
|
345
337
|
end
|
346
338
|
end
|
347
339
|
|
348
|
-
|
349
340
|
out_pipes.each do |sout|
|
350
341
|
ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :pair => stream
|
351
342
|
end
|
@@ -401,4 +392,38 @@ module Open
|
|
401
392
|
str
|
402
393
|
end
|
403
394
|
|
395
|
+
def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
|
396
|
+
Open.open_pipe do |sin|
|
397
|
+
line = stream.gets
|
398
|
+
while line =~ /^#{header_hash}/ do
|
399
|
+
sin.puts line
|
400
|
+
line = stream.gets
|
401
|
+
end
|
402
|
+
|
403
|
+
line_stream = Open.open_pipe do |line_stream_in|
|
404
|
+
line_stream_in.puts line
|
405
|
+
begin
|
406
|
+
Open.consume_stream(stream, false, line_stream_in)
|
407
|
+
rescue
|
408
|
+
raise $!
|
409
|
+
end
|
410
|
+
end
|
411
|
+
|
412
|
+
sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
|
413
|
+
|
414
|
+
begin
|
415
|
+
Open.consume_stream(sorted, false, sin)
|
416
|
+
rescue
|
417
|
+
Log.exception $!
|
418
|
+
begin
|
419
|
+
sorted.raise($!) if sorted.respond_to? :raise
|
420
|
+
stream.raise($!) if stream.respond_to? :raise
|
421
|
+
ensure
|
422
|
+
raise $!
|
423
|
+
end
|
424
|
+
end
|
425
|
+
end
|
426
|
+
end
|
427
|
+
|
428
|
+
|
404
429
|
end
|
data/lib/scout/path/find.rb
CHANGED
@@ -9,7 +9,8 @@ module Path
|
|
9
9
|
break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
|
10
10
|
file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
|
11
11
|
file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
|
12
|
-
file =~ /(?:scout|rbbt)\/persist.rb/
|
12
|
+
file =~ /(?:scout|rbbt)\/persist.rb/ or
|
13
|
+
file =~ /modules\/rbbt-util/
|
13
14
|
end
|
14
15
|
file = file.sub(/\.rb[^\w].*/,'.rb')
|
15
16
|
end
|
@@ -136,15 +137,38 @@ module Path
|
|
136
137
|
found
|
137
138
|
end
|
138
139
|
|
140
|
+
def self.exists_file_or_alternatives(file)
|
141
|
+
return file if File.exist?(file) or File.directory?(file)
|
142
|
+
%w(gz bgz zip).each do |extension|
|
143
|
+
alt_file = file + '.' + extension
|
144
|
+
return alt_file if File.exist?(alt_file) or File.directory?(alt_file)
|
145
|
+
end
|
146
|
+
nil
|
147
|
+
end
|
148
|
+
|
139
149
|
def find(where = nil)
|
140
|
-
|
150
|
+
if located?
|
151
|
+
if File.exist?(self)
|
152
|
+
return self if located?
|
153
|
+
else
|
154
|
+
found = Path.exists_file_or_alternatives(self)
|
155
|
+
if found
|
156
|
+
return self.annotate(found)
|
157
|
+
else
|
158
|
+
return self if located?
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
141
163
|
return find_all if where == 'all' || where == :all
|
164
|
+
|
142
165
|
return follow(where) if where
|
143
166
|
|
144
167
|
map_order.each do |map_name|
|
145
168
|
found = follow(map_name, false)
|
146
169
|
|
147
|
-
|
170
|
+
found = Path.exists_file_or_alternatives(found)
|
171
|
+
return annotate_found_where(found, map_name) if found
|
148
172
|
end
|
149
173
|
|
150
174
|
return follow(:default)
|
data/lib/scout/path/util.rb
CHANGED
@@ -1,4 +1,9 @@
|
|
1
1
|
module Path
|
2
|
+
def no_method_missing
|
3
|
+
class << self
|
4
|
+
undef_method :method_missing
|
5
|
+
end
|
6
|
+
end
|
2
7
|
|
3
8
|
def self.is_filename?(string, need_to_exists = true)
|
4
9
|
return false if string.nil?
|
@@ -56,9 +61,7 @@ module Path
|
|
56
61
|
end.flatten.uniq
|
57
62
|
end
|
58
63
|
|
59
|
-
def
|
60
|
-
|
61
|
-
undef_method :method_missing
|
62
|
-
end
|
64
|
+
def set_extension(extension)
|
65
|
+
self.annotate(self + ".#{extension}")
|
63
66
|
end
|
64
67
|
end
|
@@ -93,22 +93,15 @@ module Persist
|
|
93
93
|
|
94
94
|
Log.debug "Save #{Log.fingerprint type} on #{file}"
|
95
95
|
if save_drivers[type]
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
if IO === content || StringIO === content
|
101
|
-
main, copy = Open.tee_stream_thread content
|
102
|
-
t = Thread.new do
|
103
|
-
Thread.current["name"] = "file saver: " + file
|
104
|
-
Open.sensible_write(file, main)
|
96
|
+
if save_drivers[type].arity == 1
|
97
|
+
return Open.sensible_write(file, save_drivers[type].call(content))
|
98
|
+
else
|
99
|
+
return save_drivers[type].call(file, content)
|
105
100
|
end
|
106
|
-
ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
|
107
|
-
else
|
108
|
-
serialized = serialize(content, type)
|
109
|
-
Open.sensible_write(file, serialized, :force => true)
|
110
|
-
content
|
111
101
|
end
|
102
|
+
serialized = serialize(content, type)
|
103
|
+
Open.sensible_write(file, serialized, :force => true)
|
104
|
+
return nil
|
112
105
|
end
|
113
106
|
|
114
107
|
def self.load(file, type = :serializer)
|
data/lib/scout/persist.rb
CHANGED
@@ -14,7 +14,7 @@ module Persist
|
|
14
14
|
|
15
15
|
attr_writer :lock_dir
|
16
16
|
def lock_dir
|
17
|
-
@lock_dir ||= Path.setup("
|
17
|
+
@lock_dir ||= Path.setup("tmp/persist_locks").find
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
@@ -24,26 +24,60 @@ module Persist
|
|
24
24
|
TmpFile.tmp_for_file(name, options, other_options)
|
25
25
|
end
|
26
26
|
|
27
|
+
MEMORY_CACHE = {}
|
28
|
+
CONNECTIONS = {}
|
27
29
|
def self.persist(name, type = :serializer, options = {}, &block)
|
28
30
|
persist_options = IndiferentHash.pull_keys options, :persist
|
31
|
+
return yield if FalseClass === persist_options[:persist]
|
29
32
|
file = persist_options[:path] || options[:path] || persistence_path(name, options)
|
30
33
|
|
34
|
+
lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
|
35
|
+
|
31
36
|
update = options[:update] || persist_options[:update]
|
32
37
|
update = Open.mtime(update) if Path === update
|
33
38
|
update = Open.mtime(file) >= update ? false : true if Time === update
|
34
39
|
|
35
|
-
if
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
if type == :memory
|
41
|
+
repo = options[:memory] || options[:repo] || MEMORY_CACHE
|
42
|
+
repo[file] ||= yield
|
43
|
+
return repo[file]
|
44
|
+
end
|
45
|
+
|
46
|
+
Open.lock lockfile do |lock|
|
47
|
+
if Open.exist?(file) && ! update
|
48
|
+
Persist.load(file, type)
|
49
|
+
else
|
50
|
+
return yield(file) if block.arity == 1
|
51
|
+
res = yield
|
52
|
+
begin
|
53
|
+
Open.rm(file)
|
54
|
+
|
55
|
+
if IO === res || StringIO === res
|
56
|
+
tee_copies = options[:tee_copies] || 1
|
57
|
+
main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
|
58
|
+
main.lock = lock
|
59
|
+
t = Thread.new do
|
60
|
+
Thread.current.report_on_exception = false
|
61
|
+
Thread.current["name"] = "file saver: " + file
|
62
|
+
Open.sensible_write(file, main)
|
63
|
+
end
|
64
|
+
Thread.pass until t["name"]
|
65
|
+
copies.each_with_index do |copy,i|
|
66
|
+
next_stream = copies[i+1] if copies.length > i
|
67
|
+
ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
|
68
|
+
end
|
69
|
+
res = copies.first
|
70
|
+
raise KeepLocked.new(res)
|
71
|
+
else
|
72
|
+
pres = Persist.save(res, file, type)
|
73
|
+
res = pres unless pres.nil?
|
74
|
+
end
|
75
|
+
rescue
|
76
|
+
raise $! unless options[:canfail]
|
77
|
+
Log.debug "Could not persist #{type} on #{file}"
|
78
|
+
end
|
79
|
+
res
|
45
80
|
end
|
46
|
-
res
|
47
81
|
end
|
48
82
|
end
|
49
83
|
|
@@ -39,7 +39,7 @@ module Resource
|
|
39
39
|
else
|
40
40
|
ScoutRake.run(rakefile, rake_dir, task)
|
41
41
|
end
|
42
|
-
rescue
|
42
|
+
rescue ScoutRake::TaskNotFound
|
43
43
|
if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
|
44
44
|
raise $!
|
45
45
|
end
|
@@ -51,9 +51,9 @@ module Resource
|
|
51
51
|
|
52
52
|
def produce(path, force = false)
|
53
53
|
case
|
54
|
-
when @resources.include?(path)
|
54
|
+
when (@resources && @resources.include?(path))
|
55
55
|
type, content = @resources[path]
|
56
|
-
when (Path === path && @resources.include?(path.original))
|
56
|
+
when (Path === path && @resources && @resources.include?(path.original))
|
57
57
|
type, content = @resources[path.original]
|
58
58
|
when has_rake(path)
|
59
59
|
type = :rake
|
@@ -126,97 +126,10 @@ module Resource
|
|
126
126
|
when :rake
|
127
127
|
run_rake(path, content, rake_dir)
|
128
128
|
when :install
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
software_dir = path.resource.root.software.find :user
|
134
|
-
helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find(:lib, caller_lib_dir(__FILE__)))
|
135
|
-
#helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find)
|
136
|
-
|
137
|
-
preamble = <<-EOF
|
138
|
-
#!/bin/bash
|
139
|
-
|
140
|
-
RBBT_SOFTWARE_DIR="#{software_dir}"
|
141
|
-
|
142
|
-
INSTALL_HELPER_FILE="#{helper_file}"
|
143
|
-
source "$INSTALL_HELPER_FILE"
|
144
|
-
EOF
|
145
|
-
|
146
|
-
content = content.call if Proc === content
|
147
|
-
|
148
|
-
content = if content =~ /git:|\.git$/
|
149
|
-
{:git => content}
|
150
|
-
else
|
151
|
-
{:src => content}
|
152
|
-
end if String === content and Open.remote?(content)
|
153
|
-
|
154
|
-
script_text = case content
|
155
|
-
when nil
|
156
|
-
raise "No way to install #{path}"
|
157
|
-
when Path
|
158
|
-
Open.read(content)
|
159
|
-
when String
|
160
|
-
if Path.is_filename?(content) and Open.exists?(content)
|
161
|
-
Open.read(content)
|
162
|
-
else
|
163
|
-
content
|
164
|
-
end
|
165
|
-
when Hash
|
166
|
-
name = content[:name] || File.basename(path)
|
167
|
-
git = content[:git]
|
168
|
-
src = content[:src]
|
169
|
-
url = content[:url]
|
170
|
-
jar = content[:jar]
|
171
|
-
extra = content[:extra]
|
172
|
-
commands = content[:commands]
|
173
|
-
if git
|
174
|
-
<<-EOF
|
175
|
-
|
176
|
-
name='#{name}'
|
177
|
-
url='#{git}'
|
178
|
-
|
179
|
-
install_git "$name" "$url" #{extra}
|
180
|
-
|
181
|
-
#{commands}
|
182
|
-
EOF
|
183
|
-
elsif src
|
184
|
-
<<-EOF
|
185
|
-
|
186
|
-
name='#{name}'
|
187
|
-
url='#{src}'
|
188
|
-
|
189
|
-
install_src "$name" "$url" #{extra}
|
190
|
-
|
191
|
-
#{commands}
|
192
|
-
EOF
|
193
|
-
elsif jar
|
194
|
-
<<-EOF
|
195
|
-
|
196
|
-
name='#{name}'
|
197
|
-
url='#{jar}'
|
198
|
-
|
199
|
-
install_jar "$name" "$url" #{extra}
|
200
|
-
|
201
|
-
#{commands}
|
202
|
-
EOF
|
203
|
-
else
|
204
|
-
<<-EOF
|
205
|
-
|
206
|
-
name='#{name}'
|
207
|
-
url='#{url}'
|
208
|
-
|
209
|
-
#{commands}
|
210
|
-
EOF
|
211
|
-
end
|
212
|
-
end
|
213
|
-
|
214
|
-
script = preamble + "\n" + script_text
|
215
|
-
Log.debug "Installing software with script:\n" << script
|
216
|
-
CMD.cmd_log('bash', :in => script)
|
217
|
-
|
218
|
-
set_software_env(software_dir) unless $set_software_env
|
219
|
-
$set_software_env = true
|
129
|
+
software_dir = self.root.software
|
130
|
+
name = File.basename(path)
|
131
|
+
Resource.install(content, name, software_dir)
|
132
|
+
set_software_env(software_dir)
|
220
133
|
else
|
221
134
|
raise "Could not produce #{ resource }. (#{ type }, #{ content })"
|
222
135
|
end
|
@@ -0,0 +1,176 @@
|
|
1
|
+
module Resource
|
2
|
+
|
3
|
+
def self.install_helpers
|
4
|
+
File.expand_path(Scout.share.software.install_helpers.find(:lib))
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.install(content, name, software_dir = Path.setup('software'), &block)
|
8
|
+
software_dir ||= Path.setup('software')
|
9
|
+
software_dir = software_dir.find if Path === software_dir
|
10
|
+
|
11
|
+
content = block if block_given?
|
12
|
+
|
13
|
+
preamble = <<-EOF
|
14
|
+
#!/bin/bash
|
15
|
+
|
16
|
+
SOFTWARE_DIR="#{software_dir}"
|
17
|
+
|
18
|
+
INSTALL_HELPER_FILE="#{install_helpers}"
|
19
|
+
source "$INSTALL_HELPER_FILE"
|
20
|
+
EOF
|
21
|
+
|
22
|
+
content = content.call if Proc === content
|
23
|
+
|
24
|
+
name = content[:name] if Hash === content && content.include?(:name)
|
25
|
+
content =
|
26
|
+
if content =~ /git:|\.git$/
|
27
|
+
{:git => content}
|
28
|
+
else
|
29
|
+
{:src => content}
|
30
|
+
end if String === content and Open.remote?(content)
|
31
|
+
|
32
|
+
script_text =
|
33
|
+
case content
|
34
|
+
when nil
|
35
|
+
raise "No way to install #{name}"
|
36
|
+
when Path
|
37
|
+
Open.read(content)
|
38
|
+
when String
|
39
|
+
if Path.is_filename?(content) and Open.exists?(content)
|
40
|
+
Open.read(content)
|
41
|
+
else
|
42
|
+
content
|
43
|
+
end
|
44
|
+
when Hash
|
45
|
+
name = content[:name] || name
|
46
|
+
git = content[:git]
|
47
|
+
src = content[:src]
|
48
|
+
url = content[:url]
|
49
|
+
jar = content[:jar]
|
50
|
+
extra = content[:extra]
|
51
|
+
commands = content[:commands]
|
52
|
+
if git
|
53
|
+
<<-EOF
|
54
|
+
|
55
|
+
name='#{name}'
|
56
|
+
url='#{git}'
|
57
|
+
|
58
|
+
install_git "$name" "$url" #{extra}
|
59
|
+
|
60
|
+
#{commands}
|
61
|
+
EOF
|
62
|
+
elsif src
|
63
|
+
<<-EOF
|
64
|
+
|
65
|
+
name='#{name}'
|
66
|
+
url='#{src}'
|
67
|
+
|
68
|
+
install_src "$name" "$url" #{extra}
|
69
|
+
|
70
|
+
#{commands}
|
71
|
+
EOF
|
72
|
+
elsif jar
|
73
|
+
<<-EOF
|
74
|
+
|
75
|
+
name='#{name}'
|
76
|
+
url='#{jar}'
|
77
|
+
|
78
|
+
install_jar "$name" "$url" #{extra}
|
79
|
+
|
80
|
+
#{commands}
|
81
|
+
EOF
|
82
|
+
else
|
83
|
+
<<-EOF
|
84
|
+
|
85
|
+
name='#{name}'
|
86
|
+
url='#{url}'
|
87
|
+
|
88
|
+
#{commands}
|
89
|
+
EOF
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
script = preamble + "\n" + script_text
|
94
|
+
Log.debug "Installing software #{name} into #{software_dir} with script:\n" << script
|
95
|
+
CMD.cmd_log('bash', :in => script)
|
96
|
+
Resource.set_software_env(software_dir)
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.set_software_env(software_dir = Path.setup('software'))
|
100
|
+
software_dir.opt.find_all.collect{|d| d.annotate(File.dirname(d)) }.reverse.each do |software_dir|
|
101
|
+
next unless software_dir.exists?
|
102
|
+
Log.medium "Preparing software env at #{software_dir}"
|
103
|
+
|
104
|
+
software_dir = File.expand_path(software_dir)
|
105
|
+
opt_dir = File.join(software_dir, 'opt')
|
106
|
+
bin_dir = File.join(opt_dir, 'bin')
|
107
|
+
|
108
|
+
Misc.env_add 'PATH', bin_dir
|
109
|
+
|
110
|
+
FileUtils.mkdir_p opt_dir unless File.exist? opt_dir
|
111
|
+
|
112
|
+
%w(.ld-paths .c-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
|
113
|
+
filename = File.join(opt_dir, file)
|
114
|
+
begin
|
115
|
+
FileUtils.touch filename unless File.exist? filename
|
116
|
+
rescue
|
117
|
+
Log.warn("Could not touch #{ filename }")
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
Open.read(File.join opt_dir, '.c-paths').split(/\n/).each do |line|
|
122
|
+
dir = line.chomp
|
123
|
+
dir = File.join(opt_dir, dir) unless dir[0] == "/"
|
124
|
+
Misc.env_add('CPLUS_INCLUDE_PATH',dir)
|
125
|
+
Misc.env_add('C_INCLUDE_PATH',dir)
|
126
|
+
end if File.exist? File.join(opt_dir, '.c-paths')
|
127
|
+
|
128
|
+
Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
|
129
|
+
dir = line.chomp
|
130
|
+
dir = File.join(opt_dir, dir) unless dir[0] == "/"
|
131
|
+
Misc.env_add('LIBRARY_PATH',dir)
|
132
|
+
Misc.env_add('LD_LIBRARY_PATH',dir)
|
133
|
+
Misc.env_add('LD_RUN_PATH',dir)
|
134
|
+
end if File.exist? File.join(opt_dir, '.ld-paths')
|
135
|
+
|
136
|
+
Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
|
137
|
+
dir = line.chomp
|
138
|
+
dir = File.join(opt_dir, dir) unless dir[0] == "/"
|
139
|
+
Misc.env_add('PKG_CONFIG_PATH',dir)
|
140
|
+
end if File.exist? File.join(opt_dir, '.pkgconfig-paths')
|
141
|
+
|
142
|
+
Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
|
143
|
+
dir = line.chomp
|
144
|
+
dir = File.join(opt_dir, dir) unless dir[0] == "/"
|
145
|
+
Misc.env_add('ACLOCAL_FLAGS', "-I #{dir}", ' ')
|
146
|
+
end if File.exist? File.join(opt_dir, '.aclocal-paths')
|
147
|
+
|
148
|
+
Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
|
149
|
+
dir = line.chomp
|
150
|
+
dir = File.join(opt_dir, dir) unless dir[0] == "/"
|
151
|
+
Misc.env_add('CLASSPATH', "#{dir}")
|
152
|
+
end if File.exist? File.join(opt_dir, '.java-classpaths')
|
153
|
+
|
154
|
+
Dir.glob(File.join opt_dir, 'jars', '*.jar').each do |file|
|
155
|
+
Misc.env_add('CLASSPATH', "#{file}")
|
156
|
+
end
|
157
|
+
|
158
|
+
if File.exist?(File.join(opt_dir, '.post_install')) and File.directory?(File.join(opt_dir, '.post_install'))
|
159
|
+
Dir.glob(File.join(opt_dir, '.post_install','*')).each do |file|
|
160
|
+
|
161
|
+
# Load exports
|
162
|
+
Open.read(file).split("\n").each do |line|
|
163
|
+
next unless line =~ /^\s*export\s+([^=]+)=(.*)/
|
164
|
+
var = $1.strip
|
165
|
+
value = $2.strip
|
166
|
+
value.sub!(/^['"]/,'')
|
167
|
+
value.sub!(/['"]$/,'')
|
168
|
+
value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
|
169
|
+
Log.debug "Set variable export from .post_install: #{Misc.fingerprint [var,value]*"="}"
|
170
|
+
ENV[var] = value
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|