scout-gear 7.1.0 → 7.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +65 -2
  3. data/VERSION +1 -1
  4. data/bin/scout +5 -1
  5. data/lib/rbbt-scout.rb +5 -0
  6. data/lib/scout/concurrent_stream.rb +13 -8
  7. data/lib/scout/config.rb +168 -0
  8. data/lib/scout/exceptions.rb +5 -3
  9. data/lib/scout/indiferent_hash/options.rb +1 -0
  10. data/lib/scout/indiferent_hash.rb +4 -2
  11. data/lib/scout/log/color.rb +3 -2
  12. data/lib/scout/log/progress/report.rb +1 -0
  13. data/lib/scout/log/progress/util.rb +66 -1
  14. data/lib/scout/log/progress.rb +5 -3
  15. data/lib/scout/log.rb +3 -2
  16. data/lib/scout/misc/helper.rb +31 -0
  17. data/lib/scout/misc/monitor.rb +4 -1
  18. data/lib/scout/misc/system.rb +15 -0
  19. data/lib/scout/misc.rb +2 -0
  20. data/lib/scout/named_array.rb +68 -0
  21. data/lib/scout/open/stream.rb +58 -33
  22. data/lib/scout/path/find.rb +27 -3
  23. data/lib/scout/path/util.rb +7 -4
  24. data/lib/scout/persist/serialize.rb +7 -14
  25. data/lib/scout/persist.rb +46 -12
  26. data/lib/scout/resource/produce.rb +7 -94
  27. data/lib/scout/resource/software.rb +176 -0
  28. data/lib/scout/semaphore.rb +8 -1
  29. data/lib/scout/tsv/dumper.rb +112 -0
  30. data/lib/scout/tsv/index.rb +161 -0
  31. data/lib/scout/tsv/open.rb +128 -0
  32. data/lib/scout/tsv/parser.rb +230 -30
  33. data/lib/scout/tsv/path.rb +13 -0
  34. data/lib/scout/tsv/persist/adapter.rb +367 -0
  35. data/lib/scout/tsv/persist/fix_width_table.rb +324 -0
  36. data/lib/scout/tsv/persist/serialize.rb +117 -0
  37. data/lib/scout/tsv/persist/tokyocabinet.rb +113 -0
  38. data/lib/scout/tsv/persist.rb +13 -0
  39. data/lib/scout/tsv/traverse.rb +143 -0
  40. data/lib/scout/tsv/util/filter.rb +303 -0
  41. data/lib/scout/tsv/util/process.rb +73 -0
  42. data/lib/scout/tsv/util/select.rb +220 -0
  43. data/lib/scout/tsv/util.rb +82 -0
  44. data/lib/scout/tsv.rb +16 -3
  45. data/lib/scout/work_queue/worker.rb +4 -4
  46. data/lib/scout/work_queue.rb +22 -7
  47. data/lib/scout/workflow/definition.rb +101 -4
  48. data/lib/scout/workflow/step/config.rb +18 -0
  49. data/lib/scout/workflow/step/dependencies.rb +40 -0
  50. data/lib/scout/workflow/step/file.rb +15 -0
  51. data/lib/scout/workflow/step/info.rb +35 -4
  52. data/lib/scout/workflow/step/progress.rb +14 -0
  53. data/lib/scout/workflow/step/provenance.rb +148 -0
  54. data/lib/scout/workflow/step.rb +71 -17
  55. data/lib/scout/workflow/task.rb +10 -5
  56. data/lib/scout/workflow/usage.rb +3 -1
  57. data/lib/scout/workflow.rb +11 -3
  58. data/lib/scout-gear.rb +1 -0
  59. data/lib/scout.rb +1 -0
  60. data/scout-gear.gemspec +64 -10
  61. data/scout_commands/find +1 -1
  62. data/scout_commands/workflow/task +16 -9
  63. data/scout_commands/workflow/task_old +2 -2
  64. data/share/software/install_helpers +523 -0
  65. data/test/scout/log/test_progress.rb +0 -2
  66. data/test/scout/misc/test_system.rb +21 -0
  67. data/test/scout/open/test_stream.rb +160 -1
  68. data/test/scout/path/test_find.rb +14 -7
  69. data/test/scout/resource/test_software.rb +24 -0
  70. data/test/scout/test_config.rb +66 -0
  71. data/test/scout/test_meta_extension.rb +10 -0
  72. data/test/scout/test_named_array.rb +19 -0
  73. data/test/scout/test_persist.rb +96 -0
  74. data/test/scout/test_tmpfile.rb +1 -1
  75. data/test/scout/test_tsv.rb +50 -1
  76. data/test/scout/test_work_queue.rb +41 -13
  77. data/test/scout/tsv/persist/test_adapter.rb +44 -0
  78. data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
  79. data/test/scout/tsv/persist/test_tokyocabinet.rb +92 -0
  80. data/test/scout/tsv/test_dumper.rb +44 -0
  81. data/test/scout/tsv/test_index.rb +156 -0
  82. data/test/scout/tsv/test_open.rb +9 -0
  83. data/test/scout/tsv/test_parser.rb +114 -3
  84. data/test/scout/tsv/test_persist.rb +43 -0
  85. data/test/scout/tsv/test_traverse.rb +116 -0
  86. data/test/scout/tsv/test_util.rb +23 -0
  87. data/test/scout/tsv/util/test_filter.rb +188 -0
  88. data/test/scout/tsv/util/test_process.rb +47 -0
  89. data/test/scout/tsv/util/test_select.rb +44 -0
  90. data/test/scout/work_queue/test_worker.rb +66 -9
  91. data/test/scout/workflow/step/test_dependencies.rb +25 -0
  92. data/test/scout/workflow/step/test_info.rb +15 -17
  93. data/test/scout/workflow/step/test_load.rb +19 -21
  94. data/test/scout/workflow/step/test_provenance.rb +25 -0
  95. data/test/scout/workflow/test_step.rb +206 -10
  96. data/test/scout/workflow/test_task.rb +0 -3
  97. data/test/test_helper.rb +9 -1
  98. metadata +50 -6
@@ -37,7 +37,13 @@ if continue
37
37
  int ret;
38
38
  sem_t* sem;
39
39
  sem = sem_open(name, 0);
40
+ if (sem == SEM_FAILED){
41
+ return(errno);
42
+ }
40
43
  ret = sem_wait(sem);
44
+ if (ret == -1){
45
+ return(errno);
46
+ }
41
47
  sem_close(sem);
42
48
  return(ret);
43
49
  }
@@ -51,6 +57,7 @@ if continue
51
57
  sem_close(sem);
52
58
  }
53
59
  EOF
60
+
54
61
  end
55
62
 
56
63
  SEM_MUTEX = Mutex.new
@@ -66,7 +73,7 @@ if continue
66
73
 
67
74
  def self.with_semaphore(size, file = nil)
68
75
  if file.nil?
69
- file = "/" << Misc.digest(rand(1000000000000).to_s) if file.nil?
76
+ file = "/scout-" << Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
70
77
  else
71
78
  file = file.gsub('/', '_') if file
72
79
  end
@@ -0,0 +1,112 @@
1
+ module TSV
2
+ class Dumper
3
+ def self.header_lines(key_field, fields, entry_hash = nil)
4
+ if Hash === entry_hash
5
+ sep = entry_hash[:sep] ? entry_hash[:sep] : "\t"
6
+ preamble = entry_hash[:preamble]
7
+ header_hash = entry_hash[:header_hash]
8
+ end
9
+
10
+ header_hash = "#" if header_hash.nil?
11
+
12
+ preamble = "#: " << Misc.hash2string(entry_hash.merge(:key_field => nil, :fields => nil)) << "\n" if preamble.nil? and entry_hash and entry_hash.values.compact.any?
13
+
14
+ str = ""
15
+ str << preamble.strip << "\n" if preamble and not preamble.empty?
16
+ if fields
17
+ if fields.empty?
18
+ str << header_hash << (key_field || "ID").to_s << "\n"
19
+ else
20
+ str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n"
21
+ end
22
+ end
23
+
24
+ str
25
+ end
26
+
27
+ def self.header(options={})
28
+ key_field, fields, sep, header_hash, preamble = IndiferentHash.process_options options,
29
+ :key_field, :fields, :sep, :header_hash, :preamble,
30
+ :sep => "\t", :header_hash => "#", :preamble => true
31
+
32
+ if fields.nil? || key_field.nil?
33
+ fields_str = nil
34
+ else
35
+ fields_str = "#{header_hash}#{key_field}#{sep}#{fields*sep}"
36
+ end
37
+
38
+ if preamble && options.values.compact.any?
39
+ preamble_str = "#: " << IndiferentHash.hash2string(options)
40
+ else
41
+ preamble_str = nil
42
+ end
43
+
44
+ [preamble_str, fields_str].compact * "\n"
45
+ end
46
+
47
+
48
+ attr_accessor :options
49
+ def initialize(options = {})
50
+ @sep, @type = IndiferentHash.process_options options,
51
+ :sep, :type,
52
+ :sep => "\t", :type => :double
53
+ @options = options
54
+ @sout, @sin = Open.pipe
55
+ ConcurrentStream.setup(@sin, pair: @sout)
56
+ ConcurrentStream.setup(@sout, pair: @sin)
57
+ end
58
+
59
+ def init
60
+ header = Dumper.header(@options.merge(:type => @type, :sep => @sep))
61
+ @sin.puts header if header and ! header.empty?
62
+ end
63
+
64
+ def add(key, value)
65
+
66
+ case @type
67
+ when :single
68
+ @sin.puts key + @sep + value
69
+ when :list, :flat
70
+ @sin.puts key + @sep + value * @sep
71
+ when :double
72
+ @sin.puts key + @sep + value.collect{|v| v * "|" } * @sep
73
+ end
74
+ end
75
+
76
+ def close
77
+ @sin.close
78
+ @sin.join
79
+ end
80
+
81
+ def stream
82
+ @sout
83
+ end
84
+
85
+ def abort(exception=nil)
86
+ @sin.abort(exception)
87
+ end
88
+ end
89
+
90
+ def stream
91
+ dumper = TSV::Dumper.new self.extension_attr_hash
92
+ dumper.init
93
+ t = Thread.new do
94
+ begin
95
+ Thread.current.report_on_exception = true
96
+ Thread.current["name"] = "Dumper thread"
97
+ self.each do |k,v|
98
+ dumper.add k, v
99
+ end
100
+ dumper.close
101
+ rescue
102
+ dumper.abort($!)
103
+ end
104
+ end
105
+ Thread.pass until t["name"]
106
+ dumper.stream
107
+ end
108
+
109
+ def to_s
110
+ stream.read
111
+ end
112
+ end
@@ -0,0 +1,161 @@
1
+ require_relative 'parser'
2
+ require_relative 'persist/fix_width_table'
3
+ module TSV
4
+ def self.index(tsv_file, target: 0, fields: nil, order: true, **kwargs)
5
+ persist, type = IndiferentHash.process_options kwargs,
6
+ :persist, :persist_type,
7
+ :persist => false, :persist_type => "HDB"
8
+ kwargs.delete :type
9
+
10
+ Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
11
+ if filename
12
+ index = ScoutCabinet.open(filename, true, type)
13
+ TSV.setup(index, :type => :single)
14
+ index.extend TSVAdapter
15
+ else
16
+ index = TSV.setup({}, :type => :single)
17
+ end
18
+
19
+ dummy_data = TSV.setup({}, :key_field => "Key", :fields => ["Target"])
20
+ if order
21
+ tmp_index = {}
22
+ key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :double, into: dummy_data, unnamed: true, **kwargs do |k,values|
23
+ values.each_with_index do |list,i|
24
+ list.each do |e|
25
+ tmp_index[e] ||= []
26
+ tmp_index[e][i] ||= []
27
+ tmp_index[e][i] << k
28
+ end
29
+ end
30
+ end
31
+ tmp_index.each do |e,list|
32
+ index[e] = list.flatten.compact.uniq.first
33
+ end
34
+ else
35
+ key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :flat, into: dummy_data, unnamed: true, **kwargs do |k,values|
36
+ values.each do |e|
37
+ index[e] = k unless index.include?(e)
38
+ end
39
+ end
40
+ end
41
+
42
+ index.key_field = dummy_data.fields * ", "
43
+ index.fields = [dummy_data.key_field]
44
+ index
45
+ end
46
+ end
47
+
48
+ def index(*args, **kwargs, &block)
49
+ TSV.index(self, *args, **kwargs, &block)
50
+ end
51
+
52
+ def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, **kwargs)
53
+ persist, type = IndiferentHash.process_options kwargs,
54
+ :persist, :persist_type,
55
+ :persist => false, :persist_type => :fwt
56
+ kwargs.delete :type
57
+
58
+ Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
59
+
60
+ max_key_size = 0
61
+ index_data = []
62
+ TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field] do |key, values|
63
+ key_size = key.length
64
+ max_key_size = key_size if key_size > max_key_size
65
+
66
+ start_pos, end_pos = values
67
+ if Array === start_pos
68
+ start_pos.zip(end_pos).each do |s,e|
69
+ index_data << [key, [s.to_i, e.to_i]]
70
+ end
71
+ else
72
+ index_data << [key, [start_pos.to_i, end_pos.to_i]]
73
+ end
74
+ end
75
+
76
+ filename = :memory if filename.nil?
77
+ index = FixWidthTable.get(filename, max_key_size, true)
78
+ index.add_range index_data
79
+ index.read
80
+ index
81
+ end
82
+ end
83
+
84
+ def range_index(*args, **kwargs, &block)
85
+ TSV.range_index(self, *args, **kwargs, &block)
86
+ end
87
+
88
+
89
+ #def range_index(start_field = nil, end_field = nil, options = {})
90
+ # start_field ||= "Start"
91
+ # end_field ||= "End"
92
+
93
+ # options = Misc.add_defaults options,
94
+ # :persist => false, :persist_file => nil, :persist_update => false
95
+
96
+ # persist_options = Misc.pull_keys options, :persist
97
+ # persist_options[:prefix] ||= "RangeIndex[#{start_field}-#{end_field}]"
98
+
99
+ # Persist.persist(filename || self.object_id.to_s, :fwt, persist_options) do
100
+ # max_key_size = 0
101
+ # index_data = []
102
+ # with_unnamed do
103
+ # with_monitor :desc => "Creating Index Data", :step => 10000 do
104
+ # through :key, [start_field, end_field] do |key, values|
105
+ # key_size = key.length
106
+ # max_key_size = key_size if key_size > max_key_size
107
+
108
+ # start_pos, end_pos = values
109
+ # if Array === start_pos
110
+ # start_pos.zip(end_pos).each do |s,e|
111
+ # index_data << [key, [s.to_i, e.to_i]]
112
+ # end
113
+ # else
114
+ # index_data << [key, [start_pos.to_i, end_pos.to_i]]
115
+ # end
116
+ # end
117
+ # end
118
+ # end
119
+
120
+ # index = FixWidthTable.get(:memory, max_key_size, true)
121
+ # index.add_range index_data
122
+ # index.read
123
+ # index
124
+ # end
125
+ #end
126
+
127
+ #def self.range_index(file, start_field = nil, end_field = nil, options = {})
128
+ # start_field ||= "Start"
129
+ # end_field ||= "End"
130
+
131
+ # data_options = Misc.pull_keys options, :data
132
+ # filename = case
133
+ # when (String === file or Path === file)
134
+ # file
135
+ # when file.respond_to?(:filename)
136
+ # file.filename
137
+ # else
138
+ # file.object_id.to_s
139
+ # end
140
+ # persist_options = Misc.pull_keys options, :persist
141
+ # persist_options[:prefix] ||= "StaticRangeIndex[#{start_field}-#{end_field}]"
142
+
143
+ # filters = Misc.process_options options, :filters
144
+
145
+ # if filters
146
+ # filename += ":Filtered[#{filters.collect{|f| f * "="} * ", "}]"
147
+ # end
148
+
149
+ # Persist.persist(filename, :fwt, persist_options) do
150
+ # tsv = TSV.open(file, data_options)
151
+ # if filters
152
+ # tsv.filter
153
+ # filters.each do |match, value|
154
+ # tsv.add_filter match, value
155
+ # end
156
+ # end
157
+
158
+ # tsv.range_index(start_field, end_field, options)
159
+ # end
160
+ #end
161
+ end
@@ -0,0 +1,128 @@
1
+ require_relative '../open'
2
+ module Open
3
+ def self.traverse_add(into, res)
4
+ case into
5
+ when TSV::Dumper
6
+ into.add *res
7
+ when TSV, Hash
8
+ key, value = res
9
+ into[key] = value
10
+ end
11
+ end
12
+
13
+ #def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
14
+ # case obj
15
+ # when TSV
16
+ # obj.traverse options[:key_field], options[:fields], **options do |k,v|
17
+ # res = yield k, v
18
+ # end
19
+ # when String
20
+ # f = Open.open(obj)
21
+ # self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
22
+ # when Step
23
+ # self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
24
+ # when IO
25
+ # if into && (IO === into || into.respond_to?(:stream) )
26
+ # into_thread = Thread.new do
27
+ # Thread.current.report_on_exception = false
28
+ # Thread.current["name"] = "Traverse into"
29
+ # TSV.parse obj, **options do |k,v|
30
+ # begin
31
+ # res = block.call k, v
32
+ # traverse_add into, res
33
+ # rescue
34
+ # into.abort $!
35
+ # end
36
+ # nil
37
+ # end
38
+ # into.close if into.respond_to?(:close)
39
+ # end
40
+ # Thread.pass until into_thread
41
+ # into
42
+ # else
43
+ # TSV.parse obj, **options do |k,v|
44
+ # block.call k, v
45
+ # nil
46
+ # end
47
+ # end
48
+ # end
49
+ #end
50
+
51
+ def self.traverse(obj, into: nil, cpus: nil, bar: nil, callback: nil, unnamed: true, **options, &block)
52
+
53
+ if into || bar
54
+ orig_callback = callback if callback
55
+ bar = Log::ProgressBar.get_obj_bar(bar, obj)
56
+ callback = proc do |res|
57
+ bar.tick if bar
58
+ traverse_add into, res if into
59
+ orig_callback.call res if orig_callback
60
+ end
61
+
62
+ if into.respond_to?(:close)
63
+ into_thread = Thread.new do
64
+ Thread.current.report_on_exception = false
65
+ Thread.current["name"] = "Traverse into"
66
+ error = false
67
+ begin
68
+ self.traverse(obj, callback: callback, **options, &block)
69
+ into.close if into.respond_to?(:close)
70
+ bar.remove if bar
71
+ rescue Exception
72
+ into.abort($!) if into.respond_to?(:abort)
73
+ bar.remove($!) if bar
74
+ end
75
+ end
76
+ Thread.pass until into_thread
77
+ return into
78
+ end
79
+ end
80
+
81
+ begin
82
+ case obj
83
+ when TSV
84
+ obj.traverse options[:key_field], options[:fields], unnamed: unnamed, **options do |k,v|
85
+ res = block.call(k, v)
86
+ callback.call res if callback
87
+ nil
88
+ end
89
+ when Array
90
+ obj.each do |line|
91
+ res = block.call(line)
92
+ callback.call res if callback
93
+ nil
94
+ end
95
+ when String
96
+ f = Open.open(obj)
97
+ self.traverse(f, cpus: cpus, callback: callback, **options, &block)
98
+ when Step
99
+ raise obj.exception if obj.error?
100
+ self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
101
+ when IO
102
+ TSV.parse obj, **options do |k,v|
103
+ res = block.call k, v
104
+ callback.call res if callback
105
+ nil
106
+ end
107
+ else
108
+ TSV.parse obj, **options do |k,v|
109
+ res = block.call k, v
110
+ callback.call res if callback
111
+ nil
112
+ end
113
+ end
114
+ bar.remove if bar
115
+ rescue
116
+ bar.abort($!) if bar
117
+ raise $!
118
+ end
119
+
120
+ into
121
+ end
122
+ end
123
+
124
+ module TSV
125
+ def self.traverse(*args, **kwargs, &block)
126
+ Open.traverse(*args, **kwargs, &block)
127
+ end
128
+ end