rbbt-util 5.14.38 → 5.14.39
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/etc/app.d/grid_system.rb +1 -0
- data/etc/app.d/resources.rb +3 -0
- data/lib/rbbt/association/database.rb +4 -2
- data/lib/rbbt/association/index.rb +20 -10
- data/lib/rbbt/association/open.rb +7 -4
- data/lib/rbbt/association/util.rb +12 -0
- data/lib/rbbt/knowledge_base/registry.rb +6 -5
- data/lib/rbbt/knowledge_base.rb +2 -2
- data/lib/rbbt/persist.rb +12 -8
- data/lib/rbbt/tsv/change_id.rb +10 -5
- data/lib/rbbt/tsv/manipulate.rb +3 -2
- data/lib/rbbt/tsv/matrix.rb +0 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +2 -0
- data/lib/rbbt/tsv/util.rb +18 -0
- data/lib/rbbt/util/R/eval.rb +1 -1
- data/lib/rbbt/util/R/model.rb +28 -4
- data/lib/rbbt/util/R.rb +6 -0
- data/lib/rbbt/util/concurrency/processes/socket.rb +7 -1
- data/lib/rbbt/util/concurrency/processes/worker.rb +15 -7
- data/lib/rbbt/util/concurrency/processes.rb +8 -4
- data/lib/rbbt/util/log/progress/report.rb +3 -3
- data/lib/rbbt/util/log/progress.rb +20 -15
- data/lib/rbbt/util/misc/concurrent_stream.rb +2 -1
- data/lib/rbbt/util/misc/development.rb +11 -3
- data/lib/rbbt/util/misc/objects.rb +1 -1
- data/lib/rbbt/workflow/accessor.rb +19 -16
- data/lib/rbbt/workflow/step/run.rb +7 -0
- data/share/rbbt_commands/app/start +3 -2
- data/test/rbbt/knowledge_base/test_entity.rb +2 -1
- data/test/rbbt/knowledge_base/test_query.rb +2 -1
- data/test/rbbt/test_entity.rb +12 -13
- data/test/rbbt/test_knowledge_base.rb +13 -1
- data/test/rbbt/tsv/test_util.rb +15 -0
- data/test/rbbt/util/R/test_model.rb +25 -1
- data/test/rbbt/util/test_misc.rb +43 -40
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fb9eba165b567301078196458cb721d1ca1f130b
|
4
|
+
data.tar.gz: e2f067c01e6a2aff41d00d76d89fbad6c895678d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e943920dabb8863f3fc5b1b142c7c02bd1ce1131ddf4173cf42463f4d77726bca36f700b50ba67aa0d6d952c3033f96f164009a626d3a2fd050cbd0db00201b7
|
7
|
+
data.tar.gz: 77591c927621415e7030ac7f8e4d2efad8d1c7b4b977c07147d49639de2ed1b8f018a753c32453830aa6e8a5585ad079730e3e56f86896d0859b159bc7a6ed48
|
data/etc/app.d/grid_system.rb
CHANGED
data/etc/app.d/resources.rb
CHANGED
@@ -5,9 +5,12 @@ RbbtRESTHelpers.template_resources.unshift Rbbt.www.views.find if Rbbt.www.views
|
|
5
5
|
#load Rbbt.etc['app.d']['foundation.rb'].find if Rbbt.etc['app.d']['foundation.rb'].exists?
|
6
6
|
load Rbbt.etc['app.d']['grid_system.rb'].find if Rbbt.etc['app.d']['grid_system.rb'].exists?
|
7
7
|
|
8
|
+
require 'sass-css-importer'
|
9
|
+
|
8
10
|
Compass::Frameworks::ALL.each do |importer|
|
9
11
|
next unless importer.respond_to? :path
|
10
12
|
path = importer.stylesheets_directory
|
11
13
|
RbbtRESTHelpers.add_sass_load_path path
|
12
14
|
end
|
15
|
+
RbbtRESTHelpers.add_sass_load_path './www/views/compass'
|
13
16
|
|
@@ -71,8 +71,9 @@ module Association
|
|
71
71
|
info_fields = field_pos.collect{|f| f == :key ? :key : all_fields[f]}
|
72
72
|
options = options.merge({:key_field => source_field, :fields => info_fields})
|
73
73
|
|
74
|
-
|
75
|
-
|
74
|
+
tsv.with_monitor(options[:monitor]) do
|
75
|
+
tsv = tsv.reorder source_field, fields if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
|
76
|
+
end
|
76
77
|
|
77
78
|
tsv.key_field = source_header
|
78
79
|
tsv.fields = field_headers
|
@@ -124,6 +125,7 @@ module Association
|
|
124
125
|
end
|
125
126
|
|
126
127
|
open_options = options.merge(parser.options).merge(:parser => parser)
|
128
|
+
open_options = Misc.add_defaults open_options, :monitor => {:desc => "Parsing #{ Misc.fingerprint stream }"}
|
127
129
|
|
128
130
|
tsv = TSV.parse parser.stream, {}, open_options
|
129
131
|
tsv.key_field = source_header
|
@@ -7,12 +7,12 @@ module Association
|
|
7
7
|
options = options.nil? ? {} : options.dup
|
8
8
|
persist_options = persist_options.nil? ? Misc.pull_keys(options, :persist) : persist_options.dup
|
9
9
|
|
10
|
-
persist_options = Misc.add_defaults persist_options.dup, :persist => true
|
10
|
+
persist_options = Misc.add_defaults persist_options.dup, :persist => true
|
11
11
|
persist = persist_options[:persist]
|
12
12
|
|
13
13
|
file = version_file(file, options[:namespace]) if options[:namespace] and String === file
|
14
|
-
|
15
|
-
|
14
|
+
Persist.persist_tsv(file, "Association Index", options, persist_options.merge(:engine => "BDB")) do |data|
|
15
|
+
options = Misc.add_defaults options.dup, :monitor => "Building index for #{Misc.fingerprint file}"
|
16
16
|
recycle = options[:recycle]
|
17
17
|
undirected = options[:undirected]
|
18
18
|
|
@@ -117,29 +117,39 @@ module Association
|
|
117
117
|
|
118
118
|
if File.exists?(reverse_filename)
|
119
119
|
new = Persist.open_tokyocabinet(reverse_filename, false, serializer, TokyoCabinet::BDB)
|
120
|
+
raise "Index has no info: #{reverse_filename}" if new.key_field.nil?
|
121
|
+
Association::Index.setup new
|
120
122
|
new
|
121
123
|
else
|
122
|
-
FileUtils.mkdir_p File.dirname(reverse_filename) unless File.exists?(File.
|
124
|
+
FileUtils.mkdir_p File.dirname(reverse_filename) unless File.exists?(File.dirname(reverse_filename))
|
125
|
+
|
123
126
|
new = Persist.open_tokyocabinet(reverse_filename, true, serializer, TokyoCabinet::BDB)
|
124
|
-
|
127
|
+
|
125
128
|
self.with_unnamed do
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
+
self.with_monitor :desc => "Reversing #{ persistence_path }" do
|
130
|
+
self.through do |key, value|
|
131
|
+
new_key = key.split("~").reverse.join("~")
|
132
|
+
new[new_key] = value
|
133
|
+
end
|
129
134
|
end
|
130
135
|
end
|
131
136
|
annotate(new)
|
132
137
|
new.key_field = key_field.split("~").values_at(1,0,2).compact * "~"
|
138
|
+
new.read_and_close do
|
139
|
+
Association::Index.setup new
|
140
|
+
end
|
133
141
|
new.read
|
134
142
|
end
|
135
143
|
|
136
144
|
new.unnamed = true
|
137
145
|
|
138
|
-
Association::Index.setup new
|
139
|
-
|
140
146
|
new.undirected = undirected
|
141
147
|
|
142
148
|
new
|
149
|
+
rescue Exception
|
150
|
+
Log.error "Deleting after error reversing database: #{ reverse_filename }"
|
151
|
+
FileUtils.rm reverse_filename if File.exists? reverse_filename
|
152
|
+
raise $!
|
143
153
|
end
|
144
154
|
end
|
145
155
|
|
@@ -11,7 +11,7 @@ module Association
|
|
11
11
|
options = options.nil? ? {} : options.dup
|
12
12
|
persist_options = persist_options.nil? ? Misc.pull_keys(options, :persist) : persist_options.dup
|
13
13
|
|
14
|
-
options = Misc.add_defaults options, :zipped => true
|
14
|
+
options = Misc.add_defaults options, :zipped => true, :monitor => {:desc => "Opening database #{Misc.fingerprint file}"}
|
15
15
|
persist_options = Misc.add_defaults persist_options, :persist => true, :dir => Rbbt.var.associations
|
16
16
|
persist = persist_options[:persist]
|
17
17
|
|
@@ -19,15 +19,18 @@ module Association
|
|
19
19
|
file = file.call if Proc === file
|
20
20
|
|
21
21
|
data = Persist.persist_tsv(file, "Association Database", options, persist_options) do |data|
|
22
|
+
options = options.dup
|
22
23
|
tsv = Association.database(file, options.merge(:persist => persist))
|
23
24
|
tsv = tsv.to_double unless tsv.type == :double
|
24
25
|
|
25
26
|
tsv.annotate data
|
26
27
|
|
27
28
|
data.serializer = :double if data.respond_to? :serializer
|
28
|
-
tsv.
|
29
|
-
tsv.
|
30
|
-
|
29
|
+
tsv.with_unnamed do
|
30
|
+
tsv.with_monitor(options[:monitor]) do
|
31
|
+
tsv.through do |k,v|
|
32
|
+
data[k] = v
|
33
|
+
end
|
31
34
|
end
|
32
35
|
end
|
33
36
|
|
@@ -55,6 +55,18 @@ module Association
|
|
55
55
|
source_specs[2] = source_format if source_format
|
56
56
|
target_specs[2] = target_format if target_format
|
57
57
|
|
58
|
+
if source_specs.first and not all_fields.include? source_specs.first and defined? Entity and (_format = Entity.formats[source_specs.first.to_s])
|
59
|
+
_source = all_fields.select{|f| Entity.formats[f.to_s] == _format }.first
|
60
|
+
raise "Source not found #{source_specs}. Options: #{Misc.fingerprint all_fields}" if _target.nil?
|
61
|
+
source_specs[0] = _source
|
62
|
+
end
|
63
|
+
|
64
|
+
if target_specs.first and not all_fields.include? target_specs.first and defined? Entity and (_format = Entity.formats[target_specs.first.to_s])
|
65
|
+
_target = all_fields.select{|f| Entity.formats[f.to_s].to_s == _format.to_s }.first
|
66
|
+
raise "Target not found #{target_specs}. Options: #{Misc.fingerprint all_fields}" if _target.nil?
|
67
|
+
target_specs[0] = _target
|
68
|
+
end
|
69
|
+
|
58
70
|
if source_specs[0].nil? and target_specs[0].nil?
|
59
71
|
source_specs[0] = key_field
|
60
72
|
target_specs[0] = fields[0]
|
@@ -44,7 +44,7 @@ class KnowledgeBase
|
|
44
44
|
Persist.memory("Index:" << [key, dir] * "@") do
|
45
45
|
options = options.dup
|
46
46
|
persist_dir = dir
|
47
|
-
persist_file = persist_dir[key]
|
47
|
+
persist_file = persist_dir[key].find
|
48
48
|
file, registered_options = registry[name]
|
49
49
|
|
50
50
|
options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
|
@@ -79,15 +79,16 @@ class KnowledgeBase
|
|
79
79
|
|
80
80
|
def get_database(name, options = {})
|
81
81
|
name = name.to_s
|
82
|
-
|
83
|
-
@indices[key] ||=
|
82
|
+
@databases[[name, options]] ||=
|
84
83
|
begin
|
84
|
+
fp = Misc.fingerprint([name,options])
|
85
|
+
key = name.to_s + "_" + Misc.digest(fp) + '.database'
|
85
86
|
Persist.memory("Database:" << [key, dir] * "@") do
|
86
87
|
options = options.dup
|
87
|
-
|
88
|
+
persist_dir = dir
|
89
|
+
persist_file = persist_dir[key].find
|
88
90
|
file, registered_options = registry[name]
|
89
91
|
|
90
|
-
|
91
92
|
options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
|
92
93
|
options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format, :persist => true
|
93
94
|
|
data/lib/rbbt/knowledge_base.rb
CHANGED
@@ -7,9 +7,9 @@ require 'rbbt/knowledge_base/syndicate'
|
|
7
7
|
|
8
8
|
class KnowledgeBase
|
9
9
|
|
10
|
-
attr_accessor :namespace, :dir, :indices, :registry, :format, :
|
10
|
+
attr_accessor :namespace, :dir, :databases, :indices, :registry, :format, :entity_options
|
11
11
|
def initialize(dir, namespace = nil)
|
12
|
-
@dir = Path.setup(dir.dup)
|
12
|
+
@dir = Path.setup(dir.dup)
|
13
13
|
|
14
14
|
@namespace = namespace
|
15
15
|
@format = IndiferentHash.setup({})
|
data/lib/rbbt/persist.rb
CHANGED
@@ -253,16 +253,20 @@ module Persist
|
|
253
253
|
end
|
254
254
|
|
255
255
|
def self.persist_file(path, type, persist_options, &block)
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
256
|
+
Misc.insist do
|
257
|
+
begin
|
258
|
+
if is_persisted?(path, persist_options)
|
259
|
+
Log.low "Persist up-to-date: #{ path } - #{Misc.fingerprint persist_options}"
|
260
|
+
return path if persist_options[:no_load]
|
261
|
+
return load_file(path, type)
|
262
|
+
else
|
263
|
+
Open.rm path if Open.exists? path
|
264
|
+
end
|
265
|
+
rescue Exception
|
266
|
+
Log.warn "Exception loading persistence (#{ type }) #{ path }: #{$!.message}. Erase and retry."
|
262
267
|
Open.rm path if Open.exists? path
|
268
|
+
raise $!
|
263
269
|
end
|
264
|
-
rescue Exception
|
265
|
-
Open.rm path if Open.exists? path
|
266
270
|
end
|
267
271
|
|
268
272
|
lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
|
data/lib/rbbt/tsv/change_id.rb
CHANGED
@@ -98,11 +98,16 @@ module TSV
|
|
98
98
|
return file.index(options.merge(:target => target, :fields => fields, :order => true))
|
99
99
|
end
|
100
100
|
else
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
101
|
+
next unless file.exists?
|
102
|
+
begin
|
103
|
+
all_fields = TSV.parse_header(file).all_fields
|
104
|
+
target = all_fields[1] if target.nil?
|
105
|
+
if (source.nil? or all_fields.include? source) and all_fields.include? target
|
106
|
+
index = TSV.index(file, options.merge(:target => target, :fields => fields, :order => true))
|
107
|
+
return index
|
108
|
+
end
|
109
|
+
rescue Exception
|
110
|
+
Log.error "Exception reading identifier file: #{file.find}"
|
106
111
|
end
|
107
112
|
end
|
108
113
|
end
|
data/lib/rbbt/tsv/manipulate.rb
CHANGED
@@ -636,14 +636,15 @@ module TSV
|
|
636
636
|
@monitor = {:desc => "Adding field #{ names * ", " }"} if TrueClass === monitor
|
637
637
|
|
638
638
|
through do |key, values|
|
639
|
+
values ||= fields ? [nil] * fields : []
|
639
640
|
new_values = yield(key, values)
|
640
641
|
|
641
642
|
case type
|
642
643
|
when :double
|
643
644
|
new_values = new_values.collect{|v| [v] } if Array === new_values and new_values.first and not Array === new_values.first
|
644
|
-
values += new_values
|
645
|
+
values += new_values || [nil] * names.length
|
645
646
|
when :list
|
646
|
-
values += new_values
|
647
|
+
values += new_values || [nil] * names.length
|
647
648
|
end
|
648
649
|
|
649
650
|
self[key] = values
|
data/lib/rbbt/tsv/matrix.rb
CHANGED
@@ -335,7 +335,9 @@ module TSV
|
|
335
335
|
def self.traverse_cpus(num, obj, options, &block)
|
336
336
|
begin
|
337
337
|
callback, cleanup, join, respawn = Misc.process_options options, :callback, :cleanup, :join, :respawn
|
338
|
+
respawn = true if ENV["RBBT_RESPAWN"] and ENV["RBBT_RESPAWN"] == "true"
|
338
339
|
|
340
|
+
Log.low "Traversing in #{ num } cpus: #{respawn ? "respawn" : "no respawn"}"
|
339
341
|
q = RbbtProcessQueue.new num, cleanup, join, respawn
|
340
342
|
q.callback &callback
|
341
343
|
q.init &block
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -186,6 +186,24 @@ module TSV
|
|
186
186
|
self
|
187
187
|
end
|
188
188
|
|
189
|
+
def unzip_replicates
|
190
|
+
raise "Can only unzip replicates in :double TSVs" unless type == :double
|
191
|
+
|
192
|
+
new = {}
|
193
|
+
self.with_unnamed do
|
194
|
+
through do |k,vs|
|
195
|
+
Misc.zip_fields(vs).each_with_index do |v,i|
|
196
|
+
new[k + "(#{i})"] = v
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
self.annotate(new)
|
202
|
+
new.type = :list
|
203
|
+
|
204
|
+
new
|
205
|
+
end
|
206
|
+
|
189
207
|
def to_list
|
190
208
|
new = {}
|
191
209
|
case type
|
data/lib/rbbt/util/R/eval.rb
CHANGED
@@ -31,9 +31,9 @@ module R
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def self.clear
|
34
|
-
Log.warn "Clearing Rserver session #{SESSION}, PID #{@@instance_process}"
|
35
34
|
@@instance = nil
|
36
35
|
if defined? @@instance_process and @@instance_process and Misc.pid_exists? @@instance_process
|
36
|
+
Log.warn "Clearing Rserver session #{SESSION}, PID #{@@instance_process}"
|
37
37
|
begin
|
38
38
|
Process.kill :INT, @@instance_process
|
39
39
|
rescue Exception
|
data/lib/rbbt/util/R/model.rb
CHANGED
@@ -19,13 +19,18 @@ module R
|
|
19
19
|
R_METHOD = :eval
|
20
20
|
|
21
21
|
attr_accessor :name, :formula
|
22
|
-
def initialize(name, formula, options = {})
|
22
|
+
def initialize(name, formula, data = nil, options = {})
|
23
23
|
@name = name
|
24
24
|
@formula = formula
|
25
25
|
@options = options || {}
|
26
|
+
if data and not model_file.exists?
|
27
|
+
method = Misc.process_options options, :fit
|
28
|
+
fit(data, method || "lm", options)
|
29
|
+
end
|
26
30
|
end
|
27
31
|
|
28
32
|
def colClasses(tsv)
|
33
|
+
return nil unless TSV === tsv
|
29
34
|
"c('character', " <<
|
30
35
|
(tsv.fields.collect{|f| R.ruby2R(@options[f] ? @options[f].to_s : ":NA") } * ", ") <<
|
31
36
|
")"
|
@@ -63,12 +68,31 @@ data = NULL
|
|
63
68
|
end
|
64
69
|
|
65
70
|
def predict(tsv, field = "Prediction")
|
66
|
-
|
67
|
-
|
71
|
+
case tsv
|
72
|
+
when TSV
|
73
|
+
tsv = Model.groom tsv, formula
|
74
|
+
tsv.R <<-EOF, r_options(tsv)
|
68
75
|
model = rbbt.model.load('#{model_file}');
|
69
76
|
data.groomed = rbbt.model.groom(data,formula=#{formula})
|
70
77
|
data$#{field} = predict(model, data.groomed);
|
71
|
-
|
78
|
+
EOF
|
79
|
+
when Hash
|
80
|
+
res = R.eval_a <<-EOF
|
81
|
+
model = rbbt.model.load('#{model_file}');
|
82
|
+
predict(model, data.frame(#{R.ruby2R tsv}));
|
83
|
+
EOF
|
84
|
+
Array === tsv.values.first ? res : res.first
|
85
|
+
when Fixnum, Array, Float, String
|
86
|
+
field = formula.split("~").last.strip
|
87
|
+
|
88
|
+
res = R.eval_a <<-EOF
|
89
|
+
model = rbbt.model.load('#{model_file}');
|
90
|
+
predict(model, data.frame(#{field} = #{R.ruby2R tsv}));
|
91
|
+
EOF
|
92
|
+
Array === tsv ? res : res.first
|
93
|
+
else
|
94
|
+
raise "Unknown object for predict: #{Misc.fingerprint tsv}"
|
95
|
+
end
|
72
96
|
end
|
73
97
|
|
74
98
|
def exists?
|
data/lib/rbbt/util/R.rb
CHANGED
@@ -86,8 +86,12 @@ source('#{UTIL}');
|
|
86
86
|
|
87
87
|
def self.ruby2R(object)
|
88
88
|
case object
|
89
|
+
when Float::INFINITY
|
90
|
+
"Inf"
|
89
91
|
when nil
|
90
92
|
"NULL"
|
93
|
+
when "NA"
|
94
|
+
"NA"
|
91
95
|
when TSV
|
92
96
|
"matrix(#{R.ruby2R object.values},dimnames=list(#{R.ruby2R object.keys}, #{R.ruby2R object.fields}))"
|
93
97
|
when Symbol
|
@@ -102,6 +106,8 @@ source('#{UTIL}');
|
|
102
106
|
"FALSE"
|
103
107
|
when Array
|
104
108
|
"c(#{object.collect{|e| ruby2R(e) } * ", "})"
|
109
|
+
when Hash
|
110
|
+
object.collect{|k,v| [k, ruby2R(v)] * "="} * ", "
|
105
111
|
else
|
106
112
|
raise "Type of object not known: #{ object.inspect }"
|
107
113
|
end
|
@@ -55,7 +55,12 @@ class RbbtProcessQueue
|
|
55
55
|
payload = Misc.read_stream stream, size
|
56
56
|
case type
|
57
57
|
when "S"
|
58
|
-
|
58
|
+
begin
|
59
|
+
@serializer.load(payload)
|
60
|
+
rescue Exception
|
61
|
+
Log.exception $!
|
62
|
+
raise $!
|
63
|
+
end
|
59
64
|
when "C"
|
60
65
|
payload
|
61
66
|
end
|
@@ -84,6 +89,7 @@ class RbbtProcessQueue
|
|
84
89
|
|
85
90
|
def push(obj)
|
86
91
|
RbbtSemaphore.synchronize(@write_sem) do
|
92
|
+
obj = Annotated.purge(obj)
|
87
93
|
self.dump(obj, @swrite)
|
88
94
|
end
|
89
95
|
end
|
@@ -30,7 +30,7 @@ class RbbtProcessQueue
|
|
30
30
|
Kernel.exit! 28
|
31
31
|
rescue ClosedStream
|
32
32
|
rescue Aborted, Interrupt
|
33
|
-
Log.
|
33
|
+
Log.info "Worker #{Process.pid} aborted"
|
34
34
|
rescue Exception
|
35
35
|
Log.exception $!
|
36
36
|
@callback_queue.push($!) if @callback_queue
|
@@ -56,17 +56,24 @@ class RbbtProcessQueue
|
|
56
56
|
@current = Process.fork do
|
57
57
|
run
|
58
58
|
end
|
59
|
-
|
59
|
+
@asked = false
|
60
|
+
|
61
|
+
initial = Misc.memory_use(Process.pid)
|
62
|
+
memory_cap = multiplier * initial
|
63
|
+
Log.medium "Worker #{Process.pid} started with #{@current} -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap}"
|
60
64
|
|
61
65
|
@monitor_thread = Thread.new do
|
62
66
|
begin
|
63
67
|
while true
|
64
68
|
current = Misc.memory_use(@current)
|
65
|
-
|
66
|
-
|
67
|
-
|
69
|
+
if current > memory_cap and not @asked
|
70
|
+
Log.medium "Worker #{@current} for #{Process.pid} asked to respawn -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap} - current: #{current}"
|
71
|
+
RbbtSemaphore.synchronize(@callback_queue.write_sem) do
|
72
|
+
Process.kill "USR1", @current
|
73
|
+
end
|
74
|
+
@asked = true
|
68
75
|
end
|
69
|
-
sleep
|
76
|
+
sleep 3 + rand(5)
|
70
77
|
end
|
71
78
|
rescue
|
72
79
|
Log.exception $!
|
@@ -80,7 +87,8 @@ class RbbtProcessQueue
|
|
80
87
|
@current = Process.fork do
|
81
88
|
run
|
82
89
|
end
|
83
|
-
|
90
|
+
@asked = false
|
91
|
+
Log.high "Worker #{Process.pid} respawning to #{@current}"
|
84
92
|
end
|
85
93
|
rescue Aborted, Interrupt
|
86
94
|
Log.warn "Worker #{Process.pid} aborted"
|
@@ -24,8 +24,13 @@ class RbbtProcessQueue
|
|
24
24
|
begin
|
25
25
|
loop do
|
26
26
|
p = @callback_queue.pop
|
27
|
-
|
28
|
-
|
27
|
+
|
28
|
+
if Exception === p or (Array === p and Exception === p.first)
|
29
|
+
e = Array === p ? p.first : p
|
30
|
+
Log.warn "Callback recieved exception from worker: #{e.message}" unless Aborted === e or ClosedStream === e
|
31
|
+
raise e
|
32
|
+
end
|
33
|
+
|
29
34
|
@callback.call p
|
30
35
|
end
|
31
36
|
rescue Aborted
|
@@ -34,8 +39,7 @@ class RbbtProcessQueue
|
|
34
39
|
raise $!
|
35
40
|
rescue ClosedStream
|
36
41
|
rescue Exception
|
37
|
-
Log.warn "
|
38
|
-
Log.exception $!
|
42
|
+
Log.warn "Exception captured in callback: #{$!.message}"
|
39
43
|
@process_monitor.raise $!
|
40
44
|
raise $!
|
41
45
|
ensure
|
@@ -64,7 +64,7 @@ module Log
|
|
64
64
|
indicator << " #{Log.color(:blue, percent.to_s << "%")}"
|
65
65
|
|
66
66
|
used = time - @start
|
67
|
-
if @mean_max and @mean_max > 0
|
67
|
+
if @mean_max and @mean_max > 0 and @mean > 0
|
68
68
|
eta = (@max - @ticks) / @mean
|
69
69
|
else
|
70
70
|
eta = (@max - @ticks) / (@ticks/used)
|
@@ -73,7 +73,7 @@ module Log
|
|
73
73
|
used = Misc.format_seconds(used)
|
74
74
|
eta = [eta/3600, eta/60 % 60, eta % 60].map{|t| "%02i" % t }.join(':')
|
75
75
|
|
76
|
-
indicator << " #{Log.color :yellow, used} used #{Log.color :yellow, eta} left"
|
76
|
+
indicator << " #{Log.color :yellow, used} used #{Log.color :yellow, eta} left - #{Log.color :yellow, ticks.to_s} of #{Log.color :yellow, @max.to_s}"
|
77
77
|
|
78
78
|
indicator
|
79
79
|
end
|
@@ -83,7 +83,7 @@ module Log
|
|
83
83
|
return str << " " << Log.color(:yellow, "waiting") if @ticks == 0
|
84
84
|
str << " " << thr_msg
|
85
85
|
if max
|
86
|
-
str << Log.color(:blue, " -- ") << eta_msg
|
86
|
+
str << Log.color(:blue, " -- ") << eta_msg
|
87
87
|
else
|
88
88
|
str << Log.color(:blue, " -- ") << ticks.to_s
|
89
89
|
end
|
@@ -27,24 +27,29 @@ module Log
|
|
27
27
|
return if ENV["RBBT_NO_PROGRESS"] == "true"
|
28
28
|
@ticks += 1
|
29
29
|
|
30
|
-
|
31
|
-
|
32
|
-
@last_time
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
30
|
+
begin
|
31
|
+
time = Time.now
|
32
|
+
if @last_time.nil?
|
33
|
+
@last_time = time
|
34
|
+
@last_count = @ticks
|
35
|
+
@start = time
|
36
|
+
return
|
37
|
+
end
|
37
38
|
|
38
|
-
|
39
|
-
|
40
|
-
|
39
|
+
diff = time - @last_time
|
40
|
+
report and return if diff > @frequency
|
41
|
+
return unless max
|
41
42
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
43
|
+
percent = self.percent
|
44
|
+
if @last_percent.nil?
|
45
|
+
@last_percent = percent
|
46
|
+
return
|
47
|
+
end
|
48
|
+
report and return if percent > @last_percent and diff > 0.3
|
49
|
+
rescue Exception
|
50
|
+
Log.warn "Exception during report: " << $!.message
|
51
|
+
Log.exception $!
|
46
52
|
end
|
47
|
-
report and return if percent > @last_percent and diff > 0.3
|
48
53
|
end
|
49
54
|
end
|
50
55
|
end
|
@@ -6,7 +6,9 @@ module Misc
|
|
6
6
|
end
|
7
7
|
|
8
8
|
def self.pre_fork
|
9
|
-
Persist::CONNECTIONS.values.each do |db|
|
9
|
+
Persist::CONNECTIONS.values.each do |db|
|
10
|
+
db.close if db.write?
|
11
|
+
end
|
10
12
|
ObjectSpace.each_object(Mutex) do |m|
|
11
13
|
begin
|
12
14
|
m.unlock
|
@@ -276,7 +278,8 @@ module Misc
|
|
276
278
|
end
|
277
279
|
end
|
278
280
|
|
279
|
-
def self.bootstrap(elems, num = :current,
|
281
|
+
def self.bootstrap(elems, num = :current, options = {}, &block)
|
282
|
+
IndiferentHash.setup options
|
280
283
|
num = :current if num.nil?
|
281
284
|
cpus = case num
|
282
285
|
when :current
|
@@ -289,10 +292,14 @@ module Misc
|
|
289
292
|
else
|
290
293
|
32000 / num
|
291
294
|
end
|
295
|
+
else
|
296
|
+
raise "Parameter 'num' not understood: #{Misc.fingerprint num}"
|
292
297
|
end
|
293
298
|
|
294
299
|
|
295
|
-
options = Misc.add_defaults options, :
|
300
|
+
options = Misc.add_defaults options, :respawn => true, :cpus => cpus, :into => Set.new
|
301
|
+
options = Misc.add_defaults options, :bar => "Bootstrap in #{ options[:cpus] } cpus: #{ Misc.fingerprint Annotated.purge(elems) }"
|
302
|
+
respawn = options[:respawn] and options[:cpus] and options[:cpus].to_i > 1
|
296
303
|
|
297
304
|
index = (0..elems.length-1).to_a.collect{|v| v.to_s }
|
298
305
|
TSV.traverse index, options do |pos|
|
@@ -303,6 +310,7 @@ module Misc
|
|
303
310
|
rescue Interrupt
|
304
311
|
Log.warn "Process #{Process.pid} was aborted"
|
305
312
|
end
|
313
|
+
raise RbbtProcessQueue::RbbtProcessQueueWorker::Respawn if respawn == :always and cpus > 1
|
306
314
|
nil
|
307
315
|
end
|
308
316
|
end
|