scout-gear 7.2.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +51 -6
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +33 -29
- data/lib/scout/config.rb +1 -1
- data/lib/scout/exceptions.rb +1 -0
- data/lib/scout/log/color.rb +4 -2
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +71 -2
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/helper.rb +31 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +12 -1
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/misc.rb +1 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +111 -42
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +60 -30
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/semaphore.rb +8 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +85 -54
- data/lib/scout/tsv/index.rb +188 -20
- data/lib/scout/tsv/open.rb +182 -0
- data/lib/scout/tsv/parser.rb +200 -118
- data/lib/scout/tsv/path.rb +5 -6
- data/lib/scout/tsv/persist/adapter.rb +26 -37
- data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
- data/lib/scout/tsv/persist/serialize.rb +117 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
- data/lib/scout/tsv/persist.rb +4 -2
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +136 -37
- data/lib/scout/tsv/util/filter.rb +312 -0
- data/lib/scout/tsv/util/process.rb +73 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +265 -0
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +126 -19
- data/lib/scout/tsv.rb +28 -5
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +29 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +40 -5
- data/lib/scout/workflow/step/progress.rb +14 -0
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +104 -33
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +86 -47
- data/lib/scout/workflow/usage.rb +10 -6
- data/scout-gear.gemspec +30 -3
- data/scout_commands/workflow/task +37 -9
- data/scout_commands/workflow/task_old +2 -2
- data/test/scout/open/test_stream.rb +61 -59
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +67 -0
- data/test/scout/test_tmpfile.rb +1 -1
- data/test/scout/test_tsv.rb +222 -3
- data/test/scout/test_work_queue.rb +21 -18
- data/test/scout/tsv/persist/test_adapter.rb +11 -1
- data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +127 -3
- data/test/scout/tsv/test_open.rb +167 -0
- data/test/scout/tsv/test_parser.rb +45 -3
- data/test/scout/tsv/test_persist.rb +9 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +195 -3
- data/test/scout/tsv/test_util.rb +24 -0
- data/test/scout/tsv/util/test_filter.rb +188 -0
- data/test/scout/tsv/util/test_process.rb +47 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +58 -0
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/work_queue/test_worker.rb +63 -6
- data/test/scout/workflow/step/test_load.rb +3 -3
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +13 -13
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- data/test/test_helper.rb +3 -1
- metadata +29 -2
@@ -0,0 +1,141 @@
|
|
1
|
+
module TSV
|
2
|
+
class Transformer
|
3
|
+
attr_accessor :unnamed, :parser, :dumper
|
4
|
+
|
5
|
+
def initialize(parser, dumper = nil, unnamed: false)
|
6
|
+
if TSV::Parser === parser
|
7
|
+
@parser = parser
|
8
|
+
elsif TSV === parser
|
9
|
+
@parser = parser
|
10
|
+
else
|
11
|
+
@parser = TSV::Parser.new parser
|
12
|
+
end
|
13
|
+
@unnamed = unnamed
|
14
|
+
if dumper.nil?
|
15
|
+
@dumper = TSV::Dumper.new(@parser)
|
16
|
+
@dumper.sep = "\t"
|
17
|
+
else
|
18
|
+
@dumper = dumper
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def key_field=(key_field)
|
23
|
+
@dumper.key_field = key_field
|
24
|
+
end
|
25
|
+
|
26
|
+
def fields=(fields)
|
27
|
+
@dumper.fields = fields
|
28
|
+
end
|
29
|
+
|
30
|
+
def type=(type)
|
31
|
+
@dumper.type = type
|
32
|
+
end
|
33
|
+
|
34
|
+
def type
|
35
|
+
@dumper.type
|
36
|
+
end
|
37
|
+
|
38
|
+
def sep=(sep)
|
39
|
+
@dumper.sep = sep
|
40
|
+
end
|
41
|
+
|
42
|
+
def include?(*args)
|
43
|
+
false
|
44
|
+
end
|
45
|
+
|
46
|
+
def key_field
|
47
|
+
@dumper.key_field
|
48
|
+
end
|
49
|
+
|
50
|
+
def fields
|
51
|
+
@dumper.fields
|
52
|
+
end
|
53
|
+
|
54
|
+
def all_fields
|
55
|
+
return nil if fields.nil?
|
56
|
+
[key_field] + fields
|
57
|
+
end
|
58
|
+
|
59
|
+
def options
|
60
|
+
@dumper.options
|
61
|
+
end
|
62
|
+
|
63
|
+
def identify_field(name)
|
64
|
+
TSV.identify_field key_field, fields, name
|
65
|
+
end
|
66
|
+
|
67
|
+
def traverse(*args, **kwargs, &block)
|
68
|
+
kwargs[:into] = @dumper
|
69
|
+
kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
|
70
|
+
@dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
|
71
|
+
Log.debug "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @dumper}"
|
72
|
+
Open.traverse(@parser, *args, **kwargs) do |k,v|
|
73
|
+
NamedArray.setup(v, @parser.fields, k) unless @unnamed
|
74
|
+
block.call k, v
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def each(*args, **kwargs, &block)
|
79
|
+
kwargs[:into] = @dumper
|
80
|
+
kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
|
81
|
+
@dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
|
82
|
+
Open.traverse(@parser, *args, **kwargs) do |k,v|
|
83
|
+
NamedArray.setup(v, @parser.fields, k) unless @unnamed
|
84
|
+
block.call k, v
|
85
|
+
[k, v]
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def with_unnamed
|
90
|
+
begin
|
91
|
+
old_unnamed = @unnamed
|
92
|
+
@unnamed = true
|
93
|
+
yield
|
94
|
+
ensure
|
95
|
+
@unnamed = old_unnamed
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def []=(key, value)
|
100
|
+
@dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
|
101
|
+
@dumper.add key, value
|
102
|
+
end
|
103
|
+
|
104
|
+
def stream
|
105
|
+
@dumper.stream
|
106
|
+
end
|
107
|
+
|
108
|
+
def tsv(*args)
|
109
|
+
TSV === @dumper ? @dumper : TSV.open(stream, *args)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def to_list
|
114
|
+
res = self.annotate({})
|
115
|
+
transformer = Transformer.new self, res
|
116
|
+
transformer.type = :list
|
117
|
+
transformer.traverse do |k,v|
|
118
|
+
case self.type
|
119
|
+
when :single
|
120
|
+
[k, [v]]
|
121
|
+
when :double
|
122
|
+
[k, v.collect{|v| v.first }]
|
123
|
+
when :flat
|
124
|
+
[k, v.slice(0,1)]
|
125
|
+
end
|
126
|
+
end
|
127
|
+
res
|
128
|
+
end
|
129
|
+
|
130
|
+
def to_single
|
131
|
+
res = self.annotate({})
|
132
|
+
transformer = Transformer.new self, res
|
133
|
+
transformer.type = :single
|
134
|
+
transformer.traverse do |k,v|
|
135
|
+
v = v.first while Array === v
|
136
|
+
[k, v]
|
137
|
+
end
|
138
|
+
res
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
data/lib/scout/tsv/traverse.rb
CHANGED
@@ -1,48 +1,147 @@
|
|
1
1
|
require_relative 'parser'
|
2
2
|
module TSV
|
3
|
-
def
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
3
|
+
def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
|
4
|
+
key_field = key_field_pos if key_field.nil?
|
5
|
+
fields = fields_pos.dup if fields.nil?
|
6
|
+
type = @type if type.nil?
|
7
|
+
key_pos = self.identify_field(key_field)
|
8
|
+
fields = self.all_fields if fields == :all
|
9
|
+
fields = [fields] unless fields.nil? || Array === fields
|
10
|
+
positions = fields.nil? || fields == :all ? nil : self.identify_field(fields)
|
11
|
+
|
12
|
+
|
13
|
+
if key_pos == :key
|
14
|
+
key_name = @key_field
|
15
|
+
else
|
16
|
+
key_name = @fields[key_pos]
|
17
|
+
if positions.nil?
|
18
|
+
positions = (0..@fields.length-1).to_a
|
19
|
+
positions.delete_at key_pos
|
20
|
+
positions.unshift :key
|
21
|
+
end
|
10
22
|
end
|
11
|
-
end
|
12
23
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
24
|
+
if positions.nil? && key_pos == :key
|
25
|
+
field_names = @fields
|
26
|
+
elsif positions.nil? && key_pos != :key
|
27
|
+
field_names = @fields.dup
|
28
|
+
field_names.delete_at key_pos unless fields == :all
|
29
|
+
elsif positions.include?(:key)
|
30
|
+
field_names = positions.collect{|p| p == :key ? @key_field : @fields[p] }
|
31
|
+
else
|
32
|
+
field_names = @fields.values_at *positions
|
33
|
+
end
|
34
|
+
|
35
|
+
key_index = positions.index :key if positions
|
36
|
+
positions.delete :key if positions
|
37
|
+
|
38
|
+
log_message = "Traverse #{Log.fingerprint self}"
|
39
|
+
Log.debug log_message
|
40
|
+
bar = log_message if TrueClass === bar
|
41
|
+
|
42
|
+
Log::ProgressBar.with_obj_bar(self, bar) do |bar|
|
43
|
+
with_unnamed unnamed do
|
44
|
+
each do |key,values|
|
45
|
+
bar.tick if bar
|
46
|
+
values = [values] if @type == :single
|
47
|
+
if positions.nil?
|
48
|
+
if key_pos != :key
|
49
|
+
values = values.dup
|
50
|
+
key = values.delete_at(key_pos)
|
51
|
+
end
|
52
|
+
else
|
53
|
+
orig_key = key
|
54
|
+
key = values[key_pos] if key_pos != :key
|
55
|
+
|
56
|
+
values = values.values_at(*positions)
|
57
|
+
if key_index
|
58
|
+
if @type == :double
|
59
|
+
values.insert key_index, [orig_key]
|
60
|
+
else
|
61
|
+
values.insert key_index, orig_key
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
values = TSV.cast_value(values, cast) if cast
|
67
|
+
|
68
|
+
if Array === key
|
69
|
+
if @type == :double && one2one
|
70
|
+
if one2one == :strict
|
71
|
+
key.each_with_index do |key_i,i|
|
72
|
+
if type == :double
|
73
|
+
v_i = values.collect{|v| [v[i]] }
|
74
|
+
else
|
75
|
+
v_i = values.collect{|v| v[i] }
|
76
|
+
end
|
77
|
+
yield key_i, v_i
|
78
|
+
end
|
79
|
+
else
|
80
|
+
key.each_with_index do |key_i,i|
|
81
|
+
if type == :double
|
82
|
+
v_i = values.collect{|v| [v[i] || v.first] }
|
83
|
+
else
|
84
|
+
v_i = values.collect{|v| v[i] || v.first }
|
85
|
+
end
|
86
|
+
yield key_i, v_i, @fields
|
87
|
+
end
|
88
|
+
end
|
89
|
+
else
|
90
|
+
key.each_with_index do |key_i, i|
|
91
|
+
if type == :double
|
92
|
+
yield key_i, values
|
93
|
+
elsif type == :list
|
94
|
+
yield key_i, values.collect{|v| v[i] }
|
95
|
+
elsif type == :flat
|
96
|
+
yield key_i, values.flatten
|
97
|
+
elsif type == :single
|
98
|
+
yield key_i, values.first
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
else
|
103
|
+
if type == @type
|
104
|
+
if type == :single
|
105
|
+
yield key, values.first
|
106
|
+
else
|
107
|
+
yield key, values
|
108
|
+
end
|
109
|
+
else
|
110
|
+
case [type, @type]
|
111
|
+
when [:double, :list]
|
112
|
+
yield key, values.collect{|v| [v] }
|
113
|
+
when [:double, :flat]
|
114
|
+
yield key, [values]
|
115
|
+
when [:double, :single]
|
116
|
+
yield key, [values]
|
117
|
+
when [:list, :double]
|
118
|
+
yield key, values.collect{|v| v.first }
|
119
|
+
when [:list, :flat]
|
120
|
+
yield key, [values.first]
|
121
|
+
when [:list, :single]
|
122
|
+
yield key, values
|
123
|
+
when [:flat, :double]
|
124
|
+
yield key, values.flatten
|
125
|
+
when [:flat, :list]
|
126
|
+
yield key, values.flatten
|
127
|
+
when [:flat, :single]
|
128
|
+
yield key, values
|
129
|
+
when [:single, :double]
|
130
|
+
yield key, values.flatten.first
|
131
|
+
when [:single, :list]
|
132
|
+
yield key, values.first
|
133
|
+
when [:single, :flat]
|
134
|
+
yield key, values.first
|
135
|
+
end
|
33
136
|
end
|
34
|
-
nil
|
35
137
|
end
|
36
|
-
into.close if into.respond_to?(:close)
|
37
|
-
end
|
38
|
-
Thread.pass until into_thread
|
39
|
-
into
|
40
|
-
else
|
41
|
-
TSV.parse obj, **options do |k,v|
|
42
|
-
block.call k, v
|
43
|
-
nil
|
44
138
|
end
|
45
139
|
end
|
46
140
|
end
|
141
|
+
|
142
|
+
|
143
|
+
[key_name, field_names]
|
47
144
|
end
|
145
|
+
|
146
|
+
alias through traverse
|
48
147
|
end
|
@@ -0,0 +1,312 @@
|
|
1
|
+
require 'set'
|
2
|
+
module Filtered
|
3
|
+
|
4
|
+
class FilterArray
|
5
|
+
attr_accessor :filters
|
6
|
+
|
7
|
+
def ids
|
8
|
+
ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.merge_sorted_arrays(list, filter.ids.dup)}
|
9
|
+
end
|
10
|
+
|
11
|
+
def method_missing(name, *args)
|
12
|
+
filters.each do |filter|
|
13
|
+
filter.send(name, *args)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
#{{{ FILTER
|
19
|
+
|
20
|
+
class Filter
|
21
|
+
attr_accessor :data, :match, :fieldnum, :value, :list, :unsaved
|
22
|
+
attr_accessor :persistence
|
23
|
+
|
24
|
+
def initialize(data, match, value, persistence = nil)
|
25
|
+
@data = data
|
26
|
+
@match = match
|
27
|
+
@value = value
|
28
|
+
@unsaved = []
|
29
|
+
|
30
|
+
case
|
31
|
+
when Hash === persistence
|
32
|
+
@persistence = persistence
|
33
|
+
when String === persistence
|
34
|
+
@persistence = begin
|
35
|
+
file = ScoutCabinet.open(persistence, true, "HDB")
|
36
|
+
TSV.setup file, :type => :list
|
37
|
+
file.extend TSVAdapter
|
38
|
+
file
|
39
|
+
end
|
40
|
+
@persistence.read
|
41
|
+
end
|
42
|
+
|
43
|
+
@list = nil
|
44
|
+
case
|
45
|
+
when @match == :key
|
46
|
+
@value = Set.new(@value)
|
47
|
+
class << self
|
48
|
+
self
|
49
|
+
end.class_eval <<-EOC
|
50
|
+
def match_entry(key, entry)
|
51
|
+
key == @value or (Set === @value and @value.include? key)
|
52
|
+
end
|
53
|
+
EOC
|
54
|
+
when @match.match(/field:(.*)/)
|
55
|
+
@fieldnum = data.identify_field $1
|
56
|
+
class << self
|
57
|
+
self
|
58
|
+
end.class_eval <<-EOC
|
59
|
+
def match_entry(key, entry)
|
60
|
+
value = entry[@fieldnum]
|
61
|
+
value == @value or (Array === value and value.include? @value)
|
62
|
+
end
|
63
|
+
EOC
|
64
|
+
else
|
65
|
+
raise "Unknown match: #{ @match }"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def key
|
70
|
+
case
|
71
|
+
when String === value
|
72
|
+
value
|
73
|
+
else
|
74
|
+
Marshal.dump(value)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def save(ids)
|
79
|
+
if persistence
|
80
|
+
persistence.write_and_close do
|
81
|
+
persistence[self.key] = ids
|
82
|
+
end
|
83
|
+
else
|
84
|
+
if @list.nil?
|
85
|
+
@list = ids
|
86
|
+
else
|
87
|
+
@list.replace ids
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def update
|
93
|
+
ids = []
|
94
|
+
|
95
|
+
data.with_unnamed do
|
96
|
+
data.unfiltered_each do |key, entry|
|
97
|
+
ids << key if match_entry(key, entry)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
save(ids.sort)
|
102
|
+
end
|
103
|
+
|
104
|
+
def saved
|
105
|
+
if persistence.nil?
|
106
|
+
return nil if list.nil?
|
107
|
+
list
|
108
|
+
else
|
109
|
+
return nil if not persistence.include?(self.key)
|
110
|
+
persistence.write_and_close do
|
111
|
+
persistence[self.key]
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def add_unsaved
|
117
|
+
save(Misc.merge_sorted_arrays(unsaved.sort, saved || [])) if unsaved.any?
|
118
|
+
unsaved.clear
|
119
|
+
end
|
120
|
+
|
121
|
+
def ids
|
122
|
+
add_unsaved
|
123
|
+
|
124
|
+
list = saved
|
125
|
+
if list.nil?
|
126
|
+
update
|
127
|
+
list = saved
|
128
|
+
end
|
129
|
+
list
|
130
|
+
end
|
131
|
+
|
132
|
+
def add(id)
|
133
|
+
unsaved.push id
|
134
|
+
end
|
135
|
+
|
136
|
+
def clean
|
137
|
+
add_unsaved
|
138
|
+
if persistence and persistence.include? self.key
|
139
|
+
persistence.write_and_close do
|
140
|
+
persistence.delete self.key
|
141
|
+
end
|
142
|
+
else
|
143
|
+
@list = nil
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def reset
|
148
|
+
add_unsaved
|
149
|
+
if persistence
|
150
|
+
persistence.write_and_close do
|
151
|
+
persistence.clear
|
152
|
+
end
|
153
|
+
else
|
154
|
+
@list = nil
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
#}}} FILTER
|
160
|
+
|
161
|
+
def self.extended(base)
|
162
|
+
if not base.respond_to? :unfiltered_set
|
163
|
+
class << base
|
164
|
+
attr_accessor :filter_dir, :filters
|
165
|
+
|
166
|
+
alias unfiltered_set []=
|
167
|
+
alias []= filtered_set
|
168
|
+
|
169
|
+
alias unfiltered_filename filename
|
170
|
+
alias filename filtered_filename
|
171
|
+
|
172
|
+
alias unfiltered_keys keys
|
173
|
+
alias keys filtered_keys
|
174
|
+
|
175
|
+
alias unfiltered_values values
|
176
|
+
alias values filtered_values
|
177
|
+
|
178
|
+
alias unfiltered_each each
|
179
|
+
alias each filtered_each
|
180
|
+
|
181
|
+
alias unfiltered_collect collect
|
182
|
+
alias collect filtered_collect
|
183
|
+
|
184
|
+
alias unfiltered_delete delete
|
185
|
+
alias delete filtered_delete
|
186
|
+
end
|
187
|
+
end
|
188
|
+
base.filters = []
|
189
|
+
end
|
190
|
+
|
191
|
+
def filtered_filename
|
192
|
+
if filters.empty?
|
193
|
+
unfiltered_filename
|
194
|
+
else
|
195
|
+
unfiltered_filename + ":Filtered[#{filters.collect{|f| [f.match, Array === f.value ? Misc.digest(:values => f.value) : f.value] * "="} * ", "}]"
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def filtered_set(key, value, clean = false)
|
200
|
+
if filters.empty?
|
201
|
+
self.send(:unfiltered_set, key, value, clean)
|
202
|
+
else
|
203
|
+
filters.each do |filter|
|
204
|
+
filter.add key if filter.match_entry key, value
|
205
|
+
end
|
206
|
+
self.send(:unfiltered_set, key, value, clean)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
def filtered_keys
|
211
|
+
if filters.empty?
|
212
|
+
self.send(:unfiltered_keys)
|
213
|
+
else
|
214
|
+
filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
def filtered_values
|
219
|
+
if filters.empty?
|
220
|
+
self.send(:unfiltered_values)
|
221
|
+
else
|
222
|
+
ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
|
223
|
+
self.send :values_at, *ids
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
def filtered_each(&block)
|
228
|
+
if filters.empty?
|
229
|
+
self.send(:unfiltered_each, &block)
|
230
|
+
else
|
231
|
+
ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
|
232
|
+
|
233
|
+
ids.each do |id|
|
234
|
+
value = self[id]
|
235
|
+
yield id, value if block_given?
|
236
|
+
[id, value]
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
def filtered_collect(&block)
|
242
|
+
if filters.empty?
|
243
|
+
self.send(:unfiltered_collect, &block)
|
244
|
+
else
|
245
|
+
ids = filters.inject(nil){|list,filter| list = (list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup))}
|
246
|
+
|
247
|
+
new = self.annotate({})
|
248
|
+
|
249
|
+
ids.zip(self.send(:values_at, *ids)).each do |id, values|
|
250
|
+
new[id] = values
|
251
|
+
end
|
252
|
+
new.send :collect, &block
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
def filtered_delete(key)
|
257
|
+
if filters.empty?
|
258
|
+
self.send(:unfiltered_delete, key)
|
259
|
+
else
|
260
|
+
reset_filters
|
261
|
+
self.send :unfiltered_delete, key
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
def add_filter(match, value, persistence = nil)
|
266
|
+
if persistence.nil? and filter_dir
|
267
|
+
persistence = File.join(filter_dir, match.to_s + '.filter')
|
268
|
+
end
|
269
|
+
|
270
|
+
filter = Filter.new self, match, value, persistence
|
271
|
+
filters.push filter
|
272
|
+
end
|
273
|
+
|
274
|
+
def pop_filter
|
275
|
+
filters.pop.add_unsaved if filters.any?
|
276
|
+
end
|
277
|
+
|
278
|
+
def size
|
279
|
+
filters.empty? ? super : filters.collect{|f| f.ids.length }.min
|
280
|
+
end
|
281
|
+
|
282
|
+
end
|
283
|
+
|
284
|
+
module TSV
|
285
|
+
def filter(filter_dir = nil)
|
286
|
+
self.extend Filtered
|
287
|
+
self.filter_dir = filter_dir
|
288
|
+
self.filters = []
|
289
|
+
self
|
290
|
+
end
|
291
|
+
|
292
|
+
def reset_filters
|
293
|
+
if @filter_dir.nil? or @filter_dir.empty?
|
294
|
+
@filters.each do |filter| filter.reset end if Array === @filters
|
295
|
+
return
|
296
|
+
end
|
297
|
+
|
298
|
+
Dir.glob(File.join(@filter_dir, '*.filter')).each do |f|
|
299
|
+
FileUtils.rm f
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
def with_filters(filters, &block)
|
304
|
+
filter
|
305
|
+
begin
|
306
|
+
filters.each{|field,value| add_filter field, value }
|
307
|
+
ensure
|
308
|
+
reset_filters
|
309
|
+
end
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module TSV
|
2
|
+
def process(field, &block)
|
3
|
+
field_pos = identify_field field
|
4
|
+
|
5
|
+
through do |key, values|
|
6
|
+
case
|
7
|
+
when type == :single
|
8
|
+
field_values = values
|
9
|
+
when type == :flat
|
10
|
+
field_values = values
|
11
|
+
else
|
12
|
+
next if values.nil?
|
13
|
+
field_values = values[field_pos]
|
14
|
+
end
|
15
|
+
|
16
|
+
new_values = case
|
17
|
+
when block.arity == 1
|
18
|
+
yield(field_values)
|
19
|
+
when block.arity == 2
|
20
|
+
yield(field_values, key)
|
21
|
+
when block.arity == 3
|
22
|
+
yield(field_values, key, values)
|
23
|
+
else
|
24
|
+
raise "Unexpected arity in block, must be 1, 2 or 3: #{block.arity}"
|
25
|
+
end
|
26
|
+
|
27
|
+
case
|
28
|
+
when type == :single
|
29
|
+
self[key] = new_values
|
30
|
+
when type == :flat
|
31
|
+
self[key] = new_values
|
32
|
+
else
|
33
|
+
if ! values[field_pos].frozen? && ((String === values[field_pos] && String === new_values) ||
|
34
|
+
(Array === values[field_pos] && Array === new_values))
|
35
|
+
values[field_pos].replace new_values
|
36
|
+
else
|
37
|
+
values[field_pos] = new_values
|
38
|
+
end
|
39
|
+
self[key] = values
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
self
|
44
|
+
end
|
45
|
+
|
46
|
+
def add_field(name = nil)
|
47
|
+
through do |key, values|
|
48
|
+
new_values = yield(key, values)
|
49
|
+
new_values = [new_values] if type == :double and not Array === new_values
|
50
|
+
|
51
|
+
case
|
52
|
+
when (values.nil? and (fields.nil? or fields.empty?))
|
53
|
+
values = [new_values]
|
54
|
+
when values.nil?
|
55
|
+
values = [nil] * fields.length + [new_values]
|
56
|
+
when Array === values
|
57
|
+
values += [new_values]
|
58
|
+
else
|
59
|
+
values << new_values
|
60
|
+
end
|
61
|
+
|
62
|
+
self[key] = values
|
63
|
+
end
|
64
|
+
|
65
|
+
if not fields.nil? and not name.nil?
|
66
|
+
new_fields = self.fields + [name]
|
67
|
+
self.fields = new_fields
|
68
|
+
end
|
69
|
+
|
70
|
+
self
|
71
|
+
end
|
72
|
+
|
73
|
+
end
|