scout-gear 7.2.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +51 -6
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +33 -29
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/exceptions.rb +1 -0
  10. data/lib/scout/log/color.rb +4 -2
  11. data/lib/scout/log/progress/report.rb +1 -1
  12. data/lib/scout/log/progress/util.rb +71 -2
  13. data/lib/scout/log/progress.rb +1 -1
  14. data/lib/scout/log/trap.rb +107 -0
  15. data/lib/scout/log.rb +56 -21
  16. data/lib/scout/meta_extension.rb +13 -6
  17. data/lib/scout/misc/digest.rb +1 -1
  18. data/lib/scout/misc/format.rb +12 -0
  19. data/lib/scout/misc/helper.rb +31 -0
  20. data/lib/scout/misc/insist.rb +1 -1
  21. data/lib/scout/misc/monitor.rb +12 -1
  22. data/lib/scout/misc/system.rb +10 -0
  23. data/lib/scout/misc.rb +1 -0
  24. data/lib/scout/named_array.rb +65 -3
  25. data/lib/scout/open/lock/lockfile.rb +587 -0
  26. data/lib/scout/open/lock.rb +28 -2
  27. data/lib/scout/open/remote.rb +4 -0
  28. data/lib/scout/open/stream.rb +111 -42
  29. data/lib/scout/open/util.rb +13 -3
  30. data/lib/scout/path/find.rb +9 -1
  31. data/lib/scout/path/util.rb +35 -0
  32. data/lib/scout/persist/serialize.rb +18 -5
  33. data/lib/scout/persist.rb +60 -30
  34. data/lib/scout/resource/path.rb +53 -0
  35. data/lib/scout/resource/produce.rb +0 -8
  36. data/lib/scout/resource/util.rb +2 -1
  37. data/lib/scout/semaphore.rb +8 -1
  38. data/lib/scout/tmpfile.rb +7 -8
  39. data/lib/scout/tsv/attach.rb +177 -0
  40. data/lib/scout/tsv/change_id.rb +40 -0
  41. data/lib/scout/tsv/dumper.rb +85 -54
  42. data/lib/scout/tsv/index.rb +188 -20
  43. data/lib/scout/tsv/open.rb +182 -0
  44. data/lib/scout/tsv/parser.rb +200 -118
  45. data/lib/scout/tsv/path.rb +5 -6
  46. data/lib/scout/tsv/persist/adapter.rb +26 -37
  47. data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
  48. data/lib/scout/tsv/persist/serialize.rb +117 -0
  49. data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
  50. data/lib/scout/tsv/persist.rb +4 -2
  51. data/lib/scout/tsv/transformer.rb +141 -0
  52. data/lib/scout/tsv/traverse.rb +136 -37
  53. data/lib/scout/tsv/util/filter.rb +312 -0
  54. data/lib/scout/tsv/util/process.rb +73 -0
  55. data/lib/scout/tsv/util/reorder.rb +81 -0
  56. data/lib/scout/tsv/util/select.rb +265 -0
  57. data/lib/scout/tsv/util/unzip.rb +86 -0
  58. data/lib/scout/tsv/util.rb +126 -19
  59. data/lib/scout/tsv.rb +28 -5
  60. data/lib/scout/work_queue/socket.rb +6 -1
  61. data/lib/scout/work_queue/worker.rb +5 -2
  62. data/lib/scout/work_queue.rb +15 -8
  63. data/lib/scout/workflow/definition.rb +29 -2
  64. data/lib/scout/workflow/step/dependencies.rb +24 -4
  65. data/lib/scout/workflow/step/info.rb +40 -5
  66. data/lib/scout/workflow/step/progress.rb +14 -0
  67. data/lib/scout/workflow/step/provenance.rb +8 -7
  68. data/lib/scout/workflow/step/status.rb +45 -0
  69. data/lib/scout/workflow/step.rb +104 -33
  70. data/lib/scout/workflow/task/inputs.rb +14 -20
  71. data/lib/scout/workflow/task.rb +86 -47
  72. data/lib/scout/workflow/usage.rb +10 -6
  73. data/scout-gear.gemspec +30 -3
  74. data/scout_commands/workflow/task +37 -9
  75. data/scout_commands/workflow/task_old +2 -2
  76. data/test/scout/open/test_stream.rb +61 -59
  77. data/test/scout/path/test_find.rb +10 -1
  78. data/test/scout/resource/test_produce.rb +15 -0
  79. data/test/scout/test_meta_extension.rb +25 -0
  80. data/test/scout/test_named_array.rb +18 -0
  81. data/test/scout/test_persist.rb +67 -0
  82. data/test/scout/test_tmpfile.rb +1 -1
  83. data/test/scout/test_tsv.rb +222 -3
  84. data/test/scout/test_work_queue.rb +21 -18
  85. data/test/scout/tsv/persist/test_adapter.rb +11 -1
  86. data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
  87. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  88. data/test/scout/tsv/test_attach.rb +227 -0
  89. data/test/scout/tsv/test_change_id.rb +98 -0
  90. data/test/scout/tsv/test_dumper.rb +1 -1
  91. data/test/scout/tsv/test_index.rb +127 -3
  92. data/test/scout/tsv/test_open.rb +167 -0
  93. data/test/scout/tsv/test_parser.rb +45 -3
  94. data/test/scout/tsv/test_persist.rb +9 -0
  95. data/test/scout/tsv/test_transformer.rb +108 -0
  96. data/test/scout/tsv/test_traverse.rb +195 -3
  97. data/test/scout/tsv/test_util.rb +24 -0
  98. data/test/scout/tsv/util/test_filter.rb +188 -0
  99. data/test/scout/tsv/util/test_process.rb +47 -0
  100. data/test/scout/tsv/util/test_reorder.rb +94 -0
  101. data/test/scout/tsv/util/test_select.rb +58 -0
  102. data/test/scout/tsv/util/test_unzip.rb +112 -0
  103. data/test/scout/work_queue/test_socket.rb +0 -1
  104. data/test/scout/work_queue/test_worker.rb +63 -6
  105. data/test/scout/workflow/step/test_load.rb +3 -3
  106. data/test/scout/workflow/step/test_status.rb +31 -0
  107. data/test/scout/workflow/task/test_inputs.rb +14 -14
  108. data/test/scout/workflow/test_step.rb +13 -13
  109. data/test/scout/workflow/test_task.rb +168 -32
  110. data/test/scout/workflow/test_usage.rb +33 -6
  111. data/test/test_helper.rb +3 -1
  112. metadata +29 -2
@@ -0,0 +1,141 @@
1
+ module TSV
2
+ class Transformer
3
+ attr_accessor :unnamed, :parser, :dumper
4
+
5
+ def initialize(parser, dumper = nil, unnamed: false)
6
+ if TSV::Parser === parser
7
+ @parser = parser
8
+ elsif TSV === parser
9
+ @parser = parser
10
+ else
11
+ @parser = TSV::Parser.new parser
12
+ end
13
+ @unnamed = unnamed
14
+ if dumper.nil?
15
+ @dumper = TSV::Dumper.new(@parser)
16
+ @dumper.sep = "\t"
17
+ else
18
+ @dumper = dumper
19
+ end
20
+ end
21
+
22
+ def key_field=(key_field)
23
+ @dumper.key_field = key_field
24
+ end
25
+
26
+ def fields=(fields)
27
+ @dumper.fields = fields
28
+ end
29
+
30
+ def type=(type)
31
+ @dumper.type = type
32
+ end
33
+
34
+ def type
35
+ @dumper.type
36
+ end
37
+
38
+ def sep=(sep)
39
+ @dumper.sep = sep
40
+ end
41
+
42
+ def include?(*args)
43
+ false
44
+ end
45
+
46
+ def key_field
47
+ @dumper.key_field
48
+ end
49
+
50
+ def fields
51
+ @dumper.fields
52
+ end
53
+
54
+ def all_fields
55
+ return nil if fields.nil?
56
+ [key_field] + fields
57
+ end
58
+
59
+ def options
60
+ @dumper.options
61
+ end
62
+
63
+ def identify_field(name)
64
+ TSV.identify_field key_field, fields, name
65
+ end
66
+
67
+ def traverse(*args, **kwargs, &block)
68
+ kwargs[:into] = @dumper
69
+ kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
70
+ @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
71
+ Log.debug "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @dumper}"
72
+ Open.traverse(@parser, *args, **kwargs) do |k,v|
73
+ NamedArray.setup(v, @parser.fields, k) unless @unnamed
74
+ block.call k, v
75
+ end
76
+ end
77
+
78
+ def each(*args, **kwargs, &block)
79
+ kwargs[:into] = @dumper
80
+ kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
81
+ @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
82
+ Open.traverse(@parser, *args, **kwargs) do |k,v|
83
+ NamedArray.setup(v, @parser.fields, k) unless @unnamed
84
+ block.call k, v
85
+ [k, v]
86
+ end
87
+ end
88
+
89
+ def with_unnamed
90
+ begin
91
+ old_unnamed = @unnamed
92
+ @unnamed = true
93
+ yield
94
+ ensure
95
+ @unnamed = old_unnamed
96
+ end
97
+ end
98
+
99
+ def []=(key, value)
100
+ @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
101
+ @dumper.add key, value
102
+ end
103
+
104
+ def stream
105
+ @dumper.stream
106
+ end
107
+
108
+ def tsv(*args)
109
+ TSV === @dumper ? @dumper : TSV.open(stream, *args)
110
+ end
111
+ end
112
+
113
+ def to_list
114
+ res = self.annotate({})
115
+ transformer = Transformer.new self, res
116
+ transformer.type = :list
117
+ transformer.traverse do |k,v|
118
+ case self.type
119
+ when :single
120
+ [k, [v]]
121
+ when :double
122
+ [k, v.collect{|v| v.first }]
123
+ when :flat
124
+ [k, v.slice(0,1)]
125
+ end
126
+ end
127
+ res
128
+ end
129
+
130
+ def to_single
131
+ res = self.annotate({})
132
+ transformer = Transformer.new self, res
133
+ transformer.type = :single
134
+ transformer.traverse do |k,v|
135
+ v = v.first while Array === v
136
+ [k, v]
137
+ end
138
+ res
139
+ end
140
+ end
141
+
@@ -1,48 +1,147 @@
1
1
  require_relative 'parser'
2
2
  module TSV
3
- def self.traverse_add(into, res)
4
- case into
5
- when TSV::Dumper
6
- into.add *res
7
- when TSV, Hash
8
- key, value = res
9
- into[key] = value
3
+ def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
4
+ key_field = key_field_pos if key_field.nil?
5
+ fields = fields_pos.dup if fields.nil?
6
+ type = @type if type.nil?
7
+ key_pos = self.identify_field(key_field)
8
+ fields = self.all_fields if fields == :all
9
+ fields = [fields] unless fields.nil? || Array === fields
10
+ positions = fields.nil? || fields == :all ? nil : self.identify_field(fields)
11
+
12
+
13
+ if key_pos == :key
14
+ key_name = @key_field
15
+ else
16
+ key_name = @fields[key_pos]
17
+ if positions.nil?
18
+ positions = (0..@fields.length-1).to_a
19
+ positions.delete_at key_pos
20
+ positions.unshift :key
21
+ end
10
22
  end
11
- end
12
23
 
13
- def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
14
- case obj
15
- when TSV
16
- self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
17
- when String
18
- f = Open.open(obj)
19
- self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
20
- when Step
21
- self.traverse(obj.get_stream, into: into, cpus: cpus, bar: bar, **options, &block)
22
- when IO
23
- if into
24
- into_thread = Thread.new do
25
- Thread.current.report_on_exception = false
26
- Thread.current["name"] = "Traverse into"
27
- TSV.parse obj, **options do |k,v|
28
- begin
29
- res = block.call k, v
30
- traverse_add into, res
31
- rescue
32
- into.abort $!
24
+ if positions.nil? && key_pos == :key
25
+ field_names = @fields
26
+ elsif positions.nil? && key_pos != :key
27
+ field_names = @fields.dup
28
+ field_names.delete_at key_pos unless fields == :all
29
+ elsif positions.include?(:key)
30
+ field_names = positions.collect{|p| p == :key ? @key_field : @fields[p] }
31
+ else
32
+ field_names = @fields.values_at *positions
33
+ end
34
+
35
+ key_index = positions.index :key if positions
36
+ positions.delete :key if positions
37
+
38
+ log_message = "Traverse #{Log.fingerprint self}"
39
+ Log.debug log_message
40
+ bar = log_message if TrueClass === bar
41
+
42
+ Log::ProgressBar.with_obj_bar(self, bar) do |bar|
43
+ with_unnamed unnamed do
44
+ each do |key,values|
45
+ bar.tick if bar
46
+ values = [values] if @type == :single
47
+ if positions.nil?
48
+ if key_pos != :key
49
+ values = values.dup
50
+ key = values.delete_at(key_pos)
51
+ end
52
+ else
53
+ orig_key = key
54
+ key = values[key_pos] if key_pos != :key
55
+
56
+ values = values.values_at(*positions)
57
+ if key_index
58
+ if @type == :double
59
+ values.insert key_index, [orig_key]
60
+ else
61
+ values.insert key_index, orig_key
62
+ end
63
+ end
64
+ end
65
+
66
+ values = TSV.cast_value(values, cast) if cast
67
+
68
+ if Array === key
69
+ if @type == :double && one2one
70
+ if one2one == :strict
71
+ key.each_with_index do |key_i,i|
72
+ if type == :double
73
+ v_i = values.collect{|v| [v[i]] }
74
+ else
75
+ v_i = values.collect{|v| v[i] }
76
+ end
77
+ yield key_i, v_i
78
+ end
79
+ else
80
+ key.each_with_index do |key_i,i|
81
+ if type == :double
82
+ v_i = values.collect{|v| [v[i] || v.first] }
83
+ else
84
+ v_i = values.collect{|v| v[i] || v.first }
85
+ end
86
+ yield key_i, v_i, @fields
87
+ end
88
+ end
89
+ else
90
+ key.each_with_index do |key_i, i|
91
+ if type == :double
92
+ yield key_i, values
93
+ elsif type == :list
94
+ yield key_i, values.collect{|v| v[i] }
95
+ elsif type == :flat
96
+ yield key_i, values.flatten
97
+ elsif type == :single
98
+ yield key_i, values.first
99
+ end
100
+ end
101
+ end
102
+ else
103
+ if type == @type
104
+ if type == :single
105
+ yield key, values.first
106
+ else
107
+ yield key, values
108
+ end
109
+ else
110
+ case [type, @type]
111
+ when [:double, :list]
112
+ yield key, values.collect{|v| [v] }
113
+ when [:double, :flat]
114
+ yield key, [values]
115
+ when [:double, :single]
116
+ yield key, [values]
117
+ when [:list, :double]
118
+ yield key, values.collect{|v| v.first }
119
+ when [:list, :flat]
120
+ yield key, [values.first]
121
+ when [:list, :single]
122
+ yield key, values
123
+ when [:flat, :double]
124
+ yield key, values.flatten
125
+ when [:flat, :list]
126
+ yield key, values.flatten
127
+ when [:flat, :single]
128
+ yield key, values
129
+ when [:single, :double]
130
+ yield key, values.flatten.first
131
+ when [:single, :list]
132
+ yield key, values.first
133
+ when [:single, :flat]
134
+ yield key, values.first
135
+ end
33
136
  end
34
- nil
35
137
  end
36
- into.close if into.respond_to?(:close)
37
- end
38
- Thread.pass until into_thread
39
- into
40
- else
41
- TSV.parse obj, **options do |k,v|
42
- block.call k, v
43
- nil
44
138
  end
45
139
  end
46
140
  end
141
+
142
+
143
+ [key_name, field_names]
47
144
  end
145
+
146
+ alias through traverse
48
147
  end
@@ -0,0 +1,312 @@
1
+ require 'set'
2
+ module Filtered
3
+
4
+ class FilterArray
5
+ attr_accessor :filters
6
+
7
+ def ids
8
+ ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.merge_sorted_arrays(list, filter.ids.dup)}
9
+ end
10
+
11
+ def method_missing(name, *args)
12
+ filters.each do |filter|
13
+ filter.send(name, *args)
14
+ end
15
+ end
16
+ end
17
+
18
+ #{{{ FILTER
19
+
20
+ class Filter
21
+ attr_accessor :data, :match, :fieldnum, :value, :list, :unsaved
22
+ attr_accessor :persistence
23
+
24
+ def initialize(data, match, value, persistence = nil)
25
+ @data = data
26
+ @match = match
27
+ @value = value
28
+ @unsaved = []
29
+
30
+ case
31
+ when Hash === persistence
32
+ @persistence = persistence
33
+ when String === persistence
34
+ @persistence = begin
35
+ file = ScoutCabinet.open(persistence, true, "HDB")
36
+ TSV.setup file, :type => :list
37
+ file.extend TSVAdapter
38
+ file
39
+ end
40
+ @persistence.read
41
+ end
42
+
43
+ @list = nil
44
+ case
45
+ when @match == :key
46
+ @value = Set.new(@value)
47
+ class << self
48
+ self
49
+ end.class_eval <<-EOC
50
+ def match_entry(key, entry)
51
+ key == @value or (Set === @value and @value.include? key)
52
+ end
53
+ EOC
54
+ when @match.match(/field:(.*)/)
55
+ @fieldnum = data.identify_field $1
56
+ class << self
57
+ self
58
+ end.class_eval <<-EOC
59
+ def match_entry(key, entry)
60
+ value = entry[@fieldnum]
61
+ value == @value or (Array === value and value.include? @value)
62
+ end
63
+ EOC
64
+ else
65
+ raise "Unknown match: #{ @match }"
66
+ end
67
+ end
68
+
69
+ def key
70
+ case
71
+ when String === value
72
+ value
73
+ else
74
+ Marshal.dump(value)
75
+ end
76
+ end
77
+
78
+ def save(ids)
79
+ if persistence
80
+ persistence.write_and_close do
81
+ persistence[self.key] = ids
82
+ end
83
+ else
84
+ if @list.nil?
85
+ @list = ids
86
+ else
87
+ @list.replace ids
88
+ end
89
+ end
90
+ end
91
+
92
+ def update
93
+ ids = []
94
+
95
+ data.with_unnamed do
96
+ data.unfiltered_each do |key, entry|
97
+ ids << key if match_entry(key, entry)
98
+ end
99
+ end
100
+
101
+ save(ids.sort)
102
+ end
103
+
104
+ def saved
105
+ if persistence.nil?
106
+ return nil if list.nil?
107
+ list
108
+ else
109
+ return nil if not persistence.include?(self.key)
110
+ persistence.write_and_close do
111
+ persistence[self.key]
112
+ end
113
+ end
114
+ end
115
+
116
+ def add_unsaved
117
+ save(Misc.merge_sorted_arrays(unsaved.sort, saved || [])) if unsaved.any?
118
+ unsaved.clear
119
+ end
120
+
121
+ def ids
122
+ add_unsaved
123
+
124
+ list = saved
125
+ if list.nil?
126
+ update
127
+ list = saved
128
+ end
129
+ list
130
+ end
131
+
132
+ def add(id)
133
+ unsaved.push id
134
+ end
135
+
136
+ def clean
137
+ add_unsaved
138
+ if persistence and persistence.include? self.key
139
+ persistence.write_and_close do
140
+ persistence.delete self.key
141
+ end
142
+ else
143
+ @list = nil
144
+ end
145
+ end
146
+
147
+ def reset
148
+ add_unsaved
149
+ if persistence
150
+ persistence.write_and_close do
151
+ persistence.clear
152
+ end
153
+ else
154
+ @list = nil
155
+ end
156
+ end
157
+ end
158
+
159
+ #}}} FILTER
160
+
161
+ def self.extended(base)
162
+ if not base.respond_to? :unfiltered_set
163
+ class << base
164
+ attr_accessor :filter_dir, :filters
165
+
166
+ alias unfiltered_set []=
167
+ alias []= filtered_set
168
+
169
+ alias unfiltered_filename filename
170
+ alias filename filtered_filename
171
+
172
+ alias unfiltered_keys keys
173
+ alias keys filtered_keys
174
+
175
+ alias unfiltered_values values
176
+ alias values filtered_values
177
+
178
+ alias unfiltered_each each
179
+ alias each filtered_each
180
+
181
+ alias unfiltered_collect collect
182
+ alias collect filtered_collect
183
+
184
+ alias unfiltered_delete delete
185
+ alias delete filtered_delete
186
+ end
187
+ end
188
+ base.filters = []
189
+ end
190
+
191
+ def filtered_filename
192
+ if filters.empty?
193
+ unfiltered_filename
194
+ else
195
+ unfiltered_filename + ":Filtered[#{filters.collect{|f| [f.match, Array === f.value ? Misc.digest(:values => f.value) : f.value] * "="} * ", "}]"
196
+ end
197
+ end
198
+
199
+ def filtered_set(key, value, clean = false)
200
+ if filters.empty?
201
+ self.send(:unfiltered_set, key, value, clean)
202
+ else
203
+ filters.each do |filter|
204
+ filter.add key if filter.match_entry key, value
205
+ end
206
+ self.send(:unfiltered_set, key, value, clean)
207
+ end
208
+ end
209
+
210
+ def filtered_keys
211
+ if filters.empty?
212
+ self.send(:unfiltered_keys)
213
+ else
214
+ filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
215
+ end
216
+ end
217
+
218
+ def filtered_values
219
+ if filters.empty?
220
+ self.send(:unfiltered_values)
221
+ else
222
+ ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
223
+ self.send :values_at, *ids
224
+ end
225
+ end
226
+
227
+ def filtered_each(&block)
228
+ if filters.empty?
229
+ self.send(:unfiltered_each, &block)
230
+ else
231
+ ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
232
+
233
+ ids.each do |id|
234
+ value = self[id]
235
+ yield id, value if block_given?
236
+ [id, value]
237
+ end
238
+ end
239
+ end
240
+
241
+ def filtered_collect(&block)
242
+ if filters.empty?
243
+ self.send(:unfiltered_collect, &block)
244
+ else
245
+ ids = filters.inject(nil){|list,filter| list = (list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup))}
246
+
247
+ new = self.annotate({})
248
+
249
+ ids.zip(self.send(:values_at, *ids)).each do |id, values|
250
+ new[id] = values
251
+ end
252
+ new.send :collect, &block
253
+ end
254
+ end
255
+
256
+ def filtered_delete(key)
257
+ if filters.empty?
258
+ self.send(:unfiltered_delete, key)
259
+ else
260
+ reset_filters
261
+ self.send :unfiltered_delete, key
262
+ end
263
+ end
264
+
265
+ def add_filter(match, value, persistence = nil)
266
+ if persistence.nil? and filter_dir
267
+ persistence = File.join(filter_dir, match.to_s + '.filter')
268
+ end
269
+
270
+ filter = Filter.new self, match, value, persistence
271
+ filters.push filter
272
+ end
273
+
274
+ def pop_filter
275
+ filters.pop.add_unsaved if filters.any?
276
+ end
277
+
278
+ def size
279
+ filters.empty? ? super : filters.collect{|f| f.ids.length }.min
280
+ end
281
+
282
+ end
283
+
284
+ module TSV
285
+ def filter(filter_dir = nil)
286
+ self.extend Filtered
287
+ self.filter_dir = filter_dir
288
+ self.filters = []
289
+ self
290
+ end
291
+
292
+ def reset_filters
293
+ if @filter_dir.nil? or @filter_dir.empty?
294
+ @filters.each do |filter| filter.reset end if Array === @filters
295
+ return
296
+ end
297
+
298
+ Dir.glob(File.join(@filter_dir, '*.filter')).each do |f|
299
+ FileUtils.rm f
300
+ end
301
+ end
302
+
303
+ def with_filters(filters, &block)
304
+ filter
305
+ begin
306
+ filters.each{|field,value| add_filter field, value }
307
+ ensure
308
+ reset_filters
309
+ end
310
+ end
311
+ end
312
+
@@ -0,0 +1,73 @@
1
+ module TSV
2
+ def process(field, &block)
3
+ field_pos = identify_field field
4
+
5
+ through do |key, values|
6
+ case
7
+ when type == :single
8
+ field_values = values
9
+ when type == :flat
10
+ field_values = values
11
+ else
12
+ next if values.nil?
13
+ field_values = values[field_pos]
14
+ end
15
+
16
+ new_values = case
17
+ when block.arity == 1
18
+ yield(field_values)
19
+ when block.arity == 2
20
+ yield(field_values, key)
21
+ when block.arity == 3
22
+ yield(field_values, key, values)
23
+ else
24
+ raise "Unexpected arity in block, must be 1, 2 or 3: #{block.arity}"
25
+ end
26
+
27
+ case
28
+ when type == :single
29
+ self[key] = new_values
30
+ when type == :flat
31
+ self[key] = new_values
32
+ else
33
+ if ! values[field_pos].frozen? && ((String === values[field_pos] && String === new_values) ||
34
+ (Array === values[field_pos] && Array === new_values))
35
+ values[field_pos].replace new_values
36
+ else
37
+ values[field_pos] = new_values
38
+ end
39
+ self[key] = values
40
+ end
41
+ end
42
+
43
+ self
44
+ end
45
+
46
+ def add_field(name = nil)
47
+ through do |key, values|
48
+ new_values = yield(key, values)
49
+ new_values = [new_values] if type == :double and not Array === new_values
50
+
51
+ case
52
+ when (values.nil? and (fields.nil? or fields.empty?))
53
+ values = [new_values]
54
+ when values.nil?
55
+ values = [nil] * fields.length + [new_values]
56
+ when Array === values
57
+ values += [new_values]
58
+ else
59
+ values << new_values
60
+ end
61
+
62
+ self[key] = values
63
+ end
64
+
65
+ if not fields.nil? and not name.nil?
66
+ new_fields = self.fields + [name]
67
+ self.fields = new_fields
68
+ end
69
+
70
+ self
71
+ end
72
+
73
+ end