rbbt-util 4.4.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,11 @@
1
1
  require 'rbbt/util/chain_methods'
2
+ require 'yaml'
2
3
  module TSV
3
4
  extend ChainMethods
4
5
  self.chain_prefix = :tsv
5
6
 
7
+ NIL_YAML = "--- \n"
8
+
6
9
  attr_accessor :unnamed, :serializer_module, :entity_options
7
10
 
8
11
  def entity_options
@@ -101,7 +104,7 @@ module TSV
101
104
  end
102
105
 
103
106
  def tsv_values
104
- values = values_at(*keys)
107
+ values = chunked_values_at(keys)
105
108
  return values if @unnamed or fields.nil?
106
109
 
107
110
  case type
@@ -188,6 +191,16 @@ module TSV
188
191
  end
189
192
  end
190
193
 
194
+ def chunked_values_at(keys, max = 5000)
195
+ Misc.ordered_divide(keys, max).inject([]) do |acc,c|
196
+ new = self.values_at(*c)
197
+ new.annotate acc if new.respond_to? :annotate and acc.empty?
198
+ acc.concat(new)
199
+ end
200
+ end
201
+
202
+
203
+
191
204
  #{{{ Sorting
192
205
 
193
206
  def tsv_sort_by(field = nil, just_keys = false, &block)
@@ -278,7 +291,7 @@ attr_accessor :#{entry}
278
291
 
279
292
  def #{ entry }
280
293
  if not defined? @#{entry}
281
- @#{entry} = YAML.load(self.tsv_clean_get_brackets('#{key}') || nil.to_yaml)
294
+ @#{entry} = (value = self.tsv_clean_get_brackets('#{key}')).nil? ? nil : YAML.load(value)
282
295
  end
283
296
  @#{entry}
284
297
  end
@@ -288,7 +301,7 @@ if '#{entry}' == 'serializer'
288
301
 
289
302
  def #{ entry }=(value)
290
303
  @#{entry} = value
291
- self.tsv_clean_set_brackets '#{key}', value.to_yaml
304
+ self.tsv_clean_set_brackets '#{key}', value.nil? ? NIL_YAML : value.to_yaml
292
305
 
293
306
  return if value.nil?
294
307
 
@@ -324,7 +337,7 @@ if '#{entry}' == 'serializer'
324
337
  else
325
338
  def #{ entry }=(value)
326
339
  @#{entry} = value
327
- self.tsv_clean_set_brackets '#{key}', value.to_yaml
340
+ self.tsv_clean_set_brackets '#{key}', value.nil? ? NIL_YAML : value.to_yaml
328
341
  end
329
342
  end
330
343
  "
@@ -341,7 +354,7 @@ end
341
354
  :serializer
342
355
 
343
356
  def fields
344
- @fields ||= YAML.load(self.tsv_clean_get_brackets("__tsv_hash_fields") || nil.to_yaml)
357
+ @fields ||= YAML.load(self.tsv_clean_get_brackets("__tsv_hash_fields") || "--- \n")
345
358
  if @fields.nil? or @unnamed
346
359
  @fields
347
360
  else
@@ -9,10 +9,20 @@ module TSV
9
9
  other.with_unnamed do
10
10
  with_unnamed do
11
11
  through do |key, values|
12
+ self[key] = [] if self[key].nil?
12
13
  if other.include? key
13
- new_values = other[key].values_at *field_positions
14
+ case
15
+ when other.type == :flat
16
+ new_values = [other[key]]
17
+ when other.type == :single
18
+ new_values = [other[key]]
19
+ else
20
+ new_values = other[key].values_at *field_positions
21
+ end
22
+
14
23
  new_values.collect!{|v| [v]} if type == :double and not other.type == :double
15
24
  new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and other.type == :double
25
+
16
26
  self[key] = self[key].concat new_values
17
27
  else
18
28
  if type == :double
@@ -248,7 +258,7 @@ module TSV
248
258
  if values.nil?
249
259
  nil
250
260
  else
251
- next_index.values_at(*values).flatten.collect
261
+ next_index.values_at(*values).flatten.collect.to_a
252
262
  end
253
263
  end
254
264
  current_index.fields = [next_key]
@@ -181,7 +181,14 @@ module TSV
181
181
  if not traverser.new_field_names.nil?
182
182
  case type
183
183
  when :double, :list
184
- NamedArray.setup value, traverser.new_field_names, key, entity_options
184
+ if value.frozen?
185
+ Log.warn "Value frozen: #{ value }"
186
+ end
187
+ if value.nil?
188
+ nil
189
+ else
190
+ NamedArray.setup value, traverser.new_field_names, key, entity_options
191
+ end
185
192
  when :flat, :single
186
193
  Misc.prepare_entity(value, traverser.new_field_names.first, entity_options)
187
194
  end
@@ -280,7 +287,7 @@ module TSV
280
287
  elems.sort_by{|k,v| v}.collect{|k,v| k}
281
288
  end
282
289
 
283
- def select(method = nil)
290
+ def select(method = nil, &block)
284
291
  new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
285
292
 
286
293
  new.key_field = key_field
@@ -320,24 +327,48 @@ module TSV
320
327
  end
321
328
  when String === method
322
329
  if block_given?
323
- with_unnamed do
324
- case
325
- when (method == key_field or method == :key)
326
- through do |key, values|
327
- new[key] = values if yield(key)
330
+ case
331
+ when block.arity == 1
332
+ with_unnamed do
333
+ case
334
+ when (method == key_field or method == :key)
335
+ through do |key, values|
336
+ new[key] = values if yield(key)
337
+ end
338
+ when (type == :single or type == :flat)
339
+ through do |key, value|
340
+ new[key] = value if yield(value)
341
+ end
342
+ else
343
+ pos = identify_field method
344
+ through do |key, values|
345
+ new[key] = values if yield(values[pos])
346
+ end
328
347
  end
329
- when (type == :single or type == :flat)
330
- through do |key, value|
331
- new[key] = value if yield(value)
332
- end
333
- else
334
- pos = identify_field method
335
- through do |key, values|
336
- new[key] = values if yield(values[pos])
348
+ end
349
+ when block.arity == 2
350
+ with_unnamed do
351
+ case
352
+ when (method == key_field or method == :key)
353
+ through do |key, values|
354
+ new[key] = values if yield(key, key)
355
+ end
356
+ when (type == :single or type == :flat)
357
+ through do |key, value|
358
+ new[key] = value if yield(key, value)
359
+ end
360
+ else
361
+ pos = identify_field method
362
+ through do |key, values|
363
+ new[key] = values if yield(key, values[pos])
364
+ end
337
365
  end
366
+
338
367
  end
339
368
 
340
369
  end
370
+
371
+
341
372
  else
342
373
  with_unnamed do
343
374
  through do |key, values|
@@ -405,6 +436,22 @@ module TSV
405
436
  new
406
437
  end
407
438
 
439
+ def column(field, cast = nil)
440
+ new = slice(field)
441
+
442
+ new.with_unnamed do
443
+ new.each do |k,v|
444
+ nv = v.first
445
+ nv = nv.send(cast) unless cast.nil?
446
+ new[k] = nv
447
+ end
448
+ end
449
+
450
+ new.type = :single
451
+
452
+ new
453
+ end
454
+
408
455
  def process(field, &block)
409
456
  field_pos = identify_field field
410
457
 
@@ -438,10 +485,16 @@ module TSV
438
485
  when type == :flat
439
486
  self[key] = new_values
440
487
  else
441
- values[field_pos].replace new_values
488
+ if (String === values[field_pos] and String === new_values) or
489
+ (Array === values[field_pos] and Array === new_values)
490
+ values[field_pos].replace new_values
491
+ else
492
+ values[field_pos] = new_values
493
+ end
442
494
  self[key] = values
443
495
  end
444
496
  end
497
+ self
445
498
  end
446
499
 
447
500
  def add_field(name = nil)
@@ -65,11 +65,12 @@ module TSV
65
65
  def get_values_single_from_flat(parts)
66
66
  return parts.shift, parts.first if field_positions.nil? and key_position.nil?
67
67
  if key_position == 0
68
- [parts.shift, parts]
68
+ [parts.shift, parts.first]
69
69
  else
70
70
  key = parts.shift
71
- [parts, [key]]
71
+ [parts, key]
72
72
  end
73
+
73
74
  end
74
75
 
75
76
  def get_values_single(parts)
data/lib/rbbt/tsv/util.rb CHANGED
@@ -35,13 +35,14 @@ module TSV
35
35
  def self.get_filename(file)
36
36
  case
37
37
  when String === file
38
- filename = file
38
+ file
39
+ when file.respond_to?(:filename)
40
+ file.filename
39
41
  when file.respond_to?(:gets)
40
- filename = file.filename if file.respond_to? :filename
42
+ nil
41
43
  else
42
- raise "Cannot get stream from: #{file.inspect}"
44
+ raise "Cannot get filename from: #{file.inspect}"
43
45
  end
44
- filename
45
46
  end
46
47
 
47
48
  def self.get_stream(file, open_options = {})
@@ -32,8 +32,8 @@ module ChainMethods
32
32
 
33
33
  class << base; self; end.module_eval do
34
34
  methods.each do |new_method|
35
- original = new_method.to_s.sub(prefix.to_s + '_', '')
36
- clean_method = prefix.to_s + '_clean_' + original
35
+ original = new_method.to_s.sub(prefix.to_s << '_', '')
36
+ clean_method = prefix.to_s << '_clean_' << original
37
37
 
38
38
  original = "[]" if original == "get_brackets"
39
39
  original = "[]=" if original == "set_brackets"
@@ -66,12 +66,12 @@ module ChainMethods
66
66
  end
67
67
  end
68
68
  end
69
- end
69
+ end
70
70
 
71
- base.chain_prefix = base.to_s.downcase.to_sym
72
- end
71
+ base.chain_prefix = base.to_s.downcase.to_sym
72
+ end
73
73
 
74
- def self.extended(base)
75
- chain_methods_extended(base)
76
- end
74
+ def self.extended(base)
75
+ chain_methods_extended(base)
76
+ end
77
77
  end
data/lib/rbbt/util/cmd.rb CHANGED
@@ -159,7 +159,7 @@ module CMD
159
159
  exit(-1)
160
160
  rescue Exception
161
161
  Log.debug("CMDError: #{$!.message}") if log
162
- ddd $!.backtrace if log
162
+ Log.debug("Backtrace: \n" + $!.backtrace * "\n") if log
163
163
  raise CMDError, $!.message
164
164
  end
165
165
  }
@@ -9,7 +9,12 @@ require 'digest/md5'
9
9
  module Misc
10
10
  class FieldNotFoundError < StandardError;end
11
11
 
12
- COLOR_LIST = %w(red green blue black yellow pink purple)
12
+ def self.humanize(string)
13
+ string.gsub(/([a-z])([A-Z])/,'\1_\2').downcase
14
+ end
15
+
16
+ COLOR_LIST = %w(#BC80BD #CCEBC5 #FFED6F #8DD3C7 #FFFFB3 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9)
17
+
13
18
  def self.colors_for(list)
14
19
  unused = COLOR_LIST.dup
15
20
 
@@ -98,8 +103,21 @@ module Misc
98
103
  def self.prepare_entity(entity, field, options = {})
99
104
  return entity unless String === entity or Array === entity
100
105
  options ||= {}
106
+
101
107
  dup_array = options.delete :dup_array
102
- entity = Entity.formats[field].setup(((entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) ), options.merge({:format => field})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? field
108
+
109
+ if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? field
110
+ params = options.dup
111
+
112
+ params[:format] ||= params.delete "format"
113
+ params.merge!(:format => field) unless params.include?(:format) and not ((f = params[:format]).nil? or (String === f and f.empty?))
114
+
115
+ entity = Entity.formats[field].setup(
116
+ ((entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) ),
117
+ params
118
+ )
119
+ end
120
+
103
121
  entity
104
122
  end
105
123
 
@@ -128,7 +146,7 @@ module Misc
128
146
  when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
129
147
  remove_long_items(obj[0..ARRAY_MAX_LENGTH-2] << "TRUNCATED at #{ ARRAY_MAX_LENGTH } (#{obj.length})")
130
148
  when (Hash === obj and obj.length > ARRAY_MAX_LENGTH)
131
- remove_long_items(obj.compact[0..ARRAY_MAX_LENGTH-2] << ["TRUNCATED", "at #{ ARRAY_MAX_LENGTH } (#{obj.length})"])
149
+ remove_long_items(obj.collect.compact[0..ARRAY_MAX_LENGTH-2] << ["TRUNCATED", "at #{ ARRAY_MAX_LENGTH } (#{obj.length})"])
132
150
  when (String === obj and obj.length > STRING_MAX_LENGTH)
133
151
  obj[0..STRING_MAX_LENGTH-1] << " TRUNCATED at #{STRING_MAX_LENGTH} (#{obj.length})"
134
152
  when Hash === obj
@@ -218,9 +236,10 @@ module Misc
218
236
  if Hash === values.last
219
237
  extra = values.pop
220
238
  inputs = Misc.zip2hash(keys, values)
221
- inputs.delete_if{|k,v| v.nil?}
239
+ inputs.delete_if{|k,v| v.nil? or (String === v and v.empty?)}
222
240
  inputs = Misc.add_defaults inputs, extra
223
- inputs.delete_if{|k,v| not keys.include? k}
241
+ inputs.delete_if{|k,v| not keys.include?(k) and not (Symbol === k ? keys.include?(k.to_s) : keys.include?(k.to_sym))}
242
+ inputs
224
243
  else
225
244
  Misc.zip2hash(keys, values)
226
245
  end
@@ -461,16 +480,21 @@ end
461
480
  new
462
481
  end
463
482
 
464
- def self.array2hash(array)
483
+ def self.array2hash(array, default = nil)
465
484
  hash = {}
466
485
  array.each do |key, value|
486
+ value = default.dup if value.nil? and not default.nil?
467
487
  hash[key] = value
468
488
  end
469
489
  hash
470
490
  end
471
491
 
472
492
  def self.zip2hash(list1, list2)
473
- array2hash(list1.zip(list2))
493
+ hash = {}
494
+ list1.each_with_index do |e,i|
495
+ hash[e] = list2[i]
496
+ end
497
+ hash
474
498
  end
475
499
 
476
500
  def self.process_to_hash(list)
@@ -506,7 +530,46 @@ end
506
530
  res
507
531
  end
508
532
 
509
- def self.profile
533
+ def self.profile_html(options = {})
534
+ require 'ruby-prof'
535
+ RubyProf.start
536
+ begin
537
+ res = yield
538
+ rescue Exception
539
+ puts "Profiling aborted"
540
+ raise $!
541
+ ensure
542
+ result = RubyProf.stop
543
+ printer = RubyProf::MultiPrinter.new(result)
544
+ TmpFile.with_file do |dir|
545
+ FileUtils.mkdir_p dir unless File.exists? dir
546
+ printer.print(:path => dir, :profile => 'profile')
547
+ CMD.cmd("firefox -no-remote '#{ dir }'")
548
+ end
549
+ end
550
+
551
+ res
552
+ end
553
+
554
+ def self.profile_graph(options = {})
555
+ require 'ruby-prof'
556
+ RubyProf.start
557
+ begin
558
+ res = yield
559
+ rescue Exception
560
+ puts "Profiling aborted"
561
+ raise $!
562
+ ensure
563
+ result = RubyProf.stop
564
+ #result.eliminate_methods!([/annotated_array_clean_/])
565
+ printer = RubyProf::GraphPrinter.new(result)
566
+ printer.print(STDOUT, options)
567
+ end
568
+
569
+ res
570
+ end
571
+
572
+ def self.profile(options = {})
510
573
  require 'ruby-prof'
511
574
  RubyProf.start
512
575
  begin
@@ -516,8 +579,9 @@ end
516
579
  raise $!
517
580
  ensure
518
581
  result = RubyProf.stop
582
+ #result.eliminate_methods!([/annotated_array_clean_/])
519
583
  printer = RubyProf::FlatPrinter.new(result)
520
- printer.print(STDOUT, 0)
584
+ printer.print(STDOUT, options)
521
585
  end
522
586
 
523
587
  res
@@ -544,8 +608,10 @@ end
544
608
  begin
545
609
  yield
546
610
  rescue
611
+ Log.warn("Insisting after exception: #{$!.message}")
547
612
  try += 1
548
613
  retry if try < times
614
+ raise $!
549
615
  end
550
616
  end
551
617
 
@@ -561,6 +627,30 @@ end
561
627
  }.compact * "#"
562
628
  end
563
629
 
630
+ def self.GET_params2hash(string)
631
+ hash = {}
632
+ string.split('&').collect{|item|
633
+ key, value = item.split("=").values_at 0, 1
634
+ hash[key] = value.nil? ? "" : CGI.unescape(value)
635
+ }
636
+ hash
637
+ end
638
+
639
+ def self.hash2GET_params(hash)
640
+ hash.sort_by{|k,v| k.to_s}.collect{|k,v|
641
+ next unless %w(Symbol String Float Fixnum Integer TrueClass FalseClass Module Class Object Array).include? v.class.to_s
642
+ v = case
643
+ when Symbol === v
644
+ ":" << v.to_s
645
+ when Array === v
646
+ v * ","
647
+ else
648
+ CGI.escape(v.to_s)
649
+ end
650
+ [ Symbol === k ? k.to_s : k, v] * "="
651
+ }.compact * "&"
652
+ end
653
+
564
654
  def self.path_relative_to(basedir, path)
565
655
  path = File.expand_path(path)
566
656
  basedir = File.expand_path(basedir)
@@ -662,6 +752,7 @@ end
662
752
 
663
753
  new_options[key] = value
664
754
  end
755
+
665
756
  new_options
666
757
  end
667
758
 
@@ -694,7 +785,7 @@ end
694
785
  when String === v
695
786
  str << k.to_s << "=>" << v
696
787
  when (Array === v and v.length > HASH2MD5_MAX_ARRAY_LENGTH)
697
- str << k.to_s << "=>[" << v[0..HASH2MD5_MAX_ARRAY_LENGTH] * "," << "]"
788
+ str << k.to_s << "=>[" << v[0..HASH2MD5_MAX_ARRAY_LENGTH] * "," << "; #{ v.length }]"
698
789
  when Array === v
699
790
  str << k.to_s << "=>[" << v * "," << "]"
700
791
  else
@@ -719,9 +810,9 @@ end
719
810
 
720
811
  def self.process_options(hash, *keys)
721
812
  if keys.length == 1
722
- hash.delete keys.first.to_sym
813
+ hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s)
723
814
  else
724
- keys.collect do |key| hash.delete(key.to_sym) || hash.delete(key.to_s) end
815
+ keys.collect do |key| hash.include?(key.to_sym) ? hash.delete(key.to_sym) : hash.delete(key.to_s) end
725
816
  end
726
817
  end
727
818
 
@@ -815,7 +906,7 @@ end
815
906
  # element in each chunk iteratively.
816
907
  def self.divide(array, num)
817
908
  chunks = []
818
- num.times do chunks << [] end
909
+ num.to_i.times do chunks << [] end
819
910
  array.each_with_index{|e, i|
820
911
  c = i % num
821
912
  chunks[c] << e
@@ -823,6 +914,20 @@ end
823
914
  chunks
824
915
  end
825
916
 
917
+ # Divides the array into +num+ chunks of the same size by placing one
918
+ # element in each chunk iteratively.
919
+ def self.ordered_divide(array, num)
920
+ last = array.length - 1
921
+ chunks = []
922
+ current = 0
923
+ while current <= last
924
+ next_current = [last, current + num - 1].min
925
+ chunks << array[current..next_current]
926
+ current = next_current + 1
927
+ end
928
+ chunks
929
+ end
930
+
826
931
  def self.zip_fields(array)
827
932
  return [] if array.empty?
828
933
  array[0].zip(*array[1..-1])