rbbt-util 5.11.9 → 5.12.0

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -26,6 +26,20 @@ module ConcurrentStream
     stream
   end
 
+  def annotate(stream)
+    ConcurrentStream.setup stream
+    stream.threads = threads
+    stream.pids = pids
+    stream.callback = callback
+    stream.abort_callback = abort_callback
+    stream.filename = filename
+    stream.joined = joined
+  end
+
+  def clear
+    threads, pids, callback, abort_callback = nil
+  end
+
   def joined?
     @joined
   end
@@ -69,6 +83,7 @@ module ConcurrentStream
     join_callback
 
     @joined = true
+    close unless closed?
   end
 
   def abort_threads

data/lib/rbbt/util/misc/development.rb

@@ -1,4 +1,16 @@
 module Misc
+
+  def self.string2const(string)
+    return nil if string.nil?
+    mod = Kernel
+
+    string.to_s.split('::').each do |str|
+      mod = mod.const_get str
+    end
+
+    mod
+  end
+
   def self.benchmark(repeats = 1, message = nil)
     require 'benchmark'
     res = nil
@@ -92,4 +104,114 @@ module Misc
 
     res
   end
+
+  def self.do_once(&block)
+    return nil if $__did_once
+    $__did_once = true
+    yield
+    nil
+  end
+
+  def self.reset_do_once
+    $__did_once = false
+  end
+
+  def self.insist(times = 3, sleep = nil, msg = nil)
+    if Array === times
+      sleep_array = times
+      times = sleep_array.length
+      sleep = sleep_array.shift
+    end
+    try = 0
+    begin
+      yield
+    rescue
+      if msg
+        Log.warn("Insisting after exception: #{$!.message} -- #{msg}")
+      else
+        Log.warn("Insisting after exception: #{$!.message}")
+      end
+      if sleep and try > 0
+        sleep sleep
+        sleep = sleep_array.shift if sleep_array
+      else
+        Thread.pass
+      end
+      try += 1
+      retry if try < times
+      raise $!
+    end
+  end
+
+  def self.try3times(&block)
+    insist(3, &block)
+  end
+
+  # Divides the array into +num+ chunks of the same size by placing one
+  # element in each chunk iteratively.
+  def self.divide(array, num)
+    num = 1 if num == 0
+    chunks = []
+    num.to_i.times do chunks << [] end
+    array.each_with_index{|e, i|
+      c = i % num
+      chunks[c] << e
+    }
+    chunks
+  end
+
+  # Divides the array into chunks of +num+ same size by placing one
+  # element in each chunk iteratively.
+  def self.ordered_divide(array, num)
+    last = array.length - 1
+    chunks = []
+    current = 0
+    while current <= last
+      next_current = [last, current + num - 1].min
+      chunks << array[current..next_current]
+      current = next_current + 1
+    end
+    chunks
+  end
+
+  def self.random_sample_in_range(total, size)
+    p = Set.new
+
+    if size > total / 10
+      template = (0..total - 1).to_a
+      size.times do |i|
+        pos = (rand * (total - i)).floor
+        if pos == template.length - 1
+          v = template.pop
+        else
+          v, n = template[pos], template[-1]
+          template.pop
+          template[pos] = n 
+        end
+        p << v
+      end
+    else
+      size.times do 
+        pos = nil
+        while pos.nil? 
+          pos = (rand * total).floor
+          if p.include? pos
+            pos = nil
+          end
+        end
+        p << pos
+      end
+    end
+    p
+  end
+
+  def self.sample(ary, size, replacement = false)
+    if ary.respond_to? :sample
+      ary.sample size
+    else
+      total = ary.length
+      p = random_sample_in_range(total, size)
+      ary.values_at *p
+    end
+  end
 end

data/lib/rbbt/util/misc/inspect.rb

@@ -40,9 +40,9 @@ module Misc
     when (defined? AnnotatedArray and AnnotatedArray)
       "<A: #{fingerprint Annotated.purge(obj)} #{fingerprint obj.info}>"
     when (defined? TSV and TSV::Parser)
-      "<TSVStream:" + obj.filename + "--" << Misc.fingerprint(obj.options) << ">"
+      "<TSVStream:" + (obj.filename || "NOFILENAME") + "--" << Misc.fingerprint(obj.options) << ">"
     when IO
-      "<IO:" + (obj.respond_to?(:filename) ? obj.filename : obj.inspect) + ">"
+      "<IO:" + (obj.respond_to?(:filename) ? obj.filename || obj.inspect : obj.inspect) + ">"
     when File
       "<File:" + obj.path + ">"
     when Array
@@ -79,7 +79,7 @@ module Misc
   def self.remove_long_items(obj)
     case
     when IO === obj
-      remove_long_items("IO: " + obj.filename)
+      remove_long_items("IO: " + (obj.respond_to?(:filename) ? (obj.filename || obj.inspect) : obj.inspect ))
     when obj.respond_to?(:path)
       remove_long_items("File: " + obj.path)
     when TSV::Parser === obj

data/lib/rbbt/util/misc/manipulation.rb

@@ -0,0 +1,136 @@
+module Misc
+  def self.collapse_ranges(ranges)
+    processed = []
+    last = nil
+    final = []
+    ranges.sort_by{|range| range.begin }.each do |range|
+      rbegin = range.begin
+      rend = range.end
+      if last.nil? or rbegin > last
+        processed << [rbegin, rend]
+        last = rend
+      else
+       new_processed = []
+        processed.each do |pbegin,pend|
+          if pend < rbegin
+            final << [pbegin, pend]
+          else
+            eend = [rend, pend].max
+            new_processed << [pbegin, eend]
+            break
+          end
+        end
+        processed = new_processed
+        last = rend if rend > last
+      end
+    end
+
+    final.concat processed
+    final.collect{|b,e| (b..e)}
+  end
+
+  def self.total_length(ranges)
+    self.collapse_ranges(ranges).inject(0) do |total,range| total += range.end - range.begin + 1 end
+  end
+
+  def self.sorted_array_hits(a1, a2)
+    e1, e2 = a1.shift, a2.shift
+    counter = 0
+    match = []
+    while true
+      break if e1.nil? or e2.nil?
+      case e1 <=> e2
+      when 0
+        match << counter
+        e1, e2 = a1.shift, a2.shift
+        counter += 1
+      when -1
+        while not e1.nil? and e1 < e2
+          e1 = a1.shift 
+          counter += 1
+        end
+      when 1
+        e2 = a2.shift
+        e2 = a2.shift while not e2.nil? and e2 < e1
+      end
+    end
+    match
+  end
+
+  def self.intersect_sorted_arrays(a1, a2)
+    e1, e2 = a1.shift, a2.shift
+    intersect = []
+    while true
+      break if e1.nil? or e2.nil?
+      case e1 <=> e2
+      when 0
+        intersect << e1
+        e1, e2 = a1.shift, a2.shift
+      when -1
+        e1 = a1.shift while not e1.nil? and e1 < e2
+      when 1
+        e2 = a2.shift
+        e2 = a2.shift while not e2.nil? and e2 < e1
+      end
+    end
+    intersect
+  end
+
+  def self.merge_sorted_arrays(a1, a2)
+    e1, e2 = a1.shift, a2.shift
+    new = []
+    while true
+      case
+      when (e1 and e2)
+        case e1 <=> e2
+        when 0
+          new << e1 
+          e1, e2 = a1.shift, a2.shift
+        when -1
+          new << e1
+          e1 = a1.shift
+        when 1
+          new << e2
+          e2 = a2.shift
+        end
+      when e2
+        new << e2
+        new.concat a2
+        break
+      when e1
+        new << e1
+        new.concat a1
+        break
+      else
+        break
+      end
+    end
+    new
+  end
+
+  def self.binary_include?(array, elem)
+    upper = array.size - 1
+    lower = 0
+
+    return -1 if upper < lower
+
+    while(upper >= lower) do
+      idx = lower + (upper - lower) / 2
+      value = array[idx]
+
+      case elem <=> value
+      when 0
+        return true
+      when -1
+        upper = idx - 1
+      when 1
+        lower = idx + 1
+      else
+        raise "Cannot compare #{[elem.inspect, value.inspect] * " with "}"
+      end
+    end
+
+    return false
+  end
+
+end

data/lib/rbbt/util/misc/math.rb

@@ -29,4 +29,54 @@ module Misc
     Math.sqrt(list.compact.inject(0.0){|acc,e| d = e - mean; acc += d * d}) / (list.compact.length - 1)
   end
 
+  def self.counts(array)
+    counts = {}
+    array.each do |e|
+      counts[e] ||= 0
+      counts[e] += 1
+    end
+
+    counts
+  end
+
+  def self.proportions(array)
+    total = array.length
+
+    proportions = Hash.new 0
+
+    array.each do |e|
+      proportions[e] += 1.0 / total
+    end
+
+    class << proportions; self;end.class_eval do
+      def to_s
+        sort{|a,b| a[1] == b[1] ? a[0] <=> b[0] : a[1] <=> b[1]}.collect{|k,c| "%3d\t%s" % [c, k]} * "\n"
+      end
+    end
+
+    proportions
+  end
+
+  def self.google_venn(list1, list2, list3, name1 = nil, name2 = nil, name3 = nil, total = nil)
+    name1 ||= "list 1"
+    name2 ||= "list 2"
+    name3 ||= "list 3"
+
+    sizes = [list1, list2, list3, list1 & list2, list1 & list3, list2 & list3, list1 & list2 & list3].collect{|l| l.length}
+
+    total = total.length if Array === total
+
+    label = "#{name1}: #{sizes[0]} (#{name2}: #{sizes[3]}, #{name3}: #{sizes[4]})"
+    label << "|#{name2}: #{sizes[1]} (#{name1}: #{sizes[3]}, #{name3}: #{sizes[5]})"
+      label << "|#{name3}: #{sizes[2]} (#{name1}: #{sizes[4]}, #{name2}: #{sizes[5]})"
+      if total
+        label << "| INTERSECTION: #{sizes[6]} TOTAL: #{total}"
+      else
+        label << "| INTERSECTION: #{sizes[6]}"
+      end
+
+    max = total || sizes.max
+    sizes = sizes.collect{|v| (v.to_f/max * 100).to_i.to_f / 100}
+    url = "https://chart.googleapis.com/chart?cht=v&chs=500x300&chd=t:#{sizes * ","}&chco=FF6342,ADDE63,63C6DE,FFFFFF&chdl=#{label}"
+  end
 end

data/lib/rbbt/util/misc/objects.rb

@@ -0,0 +1,79 @@
+module Misc
+
+  def self.prepare_entity(entity, field, options = {})
+    return entity unless defined? Entity
+    return entity unless String === entity or Array === entity
+    options ||= {}
+
+    dup_array = options.delete :dup_array
+
+    if Annotated === field or Entity.respond_to?(:formats) and Entity.formats.include? field
+      params = options.dup
+
+      params[:format] ||= params.delete "format"
+      params.merge!(:format => field) unless params.include?(:format) and not ((f = params[:format]).nil? or (String === f and f.empty?))
+
+      mod = Entity === field ? field : Entity.formats[field]
+      entity = mod.setup(
+        ((entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) ),
+        params
+      ) 
+    end
+
+    entity
+  end
+
+  def self.consolidate(list)
+    list.inject(nil){|acc,e|
+      if acc.nil?
+        acc = e
+      else
+        acc.concat e
+        acc
+      end
+    }
+  end
+
+  def self.append_zipped(current, new)
+    current.each do |v|
+      n = new.shift
+      if Array === n
+        v.concat new
+      else
+        v << n
+      end
+    end
+    current
+  end
+
+  def self.zip_fields(array)
+    return [] if array.empty? or (first = array.first).nil?
+    first.zip(*array[1..-1])
+  end
+
+  def self.field_position(fields, field, quiet = false)
+    return field if Integer === field or Range === field
+    raise FieldNotFoundError, "Field information missing" if fields.nil? && ! quiet
+    fields.each_with_index{|f,i| return i if f == field}
+    field_re = Regexp.new /^#{field}$/i
+    fields.each_with_index{|f,i| return i if f =~ field_re}
+    raise FieldNotFoundError, "Field #{ field.inspect } was not found" unless quiet
+  end
+end
+
+class Hash
+  def chunked_values_at(keys, max = 5000)
+    Misc.ordered_divide(keys, max).inject([]) do |acc,c|
+      new = self.values_at(*c)
+      new.annotate acc if new.respond_to? :annotate and acc.empty?
+      acc.concat(new)
+    end
+  end
+end
+
+module LaterString
+  def to_s
+    yield
+  end
+end
+