rbbt-util 5.44.1 → 6.0.3

data/lib/rbbt/tsv/csv.rb

@@ -1,85 +1,87 @@
-require 'csv'
-
-module TSV
-  def self.csv(obj, options = {}) 
-    options = Misc.add_defaults IndiferentHash.setup(options.dup), :headers => true, :type => :list
-    headers = options[:headers]
-
-    noheaders = ! headers
-
-    type = options.delete :type
-    cast = options.delete :cast
-    merge = options.delete :merge
-    key_field = options.delete :key_field
-    fields = options.delete :fields
-    
-    if key_field || fields
-      orig_type = type
-      type = :double
-      merge = true
-    end
-
-    options[:headers] = false
-
-    csv = case obj
-          when Path
-            CSV.read obj.find.open, options
-          when String
-            if Open.remote?(obj)
-              CSV.read Open.open(obj), options
-            elsif Misc.is_filename?(obj)
-              CSV.read obj, options
-            else
-              CSV.new obj, **options
-            end
-          else
-            CSV.new obj, **options
-          end
-
-    tsv = if noheaders
-            TSV.setup({}, :key_field => nil, :fields => nil, :type => type)
-          else
-            key, *csv_fields = csv.shift
-            TSV.setup({}, :key_field => key, :fields => csv_fields, :type => type)
-          end
-
-    csv.each_with_index do |row,i|
-      if noheaders
-        key, values = ["row-#{i}", row]
-      else
-        key, *values = row
-      end
-      
-      if cast
-        values = values.collect{|v| v.send cast }
-      end
-
-      case type
-      when :double, :flat
-        tsv.zip_new(key, values)
-      when :single
-        tsv[key] = values.first
-      when :list
-        tsv[key] = values
-      end
-    end
-
-    if key_field || fields
-      tsv = tsv.reorder(key_field, fields, :zipped => true, :merge => true)
-      if tsv.type != orig_type
-        tsv = case orig_type
-              when :list
-                tsv.to_list
-              when :single
-                tsv.to_single
-              when :list
-                tsv.to_list
-              when :flat
-                tsv.to_flat
-              end
-      end
-    end
-
-    tsv
-  end
-end
+require_relative '../refactor'
+Rbbt.require_instead 'scout/tsv'
+#require 'csv'
+#
+#module TSV
+#  def self.csv(obj, options = {}) 
+#    options = Misc.add_defaults IndiferentHash.setup(options.dup), :headers => true, :type => :list
+#    headers = options[:headers]
+#
+#    noheaders = ! headers
+#
+#    type = options.delete :type
+#    cast = options.delete :cast
+#    merge = options.delete :merge
+#    key_field = options.delete :key_field
+#    fields = options.delete :fields
+#    
+#    if key_field || fields
+#      orig_type = type
+#      type = :double
+#      merge = true
+#    end
+#
+#    options[:headers] = false
+#
+#    csv = case obj
+#          when Path
+#            CSV.read obj.find.open, **options
+#          when String
+#            if Open.remote?(obj)
+#              CSV.read Open.open(obj), **options
+#            elsif Misc.is_filename?(obj)
+#              CSV.read obj, **options
+#            else
+#              CSV.new obj, **options
+#            end
+#          else
+#            CSV.new obj, **options
+#          end
+#
+#    tsv = if noheaders
+#            TSV.setup({}, :key_field => nil, :fields => nil, :type => type)
+#          else
+#            key, *csv_fields = csv.shift
+#            TSV.setup({}, :key_field => key, :fields => csv_fields, :type => type)
+#          end
+#
+#    csv.each_with_index do |row,i|
+#      if noheaders
+#        key, values = ["row-#{i}", row]
+#      else
+#        key, *values = row
+#      end
+#      
+#      if cast
+#        values = values.collect{|v| v.send cast }
+#      end
+#
+#      case type
+#      when :double, :flat
+#        tsv.zip_new(key, values)
+#      when :single
+#        tsv[key] = values.first
+#      when :list
+#        tsv[key] = values
+#      end
+#    end
+#
+#    if key_field || fields
+#      tsv = tsv.reorder(key_field, fields, :zipped => true, :merge => true)
+#      if tsv.type != orig_type
+#        tsv = case orig_type
+#              when :list
+#                tsv.to_list
+#              when :single
+#                tsv.to_single
+#              when :list
+#                tsv.to_list
+#              when :flat
+#                tsv.to_flat
+#              end
+#      end
+#    end
+#
+#    tsv
+#  end
+#end

data/lib/rbbt/tsv/dumper.rb

@@ -1,100 +1,2 @@
-module TSV
-  class Dumper
-    attr_accessor :in_stream, :stream, :options, :filename, :sep
-    def self.stream(options = {}, filename = nil, stream = nil, &block)
-      dumper = TSV::Dumper.new options, filename
-      if stream
-        dumper.set_stream stream if stream
-        yield dumper
-        stream
-      else
-        thread = Thread.new(Thread.current) do |parent|
-          yield dumper
-        end
-        ConcurrentStream.setup(dumper.stream, threads: thread)
-      end
-    end
-
-    def set_stream(stream)
-      @stream.close
-      @in_stream.close
-      @in_stream = @stream = stream
-    end
-
-    def initialize(options, filename = nil)
-      if TSV  === options
-        @options = options.options.merge(:key_field => options.key_field, :fields => options.fields)
-        @filename ||= options.filename
-      else
-        @options = options
-        @filename = filename
-      end
-      @filename ||= Misc.fingerprint options
-      @stream, @in_stream = Misc.pipe
-    end
-
-    def self.values_to_s(values, fields = nil, sep = "\t")
-      sep = "\t" if sep.nil?
-      case values
-      when nil
-        if fields.nil? or fields.empty?
-          "\n"
-        else
-          sep + ([""] * fields.length) * sep << "\n"
-        end
-      when Array
-        if fields.nil? 
-          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
-        elsif fields.empty?
-          "\n"
-        else
-          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
-        end
-      else
-        if fields.nil?
-          sep + values.to_s + "\n"
-        elsif fields.empty?
-          "\n"
-        else
-          sep + values.to_s << "\n"
-        end
-      end
-    end
-
-    def init(init_options = {})
-      options = @options.dup
-      key_field, fields = Misc.process_options options, :key_field, :fields
-
-      str = TSV.header_lines(key_field, fields, options.merge(init_options || {}))
-
-      Thread.pass while IO.select(nil, [@in_stream],nil,1).nil? if IO === @in_stream
-
-      @in_stream << str
-    end
-
-    def add(k,v)
-      @fields ||= @options[:fields]
-      @sep ||= @options[:sep]
-      begin
-        Thread.pass while IO.select(nil, [@in_stream],nil,1).nil? if IO === @in_stream
-        @in_stream << k << TSV::Dumper.values_to_s(v, @fields, @sep)
-      rescue IOError
-      rescue Exception
-        raise $!
-      end
-    end
-
-    def close_out
-      @stream.close unless StringIO === @stream || @stream.closed?
-    end
-
-    def close_in
-      @in_stream.join if @in_stream.respond_to?(:join) && ! @in_stream.joined?
-      @in_stream.close if @in_stream.respond_to?(:close) && ! @in_stream.closed? 
-    end
-
-    def close
-      close_in unless @in_stream == @stream
-    end
-  end
-end
+require_relative '../refactor'
+Rbbt.require_instead 'scout/tsv'

data/lib/rbbt/tsv/excel.rb

@@ -1,10 +1,12 @@
+require_relative '../refactor'
+Rbbt.require_instead 'scout/tsv'
 require 'spreadsheet'
 require 'rubyXL'
 
 module TSV
 
   def self._remove_link(value)
-    if value =~ /<([\w]+)[^>]*>(.*?)<\/\1>/
+    if String === value && value =~ /<([\w]+)[^>]*>(.*?)<\/\1>/
       $2
     else
       value
@@ -28,14 +30,14 @@ module TSV
 
 
   def self._excel_data(tsv, options ={})
-    options = Misc.add_defaults options, :sep2 => ', '
+    options = IndiferentHash.add_defaults options, :sep2 => ', '
 
-    name = Misc.process_options options, :name
-    sep2 = Misc.process_options options, :sep2
-    unmerge = Misc.process_options options, :unmerge
-    sort_by = Misc.process_options options, :sort_by
-    sort_by_cast = Misc.process_options options, :sort_by_cast
-    remove_links = Misc.process_options options, :remove_links
+    name = IndiferentHash.process_options options, :name
+    sep2 = IndiferentHash.process_options options, :sep2
+    unmerge = IndiferentHash.process_options options, :unmerge
+    sort_by = IndiferentHash.process_options options, :sort_by
+    sort_by_cast = IndiferentHash.process_options options, :sort_by_cast
+    remove_links = IndiferentHash.process_options options, :remove_links
 
     i = 1
     if sort_by
@@ -83,7 +85,7 @@ module TSV
       new_rows = []
       rows.each do |row|
         header = row.shift
-        Misc.zip_fields(row).each do |values|
+        NamedArray.zip_fields(row).each do |values|
           new_rows << [header] + values
         end
       end
@@ -94,11 +96,11 @@ module TSV
 
   module XLS
     def self.read(file, options = {})
-      options = Misc.add_defaults options, :sep2 => /[,|]\s?/, :skip_rows => 0
-      sheet = Misc.process_options options, :sheet
-      header = Misc.process_options options, :header
-      text = Misc.process_options options, :text
-      skip_rows = Misc.process_options options, :skip_rows
+      options = IndiferentHash.add_defaults options, :sep2 => /[,|]\s?/, :skip_rows => 0
+      sheet = IndiferentHash.process_options options, :sheet
+      header = IndiferentHash.process_options options, :header
+      text = IndiferentHash.process_options options, :text
+      skip_rows = IndiferentHash.process_options options, :skip_rows
       skip_rows = skip_rows.to_i
 
       header = true unless header == false
@@ -112,7 +114,7 @@ module TSV
       sheet_name = sheet
       Log.debug "Opening LSX #{file} sheet #{ sheet_name }"
 
-      TmpFile.with_file :extension => Misc.sanitize_filename(sheet_name.to_s) do |filename|
+      TmpFile.with_file :extension => Path.sanitize_filename(sheet_name.to_s) do |filename|
 
         sheet    = workbook.worksheet sheet
 
@@ -146,8 +148,8 @@ module TSV
     end
 
     def self.write(tsv, file, options = {})
-      options = Misc.add_defaults options, :sheet => "Sheet1"
-      sheet = Misc.process_options options, :sheet
+      options = IndiferentHash.add_defaults options, :sheet => "Sheet1"
+      sheet = IndiferentHash.process_options options, :sheet
       fields, rows = TSV._excel_data(tsv, options)
 
       book = Spreadsheet::Workbook.new
@@ -170,11 +172,11 @@ module TSV
 
   module XLSX
     def self.read(file, options = {})
-      options = Misc.add_defaults options, :sep2 => /[,|]\s?/, :skip_rows => 0
-      sheet = Misc.process_options options, :sheet
-      header = Misc.process_options options, :header
-      text = Misc.process_options options, :text
-      skip_rows = Misc.process_options options, :skip_rows
+      options = IndiferentHash.add_defaults options, :sep2 => /[,|]\s?/, :skip_rows => 0
+      sheet = IndiferentHash.process_options options, :sheet
+      header = IndiferentHash.process_options options, :header
+      text = IndiferentHash.process_options options, :text
+      skip_rows = IndiferentHash.process_options options, :skip_rows
       skip_rows = skip_rows.to_i
 
       header = true unless header == false
@@ -187,7 +189,7 @@ module TSV
       sheet_name = sheet
       Log.debug "Opening XLSX #{file} sheet #{ sheet_name }"
 
-      TmpFile.with_file :extension => Misc.sanitize_filename(sheet_name.to_s) do |filename|
+      TmpFile.with_file :extension => Path.sanitize_filename(sheet_name.to_s) do |filename|
 
         sheet    = sheet_name ? workbook[sheet_name] : workbook.worksheets.first
 
@@ -223,7 +225,7 @@ module TSV
     end
 
     def self.write(tsv, file, options = {})
-      sheet, add_sheet = Misc.process_options options, :sheet, :add_sheet
+      sheet, add_sheet = IndiferentHash.process_options options, :sheet, :add_sheet
 
       fields, rows = TSV._excel_data(tsv, options)
 

data/lib/rbbt/tsv/field_index.rb

@@ -13,6 +13,8 @@ module TSV
   def field_index(field)
     @field_indices ||= {}
     @field_indices[field] ||= Persist.persist_tsv(self, filename, {:field => field}, :prefix => "FieldIndex", :dir => TSV.field_index_dir, :persist => true, :serializer => :list, :engine => "BDB" ) do |data|
+      data.serializer = :flat
+
       tsv = {}
       case type 
       when :single, :list
@@ -30,6 +32,7 @@ module TSV
         end
       end
 
+      TSV.setup(data, :key_field => field, :fields => ["Keys"], :type => :flat)
       tsv.each do |v,keys|
         data[v] = keys.sort
       end
@@ -44,7 +47,7 @@ module TSV
       i = field_index(field)
 
       if Array === values
-        keys = values.inject([]){|acc,value| m = i[value]; acc = m.nil? ? acc : Misc.merge_sorted_arrays( acc, m) }
+        keys = values.inject([]){|acc,value| m = i[value]; acc = m.nil? ? acc : Misc.merge_sorted_arrays(acc, m) }
       else
         keys = i[values] || []
       end

data/lib/rbbt/tsv/filter.rb

@@ -32,7 +32,8 @@ module Filtered
       when Hash === persistence
         @persistence = persistence
       when String === persistence
-        @persistence = TSV.setup Persist.open_tokyocabinet(persistence, false, :list)
+        @persistence = Persist.open_tokyocabinet(persistence, false, :list)
+        @persistence = TSV.setup(@persistence)
         @persistence.read
       end
 
@@ -192,7 +193,7 @@ module Filtered
     if filters.empty?
       unfiltered_filename
     else
-      unfiltered_filename + ":Filtered[#{filters.collect{|f| [f.match, Array === f.value ? Misc.hash2md5(:values => f.value) : f.value] * "="} * ", "}]"
+      unfiltered_filename + ":Filtered[#{filters.collect{|f| [f.match, Array === f.value ? Misc.digest(:values => f.value) : f.value] * "="} * ", "}]"
     end
   end