rbbt-util 5.26.169 → 5.26.170

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9518793ebcf5bad66987d7d3c57627708a5821eb6b9680b3e26be2c7ef210360
-  data.tar.gz: 8fabc19346ad08c78f898d1e377c652b38b71edfe51a81f831fc14f114cb78a4
+  metadata.gz: ec58135152ab8da64225ec6ee2c82630300f745e9a97da8bfbce6d1286792f28
+  data.tar.gz: 7ddc25495f7f37536aac591c645095acd176f36dcb68c7e2f24f53d8a8ad7354
 SHA512:
-  metadata.gz: 7c5b12556e4c8b4b844982378c7478916b2f6cef846c9f0f864405c1753440688593b9c0ce767ae01c32c0f5708df6fe09f0e8eb147037d098f3d029220fe972
-  data.tar.gz: 8be6d8482a4a1ce953f9fe9a73d66318a7dd0e2e7c03d665a1ccfaf43165239ea37c0823eace9448051d646cd2d493219aefbaf404633c5edbd2271d132cace7
+  metadata.gz: 11bd33815e5d784ef4c2fe8dca4580a6896a76aaa14fd7e67251ceb193e46472e25d292e5fec23116a39afe15c3951b83cd9696a3050b669c60ee7af6fd759bd
+  data.tar.gz: 87d3e4dfbf4286c8303b9bedd83462c5cb6f276802fd54068c9628a64bcff5d4a6cbde520de9495763f4cf008acc8465b1d4df1cc2955bdfe99c34cd42c9ae40

data/lib/rbbt/persist/tsv/adapter.rb

@@ -71,7 +71,7 @@ module Persist
       lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
       #mutex.synchronize do
       Misc.lock(lock_filename, true) do
-        write if closed? or not write?
+        write if closed? || ! write?
         res = begin
                 yield
               rescue Exception
@@ -86,7 +86,7 @@ module Persist
     end
 
     def read_and_close
-      read if closed? or not read?
+      read if closed? || ! read?
       res = begin
               yield
             ensure
@@ -106,5 +106,10 @@ module Persist
     def range(*args)
       super(*args) #- TSV::ENTRY_KEYS.to_a
     end
+
+    def include?(*args)
+      read if closed?
+      super(*args) #- TSV::ENTRY_KEYS.to_a
+    end
   end
 end

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -7,20 +7,23 @@ module Persist
 
     attr_accessor :tokyocabinet_class
 
-    def self.open(path, write, serializer, tokyocabinet_class = TokyoCabinet::HDB)
-      if String === tokyocabinet_class and tokyocabinet_class.include? ":big" 
+    def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
+      if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
         big = true
         tokyocabinet_class = tokyocabinet_class.split(":").first
       else
         big = false
       end
 
+      dir = File.dirname(File.expand_path(path))
+      Open.mkdir(dir) unless Open.exists?(dir)
+
       tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
       tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
       database = CONNECTIONS[path] ||= tokyocabinet_class.new
 
-      if big and not Open.exists? path
+      if big and not Open.exists?(path)
         database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
       end
 
@@ -32,34 +35,40 @@ module Persist
         raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
       end
 
-
       database.extend Persist::TCAdapter unless Persist::TCAdapter === database
       database.persistence_path ||= path
       database.tokyocabinet_class = tokyocabinet_class
 
       database.mutex = Mutex.new
+      database.read
+
+      CONNECTIONS[path] = database
+
       database
     end
 
     def close
       @closed = true
+      @writable = false
       super
     end
 
     def read(force = false)
-      return if not write? and not closed and not force
+      return if ! write? && ! closed && ! force
       self.close
       if !self.open(@persistence_path, tokyocabinet_class::OREADER)
         ecode = self.ecode
         raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
       end
+
       @writable = false
       @closed = false
+
       self
     end
 
     def write(force = true)
-      return if write? and not closed and not force
+      return if write? && ! closed && ! force
       self.close
 
       if !self.open(@persistence_path, tokyocabinet_class::OWRITER)
@@ -69,16 +78,18 @@ module Persist
 
       @writable = true
       @closed = false
+
       self
     end
+
   end
 
   def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
     write = true unless File.exist? path
 
-    FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
+    FileUtils.mkdir_p File.dirname(path) unless Open.exist?(File.dirname(path))
 
-    database = Persist::TCAdapter.open(path, write, serializer, tokyocabinet_class)
+    database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
 
     unless serializer == :clean
       TSV.setup database

data/lib/rbbt/tsv/csv.rb

@@ -0,0 +1,78 @@
+require 'csv'
+
+module TSV
+  def self.csv(obj, options = {}) 
+    options = Misc.add_defaults IndiferentHash.setup(options.dup), :headers => true, :type => :list
+    headers = options[:headers]
+
+    noheaders = ! headers
+
+    type = options.delete :type
+    merge = options.delete :merge
+    key_field = options.delete :key_field
+    fields = options.delete :fields
+    
+    if key_field || fields
+      orig_type = type
+      type = :double
+      merge = true
+    end
+
+    options[:headers] = false
+
+    csv = case obj
+          when Path
+            CSV.read obj.find.open, options
+          when String
+            if Misc.is_filename?(obj)
+              CSV.read obj, options
+            else
+              CSV.new obj, options
+            end
+          else
+            CSV.new obj, options
+          end
+
+    tsv = if noheaders
+            TSV.setup({}, :key_field => nil, :fields => nil, :type => type)
+          else
+            key, *csv_fields = csv.shift
+            TSV.setup({}, :key_field => key, :fields => csv_fields, :type => type)
+          end
+
+    csv.each_with_index do |row,i|
+      if noheaders
+        key, values = ["row-#{i}", row]
+      else
+        key, *values = row
+      end
+
+      case type
+      when :double, :flat
+        tsv.zip_new(key, values)
+      when :single
+        tsv[key] = values.first
+      when :list
+        tsv[key] = values
+      end
+    end
+
+    if key_field || fields
+      tsv = tsv.reorder(key_field, fields, :zipped => true, :merge => true)
+      if tsv.type != orig_type
+        tsv = case orig_type
+              when :list
+                tsv.to_list
+              when :single
+                tsv.to_single
+              when :list
+                tsv.to_list
+              when :flat
+                tsv.to_flat
+              end
+      end
+    end
+
+    tsv
+  end
+end

data/lib/rbbt/tsv/manipulate.rb

@@ -156,7 +156,6 @@ module TSV
         end
       end
     end
-
   end
 
   #{{{ Methods
@@ -620,8 +619,8 @@ module TSV
       when type == :flat
         self[key] = new_values
       else
-        if (String === values[field_pos] and String === new_values) or
-          (Array === values[field_pos] and Array === new_values) 
+        if ! values[field_pos].frozen? && ((String === values[field_pos] && String === new_values) ||
+          (Array === values[field_pos] && Array === new_values))
            values[field_pos].replace new_values
         else
           values[field_pos] = new_values

data/lib/rbbt/tsv/parser.rb

@@ -214,6 +214,7 @@ module TSV
     end
 
     def add_to_data_flat_merge_double(data, keys, values)
+      data.write
       keys.each do |key|
         if data.include? key
           data[key] = data[key].concat values
@@ -538,7 +539,7 @@ module TSV
     end
 
     def traverse(options = {})
-      monitor, bar, grep, invert_grep, head = Misc.process_options options, :monitor, :bar, :grep, :invert_grep, :head
+      monitor, bar, grep, invert_grep, head, fixed_grep = Misc.process_options options, :monitor, :bar, :grep, :invert_grep, :head, :fixed_grep
       monitor = bar if bar and monitor.nil?
       raise "No block given in TSV::Parser#traverse" unless block_given?
 
@@ -551,10 +552,10 @@ module TSV
 
       if @tsv_grep
 
-        stream = Open.grep(stream, @tsv_grep, invert_grep)
+        stream = Open.grep(stream, @tsv_grep, invert_grep, fixed_grep)
         stream.no_fail = true
         begin
-          match = Open.grep(StringIO.new(line), @tsv_grep, invert_grep).read
+          match = Open.grep(StringIO.new(line), @tsv_grep, invert_grep, fixed_grep).read
           line = stream.gets if match.empty?
         rescue Exception
           Log.exception $!

data/lib/rbbt/util/log/progress.rb

@@ -14,7 +14,12 @@ module Log
 
   class ProgressBar
 
+    class << self
+      attr_accessor :default_file
+    end
+
     attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes
+
     def initialize(max = nil, options = {})
       options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
       depth, num_reports, desc, io, severity, file, bytes, frequency = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency
@@ -37,6 +42,10 @@ module Log
       (@ticks * 100) / @max
     end
 
+    def file
+      @file || ProgressBar.default_file
+    end
+
     def init
       @start = @last_time = Time.now
       @last_count = 0

data/lib/rbbt/util/log/progress/report.rb

@@ -127,7 +127,7 @@ module Log
     def save
       info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
       info.delete_if{|k,v| v.nil?}
-      Open.write(@file, info.to_yaml)
+      Open.write(file, info.to_yaml)
     end
 
     def report(io = STDERR)
@@ -158,7 +158,7 @@ module Log
       @last_time = Time.now
       @last_count = ticks
       @last_percent = percent if max and max > 0
-      save if @file
+      save if file
     end
 
     def done(io = STDERR)
@@ -175,7 +175,7 @@ module Log
       done_msg << " - " << thr_msg 
       done_msg << Log.color(:magenta, " · " << desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
-      Open.rm @file if @file and Open.exists? @file
+      Open.rm file if file and Open.exists? file
     end
 
     def error(io = STDERR)
@@ -192,7 +192,7 @@ module Log
       done_msg << " - " << thr_msg
       done_msg << Log.color(:magenta, " · " << desc)      
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
-      Open.rm @file if @file and Open.exists? @file
+      Open.rm file if file and Open.exists? file
     end
   end
 end

data/lib/rbbt/util/misc/format.rb

@@ -163,11 +163,10 @@ module Misc
 
   def self.fixutf8(string)
     return nil if string.nil?
-    return string if (string.respond_to? :valid_encoding? and string.valid_encoding?) or
-    (string.respond_to? :valid_encoding and string.valid_encoding)
+    return string if string.respond_to?(:encoding) && string.encoding == "UTF-8" && (string.respond_to?(:valid_encoding?) && string.valid_encoding?) ||
+                     (string.respond_to?(:valid_encoding) && string.valid_encoding)
 
     if string.respond_to?(:encode)
-      #string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
       string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
     else
       require 'iconv'

data/lib/rbbt/util/open.rb

@@ -138,11 +138,15 @@ module Open
 
   # Grep
   
-  def self.grep(stream, grep, invert = false)
+  def self.grep(stream, grep, invert = false, fixed = nil)
     case 
     when Array === grep
       TmpFile.with_file(grep * "\n", false) do |f|
-        CMD.cmd("#{GREP_CMD} #{invert ? '-v' : ''} -", "-w" => true, "-F" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        if FalseClass === fixed
+          CMD.cmd("#{GREP_CMD} #{invert ? '-v' : ''} -", "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        else
+          CMD.cmd("#{GREP_CMD} #{invert ? '-v' : ''} -", "-w" => true, "-F" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        end
       end
     else
       CMD.cmd("#{GREP_CMD} #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})

data/share/rbbt_commands/tsv/keys

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+options = SOPT.setup <<EOF
+Read a TSV file and dump it
+
+$ rbbt tsv keys [options] <filename.tsv|->
+
+Use - to read from STDIN
+
+-k--key_field* Key field
+-h--help Print this help
+-tch--tokyocabinet File is a tokyocabinet hash database
+-tcb--tokyocabinet_bd File is a tokyocabinet B database
+
+EOF
+rbbt_usage and exit 0 if options[:help]
+
+file = ARGV.shift
+
+file = STDIN if file == '-'
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+  tsv.keys.each do |k|
+    puts k
+  end
+  exit
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+  tsv.keys.each do |k|
+    puts k
+  end
+  exit
+end
+
+parser = TSV::Parser.new TSV.get_stream(file), options.merge(:fields => [])
+
+options[:merge] = false if options[:merge] == "false"
+
+Thread.new do
+  line = parser.first_line
+  bar = Log::ProgressBar.new
+  while line 
+    bar.tick
+
+    line = Misc.fixutf8(line)
+    line = parser.process line
+    raise SKIP_LINE if line.empty?
+    parts = parser.chop_line line
+    key, values = parser.get_values parts
+    values = parser.cast_values values if parser.cast?
+    
+    puts key
+  end
+end

data/test/rbbt/tsv/test_csv.rb

@@ -0,0 +1,49 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/tsv/csv'
+
+class TestCSV < Test::Unit::TestCase
+  def test_csv
+    text =<<-EOF
+Key,FieldA,FieldB
+k1,a,b
+k2,aa,bb
+    EOF
+
+    tsv = TSV.csv(text)
+    assert_equal 'bb', tsv['k2']['FieldB']
+  end
+
+  def test_csv_key_field
+    text =<<-EOF
+Key,FieldA,FieldB
+k1,a,b
+k2,aa,bb
+    EOF
+
+    tsv = TSV.csv(text, :key_field => 'FieldA', :type => :list)
+    assert_equal 'bb', tsv['aa']['FieldB']
+  end
+
+  def test_csv_double
+    text =<<-EOF
+Key,FieldA,FieldB
+k1,a,b
+k2,aa,bb
+    EOF
+
+    tsv = TSV.csv(text, :key_field => 'FieldA', :type => :double)
+    assert_equal ['bb'], tsv['aa']['FieldB']
+  end
+
+  def test_csv_noheader
+    text =<<-EOF
+k1,a,b
+k2,aa,bb
+    EOF
+
+    tsv = TSV.csv(text, :headers => false)
+    assert_equal %w(k2 aa bb), tsv['row-1']
+  end
+
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.169
+  version: 5.26.170
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-09 00:00:00.000000000 Z
+date: 2020-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -241,6 +241,7 @@ files:
 - lib/rbbt/tsv/attach.rb
 - lib/rbbt/tsv/attach/util.rb
 - lib/rbbt/tsv/change_id.rb
+- lib/rbbt/tsv/csv.rb
 - lib/rbbt/tsv/dumper.rb
 - lib/rbbt/tsv/excel.rb
 - lib/rbbt/tsv/field_index.rb
@@ -394,6 +395,7 @@ files:
 - share/rbbt_commands/tsv/head
 - share/rbbt_commands/tsv/info
 - share/rbbt_commands/tsv/json
+- share/rbbt_commands/tsv/keys
 - share/rbbt_commands/tsv/query
 - share/rbbt_commands/tsv/read
 - share/rbbt_commands/tsv/read_excel
@@ -466,6 +468,7 @@ files:
 - test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_csv.rb
 - test/rbbt/tsv/test_excel.rb
 - test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_filter.rb
@@ -583,6 +586,7 @@ test_files:
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_parser.rb
+- test/rbbt/tsv/test_csv.rb
 - test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_field_index.rb