rbbt-util 5.4.1 → 5.5.0

data/lib/rbbt/persist.rb

@@ -43,7 +43,7 @@ module Persist
 
   def self.persistence_path(file, persist_options = {}, options = {})
     persistence_file = Misc.process_options persist_options, :file
-    return persistence_file if not persistence_file.nil?
+    return persistence_file unless persistence_file.nil?
 
     prefix = Misc.process_options persist_options, :prefix
 

data/lib/rbbt/persist/tsv.rb

@@ -1,4 +1,8 @@
-require 'tokyocabinet'
+begin
+  require 'tokyocabinet'
+rescue Exception
+  Log.warn "The tokyocabinet gem could not be loaded: persistence over TSV files will fail"
+end
 
 module Persist
   TC_CONNECTIONS = {}
@@ -85,6 +89,20 @@ module Persist
           out(key)
         end
 
+        def write_and_read
+          lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+          Misc.lock(lock_filename) do
+            write if @closed or not write?
+            res = begin
+                    yield
+                  ensure
+                    read
+                  end
+            res
+          end
+        end
+
+
         def write_and_close
           lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
           Misc.lock(lock_filename) do
@@ -148,10 +166,12 @@ module Persist
              filename ||= case
                           when Path === source
                             source
-                          when source.respond_to?(:filename)
+                          when (source.respond_to?(:filename) and source.filename)
                             source.filename
                           when source.respond_to?(:cmd)
                             "CMD-#{Misc.digest(source.cmd)}"
+                          when TSV === source
+                            "TSV[#{Misc.digest Misc.fingerprint(source)}]"
                           else
                             source.object_id.to_s
                           end

data/lib/rbbt/resource.rb

@@ -1,6 +1,5 @@
 require 'rbbt/util/open'
 require 'rbbt/util/log'
-require 'rbbt/util/chain_methods'
 require 'rbbt/resource/path'
  
 module Resource

data/lib/rbbt/resource/path.rb

@@ -1,5 +1,5 @@
 require 'rbbt/resource/util'
-require 'rbbt/tsv'
+require 'yaml'
 
 module Path
   attr_accessor :resource, :pkgdir

data/lib/rbbt/resource/util.rb

@@ -1,4 +1,3 @@
-
 module Path
 
   def self.caller_lib_dir(file = nil, relative_to = 'lib')

data/lib/rbbt/tsv.rb

@@ -1,10 +1,9 @@
-require 'yaml'
+require 'rbbt/persist'
+require 'rbbt/persist/tsv'
 
+require 'rbbt/util/log'
 require 'rbbt/util/misc'
 require 'rbbt/util/named_array'
-require 'rbbt/util/log'
-
-require 'rbbt/persist'
 
 require 'rbbt/tsv/util'
 require 'rbbt/tsv/serializers'
@@ -36,6 +35,8 @@ module TSV
       hash.send("#{ entry }=", options[entry.to_sym]) if options.include? entry.to_sym
     end
 
+    hash.unnamed = options[:unnamed]
+
     hash
   end
 
@@ -95,17 +96,19 @@ module TSV
       stream.open do |f|
         Parser.new f, options
       end
-    when (String === stream and stream.length < 300 and Open.exists? stream)
+    when (String === stream and stream.length < 300 and Open.exists? stream or Open.remote? stream)
       Open.open(stream) do |f|
         Parser.new f, options
       end
     else
+      filename = stream.respond_to?(:filename) ? stream.filename : Misc.fingerprint(stream)
+      Log.debug("Parsing header of open stream: #{filename}")
       Parser.new stream, options
     end
   end
 
   def self.parse(stream, data, options = {})
-    monitor, grep, invert_grep = Misc.process_options options, :monitor, :grep, :invert_grep
+    monitor, grep, invert_grep, head = Misc.process_options options, :monitor, :grep, :invert_grep, :head
 
     parser = Parser.new stream, options
 
@@ -117,19 +120,14 @@ module TSV
 
     line = parser.rescue_first_line
 
-    if TokyoCabinet::HDB === data and parser.straight
+    if TokyoCabinet::HDB === data and parser.straight and
       data.close
-      pos = stream.pos if stream.respond_to? :pos
       begin
+        CMD.cmd('tchmgr', :log => false)
+        FileUtils.mkdir_p File.dirname(data.persistence_path)
         CMD.cmd("tchmgr importtsv '#{data.persistence_path}'", :in => stream, :log => false, :dont_close_in => true)
       rescue
         Log.debug("tchmgr importtsv failed for: #{data.persistence_path}")
-        Log.debug($!.message)
-        if stream.respond_to? :seek 
-          stream.seek pos
-        else
-          #raise "tchmgr import failed and cannot restore stream"
-        end
       end
       data.write
     end
@@ -170,6 +168,7 @@ module TSV
       progress_monitor = nil
     end
 
+    line_num = 1
     while not line.nil? 
       begin
         progress_monitor.tick(stream.pos) if progress_monitor 
@@ -183,6 +182,8 @@ module TSV
         values = parser.cast_values values if parser.cast?
         parser.add_to_data data, key, values
         line = stream.gets
+        line_num += 1
+        raise Parser::END_PARSING if head and line_num > head.to_i
       rescue Parser::SKIP_LINE
         begin
           line = stream.gets

data/lib/rbbt/tsv/accessor.rb

@@ -1,4 +1,3 @@
-#require 'rbbt/util/chain_methods'
 require 'yaml'
 module TSV
   #extend ChainMethods
@@ -35,7 +34,6 @@ module TSV
     return entity if entity.nil?
     return entity unless defined? Entity
     entity = entity if options.delete :dup_array
-    entity_templates
     if (template = entity_templates[field])
       entity = template.annotate(entity.frozen? ? entity.dup : entity)
       entity.extend AnnotatedArray if Array === entity
@@ -263,7 +261,7 @@ module TSV
         end
       when :list, :flat
         through :key, field do |key, fields|
-          elems << [key, fields]
+          elems << [key, fields.first]
         end
       when :double
         through :key, field do |key, fields|
@@ -481,19 +479,22 @@ end
 
 
   def all_fields
+    return nil if key_field.nil? or fields.nil?
     [key_field] + fields
   end
 
   def values_to_s(values)
-    case
-    when (values.nil? and fields.nil?)
-      "\n"
-    when (values.nil? and not fields.nil?)
-      "\t" << ([""] * fields.length) * "\t" << "\n"
-    when (not Array === values)
-      "\t" << values.to_s << "\n"
-    else
+    case values
+    when nil
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << ([""] * fields.length) * "\t" << "\n"
+      end
+    when Array
       "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+    else
+      "\t" << values.to_s << "\n"
     end
   end
 
@@ -541,14 +542,18 @@ end
   end
 
   def summary
-    <<-EOF
-Key field = #{key_field}
-Fields = #{fields * ", "}
+    with_unnamed do
+      <<-EOF
+Key field = #{key_field || "*No key field*"}
+Fields = #{fields ? Misc.fingerprint(fields) : "*No field info*"}
 Type = #{type}
+Size = #{size}
+namespace = #{namespace}
 Example:
-  - #{key = keys.first}: #{self[key].inspect}
+  - #{key = keys.first}: #{Misc.fingerprint self[key] }
 
-    EOF
+  EOF
+    end
   end
 
   def to_hash

data/lib/rbbt/tsv/attach.rb

@@ -144,9 +144,9 @@ module TSV
     in_namespace = options[:in_namespace]
 
     unless TSV === other
-      other_identifier_files = other.identifier_files if other.respond_to? :identifier_files
+      other_identifier_file = other.identifier_files.first if other.respond_to? :identifier_files
       other = TSV.open(other, :persist => options[:persist_input] == true)
-      other.identifiers = other_identifier_files 
+      other.identifiers = other_identifier_file
     end
 
     fields = other.fields - [key_field].concat(self.fields) if fields.nil?  or fields == :all 
@@ -157,10 +157,10 @@ module TSV
     end
 
     other_filename = other.respond_to?(:filename) ? other.filename : other.inspect
-    Log.low("Attaching fields:#{fields.inspect} from #{other_filename}.")
+    Log.low("Attaching fields:#{Misc.fingerprint fields } from #{other_filename}.")
 
     case
-    when key_field == other.key_field
+    when key_field == other.key_field 
       Log.debug "Attachment with same key: #{other.key_field}"
       attach_same_key other, fields
     when (not in_namespace and self.fields.include?(other.key_field))
@@ -175,7 +175,7 @@ module TSV
       Log.debug "Attachment with index: #{other.key_field}"
       attach_index other, index, fields
     end
-    Log.debug("Attachment of fields:#{fields.inspect} from #{other.filename.inspect} finished.")
+    Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
 
     self
   end

data/lib/rbbt/tsv/attach/util.rb

@@ -36,7 +36,7 @@ module TSV
             if type == :double
               self[key] = current.concat [[]] * fields.length
             else
-              self[key] = current.concat [""] * fields.length
+              self[key] = current.concat [nil] * fields.length
             end
           end
         end
@@ -46,6 +46,8 @@ module TSV
     self.type = :list if self.type == :single
 
     self.fields = self.fields.concat fields
+
+    self
   end
 
   def attach_source_key(other, source, options = {})
@@ -169,7 +171,7 @@ module TSV
               if type == :double
                 all_new_values = [[[]] * field_positions.length]
               else
-                all_new_values = [[""] * field_positions.length]
+                all_new_values = [[nil] * field_positions.length]
               end
             end
 

data/lib/rbbt/tsv/change_id.rb

@@ -0,0 +1,67 @@
+require 'rbbt/tsv'
+
+module TSV
+  def self.change_key(tsv, format, options = {})
+    options = Misc.add_defaults options, :persist => false, :identifiers => tsv.identifiers
+
+    identifiers, persist_input = Misc.process_options options, :identifiers, :persist_input
+
+    if not tsv.fields.include? format
+      tsv = tsv.annotate(Hash[*tsv.keys.zip(tsv.values.collect{|l| l.dup}).flatten(1)]) 
+
+      orig_type = tsv.type 
+      tsv = tsv.to_double if orig_type != :double
+
+      tsv = tsv.attach identifiers, :fields => [format], :persist_input => true
+      tsv = tsv.reorder(format, tsv.fields - [format])
+
+      tsv = tsv.to_flat  if orig_type == :flat
+
+      tsv
+    else
+      tsv.reorder(format)
+    end
+  end
+
+  def change_key(*args)
+    TSV.change_key(self, *args)
+  end
+
+  def self.swap_id(tsv, field, format, options = {})
+    options = Misc.add_defaults options, :persist => false, :identifiers => tsv.identifiers
+
+    identifiers, persist_input = Misc.process_options options, :identifiers, :persist
+
+    index = identifiers.index :target => format, :fields => [field], :persist => persist_input
+
+    orig_type = tsv.type 
+    tsv = tsv.to_double if orig_type != :double
+
+    pos = tsv.fields.index field
+    tsv.with_unnamed do
+      if tsv.type == :list or tsv.type == :single
+        tsv.through do |k,v|
+          v[pos] = index[v[pos]]
+          tsv[k] = v
+        end
+      else
+        tsv.through do |k,v|
+          v[pos] = index.values_at(*v[pos])
+          tsv[k] = v
+        end
+      end
+      
+      tsv.fields = tsv.fields.collect{|f| f == field ? format : f}
+    end
+
+    tsv = tsv.to_flat  if orig_type == :flat
+
+    tsv
+  end
+
+  def swap_id(*args)
+    TSV.swap_id(self, *args)
+  end
+
+
+end

data/lib/rbbt/tsv/index.rb

@@ -1,9 +1,11 @@
-require 'rbbt/util/misc'
 require 'rbbt/fix_width_table'
+require 'rbbt/util/misc'
+
+require 'rbbt/persist'
+require 'rbbt/persist/tsv'
+
 require 'rbbt/tsv/manipulate'
 require 'rbbt/tsv/filter'
-require 'rbbt/persist/tsv'
-require 'rbbt/persist'
 
 module TSV
 

data/lib/rbbt/tsv/manipulate.rb

@@ -1,7 +1,4 @@
 require 'progress-bar'
-require 'rbbt/persist'
-require 'rbbt/tsv/util'
-require 'set'
 
 module TSV
   
@@ -59,6 +56,7 @@ module TSV
           [key] : values[field] }.flatten
       ]
     end
+
     def initialize(key_field, fields, new_key_field, new_fields, type, uniq)
       @new_key_field = TSV.identify_field(key_field, fields, new_key_field)
 
@@ -235,28 +233,55 @@ module TSV
     persist_options[:prefix] = "Reorder"
 
     Persist.persist_tsv self, self.filename, {:key_field => new_key_field, :fields => new_fields}, persist_options do |data|
+      if data.respond_to? :persistence_path
+        real_data = data 
+        data = {}
+      end
 
       with_unnamed do
-        new_key_field_name, new_field_names = through new_key_field, new_fields, uniq, zipped do |key, value|
-          if data.include?(key) and not zipped
-            case type 
-            when :double
-              data[key] = data[key].zip(value).collect do |old_list, new_list| old_list + new_list end
-            when :flat
+        if zipped or (type != :double and type != :flat)
+          new_key_field_name, new_field_names = through new_key_field, new_fields, uniq, zipped do |key, value|
+            data[key] = value.clone if Array === value
+          end
+        else
+          case type 
+          when :double
+            new_key_field_name, new_field_names = through new_key_field, new_fields, uniq, zipped do |key, value|
+              if data[key].nil?
+                #data[key] = value.collect{|v| v.dup}
+                data[key] = value.collect{|v| v.dup}
+              else
+                current = data[key]
+                value.each_with_index do |v, i|
+                  current[i].concat v
+                end
+                data[key] = current if data.respond_to? :tokyocabinet_class
+              end
+            end
+          when :flat
+            new_key_field_name, new_field_names = through new_key_field, new_fields, uniq, zipped do |key, value|
+              data[key] ||= []
               data[key].concat value
             end
-          else
-            data[key] = value.dup
           end
         end
 
+        if real_data and real_data.respond_to? :persistence_path
+          real_data.serializer = type if real_data.respond_to? :serializer
+          real_data.merge!(data)
+          data = real_data
+        end
+
         data.extend TSV unless TSV === data
         data.key_field = new_key_field_name
         data.fields = new_field_names
         data.filename = filename
         data.namespace = namespace
         data.entity_options = entity_options
-        data.entity_templates = entity_templates
+        data.entity_templates = {}
+        data.fields.each do |field|
+          data.entity_templates[field] = entity_templates[field] if entity_templates.include? field
+        end
         data.type = zipped ? :list : type
       end
     end
@@ -289,7 +314,7 @@ module TSV
     elems.sort_by{|k,v| v}.collect{|k,v| k}
   end
 
-  def select(method = nil, &block)
+  def select(method = nil, invert = false, &block)
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 
     new.key_field = key_field
@@ -303,7 +328,7 @@ module TSV
    case
     when (method.nil? and block_given?)
       through do |key, values|
-        new[key] = values if yield key, values
+        new[key] = values if invert ^ (yield key, values)
       end
     when Array === method
       method = Set.new method
@@ -311,22 +336,22 @@ module TSV
         case type
         when :single
           through do |key, value|
-            new[key] = value if method.include? key or method.include? value
+            new[key] = value if invert ^ (method.include? key or method.include? value)
           end
         when :list, :flat
           through do |key, values|
-            new[key] = values if method.include? key or (method & values).any?
+            new[key] = values if invert ^ (method.include? key or (method & values).any?)
           end
         else
           through do |key, values|
-            new[key] = values if method.include? key or (method & values.flatten).any?
+            new[key] = values if invert ^ (method.include? key or (method & values.flatten).any?)
           end
         end
       end
     when Regexp === method
       with_unnamed do
         through do |key, values|
-          new[key] = values if [key,values].flatten.select{|v| v =~ method}.any?
+          new[key] = values if invert ^ ([key,values].flatten.select{|v| v =~ method}.any?)
         end
       end
     when String === method
@@ -337,18 +362,18 @@ module TSV
             case
             when (method == key_field or method == :key)
               through do |key, values|
-                new[key] = values if yield(key)
+                new[key] = values if invert ^ (yield(key))
               end
             when (type == :single or type == :flat)
               through do |key, value|
-                new[key] = value if yield(value)
+                new[key] = value if invert ^ (yield(value))
               end
             else
               pos = identify_field method
               raise "Field #{ method } not identified. Available: #{ fields * ", " }" if pos.nil?
 
               through do |key, values|
-                new[key] = values if yield(values[pos])
+                new[key] = values if invert ^ (yield(values[pos]))
               end
             end
           end
@@ -357,28 +382,26 @@ module TSV
             case
             when (method == key_field or method == :key)
               through do |key, values|
-                new[key] = values if yield(key, key)
+                new[key] = values if invert ^ (yield(key, key))
               end
             when (type == :single or type == :flat)
               through do |key, value|
-                new[key] = value if yield(key, value)
+                new[key] = value if invert ^ (yield(key, value))
               end
             else
               pos = identify_field method
               through do |key, values|
-                new[key] = values if yield(key, values[pos])
+                new[key] = values if invert ^ (yield(key, values[pos]))
               end
             end
 
           end
-
         end
 
-
       else
         with_unnamed do
           through do |key, values|
-            new[key] = values if [key,values].flatten.select{|v| v == method}.any?
+            new[key] = values if invert ^ ([key,values].flatten.select{|v| v == method}.any?)
           end
         end
       end
@@ -389,7 +412,7 @@ module TSV
       when (Array === method and (key == :key or key_field == key))
         with_unnamed do
           method.each{|key| 
-            new[key] = self[key] if self.include? key
+            new[key] = self[key] if invert ^ (self.include? key)
           }
         end
       when Array === method
@@ -398,45 +421,68 @@ module TSV
           case type
           when :single
             through :key, key do |key, value|
-              new[key] = self[key] if method.include? value
+              new[key] = self[key] if invert ^ (method.include? value)
             end
           when :list
             through :key, key do |key, values|
-              new[key] = self[key] if method.include? values.first
+              new[key] = self[key] if invert ^ (method.include? values.first)
             end
           when :flat #untested
             through :key, key do |key, values|
-              new[key] = self[key] if (method & values.flatten).any?
+              new[key] = self[key] if invert ^ ((method & values.flatten).any?)
             end
           else
             through :key, key do |key, values|
-              new[key] = self[key] if (method & values.flatten).any?
+              new[key] = self[key] if invert ^ ((method & values.flatten).any?)
             end
           end
         end
+
       when Regexp === method
         with_unnamed do
           through :key, key do |key, values|
             values = [values] if type == :single
-            new[key] = self[key] if values.flatten.select{|v| v =~ method}.any?
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| v =~ method}.any?)
           end
         end
+
+      when (String === method and method =~ /name:(.*)/)
+        name = $1
+        if name.strip =~ /^\/(.*)\/$/
+          regexp = Regexp.new $1
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| v.name =~ regexp}.any?)
+          end
+        else
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| v.name == name}.any?)
+          end
+        end
+
       when String === method
         with_unnamed do
           through :key, key do |key, values|
             values = [values] if type == :single
-            new[key] = self[key] if values.flatten.select{|v| v == method}.any?
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| v == method}.any?)
+          end
+        end
+
+      when Fixnum === method
+        with_unnamed do
+          through :key, key do |key, values|
+            new[key] = self[key] if invert ^ (values.flatten.length >= method)
           end
         end
       when Proc === method
         with_unnamed do
           through :key, key do |key, values|
             values = [values] if type == :single
-            new[key] = self[key] if values.flatten.select{|v| method.call(v)}.any?
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| method.call(v)}.any?)
           end
         end
       end
- 
     end
 
     new
@@ -538,7 +584,7 @@ module TSV
         values = [new_values]
       when values.nil?  
         values = [nil] * fields.length + [new_values]
-      when NamedArray === values
+      when Array === values
         values += [new_values]
       else
         values << new_values