rbbt-util 4.0.2 → 4.1.0

data/lib/rbbt/tsv/attach/util.rb

@@ -3,17 +3,22 @@ module TSV
   def attach_same_key(other, fields = nil)
     fields = other.fields - [key_field].concat(self.fields) if fields.nil?
 
-    through do |key, values|
-      if other.include? key
-        new_values = other[key].values_at *fields
-        new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
-        new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
-        self[key] = self[key].concat new_values
-      else
-        if type == :double
-          self[key] = self[key].concat [[]] * fields.length
-        else
-          self[key] = self[key].concat [""] * fields.length
+    field_positions = fields.collect{|field| other.identify_field field}
+    other.with_unnamed do
+      with_unnamed do
+        through do |key, values|
+          if other.include? key
+            new_values = other[key].values_at *field_positions
+            new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
+            new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
+            self[key] = self[key].concat new_values
+          else
+            if type == :double
+              self[key] = self[key].concat [[]] * fields.length
+            else
+              self[key] = self[key].concat [""] * fields.length
+            end
+          end
         end
       end
     end
@@ -33,59 +38,61 @@ module TSV
 
     source_pos = identify_field source
 
-    with_unnamed do
-      through do |key, values|
-        source_keys = values[source_pos]
+    other.with_unnamed do
+      with_unnamed do
+        through do |key, values|
+          source_keys = values[source_pos]
+
+          case
+          when (source_keys.nil? or (Array === source_keys and source_keys.empty?))
+            if type == :double
+              self[key] = values.concat field_positions.collect{|v| []}
+            else
+              self[key] = values.concat [nil] * field_positions
+            end
+          when Array === source_keys
+            all_new_values = source_keys.collect do |source_key|
+              positions = field_positions.collect do |pos|
+                if pos == :key
+                  [source_key]
+                else
+                  if other.include? source_key
+                    v = other[source_key][pos]
+                    Array === v ? v : [v]
+                  else
+                    [nil]
+                  end
+                end
+              end
+
+              positions.collect!{|v| v[0..0]} if one2one
+              positions
+            end
+
+            new = Misc.zip_fields(all_new_values).each{|field_entry|
+              field_entry.flatten!
+            }
 
-        case
-        when (source_keys.nil? or (Array === source_keys and source_keys.empty?))
-          if type == :double
-            self[key] = values.concat field_positions.collect{|v| []}
+            self[key] = values.concat new
           else
-            self[key] = values.concat [nil] * field_positions
-          end
-        when Array === source_keys
-          all_new_values = source_keys.collect do |source_key|
-            positions = field_positions.collect do |pos|
+            source_key = source_keys
+            all_new_values = field_positions.collect do |pos|
               if pos == :key
-                [source_key]
+                source_key
               else
                 if other.include? source_key
                   v = other[source_key][pos]
-                  Array === v ? v : [v]
+                  Array === v ? v.first : v
                 else
-                  [nil]
+                  nil
                 end
               end
             end
 
-            positions.collect!{|v| v[0..0]} if one2one
-            positions
-          end
-
-          new = Misc.zip_fields(all_new_values).each{|field_entry|
-            field_entry.flatten!
-          }
-
-          self[key] = values.concat new
-        else
-          source_key = source_keys
-          all_new_values = field_positions.collect do |pos|
-            if pos == :key
-              source_key
-            else
-              if other.include? source_key
-                v = other[source_key][pos]
-                Array === v ? v.first : v
-              else
-                nil
-              end
-            end
+            self[key] = values.concat all_new_values
           end
 
-          self[key] = values.concat all_new_values
         end
-
       end
     end
 
@@ -102,55 +109,61 @@ module TSV
     field_names     = field_positions.collect{|pos| pos == :key ? other.key_field : other.fields[pos] }
 
     length = self.fields.length
-    through do |key, values|
-      source_keys = index[key]
-      if source_keys.nil? or source_keys.empty?
-        all_new_values = []
-      else
-        all_new_values = []
-        source_keys.each do |source_key|
-          next unless other.include? source_key
-          new_values = field_positions.collect do |pos|
-            if pos == :key
-              if other.type == :double
-                [source_key]
+    other.with_unnamed do
+      index.with_unnamed do
+        with_unnamed do
+          through do |key, values|
+            source_keys = index[key]
+            if source_keys.nil? or source_keys.empty?
+              all_new_values = []
+            else
+              all_new_values = []
+              source_keys.each do |source_key|
+                next unless other.include? source_key
+                new_values = field_positions.collect do |pos|
+                  if pos == :key
+                    if other.type == :double
+                      [source_key]
+                    else
+                      source_key
+                    end
+                  else
+                    other[source_key][pos]
+                  end
+                end
+                new_values.collect!{|v| v.nil? ? [[]] : [v]}    if     type == :double and not other.type == :double
+                new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
+                all_new_values << new_values
+              end
+            end
+
+            if all_new_values.empty?
+              if type == :double
+                all_new_values = [[[]] * field_positions.length]
               else
-                source_key
+                all_new_values = [[""] * field_positions.length]
               end
-            else
-              other[source_key][pos]
             end
-          end
-          new_values.collect!{|v| v.nil? ? [[]] : [v]}    if     type == :double and not other.type == :double
-          new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
-          all_new_values << new_values
-        end
-      end
 
-      if all_new_values.empty?
-        if type == :double
-          all_new_values = [[[]] * field_positions.length]
-        else
-          all_new_values = [[""] * field_positions.length]
-        end
-      end
+            current = self[key] || [[]] * fields.length
 
-      current = self[key] || [[]] * fields.length
+            if current.length > length
+              all_new_values << current.slice!(length..current.length - 1)
+            end
 
-      if current.length > length
-        all_new_values << current.slice!(length..current.length - 1)
-      end
+            if type == :double
+              all_new_values = TSV.zip_fields(all_new_values).collect{|l| l.flatten}
+            else
+              all_new_values = all_new_values.first
+            end
 
-      if type == :double
-        all_new_values = TSV.zip_fields(all_new_values).collect{|l| l.flatten}
-      else
-        all_new_values = all_new_values.first
-      end
-     
-      current += all_new_values
+            current += all_new_values
 
-      self[key] = current
+            self[key] = current
 
+          end
+        end
+      end
     end
 
     self.fields = self.fields.concat field_names
@@ -184,7 +197,7 @@ module TSV
       return nil if match.empty?
       id_list << match.first
     end
-    
+
     if id_list.last != files.last.all_fields.first
       id_list << files.last.all_fields.first
       id_list.zip(files)
@@ -201,18 +214,18 @@ module TSV
     path = find_path(files, options)
 
     return nil if path.nil?
-    
+
     traversal_ids = path.collect{|p| p.first}
 
     Log.low "Found Traversal: #{traversal_ids * " => "}"
-    
+
     data_key, data_file = path.shift
     data_index = if data_key == data_file.key_field
                    Log.debug "Data index not required '#{data_file.key_field}' => '#{data_key}'"
                    nil
                  else
                    Log.debug "Data index required"
-                   data_file.index :target => data_key, :fields => data_file.key_field, :persist => false
+                   data_file.index :target => data_key, :fields => [data_file.key_field], :persist => false
                  end
 
     current_index = data_index
@@ -221,27 +234,32 @@ module TSV
       next_key, next_file = path.shift
 
       if current_index.nil?
-        current_index = next_file.index(:target => next_key, :fields => current_key, :persist => persist_input)
+        current_index = next_file.index(:target => next_key, :fields => [current_key], :persist => persist_input)
       else
-        next_index = next_file.index :target => next_key, :fields => current_key, :persist => persist_input
+        next_index = next_file.index :target => next_key, :fields => [current_key], :persist => persist_input
 
         if TokyoCabinet::HDB === current_index
-          tmp = TSV.setup({}, :key_field => current_index.key_field, :fields => current_index.fields, :serializer => current_index.serializer, :type => current_index.type, :filename => current_index.filename)
-          current_index.unnamed = true
-          current_index.each do |key,value|
-            tmp.tsv_clean_set_brackets(key, current_index.tsv_clean_get_brackets(key))
+          tmp = TSV.setup({}, :key_field => current_index.key_field, :fields => [current_index.fields], :serializer => current_index.serializer, :type => current_index.type, :filename => current_index.filename)
+          current_index.with_unnamed do
+            current_index.each do |key,value|
+              tmp.tsv_clean_set_brackets(key, current_index.tsv_clean_get_brackets(key))
+            end
           end
           current_index = tmp
         end
 
-        current_index.process current_index.fields.first do |values|
-          if values.nil?
-            nil
-          else
-            next_index.values_at(*values).flatten.collect
+        next_index.with_unnamed do
+          current_index.with_unnamed do
+            current_index.process current_index.fields.first do |values|
+              if values.nil?
+                nil
+              else
+                next_index.values_at(*values).flatten.collect
+              end
+            end
+            current_index.fields = [next_key]
           end
         end
-        current_index.fields = [next_key]
       end
       current_key = next_key
     end

data/lib/rbbt/tsv/filter.rb

@@ -76,8 +76,10 @@ module Filtered
     def update
       ids = []
 
-      data.unfiltered_each do |key, entry|
-        ids << key if match_entry(entry)
+      data.with_unnamed do
+        data.unfiltered_each do |key, entry|
+          ids << key if match_entry(entry)
+        end
       end
 
       save(ids.sort)

data/lib/rbbt/tsv/manipulate.rb

@@ -53,6 +53,12 @@ module TSV
       ]
     end
 
+    def process_reorder_flat(key, values)
+      [ values,
+        @new_fields.collect{|field| field == :key ?  
+          [key] : values[field] }.flatten
+      ]
+    end
     def initialize(key_field, fields, new_key_field, new_fields, type, uniq)
       @new_key_field = TSV.identify_field(key_field, fields, new_key_field)
 
@@ -128,12 +134,15 @@ module TSV
           end
         end
       else
-        if type == :double
+        case type 
+        when :double
           if uniq
             self.instance_eval do alias process process_reorder_double_uniq end
           else
             self.instance_eval do alias process process_reorder_double end
           end
+        when :flat
+          self.instance_eval do alias process process_reorder_flat end
         else
           self.instance_eval do alias process process_reorder_list end
         end
@@ -144,9 +153,9 @@ module TSV
 
   #{{{ Methods
 
-  def through(new_key_field = nil, new_fields = nil, uniq = false)
+  def through(new_key_field = nil, new_fields = nil, uniq = false, zipped = false)
 
-    traverser = Traverser.new @key_field, @fields, new_key_field, new_fields, type, uniq
+    traverser = Traverser.new key_field, fields, new_key_field, new_fields, type, uniq
 
     if @monitor
       desc = "Iterating TSV"
@@ -164,25 +173,71 @@ module TSV
       progress_monitor.tick if progress_monitor
 
       keys, value = traverser.process(key, value)
+
+      # Annotated with Entity and NamedArray
+      if not @unnamed
+        if not traverser.new_field_names.nil? 
+          case type
+          when :double, :list
+            NamedArray.setup value, traverser.new_field_names 
+          when :flat, :single
+            Entity.formats[traverser.new_field_names.first].setup(value, :format => traverser.new_field_names.first) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? traverser.new_field_names
+          end
+        end
+      end
+
       next if keys.nil?
-      keys.each do |key|
-        NamedArray.setup value, traverser.new_field_names if Array === value and not @unnamed
-        yield key, value
+
+      if zipped
+
+        keys.each_with_index do |k,i|
+          v = value.collect{|v|
+            r = v[i]
+            r = v[0] if r.nil?
+            r
+          }
+
+          if not @unnamed and defined?(Entity) and not traverser.new_key_field_name.nil? and Entity.respond_to?(:formats) and Entity.formats.include? traverser.new_key_field_name
+            k = Entity.formats[traverser.new_key_field_name].setup(k.dup, :format => traverser.new_key_field_name) 
+          end
+          v.key = k if NamedArray === v
+          yield k, v
+ 
+        end
+
+      else
+        keys.each do |key|
+          if not @unnamed and defined?(Entity) and not traverser.new_key_field_name.nil? and Entity.respond_to?(:formats) and Entity.formats.include? traverser.new_key_field_name
+            key = Entity.formats[traverser.new_key_field_name].setup(key.dup, :format => traverser.new_key_field_name) 
+          end
+          value.key = key if NamedArray === value
+          yield key, value
+        end
       end
     end
 
     [traverser.new_key_field_name, traverser.new_field_names]
   end
 
-  def reorder(new_key_field = nil, new_fields = nil, persist = false)
-    Persist.persist_tsv self, self.filename, {:key_field => new_key_field, :fields => new_fields}, {:persist => persist, :persist_prefix => "Reorder:"} do |data|
+  def reorder(new_key_field = nil, new_fields = nil, options = {}) 
+    zipped, uniq = Misc.process_options options, :zipped, :uniq
+
+    persist_options = Misc.pull_keys options, :persist
+    persist_options[:prefix] = "Reorder"
+
+    Persist.persist_tsv self, self.filename, {:key_field => new_key_field, :fields => new_fields}, persist_options do |data|
 
       with_unnamed do
-        new_key_field_name, new_field_names = through new_key_field, new_fields do |key, value|
-          if data.include?(key) and type == :double
-            data[key] = data[key].zip(value).collect do |old_list, new_list| old_list + new_list end
+        new_key_field_name, new_field_names = through new_key_field, new_fields, uniq, zipped do |key, value|
+          if data.include?(key) and not zipped
+            case type 
+            when :double
+              data[key] = data[key].zip(value).collect do |old_list, new_list| old_list + new_list end
+            when :flat
+              data[key].concat value
+            end
           else
-            data[key] = value
+            data[key] = value.dup
           end
         end
 
@@ -367,7 +422,7 @@ module TSV
       when type == :flat
         self[key] = new_values
       else
-        values[field].replace new_values
+        values[field_pos].replace new_values
         self[key] = values
       end
     end