rbbt-util 4.0.2 → 4.1.0

data/README.rdoc

@@ -63,3 +63,10 @@
 
     echo "source(system('rbbt_Rutil.rb', intern =T));" >> $HOME/.Rprofile
 
+5 - If you plan to run workflows you might want to redefine the default directory to place your workflows (~/.workflows)
+    
+    :script
+
+    mkdir -p ~/git/workflows
+    echo "~/git/workflows/" > ~/.rbbt/etc/workflow_dir
+

data/lib/rbbt/annotations.rb

@@ -2,6 +2,8 @@ require 'rbbt/util/chain_methods'
 require 'json'
 module Annotated
   attr_accessor :annotation_types
+  attr_accessor :context
+  attr_accessor :container
 
   def self.extended(base)
     base.annotation_types ||= []
@@ -23,7 +25,7 @@ module Annotated
   end
 
   def id
-    Misc.hash2md5 info
+    Misc.hash2md5 info.merge :self => self
   end
 
   def self.load(object, info)
@@ -97,7 +99,7 @@ module Annotated
              end
     fields = fields.collect{|f| f.to_s}
 
-    tsv = TSV.setup({}, :key_field => "ID", :fields => fields)
+    tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :list )
 
     annotations.each do |annotation|
       tsv[annotation.id] = annotation.tsv_values(fields)
@@ -111,6 +113,15 @@ module Annotated
       Annotated.load_tsv_values(id, values, tsv.fields)
     end
   end
+
+  def make_list
+    new = [self]
+    annotation_types.each do |mod|
+      mod.setup(new, *info.values_at(*mod.annotations))
+    end
+    new.context = self.context
+    new
+  end
 end
 
 
@@ -177,13 +188,13 @@ module Annotation
   def setup(object, *values)
     object.extend self
 
-    all_annotations.zip(values).each do |name, value|
+    inputs = Misc.positional2hash(all_annotations, *values)
+    inputs.each do |name, value|
       object.send(name.to_s + '=', value)
     end
 
     object
   end
-
 end
 
 module AnnotatedArray
@@ -195,25 +206,60 @@ module AnnotatedArray
     annotation_types.each do |mod|
       mod.setup(value, *info.values_at(*mod.annotations))
     end
+    value.context = self.context
+    value.container = self
     value
   end
 
   def annotated_array_each
     annotated_array_clean_each do |value|
       annotation_types.each do |mod|
-        mod.setup(value, *info.values_at(*mod.annotations))
+        mod.setup(value, info)
       end
+      value.context = self.context
+      value.container = self
       yield value
     end
   end
 
   def annotated_array_collect
-    annotated_array_clean_collect do |value|
-      annotation_types.each do |mod|
-        mod.setup(value, *info.values_at(*mod.annotations))
-      end
-      yield value
+    res = []
+
+    annotated_array_each do |value|
+      res << yield(value)
     end
+
+    res
+  end
+
+  def annotated_array_select
+    res = []
+    annotated_array_each do |value|
+      res << value if yield(value)
+    end
+ 
+    annotation_types.each do |mod|
+      mod.setup(res, *info.values_at(*mod.annotations))
+    end
+    res.context = self.context
+    res.container = self.container
+
+    res
+  end
+
+  def annotated_array_reject
+    res = []
+    annotated_array_each do |value|
+      res << value unless yield(value)
+    end
+ 
+    annotation_types.each do |mod|
+      mod.setup(res, *info.values_at(*mod.annotations))
+    end
+    res.context = self.context
+    res.container = self.container
+
+    res
   end
 
   def annotated_array_subset(list)
@@ -221,6 +267,97 @@ module AnnotatedArray
     annotation_types.each do |mod|
       mod.setup(value, *info.values_at(*mod.annotations))
     end
+    value.context = self.context
+    value.container = self.container
+    value
+  end
+
+  def annotated_array_remove(list)
+    value = (self - list)
+    annotation_types.each do |mod|
+      mod.setup(value, *info.values_at(*mod.annotations))
+    end
+    value.context = self.context
+    value.container = self.container
+    value
+  end
+
+  def annotated_array_uniq
+    value = self.annotated_array_clean_uniq
+
+    annotation_types.each do |mod|
+      mod.setup(value, *info.values_at(*mod.annotations))
+    end
+
+    value.context = self.context
+    value.container = self.container
+    value
+  end
+  def annotated_array_flatten
+    value = self.annotated_array_clean_flatten.dup
+
+    annotation_types.each do |mod|
+      mod.setup(value, *info.values_at(*mod.annotations))
+    end
+
+    value.context = self.context
+    value.container = self.container
+    value
+  end
+
+  def annotated_array_reverse
+    value = self.annotated_array_clean_reverse
+    annotation_types.each do |mod|
+      mod.setup(value, *info.values_at(*mod.annotations))
+    end
+    value.context = self.context
+    value.container = self.container
+    value
+  end
+
+
+  def annotated_array_sort_by(&block)
+    value = self.annotated_array_clean_sort_by &block
+    annotation_types.each do |mod|
+      mod.setup(value, *info.values_at(*mod.annotations))
+    end
+    value.context = self.context
+    value.container = self.container
+    value
+  end
+
+  def annotated_array_sort
+    value = self.annotated_array_clean_sort
+
+    annotation_types.each do |mod|
+      mod.setup(value, *info.values_at(*mod.annotations))
+    end
+    value.context = self.context
+    value.container = self.container
+    value
+  end
+
+  def annotated_array_select_by(method)
+    case
+    when (Symbol === method or String === method)
+      method = self.send(method) 
+    when Array === method
+      method = method.dup
+    else
+      raise "Unknown format of method: of class #{method.class}"
+    end
+
+    value = []
+    
+    self.annotated_array_clean_each do |e|
+      value << e if method.shift
+    end
+
+    #annotation_types.each do |mod|
+    #  mod.setup(value, *info.values_at(*mod.annotations))
+    #end
+    #value.context = self.context
+    #value.container = self.container
     value
   end
 end

data/lib/rbbt/persist.rb

@@ -119,7 +119,11 @@ module Persist
     when :string, :text
       Open.write(path, content)
     when :array
-      Open.write(path, content * "\n")
+      if content.any?
+        Open.write(path, content * "\n" + "\n")
+      else
+        Open.write(path, "")
+      end
     when :marshal_tsv
       Open.write(path, Marshal.dump(content.dup))
     when :marshal

data/lib/rbbt/persist/tsv.rb

@@ -83,7 +83,7 @@ module Persist
     database.persistence_path ||= path
 
     TSV.setup database
-    database.serializer = serializer unless serializer.nil?
+    database.serializer = serializer || database.serializer 
 
     database
   end
@@ -100,6 +100,7 @@ module Persist
              filename ||= source.object_id.to_s
 
              path = persistence_path(filename, persist_options, options)
+
              if is_persisted? path
                Log.debug "TSV persistence up-to-date: #{ path }"
                return open_tokyocabinet(path, false) 
@@ -109,8 +110,8 @@ module Persist
 
              FileUtils.rm path if File.exists? path
 
-             data = open_tokyocabinet(path, true)
-             data.serializer = :type
+             data = open_tokyocabinet(path, true, persist_options[:serializer])
+             data.serializer = :type unless data.serializer
              data
            else
              data = {}

data/lib/rbbt/resource/path.rb

@@ -133,6 +133,14 @@ module Path
     TSV.open(self.produce, *args)
   end
 
+  def list
+    Open.read(self.produce.find).split "\n"
+  end
+
+  def keys(field = 0, sep = "\t")
+    Open.read(self.produce.find).split("\n").collect{|l| next if l =~ /^#/; l.split(sep, -1)[field]}.compact
+  end
+
   def yaml
     YAML.load self.open
   end
@@ -149,7 +157,6 @@ module Path
     TSV.pos_index(self.produce.find, pos, options)
   end
 
-
   def to_yaml(*args)
     self.to_s.to_yaml(*args)
   end

data/lib/rbbt/tsv.rb

@@ -90,12 +90,14 @@ module TSV
       data.serializer = case
                         when parser.cast.nil?
                           data.serializer = parser.type
-                        when (parser.cast == :to_i and parser.type == :list)
+                        when (parser.cast == :to_i and (parser.type == :list or parser.type == :flat))
                           data.serializer = :integer_array
                         when (parser.cast == :to_i and parser.type == :single)
                           data.serializer = :integer
                         when (parser.cast == :to_f and parser.type == :single)
                           data.serializer = :float
+                        when (parser.cast == :to_f and (parser.type == :list or parser.type == :flat))
+                          data.serializer = :float_array
                         end
     end
 
@@ -119,7 +121,6 @@ module TSV
 
     while not line.nil?
       begin
-
         progress_monitor.tick(stream.pos) if progress_monitor 
 
         line = parser.process line

data/lib/rbbt/tsv/accessor.rb

@@ -4,7 +4,7 @@ module TSV
   extend ChainMethods
   self.chain_prefix = :tsv
 
-  attr_accessor :unnamed
+  attr_accessor :unnamed, :serializer_module
 
   def with_unnamed
     saved_unnamed = @unnamed 
@@ -47,6 +47,13 @@ module TSV
         end
       end
     end
+
+    if not data.respond_to? :serialized_get
+      class << data
+        alias serialized_get tsv_clean_get_brackets
+        alias serialized_set tsv_clean_set_brackets
+      end
+    end
   end
 
   KEY_PREFIX = "__tsv_hash_"
@@ -54,67 +61,111 @@ module TSV
   ENTRIES = []
   ENTRY_KEYS = []
 
-  def serialized_get(key)
-    raise "Uninitialized serializer" if serializer == :type
-    serialized_value = tsv_clean_get_brackets(key) 
-    SERIALIZER_ALIAS[serializer.to_sym].load(serialized_value) unless serialized_value.nil?
-  end
-
-  def serialized_set(key, value)
-    raise "Uninitialized serializer" if serializer == :type
-    if value.nil?
-      tsv_clean_set_brackets(key, nil)
-    else
-      tsv_clean_set_brackets(key, SERIALIZER_ALIAS[serializer.to_sym].dump(value))
-    end
+  #{{{ Chained Methods
+  def tsv_empty?
+    length == 0
   end
 
-  #{{{ Chained Methods
   def tsv_get_brackets(key)
-    value = if serializer.nil?
-            tsv_clean_get_brackets(key)
-          else
-            serialized_get(key)
-          end
-
-    NamedArray.setup value, fields if Array === value and not @unnamed
+    value = serialized_get(key)
+    return value if @unnamed or fields.nil?
+
+    case type
+    when :double, :list
+      NamedArray.setup value, fields, key 
+    when :flat, :single
+      Entity.formats[fields.first].setup(value, :format => fields.first) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? fields.first
+    end
     value
   end
 
   def tsv_set_brackets(key,value)
-    if serializer.nil?
-      tsv_clean_set_brackets(key, value)
-    else
-      serialized_set(key, value)
-    end
+    serialized_set(key, value)
   end
 
   def tsv_keys
-    tsv_clean_keys - ENTRY_KEYS
+    keys = tsv_clean_keys - ENTRY_KEYS
+    return keys if @unnamed or key_field.nil?
+
+    if defined?(Entity) and  Entity.respond_to?(:formats) and Entity.formats.include? key_field
+      Entity.formats[key_field].setup(keys.collect{|k| k.dup}, :format => key_field)
+    else
+      keys
+    end
   end
 
   def tsv_values
     values = values_at(*keys)
-    values.each{|value| NamedArray.setup value, fields} if Array === values.first and not @unnamed
+    return values if @unnamed or fields.nil?
+
+    case type
+    when :double, :list
+      values.each{|value| NamedArray.setup value, fields }
+    when :flat, :single
+      values.each{|value| 
+        Entity.formats[fields.first].setup(value, :format => fields.first)
+      } if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? fields.first
+    end
+      
     values
   end
 
   def tsv_each
+    fields = self.fields
+
+    serializer = self.serializer
+    serializer_module = SERIALIZER_ALIAS[serializer] unless serializer.nil?
     tsv_clean_each do |key, value|
       next if ENTRY_KEYS.include? key
 
-      value = SERIALIZER_ALIAS[serializer].load(value) unless serializer.nil?
-      NamedArray.setup value, fields if Array === value and not @unnamed
+      # TODO Update this to be more efficient
+      value = serializer_module.load(value) unless serializer.nil?
+
+      # Annotated with Entity and NamedArray
+      if not @unnamed
+        if not fields.nil? 
+          case type
+          when :double, :list
+            NamedArray.setup value, fields, key if Array === value 
+          when :flat, :single
+            Entity.formats[fields.first].setup(value, :format => fields.first) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? fields.first
+          end
+        end
+        if defined?(Entity) and not key_field.nil? and Entity.respond_to?(:formats) and Entity.formats.include? key_field
+          key = Entity.formats[key_field].setup(key.dup, :format => key_field) 
+        end
+      end
+
       yield key, value if block_given?
       [key, value]
     end
   end
 
   def tsv_collect
+    serializer = self.serializer
+    serializer_module = SERIALIZER_ALIAS[serializer] unless serializer.nil?
     tsv_clean_collect do |key, value|
       next if ENTRY_KEYS.include? key
-      value = SERIALIZER_ALIAS[serializer].load(value) unless serializer.nil? or not String === value 
-      NamedArray.setup value, fields if Array === value and not @unnamed
+
+      # TODO Update this to be more efficient
+      value = serializer_module.load(value) unless serializer.nil?
+
+      # Annotated with Entity and NamedArray
+      if not @unnamed
+        if not fields.nil? 
+          case type
+          when :double, :list
+            NamedArray.setup value, fields, key if Array === value 
+          when :flat, :single
+            Entity.formats[fields.first].setup(value, :format => fields.first) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? fields.first
+          end
+        end
+        if defined?(Entity) and not key_field.nil? and Entity.respond_to?(:formats) and Entity.formats.include? key_field
+          key = Entity.formats[key_field].setup(key.dup, :format => key_field) 
+        end
+      end
+
+
       if block_given?
         yield key, value
       else
@@ -229,9 +280,42 @@ def #{ entry }
   @#{entry}
 end
 
-def #{ entry }=(value)
-  @#{entry} = value
-  self.tsv_clean_set_brackets '#{key}', value.to_yaml
+
+if '#{entry}' == 'serializer'
+
+  def #{ entry }=(value)
+    @#{entry} = value
+    self.tsv_clean_set_brackets '#{key}', value.to_yaml
+
+    return if value.nil?
+
+    self.serializer_module = SERIALIZER_ALIAS[value.to_sym]
+    
+    if serializer_module.nil?
+      class << self
+        alias serialized_get tsv_clean_get_brackets
+        alias serialized_set tsv_clean_set_brackets
+      end
+
+    else
+      class << self
+        
+        define_method :serialized_get do |key|
+          self.serializer_module.load(tsv_clean_get_brackets(key))
+        end
+
+        define_method :serialized_set do |key, value|
+          tsv_clean_set_brackets key, self.serializer_module.dump(value)
+        end
+      end
+    end
+
+  end
+else
+  def #{ entry }=(value)
+    @#{entry} = value
+    self.tsv_clean_set_brackets '#{key}', value.to_yaml
+  end
 end
 "
     end
@@ -323,27 +407,26 @@ end
 
     str = ""
 
-    str << "#: " << Misc.hash2string(ENTRIES.collect{|key| [key.to_sym, self.send(key)]}) << "\n" unless no_options
+    str << "#: " << Misc.hash2string((ENTRIES - ["key_field", "fields"]).collect{|key| [key.to_sym, self.send(key)]}) << "\n" unless no_options
     if fields
       str << "#" << key_field << "\t" << fields * "\t" << "\n"
     end
 
-    saved_unnamed = @unnamed
-    @unnamed = false
-    if keys.nil?
-      each do |key, values|
-        key = key.to_s if Symbol === key
-        str << key.dup 
-        str << values_to_s(values)
-      end
-    else
-      keys.zip(values_at(*keys)).each do |key, values|
-        key = key.to_s if Symbol === key
-        str << key.dup << values_to_s(values)
+    with_unnamed do
+      if keys.nil?
+        each do |key, values|
+          key = key.to_s if Symbol === key
+          str << key.to_s
+          str << values_to_s(values)
+        end
+      else
+        keys.zip(values_at(*keys)).each do |key, values|
+          key = key.to_s if Symbol === key
+          str << key.to_s << values_to_s(values)
+        end
       end
-    end
 
-    @unnamed = saved_unnamed
+    end
     str
   end
 
@@ -352,5 +435,11 @@ end
     keys[0..10].zip(values[0..10]).each do |k,v| peek[k] = v end
     peek
   end
+
+  def to_hash
+    new = self.dup
+    ENTRY_KEYS.each{|entry| new.delete entry}
+    new
+  end
 end