rbbt-util 3.1.0 → 3.2.0

data/lib/rbbt/util/open.rb

@@ -101,7 +101,7 @@ module Open
     case 
     when Array === grep
       TmpFile.with_file(grep * "\n", false) do |f|
-        CMD.cmd("grep", "-E" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        CMD.cmd("grep", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
       end
     else
       CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
@@ -191,12 +191,16 @@ module Open
     f = open(file, options)
 
     if block_given?
-      while l = Misc.fixutf8(f.gets)
+      while not f.eof?
+        l = f.gets
         l = fixutf8(l) if l.respond_to?(:valid_encoding?) && ! l.valid_encoding?
         yield l
       end
+      f.close
     else
-      Misc.fixutf8(f.read)
+      text = Misc.fixutf8(f.read)
+      f.close unless f.closed?
+      text 
     end
   end
 
@@ -206,9 +210,7 @@ module Open
     when content.nil?
       begin
         File.open(file, 'w') do |f| 
-          f.flock(File::LOCK_EX)
           yield f
-          f.flock(File::LOCK_UN)
         end
       rescue Exception
         FileUtils.rm file if File.exists? file
@@ -224,8 +226,8 @@ module Open
       begin
         File.open(file, 'w') do |f| 
           f.flock(File::LOCK_EX)
-          while l = content.gets
-            f.write l
+          while not content.eof?
+            f.write content.gets
           end
           f.flock(File::LOCK_UN)
         end

data/lib/rbbt/util/persistence.rb

@@ -129,19 +129,18 @@ module Persistence
   end
 
   def self.tsv_serializer(data, extra = nil)
-    if Object::TSV === data
+    return data.serializer if Persistence::TSV === data
+    if Object::TSV === data 
       return :integer if (data.cast == "to_i" or data.cast == :to_i) and data.type == :single
       return :integer_array if (data.cast == "to_i" or data.cast == :to_i) and (data.type == :list or data.type == :flat)
 
       case
-      when data.type == :double
-        :double
-      when data.type == :list
+      when (data.type == :list or data.type == :flat)
         :list
       when data.type == :single
         :single
       else
-        :marshal
+        :double
       end
     else
       return :marshal if extra.nil?
@@ -149,14 +148,12 @@ module Persistence
       return :integer_array if (extra[:cast] == "to_i" or extra[:cast] == :to_i) and (extra[:type] == :list or extra[:type] == :flat)
 
       case
-      when extra[:type] == :double
-        :double
-      when extra[:type] == :list
+      when (extra[:type] == :list or extra[:type] == :flat)
         :list
       when extra[:type] == :single
         :single
       else
-        :marshal
+        :double
       end
     end
   end
@@ -201,7 +198,7 @@ module Persistence
     else
       Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
 
-      per = Persistence::TSV.get persistence_file, true, serializer
+      per = Persistence::TSV.get persistence_file, false, serializer
       tsv = Object::TSV.new per
       Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
         if tsv.respond_to?(key.to_sym)  and per.respond_to?(key.to_sym)
@@ -224,7 +221,7 @@ module Persistence
     filename         ||= get_filename(file)
     persistence_file ||= get_persistence_file(filename, prefix, options)
 
-    if persistence_update or not File.exists? persistence_file
+    if persistence_update or not File.exists?(persistence_file)
       Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
       res, extra = yield file, options, filename, persistence_file
 
@@ -267,7 +264,7 @@ module Persistence
       rescue Interrupt
         raise "Interrupted"
       rescue Exception
-        per.close
+        per.close unless per.nil?
         raise $!
       end
 
@@ -353,9 +350,7 @@ module Persistence
       yield file, options, filename
     else
       Log.low "Persistent Loading for #{filename}. Prefix: #{prefix}. Type #{persistence_type.to_s}"
-
       Misc.lock(persistence_file, file, prefix, options, block) do |persistence_file,file,prefix,options,block|
-
         case persistence_type.to_sym
         when :string
           persist_string(file, prefix, options, &block)
@@ -375,6 +370,14 @@ module Persistence
       end
     end
   end
+end
 
+module LocalPersist
+
+  attr_accessor :local_persistence_dir
+  def local_persist(name, prefix, type, options= {}, &block)
+    Persistence.persist(name, prefix, type, options.merge({:persistence_dir => @local_persistence_dir}), &block)
+  end
 
 end
+

data/lib/rbbt/util/resource.rb

@@ -168,6 +168,7 @@ source "$INSTALL_HELPER_FILE"
       RakeHelper.run(rakefile, file, dir)
       resource
     end
+    resource
   end
 
   def relative_to(klass, path)
@@ -226,7 +227,9 @@ source "$INSTALL_HELPER_FILE"
       join name
     end
     
-    def method_missing(name, prev = nil)
+    alias :old_method_missing :method_missing
+    def method_missing(name, prev = nil, *args)
+      old_method_missing(name, prev, *args) if name.to_s =~ /^to_/
       join prev unless prev.nil?
       join name
     end
@@ -267,7 +270,7 @@ source "$INSTALL_HELPER_FILE"
     end
     
     def define_as_proc(&block)
-      Resource.define_resource(self, :proc, &block)
+      Resource.define_resource(self, :proc, block)
     end
  
  
@@ -307,7 +310,9 @@ source "$INSTALL_HELPER_FILE"
       end
     end
 
+    alias :old_method_missing :method_missing
     def method_missing(name, *args)
+      return old_method_missing(name, *args) if name.to_s =~ /^to_/
       if key
         klass.send(name, key, *args)
       else
@@ -333,7 +338,9 @@ source "$INSTALL_HELPER_FILE"
     Path.path(name, pkgdir, namespace, lib_dir)
   end
 
-  def method_missing(name, prev = nil)
+  alias :old_method_missing :method_missing
+  def method_missing(name, prev = nil, *args)
+    return old_method_missing(name, prev, *args) if name.to_s =~ /^to_/
     if prev
       self[prev][name]
     else

data/lib/rbbt/util/task.rb

@@ -145,6 +145,7 @@ class Task
       usage << "\nMandatory options:\n"
       usage << "\tTask\tName\tType   \tDescription\n"
       usage << "\t----\t----\t----   \t-----------\n"
+
       options.each do |option|
         option_line = "\t[#{option[:source]}]\t#{option[:name]}"
         option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
@@ -154,8 +155,7 @@ class Task
     end
 
     if optional_options.any?
-      usage << "\nOptional options:"
-      usage << "Mandatory options:\n"
+      usage << "\nOptional options:\n"
       usage << "\tTask\tName\tDefault  \tType   \tDescription\n"
       usage << "\t----\t----\t-------  \t----   \t-----------\n"
       optional_options.each do |option|

data/lib/rbbt/util/task/job.rb

@@ -77,7 +77,7 @@ class Task
 
     def info
       return {} if not File.exists?(info_file)
-      info = YAML.load(File.open(info_file))
+      info = YAML.load(File.open(info_file)) || {}
       info.extend IndiferentHash
     end
 
@@ -132,6 +132,10 @@ class Task
       step == :error or step == :aborted
     end
 
+    def aborted?
+      step == :aborted 
+    end
+
     def arguments
       options.values_at *task.options
     end
@@ -142,7 +146,13 @@ class Task
 
     def run_dependencies
       required_files.each do |file| file.produce unless File.exists? file end unless required_files.nil?
-      previous_jobs.each do |job| job.start unless File.exists? job.path; job.set_info(:step, :done) end unless previous_jobs.nil?
+      previous_jobs.each do |job| 
+        if not job.recursive_done? 
+          job.clean if job.error?
+          job.start
+          job.step :done unless job.step == :error or job.step == :aborted
+        end
+      end unless previous_jobs.nil?
     end
 
     def save_dependencies
@@ -185,6 +195,7 @@ class Task
         set_info(:end_time, Time.now)
         Log.medium("[#{task.name}] Finished Job '#{ name }'. Path: '#{ path }'")
       rescue Exception
+        set_info(:exception_backtrace, $!.backtrace)
         step(:error, "#{$!.class}: #{$!.message}")
         raise $!
       end
@@ -204,7 +215,7 @@ class Task
     end
 
     def recursive_done?
-      previous_jobs.inject(true){|acc,j| acc and j.recursive_done?} and done?
+      (previous_jobs || []).inject(true){|acc,j| acc and j.recursive_done?} and done? and not error? 
     end
 
     def run
@@ -280,6 +291,8 @@ class Task
     def clean
       FileUtils.rm path if File.exists? path
       FileUtils.rm info_file if File.exists? info_file
+      FileUtils.rm_rf path + '.files' if File.exists? path + '.files'
+      self
     end
 
     def recursive_clean

data/lib/rbbt/util/tc_hash.rb

@@ -15,7 +15,6 @@ class TCHash < TokyoCabinet::HDB
     def self.load(str); str.unpack("l*"); end
   end
 
-
   class StringSerializer
     def self.dump(str); str.to_s; end
     def self.load(str); str; end
@@ -27,7 +26,7 @@ class TCHash < TokyoCabinet::HDB
     end
 
     def self.load(string)
-      string.split(/\t/)
+      string.split("\t", -1)
     end
   end
 
@@ -37,7 +36,7 @@ class TCHash < TokyoCabinet::HDB
     end
 
     def self.load(string)
-      string.split(/\t/).collect{|l| l.split("|")}
+      string.split("\t", -1).collect{|l| l.split("|", -1)}
     end
   end
 
@@ -66,17 +65,31 @@ class TCHash < TokyoCabinet::HDB
     end
   end
 
-  def serializer
-    @serializer
+  def size
+    keys.length
   end
 
-  def serializer=(value)
-    self.original_set_brackets(FIELD_INFO_ENTRIES[:serializer],value) unless value.nil?
+  def delete(key)
+    raise "Cannot deleted key: closed connection" if not write?
+    out(key) or raise "Not deleted"
+  end
+
+  attr_accessor :serializer
+  def serializer=(serializer)
+    
+    if ALIAS.include? serializer.to_sym
+      @serializer = ALIAS[serializer.to_sym]
+    else
+      @serializer = serializer
+    end
+    self.original_set_brackets(FIELD_INFO_ENTRIES[:serializer], @serializer.to_s)
   end
 
   alias original_open open
-  def open(write = false)
-    flags = write ? TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT : TokyoCabinet::BDB::OREADER
+  def open(write = false, serializer = nil)
+    flags = (write ? TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT : TokyoCabinet::BDB::OREADER)
+
+    FileUtils.mkdir_p File.dirname(@path_to_db) unless File.exists?(File.dirname(@path_to_db))
     if !self.original_open(@path_to_db, flags)
       ecode = self.ecode
       raise OpenError, "Open error: #{self.errmsg(ecode)}. Trying to open file #{@path_to_db}"
@@ -84,17 +97,17 @@ class TCHash < TokyoCabinet::HDB
 
     @write = write
 
-    if write
-      self.original_set_brackets(FIELD_INFO_ENTRIES[:serializer], @serializer.to_s) unless @serializer.nil?
-    else
+    if self.include? FIELD_INFO_ENTRIES[:serializer]
       serializer_str = self.original_get_brackets(FIELD_INFO_ENTRIES[:serializer])
 
-      if serializer_str.nil? or serializer_str.empty? 
-        @serializer = Marshal
-      else
-        mod = Misc.string2const serializer_str
-        @serializer = mod
-      end
+      mod = Misc.string2const serializer_str
+      @serializer = mod
+
+    else
+      raise "No serializer specified" if serializer.nil?
+
+      self.original_set_brackets(FIELD_INFO_ENTRIES[:serializer], serializer.to_s) unless self.include? FIELD_INFO_ENTRIES[:serializer]
+      @serializer = serializer
     end
   end
 
@@ -118,21 +131,37 @@ class TCHash < TokyoCabinet::HDB
     serializer = ALIAS[serializer] if ALIAS.include? serializer
 
     @path_to_db = path
-    @serializer = serializer
 
     if write || ! File.exists?(@path_to_db)
       self.setcache(100000) or raise "Error setting cache"
-      self.open(true)
+      self.open(true, serializer)
     else
       self.open(false)
     end
   end
 
   def self.get(path, write = false, serializer = Marshal)
-    serializer = ALIAS[serializer] if ALIAS.include? serializer
-    @serializer = serializer
-    d = CONNECTIONS[path] ||= self.new(path, false, @serializer)
-    write ? d.write : d.read
+    if ALIAS.include? serializer
+      serializer = ALIAS[serializer] 
+    else
+      serializer = serializer
+    end
+
+    case
+    when !File.exists?(path)
+      CONNECTIONS[path] = self.new(path, true, serializer)
+    when (not CONNECTIONS.include?(path))
+      CONNECTIONS[path] = self.new(path, false, serializer)
+    end
+
+    d = CONNECTIONS[path] 
+    
+    if write 
+      d.write unless d.write?
+    else
+      d.read if d.write?
+    end
+
     d
   end
 
@@ -142,14 +171,22 @@ class TCHash < TokyoCabinet::HDB
   def [](key)
     return nil unless String === key
     result = self.original_get_brackets(key)
-    result ? @serializer.load(result) : nil
+    if result.nil? or (String === result and result =~ /__Ref:/) 
+      result 
+    else
+      @serializer.load(result)
+    end
   end
 
   alias original_set_brackets []=
   def []=(key,value)
     raise KeyFormatError, "Key must be a String, its #{key.class.to_s}" unless String === key
     raise "Closed TCHash connection" unless write?
-    self.original_set_brackets(key, serializer.dump(value))
+    if String === value and value =~ /^__Ref/
+      self.original_set_brackets(key, value)
+    else
+      self.original_set_brackets(key, serializer.dump(value))
+    end
   end
 
   def values_at(*args)
@@ -204,7 +241,7 @@ class TCHash < TokyoCabinet::HDB
   def merge!(data)
     raise "Closed TCHash connection" unless write?
     serialized = 
-      data.collect{|key, values| [key.to_s, serializer.dump(values)]}
+      data.collect{|key, values| [key.to_s, serializer.dump(values)] }
     if tranbegin
       serialized.each do |key, values|
         self.putasync(key, values)

data/lib/rbbt/util/tsv.rb

@@ -14,8 +14,28 @@ require 'rbbt/util/tsv/manipulate'
 require 'rbbt/util/tsv/index'
 require 'rbbt/util/tsv/attach'
 require 'rbbt/util/tsv/resource'
+
 class TSV
 
+  ESCAPES = {
+    "\n" => "[[NL]]",
+    "\t" => "[[TAB]]",
+  }
+
+  def self.escape(text)
+    ESCAPES.each do |char,replacement|
+      text = text.gsub(char, replacement)
+    end
+    text
+  end
+
+  def self.unescape(text)
+    ESCAPES.each do |char,replacement|
+      text = text.gsub(replacement, char)
+    end
+    text
+  end
+
   def self.headers(file, options = {})
 
     ## Remove options from filename
@@ -91,7 +111,7 @@ class TSV
       extra.each do |key, values|
         self.send("#{ key }=".to_sym, values) if self.respond_to? "#{ key }=".to_sym 
       end if not extra.nil?
- 
+
     else
 
       case
@@ -99,10 +119,16 @@ class TSV
         @data = Hash[file.collect{|v| 
           [v,[]]
         }]
+        @data.key_field = key_field if key_field
+        @data.fields = fields if fields
       when Hash === file 
         @data = file
+        @data.key_field = key_field if key_field
+        @data.fields = fields if fields
       when TSV === file
         @data = file.data
+        @data.key_field = key_field if key_field
+        @data.fields = fields if fields
       when Persistence::TSV === file
         @data = file
         %w(case_insensitive namespace identifiers datadir fields key_field type filename cast).each do |key|
@@ -116,25 +142,7 @@ class TSV
           data, extra = nil
 
           if in_situ_persistence and persistence_file
-
-            cast = options[:cast]
-            type = options[:type]
-            serializer = case
-                         when ((cast == "to_i" or cast == :to_i) and type == :single)
-                           :integer 
-                         when ((cast == "to_i" or cast == :to_i) and (type == :flat or type == :list))
-                           :integer_array 
-                         when type == :double
-                           :double
-                         when type == :list
-                           :list
-                         when type == :single
-                           :single
-                         else
-                           :marshal
-                         end
-
-            options.merge! :persistence_data => Persistence::TSV.get(persistence_file, true, serializer)
+            options.merge! :persistence_data => Persistence::TSV.get(persistence_file, true, :double)
           end
 
           begin
@@ -169,6 +177,17 @@ class TSV
             raise $!
           end
 
+          if Persistence::TSV === data
+            %w(case_insensitive namespace identifiers datadir fields key_field type filename cast).each do |key| 
+              if extra.include? key.to_sym
+                if data.respond_to? "#{key}=".to_sym
+                  data.send("#{key}=".to_sym, extra[key.to_sym])
+                end
+              end
+            end 
+            data.read
+          end
+ 
           [data, extra]
         end
       end
@@ -183,7 +202,6 @@ class TSV
           #end
         end
       end 
-      @data.read if Persistence::TSV === @data
     end
   end
 
@@ -195,4 +213,9 @@ class TSV
     @data.read if @data.respond_to? :read
   end
 
+  def write?
+    @data.write? if @data.respond_to? :write
+  end
+
+
 end