rbbt-util 4.4.0 → 5.0.0

data/lib/rbbt/fix_width_table.rb

@@ -69,6 +69,10 @@ class FixWidthTable
   end
   alias << add
 
+  def last_pos
+    pos(size - 1)
+  end
+
   def pos(index)
     return nil if index < 0 or index >= size
     @file.seek(5 + (record_size) * index, IO::SEEK_SET)
@@ -156,7 +160,7 @@ class FixWidthTable
       idx = idx - 1
     end
 
-    idx
+    idx.to_i
   end
 
   def get_range(pos)

data/lib/rbbt/persist.rb

@@ -93,6 +93,7 @@ module Persist
       f = File.open(path, 'rb')
       res = f.read
       f.close
+      res.force_encoding("ASCII-8BIT") if res.respond_to? :force_encoding
       res
     when :array
       res = Open.read(path).split("\n", -1)
@@ -106,8 +107,6 @@ module Persist
       Open.read(path).to_f
     when :integer
       Open.read(path).to_i
-    when :tsv
-      TSV.open(Open.open(path))
     else
       raise "Unknown persistence: #{ type }"
     end
@@ -132,6 +131,7 @@ module Persist
     when :string, :text
       Open.write(path, content)
     when :binary
+      content.force_encoding("ASCII-8BIT") if content.respond_to? :force_encoding
       f = File.open(path, 'wb')
       f.puts content
       f.close
@@ -197,31 +197,48 @@ module Persist
             repo[key]
           end
           Annotated.load_tsv_values(key, values, "literal", "annotation_types", "JSON")
-        when keys.any?
+        when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
           repo.read_and_close do
-            keys.collect{|key|
+            keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
               Annotated.load_tsv_values(key, v, "literal", "annotation_types", "JSON")
             }
           end
+        when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
+          repo.read_and_close do
+
+            res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
+              v = repo[key]
+              Annotated.load_tsv_values(key, v, "literal", "annotation_types", "JSON")
+            }
+
+            res.first.annotate res
+            res.extend AnnotatedArray
+
+            res
+          end
         else
           entities = yield
 
-          Misc.lock(repo.persistence_path) do
-            repo.write_and_close do 
-              case
-              when entities.nil?
-                repo[subkey + "NIL"] = nil
-              when entities.empty?
-                repo[subkey + "EMPTY"] = nil
-              when (not Array === entities or AnnotatedArray === entities)
-                tsv_values = entities.tsv_values("literal", "annotation_types", "JSON") 
-                repo[subkey + entities.id << ":" << "SINGLE"] = tsv_values
-              else
-                entities.each do |e|
-                  tsv_values = e.tsv_values("literal", "annotation_types", "JSON") 
-                  repo[subkey + e.id] = tsv_values
-                end
+          repo.write_and_close do 
+            case
+            when entities.nil?
+              repo[subkey + "NIL"] = nil
+            when entities.empty?
+              repo[subkey + "EMPTY"] = nil
+            when (not Array === entities or (AnnotatedArray === entities and not Array === entities.first))
+              tsv_values = entities.tsv_values("literal", "annotation_types", "JSON") 
+              repo[subkey + entities.id << ":" << "SINGLE"] = tsv_values
+            when (not Array === entities or (AnnotatedArray === entities and AnnotatedArray === entities.first))
+              entities.each_with_index do |e,i|
+                tsv_values = e.tsv_values("literal", "annotation_types", "JSON") 
+                repo[subkey + e.id << ":ANNOTATED_DOUBLE_ARRAY:" << i.to_s] = tsv_values
+              end
+            else
+              entities.each_with_index do |e,i|
+                next if e.nil?
+                tsv_values = e.tsv_values("literal", "annotation_types", "JSON") 
+                repo[subkey + e.id << ":" << i.to_s] = tsv_values
               end
             end
           end
@@ -230,23 +247,23 @@ module Persist
         end
 
       else
-        Misc.lock(path) do
-          if is_persisted?(path, persist_options)
-            Log.debug "Persist up-to-date: #{ path } - #{persist_options.inspect[0..100]}"
-            return nil if persist_options[:no_load]
-            return load_file(path, type) 
-          else
-            Log.debug "Persist create: #{ path } - #{persist_options.inspect[0..100]}"
-          end
-          begin
-            res = yield
+        if is_persisted?(path, persist_options)
+          Log.debug "Persist up-to-date: #{ path } - #{persist_options.inspect[0..100]}"
+          return nil if persist_options[:no_load]
+          return load_file(path, type) 
+        else
+          Log.debug "Persist create: #{ path } - #{persist_options.inspect[0..100]}"
+        end
+        begin
+          res = yield
+          Misc.lock(path) do
             save_file(path, type, res)
-            res
-          rescue
-            Log.high "Error in persist. Erasing '#{ path }'"
-            FileUtils.rm path if File.exists? path
-            raise $!
           end
+          res
+        rescue
+          Log.high "Error in persist. Erasing '#{ path }'"
+          FileUtils.rm path if File.exists? path
+          raise $!
         end
       end
 

data/lib/rbbt/persist/tsv.rb

@@ -82,13 +82,15 @@ module Persist
         end
 
         def write_and_close
-          write if @closed or not write?
-          res = begin
-                  yield
-                ensure
-                  close
-                end
-          res
+          Misc.lock(persistence_path) do
+            write if @closed or not write?
+            res = begin
+                    yield
+                  ensure
+                    close
+                  end
+            res
+          end
         end
 
         def read_and_close
@@ -135,7 +137,15 @@ module Persist
              persist_options[:data]
            when persist_options[:persist]
 
-             filename ||= source.filename if source.respond_to? :filename
+             filename ||= case
+                          when source.respond_to?(:filename)
+                            source.filename
+                          when source.respond_to?(:cmd)
+                            "CMD-#{Misc.digest(source.cmd)}"
+                          else
+                            source.object_id.to_s
+                          end
+
              filename ||= source.object_id.to_s
 
              path = persistence_path(filename, persist_options, options)
@@ -161,25 +171,26 @@ module Persist
 
     begin
       if data.respond_to? :persistence_path and data != persist_options[:data]
-        Misc.lock data.persistence_path do
-          data.write_and_close do
-            yield data
-          end
+        data.write_and_close do
+          yield data
         end
       else
         yield data
       end
     rescue Exception
+      FileUtils.rm path if path and File.exists? path
+      raise $!
+    ensure
       begin
-        data.close if data.respondo_to? :close
+        data.close if data.respond_to? :close
       rescue
+        raise $!
       end
-      FileUtils.rm path if path and File.exists? path
-      raise $!
     end
 
     data.read if data.respond_to? :read and ((data.respond_to?(:write?) and data.write?) or (data.respond_to?(:closed?) and data.closed?))
 
+
     data
   end
 

data/lib/rbbt/resource.rb

@@ -29,7 +29,11 @@ module Resource
 
   def resource_method_missing(name, prev = nil, *args)
     # Fix problem with ruby 1.9 calling methods by its own initiative. ARG
-    root.send(name, prev, *args)
+    if prev.nil?
+      root.send(name, *args)
+    else
+      root.send(name, prev, *args)
+    end
   end
 
   def [](file = nil)
@@ -41,7 +45,6 @@ module Resource
   end
 
   def claim(path, type, content = nil, &block)
-    path = path.find if path.respond_to? :find
     if type == :rake
       @rake_dirs[path] = content
     else
@@ -56,28 +59,30 @@ module Resource
     end
   end
 
-  def produce(path)
+  def produce(path, force = false)
     case
     when @resources.include?(path)
       type, content = @resources[path]
     when has_rake(path)
       type = :rake
       rake_dir, content = rake_for(path)
+      rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
     else
       raise "Resource #{ path } does not seem to be claimed"
     end
 
-    if not File.exists? path
-      Misc.lock path + '.produce' do
+    final_path = path.respond_to?(:find) ? (force ? path.find(:user) : path.find) : path
+    if not File.exists? final_path or force
+      Misc.lock final_path + '.produce' do
         begin
           case type
           when :string
-            Open.write(path, content)
+            Open.write(final_path, content)
           when :url
-            Open.write(path, Open.open(content))
+            Open.write(final_path, Open.open(content))
           when :proc
             data = content.call
-            Open.write(path, data) 
+            Open.write(final_path, data) 
           when :rake
             run_rake(path, content, rake_dir)
           when :install
@@ -99,7 +104,7 @@ source "$INSTALL_HELPER_FILE"
             raise "Could not produce #{ resource }. (#{ type }, #{ content })"
           end
         rescue
-          FileUtils.rm path if File.exists? path
+          FileUtils.rm_rf final_path if File.exists? final_path
           raise $!
         end
       end

data/lib/rbbt/resource/path.rb

@@ -117,19 +117,19 @@ module Path
     end
   end
 
-  def produce
+  def produce(force = false)
     path = self.find
-    return self if File.exists? path
+    return self if File.exists?(path) and not force
 
     raise "No resource defined to produce file: #{ self }" if resource.nil?
 
-    resource.produce path
+    resource.produce self, force
 
     path
   end
 
-  def read
-    Open.read(self.produce.find)
+  def read(&block)
+    Open.read(self.produce.find, &block)
   end
 
   def open(options = {})

data/lib/rbbt/resource/rake.rb

@@ -37,12 +37,16 @@ module Rake
 
     raise TaskNotFound if Rake::Task[task].nil?
 
-    Misc.in_dir(dir) do
-      Rake::Task[task].invoke
+    t = nil
+    pid = Process.fork{
+      Misc.in_dir(dir) do
+        Rake::Task[task].invoke
 
-      Rake::Task.clear
-      Rake::FileTask.clear_files
-    end
+        Rake::Task.clear
+        Rake::FileTask.clear_files
+      end
+    }
+    Process.wait(pid)
 
   end
 end

data/lib/rbbt/resource/util.rb

@@ -8,7 +8,7 @@ module Path
       l =~ /rbbt\/resource\/path\.rb/ or
       l =~ /rbbt\/util\/misc\.rb/ or
       l =~ /progress-monitor\.rb/ 
-    }.first.sub(/\.rb.*/,'.rb') if file.nil?
+    }.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
 
     file = File.expand_path file
     return Path.setup(file) if File.exists? File.join(file, relative_to)
@@ -82,7 +82,7 @@ module Resource
 
 
   def rake_for(path)
-    entry = @rake_dirs.reject{|dir, content|
+    @rake_dirs.reject{|dir, content|
       !Misc.common_path(dir, path)
     }.sort_by{|dir, content|
       dir.length
@@ -97,6 +97,8 @@ module Resource
     task = Misc.path_relative_to rake_dir, path
     rakefile = rakefile.produce if rakefile.respond_to? :produce
 
+    rake_dir = rake_dir.find if rake_dir.respond_to? :find
+
     begin
       Rake.run(rakefile, rake_dir, task)
     rescue Rake::TaskNotFound

data/lib/rbbt/tsv.rb

@@ -17,7 +17,9 @@ require 'rbbt/tsv/filter'
 
 module TSV
   def self.setup(hash, options = {})
-    hash = Misc.array2hash hash if Array === hash
+    options = Misc.add_defaults options, :default_value => []
+    default_value = Misc.process_options options, :default_value
+    hash = Misc.array2hash(hash, default_value) if Array === hash
     hash.extend TSV
 
     IndiferentHash.setup(options)
@@ -40,9 +42,14 @@ module TSV
     filename = get_filename source
     serializer = Misc.process_options options, :serializer
     unnamed = Misc.process_options options, :unnamed
+    entity_options = Misc.process_options options, :entity_options
 
-    Log.debug "TSV open: #{ filename } - #{options.inspect}"
+    Log.debug "TSV open: #{ filename } - #{options.inspect}.#{unnamed ? " [unnamed]" : "[not unnamed]"}"
 
+    data = nil
+
+    lock_filename = filename.nil? ? nil : Persist.persistence_path(filename, {:dir => Rbbt.tmp.tsv_open_locks.find})
+    Misc.lock lock_filename  do
     data = Persist.persist_tsv source, filename, options, persist_options do |data|
       if serializer
         data.extend TSV unless TSV === data
@@ -56,14 +63,17 @@ module TSV
 
       data.filename = filename.to_s unless filename.nil?
       if data.identifiers.nil? and Path === filename and filename.identifier_file_path
-        data.identifiers = filename.identifier_file_path.dup 
+        data.identifiers = filename.identifier_file_path.to_s 
       end
 
       data
     end
+    end
 
     data.unnamed = unnamed unless unnamed.nil?
 
+    data.entity_options = entity_options
+
     data
   end
 
@@ -72,13 +82,13 @@ module TSV
   end
 
   def self.parse(stream, data, options = {})
-    monitor, grep = Misc.process_options options, :monitor, :grep
+    monitor, grep, invert_grep = Misc.process_options options, :monitor, :grep, :invert_grep
 
     parser = Parser.new stream, options
 
     if grep
       stream.rewind
-      stream = Open.grep(stream, grep)
+      stream = Open.grep(stream, grep, invert_grep)
       parser.first_line = stream.gets
     end
 
@@ -131,10 +141,11 @@ module TSV
       progress_monitor = nil
     end
 
-    while not line.nil?
+    while not line.nil? 
       begin
         progress_monitor.tick(stream.pos) if progress_monitor 
 
+        line = Misc.fixutf8(line)
         line = parser.process line
         parts = parser.chop_line line
         key, values = parser.get_values parts
@@ -142,8 +153,12 @@ module TSV
         parser.add_to_data data, key, values
         line = stream.gets
       rescue Parser::SKIP_LINE
-        line = stream.gets
-        next
+        begin
+          line = stream.gets
+          next
+        rescue IOError
+          break
+        end
       rescue Parser::END_PARSING
         break
       rescue IOError