rbbt-util 4.4.0 → 5.0.0

data/lib/rbbt/util/open.rb

@@ -109,22 +109,22 @@ module Open
 
   # Grep
   
-  def self.grep(stream, grep)
+  def self.grep(stream, grep, invert = false)
     case 
     when Array === grep
       TmpFile.with_file(grep * "\n", false) do |f|
-        CMD.cmd("grep", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        CMD.cmd("grep #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
       end
     else
-      CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
+      CMD.cmd("grep #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
     end
   end
   
-  def self.file_open(file, grep)
+  def self.file_open(file, grep, mode = 'r', invert_grep = false)
     if grep
-      grep(File.open(file), grep)
+      grep(File.open(file, mode), grep, invert_grep)
     else
-      File.open(file)
+      File.open(file, mode)
     end
   end
 
@@ -161,8 +161,28 @@ module Open
 
   # Open Read Write
 
+  def self.clean_cache(url, options = {})
+    options = Misc.add_defaults options, :noz => false, :mode => 'r'
+
+    wget_options = options[:wget_options] || {}
+    wget_options[:nice] = options.delete(:nice)
+    wget_options[:nice_key] = options.delete(:nice_key)
+    wget_options[:quiet] = options.delete(:quiet)
+    wget_options["--post-data="] = options.delete(:post) if options.include? :post
+    wget_options["--post-file"] = options.delete("--post-file") if options.include? "--post-file"
+    wget_options["--post-file="] = options.delete("--post-file=") if options.include? "--post-file="
+    wget_options[:cookies] = options.delete(:cookies)
+
+    cache_file = in_cache(url, wget_options)
+    Misc.lock(cache_file) do
+      FileUtils.rm(cache_file)
+    end if cache_file
+  end
+
   def self.open(url, options = {})
-    options = Misc.add_defaults options, :noz => false
+    options = Misc.add_defaults options, :noz => false, :mode => 'r'
+
+    mode = Misc.process_options options, :mode
 
     wget_options = options[:wget_options] || {}
     wget_options[:nice] = options.delete(:nice)
@@ -177,19 +197,19 @@ module Open
          when (IO === url or StringIO === url)
            url
          when (not remote?(url))
-           file_open(url, options[:grep])
+           file_open(url, options[:grep], mode, options[:invert_grep])
          when (options[:nocache] and options[:nocache] != :update)
            # What about grep?
            wget(url, wget_options)
          when (options[:nocache] != :update and in_cache(url, wget_options))
            Misc.lock(in_cache(url, wget_options)) do
-             file_open(in_cache(url, wget_options), options[:grep])
+             file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
            end
          else
            io = wget(url, wget_options)
            add_cache(url, io, wget_options)
            io.close
-           file_open(in_cache(url, wget_options), options[:grep])
+           file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
          end
     io = unzip(io)  if ((String === url and zip?(url))  and not options[:noz]) or options[:zip]
     io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
@@ -231,12 +251,16 @@ module Open
     end
   end
 
-  def self.write(file, content = nil)
+  def self.write(file, content = nil, options = {})
+    options = Misc.add_defaults options, :mode => 'w'
+
+    mode = Misc.process_options options, :mode
+
     FileUtils.mkdir_p File.dirname(file)
     case
     when content.nil?
       begin
-        File.open(file, 'w') do |f| 
+        File.open(file, mode) do |f| 
           yield f
         end
       rescue Exception
@@ -244,14 +268,14 @@ module Open
         raise $!
       end
     when String === content
-      File.open(file, 'w') do |f|
+      File.open(file, mode) do |f|
         f.flock(File::LOCK_EX)
         f.write content 
         f.flock(File::LOCK_UN)
       end
     else
       begin
-        File.open(file, 'w') do |f| 
+        File.open(file, mode) do |f| 
           f.flock(File::LOCK_EX)
           while not content.eof?
             f.write content.gets

data/lib/rbbt/util/simpleDSL.rb

@@ -53,7 +53,7 @@ module SimpleDSL
       
       @config[@@method_name] = File.open(actions).read
 
-      eval File.open(actions).read
+      instance_eval Open.read(actions), actions
     end
 
     unhook_method

data/lib/rbbt/workflow.rb

@@ -94,10 +94,11 @@ module Workflow
       require_local_workflow(wf_name) 
     rescue Exception
       Log.debug $!.message 
-      raise "Workflow not found: #{ wf_name }" if wf_name == wf_name.downcase
-      Log.debug "Trying with downcase: '#{wf_name.downcase}'"
+      Log.debug $!.backtrace.first
+      raise "Workflow not found: #{ wf_name }" if wf_name == Misc.humanize(wf_name)
+      Log.debug "Trying with humanized: '#{Misc.humanize wf_name}'"
       begin
-        require_local_workflow(wf_name.downcase)
+        require_local_workflow(Misc.humanize(wf_name))
       rescue Exception
         Log.debug $!.message
         raise "Workflow not found: #{ wf_name }"
@@ -213,7 +214,7 @@ module Workflow
   end
 
   def job(taskname, jobname = nil, inputs = {})
-    jobname ||= "Default"
+    jobname = "Default" if jobname.nil? or jobname.empty?
     task = tasks[taskname]
     raise "Task not found: #{ taskname }" if task.nil?
 

data/lib/rbbt/workflow/accessor.rb

@@ -23,8 +23,8 @@ class Step
       File.open(info_file) do |file|
         YAML.load(file) || {}
       end
-    rescue
-      Log.debug "Error loading yaml: " + Open.read(info_file)
+    rescue Exception
+      Log.debug "Error loading yaml: " + info_file
       raise $!
     end
   end
@@ -33,7 +33,7 @@ class Step
     Misc.lock(info_file) do
       i = info
       i[key] = value
-      Open.write(info_file, YAML.dump(i))
+      Open.write(info_file, i.to_yaml)
       value
     end
   end
@@ -54,12 +54,14 @@ class Step
     set_info(:messages, (messages || []) << message)
   end
 
-  def log(status, message = nil)
+  def log(status, message = nil, do_log = true)
+
     if message
       Log.low "[#{ status }] #{ message }: #{path}"
     else
       Log.low "[#{ status }]: #{path}"
-    end
+    end if do_log
+
     self.status = status
     message(message) unless message.nil?
   end

data/lib/rbbt/workflow/step.rb

@@ -24,6 +24,26 @@ class Step
     @inputs = inputs || []
   end
 
+  class << self
+    attr_accessor :log_relay_step
+  end
+
+  def relay_log(step)
+    return self unless Task === self.task and not self.task.name.nil?
+    if not self.respond_to? :original_log
+      class << self
+        attr_accessor :relay_step
+        alias original_log log 
+        def log(status, message = nil, do_log = true)
+          original_log(status, message, do_log)
+          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : [task.name.to_s, message] * ">", false)
+        end
+      end
+    end
+    @relay_step = step
+    self
+  end
+
   def prepare_result(value, description = nil, info = {})
     return value if description.nil?
     Entity.formats[description].setup(value, info.merge(:format => description)) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? description
@@ -54,16 +74,28 @@ class Step
 
   def run(no_load = false)
     result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq, :no_load => no_load do
+      if Step === Step.log_relay_step and not self == Step.log_relay_step
+        relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
+      end
+
       FileUtils.rm info_file if File.exists? info_file
-      log(:starting, "Starting task: #{task.name || "unnamed task"}")
 
       set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
       @dependencies.each{|dependency| 
-        log dependency.task.name || "dependency", "Processing dependency: #{ dependency.path }"
-        dependency.run true
+        begin
+          dependency.relay_log self
+          dependency.run true
+        rescue Exception
+          backtrace = $!.backtrace
+          set_info :backtrace, backtrace 
+          log(:error, "Exception processing dependency #{dependency.path}")
+          log(:error, "#{$!.class}: #{$!.message}")
+          log(:error, "backtrace: #{$!.backtrace.first}")
+          raise "Exception processing dependency #{dependency.path}"
+        end
       }
-
-      set_info :status, :started
+      
+      log(:started, "Starting task #{task.name || ""}")
 
       set_info :started, Time.now
       
@@ -72,9 +104,16 @@ class Step
       res = begin
               exec
             rescue Step::Aborted
+              log(:error, "Aborted")
               raise $!
             rescue Exception
-              set_info :backtrace, $!.backtrace
+              backtrace = $!.backtrace
+
+              # HACK: This fixes an strange behaviour in 1.9.3 where some
+              # bactrace strings are coded in ASCII-8BIT
+              backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
+
+              set_info :backtrace, backtrace 
               log(:error, "#{$!.class}: #{$!.message}")
               log(:error, "backtrace: #{$!.backtrace.first}")
               raise $!
@@ -84,7 +123,11 @@ class Step
       res
     end
 
-    prepare_result result, @task.result_description, info
+    if no_load
+      self
+    else
+      prepare_result result, @task.result_description, info
+    end
   end
 
   def fork

data/test/rbbt/test_annotations.rb

@@ -42,10 +42,25 @@ class TestAnnotations < Test::Unit::TestCase
     assert_equal annotation_str, ary[0].annotation_str
   end
 
+  def test_double_array
+    ary = ["string"]
+    annotation_str = "Annotation String"
+    ary.extend AnnotatedArray
+    ary_ary = [ary]
+    ary_ary.extend AnnotatedArray
+    AnnotatedString.setup(ary, annotation_str)
+    AnnotatedString.setup(ary_ary, annotation_str)
+    assert_equal [AnnotatedString], ary.annotation_types
+    assert_equal annotation_str, ary.annotation_str
+    assert_equal annotation_str, ary[0].annotation_str
+  end
+
+
   def test_info
     ary = ["string"]
     annotation_str = "Annotation String"
     AnnotatedString.setup(ary, annotation_str)
+    
     assert_equal({:annotation_str => annotation_str, :annotation_types => [AnnotatedString]}, ary.info)
   end
 
@@ -65,7 +80,6 @@ class TestAnnotations < Test::Unit::TestCase
     annotation_str2 = "Annotation String 2"
     AnnotatedString.setup(str1, annotation_str1)
     AnnotatedString.setup(str2, annotation_str2)
-    ddd Annotated.json(str1, true)
   end
 
   def test_tsv
@@ -75,8 +89,8 @@ class TestAnnotations < Test::Unit::TestCase
     annotation_str2 = "Annotation String 2"
     AnnotatedString.setup(str1, annotation_str1)
     AnnotatedString.setup(str2, annotation_str2)
-    assert_equal str1, Annotated.tsv([str1, str2], :all)[str1.id]["literal"] 
-    assert_equal annotation_str1, Annotated.tsv([str1, str2], :annotation_str, :JSON)[str1.id]["annotation_str"] 
+    assert_equal str1, Annotated.tsv([str1, str2], :all)[str1.id + ":0"]["literal"] 
+    assert_equal annotation_str1, Annotated.tsv([str1, str2], :annotation_str, :JSON)[str1.id + ":0"]["annotation_str"] 
   end
 
   def test_literal
@@ -128,10 +142,22 @@ class TestAnnotations < Test::Unit::TestCase
     assert_equal str + annotation_str, str.add_annot
   end
 
+  def test_double_array
+    a = ["a"]
+    b = AnnotatedString.setup([AnnotatedString.setup(["a"])])
+    AnnotatedString.setup(a)
+    a.extend AnnotatedArray
+    b.extend AnnotatedArray
+    assert AnnotatedString === b[0]
+    assert(!a.double_array)
+    assert(b.double_array)
+  end
+
   def test_annotation_positional2hash
     str = "string"
     annotation_str = "Annotation String"
     AnnotatedString.setup(str, :annotation_str => annotation_str)
     assert_equal str + annotation_str, str.add_annot
   end
+
 end

data/test/rbbt/test_persist.rb

@@ -41,6 +41,43 @@ class TestPersist < Test::Unit::TestCase
     end
   end
 
+  def test_annotation_persist_with_repeitions
+    TmpFile.with_file do |tmp|
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+      entity2bis = "Entity 2"
+
+      TestAnnotation.setup(entity1, :test_annotation => "1")
+      TestAnnotation.setup(entity2, :test_annotation => "2")
+      TestAnnotation.setup(entity2bis, :test_annotation => "2")
+
+      annotations = [entity1, entity2, entity2bis]
+
+      persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
+        annotations
+      end
+
+      assert_equal 3, persisted_annotations.length
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "2", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
+        annotations
+      end
+
+      assert_equal 3, persisted_annotations.length
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "2", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
+
   def test_bdb
     TmpFile.with_file do |tmp|
       repo = Persist.open_tokyocabinet(tmp, true, :double, TokyoCabinet::BDB)
@@ -85,6 +122,119 @@ class TestPersist < Test::Unit::TestCase
     end
   end
 
+  def test_annotation_persist_repo_annotated_array
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
+
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+
+      annotations = [entity1, entity2]
+      TestAnnotation.setup(annotations, :test_annotation => "1")
+      annotations.extend AnnotatedArray
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "1", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      persisted_annotations.extend AnnotatedArray
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "1", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
+
+  def test_annotation_persist_repo_triple_array
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
+
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+
+      annotations = [entity1, entity2]
+      TestAnnotation.setup(annotations, :test_annotation => "1")
+      annotations.extend AnnotatedArray
+
+      annotations_ary = [annotations]
+      TestAnnotation.setup(annotations_ary, :test_annotation => "1")
+      annotations_ary.extend AnnotatedArray
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations_ary
+      end
+      assert AnnotatedArray === persisted_annotations
+      ddd persisted_annotations.info
+      ddd persisted_annotations
+
+      assert_equal "Entity 1", persisted_annotations.first.first
+      assert_equal "Entity 2", persisted_annotations.first.last
+      assert_equal "1", persisted_annotations.first.first.test_annotation
+      assert_equal "1", persisted_annotations.first.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations_ary
+      end
+
+      assert AnnotatedArray === persisted_annotations
+
+      assert_equal "Entity 1", persisted_annotations.sort.first.first
+      assert_equal "Entity 2", persisted_annotations.sort.first.last
+      assert_equal "1", persisted_annotations.sort.first.first.test_annotation
+      assert_equal "1", persisted_annotations.sort.first.last.test_annotation
+
+    end
+  end
+
+  def test_annotation_persist_repo_with_repetitions
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
+
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+      entity2bis = "Entity 2"
+
+      TestAnnotation.setup(entity1, :test_annotation => "1")
+      TestAnnotation.setup(entity2, :test_annotation => "2")
+      TestAnnotation.setup(entity2bis, :test_annotation => "2")
+
+      annotations = [entity1, entity2, entity2bis]
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal 3, persisted_annotations.length
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "2", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal 3, persisted_annotations.length
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "2", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
 
   def test_array_persist
     TmpFile.with_file do |tmp|