rbbt-util 3.2.1 → 4.0.0

data/README.rdoc

@@ -0,0 +1,65 @@
+= Install
+
+1 - install rvm: https://rvm.beginrescueend.com/ (will need git and curl installed)
+
+    :script
+
+    bash < <(curl -s https://rvm.beginrescueend.com/install/rvm)
+    cd
+    cat .bash_profile > tmp.bash_profile
+    echo '[[ -s "$HOME/.rvm/scripts/rvm" ]] && . "$HOME/.rvm/scripts/rvm"' >> tmp.bash_profile
+    cp .bash_profile .bash_profile.save
+    mv tmp.bash_profile .bash_profile
+    . .bash_profile
+    rvm install 1.8.7
+    
+2 - install tokyocabinet (Intructions for user install follow. For system level, make sure we have dev package)
+    1 - download source from http://fallabs.com/tokyocabinet/tokyocabinet-1.4.47.tar.gz
+    2 - unpack and compile using a user-level prefix
+    3 - set LD_RUN_PATH and LD_LIBRARY_PATH to point there
+    
+    :script
+
+    cd
+    mkdir -p tmp/tokyocabinet   
+    cd tmp/tokyocabinet
+    wget "http://fallabs.com/tokyocabinet/tokyocabinet-1.4.47.tar.gz"
+    tar xvfz tokyocabinet-1.4.47.tar.gz
+    cd tokyocabinet-1.4.47
+        ./configure --prefix="$HOME/software/opt/tokyocabinet"
+        make
+        make install
+    cd
+    cat .bash_profile > tmp.bash_profile
+    echo "export LD_RUN_PATH='$LD_RUN_PATH:$HOME/software/opt/tokyocabinet/lib'" >> tmp.bash_profile
+    echo "export LD_LIBRARY_PATH='$LD_LIBRARY_PATH:$HOME/software/opt/tokyocabinet/lib'" >> tmp.bash_profile
+    echo "export PATH='$PATH:$HOME/software/opt/tokyocabinet/bin'" >> tmp.bash_profile
+    cp .bash_profile .bash_profile.save2
+    mv tmp.bash_profile .bash_profile
+    . .bash_profile
+    
+3 - install gems rbbt-util rbbt-sources rbbt-text rbbt-phgx
+
+    :script
+    
+    gem install rbbt-util rbbt-sources rbbt-text rbbt-phgx
+
+3.bis - Or install github repos and make ruby use them
+
+    :script
+
+    cd
+    cd git/
+    git clone git://github.com/mikisvaz/rbbt-util.git
+    git clone git://github.com/mikisvaz/rbbt-sources.git
+    git clone git://github.com/mikisvaz/rbbt-text.git
+    git clone git://github.com/mikisvaz/rbbt-phgx.git
+    alias druby="env RBBT_LOG=0 ruby $(for d in $HOME/git/rbbt-*;do echo -n "-I$d/lib ";done)"
+    alias drake="env RBBT_LOG=0 rake $(for d in $HOME/git/rbbt-*;do echo -n "-I$d/lib ";done)"
+
+4 - Set up R to find the helper lib
+
+    :script
+
+    echo "source(system('rbbt_Rutil.rb', intern =T));" >> $HOME/.Rprofile
+

data/bin/run_workflow.rb

@@ -1,8 +1,8 @@
 #!/usr/bin/env ruby
 
-require 'rbbt-util'
+require 'rbbt'
 require 'rbbt/util/simpleopt'
-require 'rbbt/util/workflow'
+require 'rbbt/workflow'
 require 'pp'
 
 def usage(task)
@@ -10,12 +10,11 @@ def usage(task)
   exit -1
 end
 
-def SOPT_options(task)
+def SOPT_options(workflow, task)
   sopt_options = []
-  task.option_summary.flatten.each do |info|
-    name = info[:name]
+  workflow.rec_inputs(task.name).each do |name|
     short = name.to_s.chars.first
-    boolean = info[:type] == :boolean
+    boolean = workflow.rec_input_types(task.name)[name].to_sym == :boolean
     
     sopt_options << "-#{short}--#{name}#{boolean ? '' : '*'}"
   end
@@ -23,28 +22,54 @@ def SOPT_options(task)
   sopt_options * ":"
 end
 
-def fix_options(task, job_options)
-  option_types = task.option_summary.flatten.inject({}){|types, new| types[new[:name]] = new[:type]; types}
+def fix_options(workflow, task, job_options)
+  option_types = workflow.rec_input_types(task.name)
+
+  workflow.resolve_locals(job_options)
 
   job_options_cleaned = {}
 
   job_options.each do |name, value|
-    value = case
-            when option_types[name] == :float
+    value = case option_types[name].to_sym
+            when :float
               value.to_f
-            when option_types[name] == :integer
+            when :integer
               value.to_i
-            when option_types[name] == :array
-              value.split(/[,|]/)
-            when option_types[name] == :tsv
-              begin
-                if value == '-'
-                  TSV.new(STDIN).to_s :sort
+            when :string, :text
+              case
+              when value == '-'
+                STDIN.read
+              when (String === value and File.exists?(value))
+                Open.read(value)
+              else
+                value
+              end
+            when :array
+              if Array === value
+                value
+              else
+                case
+                when value == '-'
+                  STDIN.read
+                when (String === value and File.exists?(value))
+                  Open.read(value)
                 else
-                  TSV.new(value).to_s :sort
-                end
-              rescue
+                  value
+                end.split(/[,|\s]/)
+              end
+            when :tsv
+              if TSV === value
                 value
+              else
+                begin
+                  if value == '-'
+                    TSV.open(STDIN).to_s :sort
+                  else
+                    TSV.new(value).to_s :sort
+                  end
+                rescue
+                  value
+                end
               end
             else
               value
@@ -55,64 +80,112 @@ def fix_options(task, job_options)
   job_options_cleaned
 end
 
-options = SOPT.get "-t--task*:-l--log*:-h--help:-n--name:-cl--clean:-rcl-recursive_clean"
+options = SOPT.get "-t--task*:-l--log*:-h--help:-n--name*:-cl--clean:-rcl-recursive_clean:-pn--printname:-srv--server"
 
-# Set log, fork, clean, recursive_clean and help
-Log.severity = options[:log].to_i if options.include? :log
-help = !!options.delete(:help)
-do_fork = !!options.delete(:fork)
-clean = !!options.delete(:clean)
-recursive_clean = !!options.delete(:recursive_clean)
-
-# Get workflow
 workflow = ARGV.first
-WorkFlow.require_workflow workflow
-
-# Set task
-namespace, task = nil, nil
-
-case 
-when (not options[:task])
-  workflow_usage if help
-  task = self.last_task
-  namespace = self
-when (options[:task] =~ /\./)
-  namespace, task = options.delete(:task).split('.')
-  namespace = Misc.string2const(namespace)
+
+if options[:server]
+  require 'rbbt/util/log'
+  require 'rbbt/workflow'
+  require 'rbbt/workflow/rest'
+  require 'sinatra'
+  require 'compass'
+
+  Workflow.require_workflow workflow
+  WorkflowREST.add_workflows Workflow.workflows.last
+
+  WorkflowREST.setup
 else
-  task_name = options.delete(:task)
-  task = self.tasks[task_name]
-end
 
-usage(task) if help
+  # Set log, fork, clean, recursive_clean and help
+  Log.severity = options[:log].to_i if options.include? :log
+  help = !!options.delete(:help)
+  do_fork = !!options.delete(:fork)
+  clean = !!options.delete(:clean)
+  recursive_clean = !!options.delete(:recursive_clean)
+
 
-name = options.delete(:name) || "Default"
+  # Get workflow
+  
+  if Rbbt.etc.remote_workflows.exists?
+    remote_workflows = Rbbt.etc.remote_workflows.yaml
+  else
+    remote_workflows = {}
+  end
+
+  if remote_workflows.include? workflow
+    require 'rbbt/workflow/rest/client'
+    workflow = RbbtRestClient.new remote_workflows[workflow]
+  else
+    Workflow.require_workflow workflow
+    workflow = Workflow.workflows.last
+  end
+
+  # Set task
+  namespace, task = nil, nil
+
+  case 
+  when (not options[:task])
+    usage if help
+    task = workflow.last_task
+    namespace = workflow
+  when (options[:task] =~ /\./)
+    namespace, task = options.delete(:task).split('.')
+    namespace = Misc.string2const(namespace)
+  else
+    task_name = options.delete(:task)
+    task = workflow.tasks[task_name]
+    raise "Task not found: #{ task_name }" if task.nil?
+  end
+
+
+  workflow.usage(task) if help
 
-# get job args
-sopt_option_string = SOPT_options(task)
-job_options = SOPT.get sopt_option_string
-job_options = fix_options(task, job_options)
+  name = options.delete(:name) || "Default"
 
-#- get job
-job = task.job(name, job_options)
+  # get job args
+  sopt_option_string = SOPT_options(workflow,task)
+  job_options = SOPT.get sopt_option_string
+  job_options = fix_options(workflow,task, job_options)
 
-# clean job
-job.clean if clean
-job.recursive_clean if recursive_clean
+  #- get job
+  job = workflow.job(task.name, name, job_options)
 
-# run
-if do_fork
-  job.fork
-  while not job.done?
-    puts "#{job.step}: #{job.messages.last}"
-    sleep 2
+  # clean job
+  job.clean if clean
+  job.recursive_clean if recursive_clean
+
+  # run
+  if do_fork
+    job.fork
+    while not job.done?
+      Log.debug "#{job.step}: #{job.messages.last}"
+      sleep 2
+    end
+    raise job.messages.last if job.error?
+    res = job.load
+  else
+    res = job.run
   end
-else
-  job.run
-end
 
-#- error
-raise job.messages.last if job.error?
+  if options.delete(:printname)
+    puts job.name
+    exit
+  else
+    Log.low "Job name: #{job.name}"
+  end
 
-#print
-pp job.load
+  #- error
+
+  #print
+  case
+  when Array === res
+    puts res * "\n"
+  when TSV === res
+    puts res
+  when Hash === res
+    puts res.to_yaml
+  else
+    puts res
+  end
+end

data/lib/rbbt-util.rb

@@ -3,11 +3,11 @@ require 'rbbt/util/open'
 require 'rbbt/util/cmd'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/filecache'
-require 'rbbt/util/tsv'
-require 'rbbt/util/persistence'
+require 'rbbt/tsv'
+require 'rbbt/persist'
 require 'rbbt/util/misc'
 
 FileCache.cachedir   = Rbbt.var.cache.filecache.find :user
 Open.cachedir        = Rbbt.var.cache["open-remote"].find :user
 TmpFile.tmpdir       = Rbbt.tmp.find :user
-Persistence.cachedir = Rbbt.var.cache.persistence.find :user
+Persist.cachedir = Rbbt.var.cache.persistence.find :user

data/lib/rbbt.rb

@@ -1,7 +1,16 @@
-require 'rbbt/util/resource'
+require 'rbbt/util/open'
+require 'rbbt/util/tmpfile'
+require 'rbbt/util/filecache'
+require 'rbbt/tsv'
+require 'rbbt/persist'
+require 'rbbt/resource'
 
 module Rbbt
   extend Resource
-end
-
+  pkgdir = 'rbbt'
 
+  FileCache.cachedir   = var.cache.filecache.find :user
+  TmpFile.tmpdir       = tmp.find :user
+  Open.cachedir        = var.cache["open-remote"].find :user
+  Persist.cachedir = var.cache.persistence.find :user
+end

data/lib/rbbt/annotations.rb

@@ -0,0 +1,215 @@
+require 'rbbt/util/chain_methods'
+require 'json'
+module Annotated
+  attr_accessor :annotation_types
+
+  def self.extended(base)
+    base.annotation_types ||= []
+  end
+
+  def annotations
+    annotation_types.collect do |mod|
+      mod.annotations
+    end.flatten.uniq
+  end
+
+  def info
+    hash = {:annotation_types => annotation_types}
+    annotations.each do |annotation|
+      value = self.send(annotation)
+      hash[annotation] = value unless value.nil?
+    end
+    hash
+  end
+
+  def id
+    Misc.hash2md5 info
+  end
+
+  def self.load(object, info)
+    annotation_types = info[:annotation_types]
+    annotation_types.each do |mod|
+      mod = Misc.string2const(mod) if String === mod
+      mod.setup_info(object, info)
+    end
+
+    object
+  end
+
+  def tsv_values(*fields)
+    fields = fields.flatten
+    info = self.info
+    values = []
+    fields.each do |field|
+      values << case
+      when field == "JSON"
+        info.to_json
+      when field == "literal"
+        self.gsub(/\n|\t/, ' ') 
+      when info.include?(field.to_sym)
+        info.delete(field.to_sym)
+      when self.respond_to?(field)
+        self.send(field)
+      end
+    end
+    values
+  end
+
+  def self.load_tsv_values(id, values, *fields)
+    fields = fields.flatten
+    info = {}
+    literal_pos = fields.index "literal"
+
+    object = if literal_pos.nil?
+               id
+             else
+               v = values[literal_pos]
+               v = v.first if Array === v
+               v
+             end
+
+    fields.each_with_index do |field,i|
+      if field == "JSON"
+        JSON.parse(values[i]).each do |key, value|
+          info[key.to_sym] = value
+        end
+      else
+        info[field.to_sym] = values[i]
+      end
+    end
+
+    self.load(object, info)
+  end
+
+  def self.tsv(annotations, *fields)
+    fields = case
+             when ((fields.compact.empty?) and not annotations.empty?)
+               fields = annotations.first.annotations
+               fields << :annotation_types
+             when (fields == [:literal] and not annotations.empty?)
+               fields = annotations.first.annotations
+               fields << :literal
+             when (fields == [:all] and not annotations.empty?)
+               fields = [:annotation_types] + annotations.first.annotations
+               fields << :literal
+             else
+               fields.flatten
+             end
+    fields = fields.collect{|f| f.to_s}
+
+    tsv = TSV.setup({}, :key_field => "ID", :fields => fields)
+
+    annotations.each do |annotation|
+      tsv[annotation.id] = annotation.tsv_values(fields)
+    end
+
+    tsv
+  end
+
+  def self.load_tsv(tsv)
+    tsv.collect do |id, values|
+      Annotated.load_tsv_values(id, values, tsv.fields)
+    end
+  end
+end
+
+
+module Annotation
+  def self.extended(base)
+    if not base.respond_to? :annotations
+      class << base
+        attr_accessor :annotations, :inheritance, :all_inheritance, :all_annotations
+        self
+      end
+
+      base.annotations = []
+      base.inheritance = []
+      base.all_annotations = []
+      base.all_inheritance = []
+
+      base.module_eval do
+        def self.extended(object)
+          object.extend Annotated
+          if not object.annotation_types.include? self
+            object.annotation_types.concat self.inheritance 
+            object.annotation_types << self
+            object.annotation_types.uniq!
+          end
+        end
+
+        def self.included(base)
+          base.inheritance << self
+          base.all_inheritance.concat self.all_inheritance if self.respond_to? :all_inheritance
+          base.all_inheritance << self
+          base.all_inheritance.uniq!
+          base.update_annotations
+        end
+      end
+
+    end
+  end
+
+  def update_annotations
+    @all_annotations = all_inheritance.inject([]){|acc,mod| acc.concat mod.annotations}.concat(@annotations)
+  end
+
+  def annotation(*values)
+    @annotations.concat values.collect{|v| v.to_sym}
+    update_annotations
+
+    module_eval do
+      attr_accessor *values
+    end
+  end
+
+ def setup_info(object, info)
+    object.extend self
+    all_annotations.each do |annotation|
+      object.send(annotation.to_s + '=', info[annotation])
+    end
+  end
+
+  def setup(object, *values)
+    object.extend self
+
+    all_annotations.zip(values).each do |name, value|
+      object.send(name.to_s + '=', value)
+    end
+
+    object
+  end
+end
+
+module AnnotatedArray
+  extend ChainMethods
+  self.chain_prefix = :annotated_array
+
+  def annotated_array_get_brackets(pos)
+    value = annotated_array_clean_get_brackets(pos)
+    annotation_types.each do |mod|
+      mod.setup(value, *info.values_at(*mod.annotations))
+    end
+    value
+  end
+
+  def annotated_array_each
+    annotation_array_clean_each do |value|
+      annotation_types.each do |mod|
+        mod.setup(value, annotation)
+      end
+      yield value
+    end
+  end
+
+  def annotated_array_collect
+    annotation_array_clean_collect do |value|
+      annotation_types.each do |mod|
+        mod.setup(value, annotation)
+      end
+      yield value
+    end
+  end
+
+end
+
+