rbbt-util 5.2.0 → 5.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/rbbt +3 -1
- data/bin/run_workflow.rb +0 -24
- data/lib/rbbt-util.rb +1 -1
- data/lib/rbbt.rb +9 -6
- data/lib/rbbt/persist.rb +12 -13
- data/lib/rbbt/persist/tsv.rb +4 -2
- data/lib/rbbt/resource.rb +1 -2
- data/lib/rbbt/tsv/attach.rb +8 -7
- data/lib/rbbt/tsv/attach/util.rb +0 -1
- data/lib/rbbt/util/misc.rb +11 -0
- data/lib/rbbt/util/named_array.rb +4 -2
- data/lib/rbbt/util/simpleopt.rb +15 -4
- data/lib/rbbt/workflow/accessor.rb +1 -1
- data/lib/rbbt/workflow/usage.rb +13 -2
- data/share/rbbt_commands/app/start +18 -0
- data/share/rbbt_commands/tsv/info +20 -0
- data/share/rbbt_commands/workflow/server +70 -0
- data/share/rbbt_commands/workflow/task +15 -4
- metadata +5 -2
data/bin/rbbt
CHANGED
data/bin/run_workflow.rb
CHANGED
@@ -86,30 +86,6 @@ workflow = ARGV.first
|
|
86
86
|
|
87
87
|
if options[:server]
|
88
88
|
|
89
|
-
#require 'rbbt/util/log'
|
90
|
-
#require 'rbbt/workflow'
|
91
|
-
#require 'rbbt/workflow/rest'
|
92
|
-
#require 'sinatra'
|
93
|
-
#require 'compass'
|
94
|
-
|
95
|
-
#if workflow
|
96
|
-
# Workflow.require_workflow workflow
|
97
|
-
# WorkflowREST.add_workflows *Workflow.workflows
|
98
|
-
#end
|
99
|
-
|
100
|
-
#WorkflowREST.setup
|
101
|
-
|
102
|
-
#Sinatra::Application.port = options[:port] || 4567
|
103
|
-
#Sinatra::Application.run = true
|
104
|
-
|
105
|
-
#if workflow and File.exists? workflow
|
106
|
-
# Sinatra::Application.views = File.join(File.dirname(workflow), 'www/views')
|
107
|
-
#end
|
108
|
-
|
109
|
-
#sinatra_file = './lib/sinatra.rb'
|
110
|
-
#if File.exists? sinatra_file
|
111
|
-
# require sinatra_file
|
112
|
-
#end
|
113
89
|
require 'zurb-foundation'
|
114
90
|
require 'modular-scale'
|
115
91
|
|
data/lib/rbbt-util.rb
CHANGED
@@ -10,4 +10,4 @@ require 'rbbt/util/misc'
|
|
10
10
|
FileCache.cachedir = Rbbt.var.cache.filecache.find :user
|
11
11
|
Open.cachedir = Rbbt.var.cache["open-remote"].find :user
|
12
12
|
TmpFile.tmpdir = Rbbt.tmp.find :user
|
13
|
-
Persist.cachedir = Rbbt.var.cache.persistence
|
13
|
+
Persist.cachedir = Rbbt.var.cache.persistence
|
data/lib/rbbt.rb
CHANGED
@@ -1,16 +1,19 @@
|
|
1
|
+
require 'rbbt/resource'
|
1
2
|
require 'rbbt/util/open'
|
3
|
+
require 'rbbt/util/cmd'
|
2
4
|
require 'rbbt/util/tmpfile'
|
3
5
|
require 'rbbt/util/filecache'
|
4
6
|
require 'rbbt/tsv'
|
5
7
|
require 'rbbt/persist'
|
6
|
-
require 'rbbt/
|
8
|
+
require 'rbbt/util/misc'
|
7
9
|
|
8
10
|
module Rbbt
|
9
11
|
extend Resource
|
10
12
|
pkgdir = 'rbbt'
|
11
|
-
|
12
|
-
FileCache.cachedir = var.cache.filecache.find :user
|
13
|
-
TmpFile.tmpdir = tmp.find :user
|
14
|
-
Open.cachedir = var.cache["open-remote"].find :user
|
15
|
-
Persist.cachedir = var.cache.persistence.find :user
|
16
13
|
end
|
14
|
+
|
15
|
+
Open.cachedir = Rbbt.var.cache["open-remote"].find :user
|
16
|
+
TmpFile.tmpdir = Rbbt.tmp.find :user
|
17
|
+
FileCache.cachedir = Rbbt.var.cache.filecache.find :user
|
18
|
+
Persist.cachedir = Rbbt.var.cache.persistence.find :user
|
19
|
+
|
data/lib/rbbt/persist.rb
CHANGED
@@ -6,20 +6,18 @@ require 'rbbt/persist/tsv'
|
|
6
6
|
require 'set'
|
7
7
|
|
8
8
|
module Persist
|
9
|
-
|
10
|
-
|
9
|
+
class << self
|
10
|
+
attr_accessor :cachedir
|
11
|
+
def self.cachedir=(cachedir)
|
12
|
+
@cachedir = Path === cachedir ? cachedir : Path.setup(cachedir)
|
13
|
+
end
|
14
|
+
def self.cachedir
|
15
|
+
@cachedir ||= Rbbt.var.cache.persistence
|
16
|
+
end
|
17
|
+
end
|
11
18
|
|
12
19
|
MEMORY = {} unless defined? MEMORY
|
13
20
|
|
14
|
-
def self.cachedir=(cachedir)
|
15
|
-
CACHEDIR.replace cachedir
|
16
|
-
FileUtils.mkdir_p CACHEDIR unless File.exist? CACHEDIR
|
17
|
-
end
|
18
|
-
|
19
|
-
def self.cachedir
|
20
|
-
CACHEDIR
|
21
|
-
end
|
22
|
-
|
23
21
|
def self.newer?(path, file)
|
24
22
|
return true if not File.exists? file
|
25
23
|
return true if File.mtime(path) < File.mtime(file)
|
@@ -60,7 +58,8 @@ module Persist
|
|
60
58
|
end
|
61
59
|
end
|
62
60
|
|
63
|
-
persistence_dir = Misc.process_options(persist_options, :dir) ||
|
61
|
+
persistence_dir = Misc.process_options(persist_options, :dir) || Persist.cachedir
|
62
|
+
Path.setup(persistence_dir) unless Path === persistence_dir
|
64
63
|
|
65
64
|
filename = perfile.gsub(/\s/,'_').gsub(/\//,'>')
|
66
65
|
clean_options = options
|
@@ -70,7 +69,7 @@ module Persist
|
|
70
69
|
options_md5 = Misc.hash2md5 clean_options
|
71
70
|
filename << ":" << options_md5 unless options_md5.empty?
|
72
71
|
|
73
|
-
|
72
|
+
persistence_dir[filename].find
|
74
73
|
end
|
75
74
|
|
76
75
|
TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
|
data/lib/rbbt/persist/tsv.rb
CHANGED
@@ -82,7 +82,8 @@ module Persist
|
|
82
82
|
end
|
83
83
|
|
84
84
|
def write_and_close
|
85
|
-
|
85
|
+
lock_filename = Persist.persistence_path(persistence_path, {:dir => Rbbt.tmp.tsv_open_locks.find})
|
86
|
+
Misc.lock(lock_filename) do
|
86
87
|
write if @closed or not write?
|
87
88
|
res = begin
|
88
89
|
yield
|
@@ -154,7 +155,8 @@ module Persist
|
|
154
155
|
|
155
156
|
if is_persisted? path
|
156
157
|
Log.debug "TSV persistence up-to-date: #{ path }"
|
157
|
-
|
158
|
+
lock_filename = Persist.persistence_path(path, {:dir => Rbbt.tmp.tsv_open_locks.find})
|
159
|
+
return Misc.lock(lock_filename) do open_tokyocabinet(path, false); end
|
158
160
|
else
|
159
161
|
Log.debug "TSV persistence creating: #{ path }"
|
160
162
|
end
|
data/lib/rbbt/resource.rb
CHANGED
data/lib/rbbt/tsv/attach.rb
CHANGED
@@ -137,6 +137,12 @@ module TSV
|
|
137
137
|
fields, one2one = Misc.process_options options, :fields, :one2one
|
138
138
|
in_namespace = options[:in_namespace]
|
139
139
|
|
140
|
+
unless TSV === other
|
141
|
+
other_identifier_files = other.identifier_files if other.respond_to? :identifier_files
|
142
|
+
other = TSV.open(other, :persist => options[:persist_input] == true) unless TSV === other
|
143
|
+
other.identifiers = other_identifier_files
|
144
|
+
end
|
145
|
+
|
140
146
|
fields = other.fields - [key_field].concat(self.fields) if fields.nil? or fields == :all
|
141
147
|
if in_namespace
|
142
148
|
fields = other.fields_in_namespace - [key_field].concat(self.fields) if fields.nil?
|
@@ -144,13 +150,8 @@ module TSV
|
|
144
150
|
fields = other.fields - [key_field].concat(self.fields) if fields.nil?
|
145
151
|
end
|
146
152
|
|
147
|
-
|
148
|
-
|
149
|
-
unless TSV === other
|
150
|
-
other_identifier_files = other.identifier_files
|
151
|
-
other = other.tsv(:persist => options[:persist_input] == true) unless TSV === other
|
152
|
-
other.identifiers = other_identifier_files
|
153
|
-
end
|
153
|
+
other_filename = other.respond_to?(:filename) ? other.filename : other.inspect
|
154
|
+
Log.medium("Attaching fields:#{fields.inspect} from #{other_filename}.")
|
154
155
|
|
155
156
|
case
|
156
157
|
when key_field == other.key_field
|
data/lib/rbbt/tsv/attach/util.rb
CHANGED
data/lib/rbbt/util/misc.rb
CHANGED
@@ -635,6 +635,17 @@ end
|
|
635
635
|
res
|
636
636
|
end
|
637
637
|
|
638
|
+
def self.do_once(&block)
|
639
|
+
return nil if $__did_once
|
640
|
+
$__did_once = true
|
641
|
+
yield
|
642
|
+
nil
|
643
|
+
end
|
644
|
+
|
645
|
+
def self.reset_do_once
|
646
|
+
$__did_once = false
|
647
|
+
end
|
648
|
+
|
638
649
|
def self.insist(times = 3, sleep = nil)
|
639
650
|
try = 0
|
640
651
|
begin
|
@@ -115,8 +115,10 @@ module NamedArray
|
|
115
115
|
end
|
116
116
|
|
117
117
|
def named_array_values_at(*keys)
|
118
|
-
keys = keys.collect{|k| Misc.field_position(fields, k) }
|
119
|
-
|
118
|
+
keys = keys.collect{|k| Misc.field_position(fields, k, true) }
|
119
|
+
keys.collect{|k|
|
120
|
+
named_array_get_brackets(k) unless k.nil?
|
121
|
+
}
|
120
122
|
end
|
121
123
|
|
122
124
|
def zip_fields
|
data/lib/rbbt/util/simpleopt.rb
CHANGED
@@ -32,18 +32,29 @@ module SOPT
|
|
32
32
|
|
33
33
|
index = 0
|
34
34
|
while index < ARGV.length do
|
35
|
-
|
35
|
+
orig_arg = ARGV[index]
|
36
|
+
|
37
|
+
if orig_arg =~ /=/
|
38
|
+
arg, value = orig_arg.match(/(.*?)=(.*)/).values_at 1, 2
|
39
|
+
else
|
40
|
+
arg = orig_arg
|
41
|
+
value = nil
|
42
|
+
end
|
43
|
+
|
36
44
|
if switches.include? arg
|
37
45
|
name = switches[arg]
|
38
46
|
i = info[name]
|
39
47
|
if i[:arg]
|
40
|
-
|
41
|
-
|
48
|
+
if value.nil?
|
49
|
+
value = ARGV[index + 1]
|
50
|
+
index += 1
|
51
|
+
end
|
52
|
+
options[name.to_sym] = value
|
42
53
|
else
|
43
54
|
options[name.to_sym] = true
|
44
55
|
end
|
45
56
|
else
|
46
|
-
rest <<
|
57
|
+
rest << orig_arg
|
47
58
|
end
|
48
59
|
index += 1
|
49
60
|
end
|
@@ -180,7 +180,7 @@ module Workflow
|
|
180
180
|
description = task.description
|
181
181
|
result_description = task.result_description
|
182
182
|
result_type = task.result_type
|
183
|
-
inputs = rec_inputs(name)
|
183
|
+
inputs = rec_inputs(name).uniq
|
184
184
|
input_types = rec_input_types(name)
|
185
185
|
input_descriptions = rec_input_descriptions(name)
|
186
186
|
input_defaults = rec_input_defaults(name)
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -13,7 +13,13 @@ module Task
|
|
13
13
|
default = input_defaults[name]
|
14
14
|
type = input_types[name]
|
15
15
|
|
16
|
-
|
16
|
+
|
17
|
+
if type.to_sym == :boolean
|
18
|
+
puts " * -#{short}, --#{name}[=<true|false>]#{default != nil ? " (default: #{default})" : ""}:"
|
19
|
+
else
|
20
|
+
puts " * -#{short}, --#{name}=<#{ type }>#{default != nil ? " (default: #{default})" : ""}:"
|
21
|
+
end
|
22
|
+
|
17
23
|
puts " " << description if description and not description.empty?
|
18
24
|
puts
|
19
25
|
end
|
@@ -32,7 +38,12 @@ module Task
|
|
32
38
|
default = dep.input_defaults[name]
|
33
39
|
type = dep.input_types[name]
|
34
40
|
|
35
|
-
|
41
|
+
if type.to_sym == :boolean
|
42
|
+
puts " * -#{short}, --#{name}[=<true|false>]#{default != nil ? " (default: #{default})" : ""}:"
|
43
|
+
else
|
44
|
+
puts " * -#{short}, --#{name}=<#{ type }>#{default != nil ? " (default: #{default})" : ""}:"
|
45
|
+
end
|
46
|
+
|
36
47
|
puts " " << description if description and not description.empty?
|
37
48
|
puts
|
38
49
|
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt-util'
|
4
|
+
require 'rbbt/util/simpleopt'
|
5
|
+
|
6
|
+
options = SOPT.get "-e--environment*:-p--port*:-s--server*"
|
7
|
+
|
8
|
+
app = ARGV.shift
|
9
|
+
|
10
|
+
app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps.find
|
11
|
+
|
12
|
+
app_dir = app_dir[app]
|
13
|
+
|
14
|
+
server = options[:server] || 'thin'
|
15
|
+
|
16
|
+
Misc.in_dir(app_dir) do
|
17
|
+
`#{options.include?(:environment)? "env RACK_ENV=#{options[:environment]}" : ""} #{server} start -p #{options[:port]} #{ARGV.collect{|a| "'#{a}'"} * " "}`
|
18
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt-util'
|
4
|
+
require 'rbbt/util/simpleopt'
|
5
|
+
|
6
|
+
|
7
|
+
file = ARGV.shift
|
8
|
+
|
9
|
+
header = TSV.parse_header(Open.open(file))
|
10
|
+
|
11
|
+
puts "File: #{ file }"
|
12
|
+
puts "Key: #{header.key_field}"
|
13
|
+
puts "Fields: "
|
14
|
+
header.fields.each_with_index do |f,i|
|
15
|
+
puts " - #{i + 1}: " << f
|
16
|
+
end
|
17
|
+
puts "Rows: #{`wc -l #{ file }|cut -f 1 -d' '`}"
|
18
|
+
puts "First line:"
|
19
|
+
puts header.first_line
|
20
|
+
|
@@ -0,0 +1,70 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt/util/simpleopt'
|
4
|
+
require 'rbbt/workflow'
|
5
|
+
|
6
|
+
require 'zurb-foundation'
|
7
|
+
require 'modular-scale'
|
8
|
+
|
9
|
+
require 'rbbt/rest/main'
|
10
|
+
require 'rbbt/rest/entity'
|
11
|
+
require 'rbbt/rest/workflow'
|
12
|
+
require 'rbbt/rest/file_server'
|
13
|
+
require 'rbbt/rest/helpers'
|
14
|
+
|
15
|
+
options = SOPT.get "--profile:-p--port*"
|
16
|
+
|
17
|
+
workflow = ARGV.first
|
18
|
+
|
19
|
+
YAML::ENGINE.yamler = 'syck' if defined? YAML::ENGINE and YAML::ENGINE.respond_to? :yamler
|
20
|
+
|
21
|
+
Workflow.require_workflow workflow
|
22
|
+
|
23
|
+
class WorkflowRest < Sinatra::Base
|
24
|
+
get '/' do
|
25
|
+
redirect to(File.join('/', Workflow.workflows.last.to_s))
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
if options[:profile]
|
30
|
+
WorkflowRest.before File.join('/', Workflow.workflows.last.to_s, '*') do
|
31
|
+
@profile = true
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
class WorkflowRest < Sinatra::Base
|
36
|
+
|
37
|
+
#{{{ MODULES AND HELPERS
|
38
|
+
register Sinatra::RbbtRESTMain
|
39
|
+
register Sinatra::RbbtRESTWorkflow
|
40
|
+
register Sinatra::RbbtRESTEntity
|
41
|
+
helpers Sinatra::RbbtMiscHelpers
|
42
|
+
|
43
|
+
#{{{ DIRECTORIES
|
44
|
+
local_var = Rbbt.var.find(:lib)
|
45
|
+
set :cache_dir , local_var.sinatra.cache.find
|
46
|
+
set :file_dir , local_var.sinatra.files.find
|
47
|
+
set :favourites_dir , local_var.sinatra.favourites.find
|
48
|
+
set :favourite_lists_dir , local_var.sinatra.favourite_lists
|
49
|
+
|
50
|
+
#{{{ SESSIONS
|
51
|
+
use Rack::Session::Cookie, :key => 'rack.session',
|
52
|
+
:path => '/',
|
53
|
+
:expire_after => 2592000,
|
54
|
+
:secret => "Workflow #{Workflow.workflows.inspect} secret!!"
|
55
|
+
|
56
|
+
#{{{ FOUNDATION RESOURCES
|
57
|
+
add_sass_load_path "#{Gem.loaded_specs['compass'].full_gem_path}/frameworks/compass/stylesheets"
|
58
|
+
add_sass_load_path "#{Gem.loaded_specs['zurb-foundation'].full_gem_path}/scss/"
|
59
|
+
add_sass_load_path "#{Gem.loaded_specs['modular-scale'].full_gem_path}/stylesheets/"
|
60
|
+
RbbtRESTHelpers.javascript_resources << Path.setup("#{Gem.loaded_specs['zurb-foundation'].full_gem_path}/js/foundation")
|
61
|
+
RbbtRESTHelpers.javascript_resources << Path.setup("#{Gem.loaded_specs['zurb-foundation'].full_gem_path}/js/vendor")
|
62
|
+
|
63
|
+
$title = "Workflow Scout"
|
64
|
+
use Rack::Deflater
|
65
|
+
end
|
66
|
+
|
67
|
+
WorkflowRest.add_workflow Workflow.workflows.last, true
|
68
|
+
|
69
|
+
WorkflowRest.port = options[:port] || 4567
|
70
|
+
WorkflowRest.run!
|
@@ -54,14 +54,20 @@ def fix_options(workflow, task, job_options)
|
|
54
54
|
if Array === value
|
55
55
|
value
|
56
56
|
else
|
57
|
-
case
|
57
|
+
str = case
|
58
58
|
when value == '-'
|
59
59
|
STDIN.read
|
60
60
|
when (String === value and File.exists?(value))
|
61
61
|
Open.read(value)
|
62
62
|
else
|
63
63
|
value
|
64
|
-
end
|
64
|
+
end
|
65
|
+
|
66
|
+
if $array_separator
|
67
|
+
str.split(/#{$array_separator}/)
|
68
|
+
else
|
69
|
+
str.split(/[,|\s]/)
|
70
|
+
end
|
65
71
|
end
|
66
72
|
when :tsv
|
67
73
|
if TSV === value
|
@@ -86,7 +92,7 @@ def fix_options(workflow, task, job_options)
|
|
86
92
|
job_options_cleaned
|
87
93
|
end
|
88
94
|
|
89
|
-
options = SOPT.get "-t--task*:--profile:-l--log*:-h--help:-n--name*:-cl--clean:-rcl-recursive_clean:-pn--printname"
|
95
|
+
options = SOPT.get "-t--task*:--profile:-l--log*:-h--help:-n--name*:-cl--clean:-rcl-recursive_clean:-pn--printname:-as--array_separator*"
|
90
96
|
|
91
97
|
workflow = ARGV.shift
|
92
98
|
usage if workflow.nil?
|
@@ -100,6 +106,7 @@ help = !!options.delete(:help)
|
|
100
106
|
do_fork = !!options.delete(:fork)
|
101
107
|
clean = !!options.delete(:clean)
|
102
108
|
recursive_clean = !!options.delete(:recursive_clean)
|
109
|
+
$array_separator = options.delete(:array_separator)
|
103
110
|
|
104
111
|
# Get workflow
|
105
112
|
|
@@ -156,7 +163,11 @@ if do_fork
|
|
156
163
|
sleep 2
|
157
164
|
end
|
158
165
|
raise job.messages.last if job.error?
|
159
|
-
|
166
|
+
if $array_separator
|
167
|
+
res = job.load(:sep2 => $array_separator)
|
168
|
+
else
|
169
|
+
res = job.load
|
170
|
+
end
|
160
171
|
else
|
161
172
|
res = job.run
|
162
173
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.2.
|
4
|
+
version: 5.2.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-05-
|
12
|
+
date: 2013-05-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -236,12 +236,15 @@ files:
|
|
236
236
|
- lib/rbbt/workflow/usage.rb
|
237
237
|
- share/install/software/lib/install_helpers
|
238
238
|
- share/lib/R/util.R
|
239
|
+
- share/rbbt_commands/app/start
|
239
240
|
- share/rbbt_commands/conf/web_user/add
|
240
241
|
- share/rbbt_commands/conf/web_user/list
|
241
242
|
- share/rbbt_commands/conf/web_user/remove
|
243
|
+
- share/rbbt_commands/tsv/info
|
242
244
|
- share/rbbt_commands/workflow/remote/add
|
243
245
|
- share/rbbt_commands/workflow/remote/list
|
244
246
|
- share/rbbt_commands/workflow/remote/remove
|
247
|
+
- share/rbbt_commands/workflow/server
|
245
248
|
- share/rbbt_commands/workflow/task
|
246
249
|
- README.rdoc
|
247
250
|
- test/rbbt/tsv/test_accessor.rb
|