rbbt-util 3.2.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +65 -0
- data/bin/run_workflow.rb +142 -69
- data/lib/rbbt-util.rb +3 -3
- data/lib/rbbt.rb +12 -3
- data/lib/rbbt/annotations.rb +215 -0
- data/lib/rbbt/{util/fix_width_table.rb → fix_width_table.rb} +17 -13
- data/lib/rbbt/persist.rb +164 -0
- data/lib/rbbt/persist/tsv.rb +135 -0
- data/lib/rbbt/resource.rb +100 -0
- data/lib/rbbt/resource/path.rb +180 -0
- data/lib/rbbt/resource/rake.rb +48 -0
- data/lib/rbbt/resource/util.rb +111 -0
- data/lib/rbbt/resource/with_key.rb +28 -0
- data/lib/rbbt/tsv.rb +134 -0
- data/lib/rbbt/tsv/accessor.rb +345 -0
- data/lib/rbbt/tsv/attach.rb +183 -0
- data/lib/rbbt/tsv/attach/util.rb +277 -0
- data/lib/rbbt/{util/tsv/filters.rb → tsv/filter.rb} +76 -37
- data/lib/rbbt/tsv/index.rb +453 -0
- data/lib/rbbt/tsv/manipulate.rb +361 -0
- data/lib/rbbt/tsv/parser.rb +231 -0
- data/lib/rbbt/tsv/serializers.rb +79 -0
- data/lib/rbbt/tsv/util.rb +67 -0
- data/lib/rbbt/util/R.rb +3 -3
- data/lib/rbbt/util/chain_methods.rb +64 -0
- data/lib/rbbt/util/cmd.rb +17 -13
- data/lib/rbbt/util/excel2tsv.rb +4 -3
- data/lib/rbbt/util/log.rb +1 -0
- data/lib/rbbt/util/misc.rb +296 -285
- data/lib/rbbt/util/open.rb +9 -2
- data/lib/rbbt/util/persistence.rb +1 -1
- data/lib/rbbt/util/task/job.rb +3 -1
- data/lib/rbbt/workflow.rb +193 -0
- data/lib/rbbt/workflow/accessor.rb +249 -0
- data/lib/rbbt/workflow/annotate.rb +60 -0
- data/lib/rbbt/workflow/soap.rb +100 -0
- data/lib/rbbt/workflow/step.rb +102 -0
- data/lib/rbbt/workflow/task.rb +76 -0
- data/test/rbbt/resource/test_path.rb +12 -0
- data/test/rbbt/test_annotations.rb +106 -0
- data/test/rbbt/{util/test_fix_width_table.rb → test_fix_width_table.rb} +8 -9
- data/test/rbbt/test_resource.rb +66 -0
- data/test/rbbt/test_tsv.rb +332 -0
- data/test/rbbt/test_workflow.rb +102 -0
- data/test/rbbt/tsv/test_accessor.rb +163 -0
- data/test/rbbt/{util/tsv → tsv}/test_attach.rb +86 -43
- data/test/rbbt/{util/tsv/test_filters.rb → tsv/test_filter.rb} +31 -13
- data/test/rbbt/tsv/test_index.rb +284 -0
- data/test/rbbt/{util/tsv → tsv}/test_manipulate.rb +35 -105
- data/test/rbbt/util/test_R.rb +1 -1
- data/test/rbbt/util/test_chain_methods.rb +22 -0
- data/test/rbbt/util/test_filecache.rb +0 -1
- data/test/rbbt/util/test_misc.rb +97 -79
- data/test/rbbt/util/test_open.rb +1 -0
- data/test/rbbt/util/test_tmpfile.rb +1 -1
- data/test/rbbt/workflow/test_soap.rb +103 -0
- data/test/rbbt/workflow/test_step.rb +142 -0
- data/test/rbbt/workflow/test_task.rb +84 -0
- data/test/test_helper.rb +7 -7
- metadata +80 -54
- data/lib/rbbt/util/rake.rb +0 -176
- data/lib/rbbt/util/resource.rb +0 -355
- data/lib/rbbt/util/task.rb +0 -183
- data/lib/rbbt/util/tc_hash.rb +0 -324
- data/lib/rbbt/util/tsv.rb +0 -236
- data/lib/rbbt/util/tsv/accessor.rb +0 -312
- data/lib/rbbt/util/tsv/attach.rb +0 -416
- data/lib/rbbt/util/tsv/index.rb +0 -419
- data/lib/rbbt/util/tsv/manipulate.rb +0 -300
- data/lib/rbbt/util/tsv/misc.rb +0 -41
- data/lib/rbbt/util/tsv/parse.rb +0 -324
- data/lib/rbbt/util/tsv/resource.rb +0 -88
- data/lib/rbbt/util/workflow.rb +0 -135
- data/lib/rbbt/util/workflow/soap.rb +0 -116
- data/test/rbbt/util/test_persistence.rb +0 -201
- data/test/rbbt/util/test_rake.rb +0 -54
- data/test/rbbt/util/test_resource.rb +0 -77
- data/test/rbbt/util/test_task.rb +0 -133
- data/test/rbbt/util/test_tc_hash.rb +0 -144
- data/test/rbbt/util/test_tsv.rb +0 -221
- data/test/rbbt/util/test_workflow.rb +0 -135
- data/test/rbbt/util/tsv/test_accessor.rb +0 -150
- data/test/rbbt/util/tsv/test_index.rb +0 -241
- data/test/rbbt/util/tsv/test_parse.rb +0 -87
- data/test/rbbt/util/tsv/test_resource.rb +0 -9
@@ -0,0 +1,180 @@
|
|
1
|
+
require 'rbbt/util/chain_methods'
|
2
|
+
require 'rbbt/resource/util'
|
3
|
+
require 'rbbt/tsv'
|
4
|
+
|
5
|
+
module Path
|
6
|
+
attr_accessor :resource, :pkgdir
|
7
|
+
|
8
|
+
extend ChainMethods
|
9
|
+
self.chain_prefix = :path
|
10
|
+
|
11
|
+
def self.setup(string, pkgdir = nil, resource = nil)
|
12
|
+
string.extend Path
|
13
|
+
string.pkgdir = pkgdir || 'rbbt'
|
14
|
+
string.resource = resource
|
15
|
+
string
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.extended(string)
|
19
|
+
setup_chains(string)
|
20
|
+
if not string.respond_to? :byte
|
21
|
+
class << string
|
22
|
+
alias byte path_clean_get_brackets
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def join(name)
|
28
|
+
if self.empty?
|
29
|
+
Path.setup name.to_s, @pkgdir, @resource
|
30
|
+
else
|
31
|
+
Path.setup File.join(self, name.to_s), @pkgdir, @resource
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def dirname
|
36
|
+
Path.setup File.dirname(self), @pkgdir, @resource
|
37
|
+
end
|
38
|
+
|
39
|
+
def path_get_brackets(name)
|
40
|
+
join name
|
41
|
+
end
|
42
|
+
|
43
|
+
def path_method_missing(name, prev = nil, *args, &block)
|
44
|
+
if block_given?
|
45
|
+
path_clean_method_missing name, prev, *args, &block
|
46
|
+
else
|
47
|
+
# Fix problem with ruby 1.9 calling methods by its own initiative. ARG
|
48
|
+
path_clean_method_missing(name, prev, *args) if name.to_s =~ /^to_/
|
49
|
+
if prev.nil?
|
50
|
+
join name
|
51
|
+
else
|
52
|
+
join(prev).join(name)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
SEARCH_PATHS = {
|
58
|
+
:user => File.join(ENV['HOME'], ".{PKGDIR}", "{TOPLEVEL}", "{SUBPATH}"),
|
59
|
+
:global => File.join('/', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
|
60
|
+
:local => File.join('/usr/local', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
|
61
|
+
:lib => File.join('{LIBDIR}', "{TOPLEVEL}", "{SUBPATH}"),
|
62
|
+
}
|
63
|
+
|
64
|
+
def find(where = nil, caller_lib = nil)
|
65
|
+
return self if located?
|
66
|
+
if self.match(/(.*?)\/(.*)/)
|
67
|
+
toplevel, subpath = self.match(/(.*?)\/(.*)/).values_at 1, 2
|
68
|
+
else
|
69
|
+
toplevel, subpath = self, ""
|
70
|
+
end
|
71
|
+
|
72
|
+
path = nil
|
73
|
+
if where.nil?
|
74
|
+
SEARCH_PATHS.keys.each do |w|
|
75
|
+
path = find(w, caller_lib)
|
76
|
+
return path if File.exists? path
|
77
|
+
end
|
78
|
+
find :user
|
79
|
+
else
|
80
|
+
libdir = where == :lib ? Path.caller_lib_dir(caller_lib) : ""
|
81
|
+
libdir ||= ""
|
82
|
+
Path.setup SEARCH_PATHS[where].sub('{PKGDIR}', pkgdir).sub('{TOPLEVEL}', toplevel).sub('{SUBPATH}', subpath).sub('{LIBDIR}', libdir), @pkgdir, @resource
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
#{{{ Methods
|
87
|
+
|
88
|
+
def in_dir?(dir)
|
89
|
+
! ! File.expand_path(self).match(/^#{Regexp.quote dir}/)
|
90
|
+
end
|
91
|
+
|
92
|
+
def to_s
|
93
|
+
self.find
|
94
|
+
end
|
95
|
+
|
96
|
+
def filename
|
97
|
+
self.find
|
98
|
+
end
|
99
|
+
|
100
|
+
def exists?
|
101
|
+
begin
|
102
|
+
self.produce
|
103
|
+
File.exists? self.find
|
104
|
+
rescue
|
105
|
+
false
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def produce
|
110
|
+
path = self.find
|
111
|
+
return self if File.exists? path
|
112
|
+
|
113
|
+
raise "No resource defined to produce file: #{ self }" if resource.nil?
|
114
|
+
|
115
|
+
resource.produce path
|
116
|
+
|
117
|
+
path
|
118
|
+
end
|
119
|
+
|
120
|
+
def read
|
121
|
+
Open.read(self.produce.find)
|
122
|
+
end
|
123
|
+
|
124
|
+
def open
|
125
|
+
Open.open(self.produce.find)
|
126
|
+
end
|
127
|
+
|
128
|
+
def to_s
|
129
|
+
"" + self
|
130
|
+
end
|
131
|
+
|
132
|
+
def tsv(*args)
|
133
|
+
TSV.open(self.produce, *args)
|
134
|
+
end
|
135
|
+
|
136
|
+
def yaml
|
137
|
+
YAML.load self.open
|
138
|
+
end
|
139
|
+
|
140
|
+
def index(options = {})
|
141
|
+
TSV.index(self.produce.find, options)
|
142
|
+
end
|
143
|
+
|
144
|
+
def range_index(start, eend, options = {})
|
145
|
+
TSV.range_index(self.produce.find, start, eend, options)
|
146
|
+
end
|
147
|
+
|
148
|
+
def pos_index(pos, options = {})
|
149
|
+
TSV.pos_index(self.produce.find, pos, options)
|
150
|
+
end
|
151
|
+
|
152
|
+
|
153
|
+
def to_yaml(*args)
|
154
|
+
self.to_s.to_yaml(*args)
|
155
|
+
end
|
156
|
+
|
157
|
+
def fields
|
158
|
+
TSV.parse_header(self.open).fields
|
159
|
+
end
|
160
|
+
|
161
|
+
def all_fields
|
162
|
+
TSV.parse_header(self.open).all_fields
|
163
|
+
end
|
164
|
+
|
165
|
+
def identifier_file_path
|
166
|
+
if self.dirname.identifiers.exists?
|
167
|
+
self.dirname.identifiers
|
168
|
+
else
|
169
|
+
nil
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def identifier_files
|
174
|
+
if identifier_file_path.nil?
|
175
|
+
[]
|
176
|
+
else
|
177
|
+
[identifier_file_path.nil?]
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'rake'
|
2
|
+
module Rake
|
3
|
+
class TaskNotFound < StandardError; end
|
4
|
+
def self.run(rakefile, dir, task)
|
5
|
+
old_pwd = FileUtils.pwd
|
6
|
+
|
7
|
+
Rake::FileTask.module_eval do
|
8
|
+
if not self.respond_to? :old_define_task
|
9
|
+
class << self
|
10
|
+
alias_method :old_define_task, :define_task
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.define_task(file, *args, &block)
|
14
|
+
@@files ||= []
|
15
|
+
@@files << file
|
16
|
+
old_define_task(file, *args, &block)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.files
|
21
|
+
@@files
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.clear_files
|
25
|
+
@@files = []
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
Rake::Task.clear
|
30
|
+
Rake::FileTask.clear_files
|
31
|
+
|
32
|
+
if block_given?
|
33
|
+
yield
|
34
|
+
else
|
35
|
+
load rakefile
|
36
|
+
end
|
37
|
+
|
38
|
+
raise TaskNotFound if Rake::Task[task].nil?
|
39
|
+
|
40
|
+
Misc.in_dir(dir) do
|
41
|
+
Rake::Task[task].invoke
|
42
|
+
|
43
|
+
Rake::Task.clear
|
44
|
+
Rake::FileTask.clear_files
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'rbbt/resource/rake'
|
2
|
+
|
3
|
+
module Path
|
4
|
+
|
5
|
+
def self.caller_lib_dir(file = nil)
|
6
|
+
file = caller.reject{|l|
|
7
|
+
l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
|
8
|
+
l =~ /rbbt\/resource\/path\.rb/ or
|
9
|
+
l =~ /rbbt\/util\/misc\.rb/ or
|
10
|
+
l =~ /progress-monitor\.rb/
|
11
|
+
}.first.sub(/\.rb.*/,'.rb') if file.nil?
|
12
|
+
|
13
|
+
file = File.expand_path file
|
14
|
+
return Path.setup(file) if File.exists? File.join(file, 'lib')
|
15
|
+
|
16
|
+
while file != '/'
|
17
|
+
dir = File.dirname file
|
18
|
+
return Path.setup(dir) if File.exists? File.join(dir, 'lib')
|
19
|
+
file = File.dirname file
|
20
|
+
end
|
21
|
+
|
22
|
+
return nil
|
23
|
+
end
|
24
|
+
|
25
|
+
SLASH = "/"[0]
|
26
|
+
DOT = "."[0]
|
27
|
+
def located?
|
28
|
+
self.byte(0) == SLASH or (self.byte(0) == DOT and self.byte(1) == SLASH)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
module Resource
|
33
|
+
def set_software_env(software_dir)
|
34
|
+
bin_dir = File.join(software_dir, 'bin')
|
35
|
+
opt_dir = File.join(software_dir, 'opt')
|
36
|
+
|
37
|
+
Misc.env_add 'PATH', bin_dir
|
38
|
+
|
39
|
+
FileUtils.mkdir_p opt_dir unless File.exists? opt_dir
|
40
|
+
%w(.ld-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
|
41
|
+
filename = File.join(opt_dir, file)
|
42
|
+
FileUtils.touch filename unless File.exists? filename
|
43
|
+
end
|
44
|
+
|
45
|
+
if not File.exists? File.join(opt_dir,'.post_install')
|
46
|
+
Open.write(File.join(opt_dir,'.post_install'),"#!/bin/bash\n")
|
47
|
+
end
|
48
|
+
|
49
|
+
Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
|
50
|
+
Misc.env_add('LD_LIBRARY_PATH',line.chomp)
|
51
|
+
Misc.env_add('LD_RUN_PATH',line.chomp)
|
52
|
+
end
|
53
|
+
|
54
|
+
Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
|
55
|
+
Misc.env_add('PKG_CONFIG_PATH',line.chomp)
|
56
|
+
end
|
57
|
+
|
58
|
+
Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
|
59
|
+
Misc.env_add('LD_LIBRARY_PATH',line.chomp)
|
60
|
+
end
|
61
|
+
|
62
|
+
Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
|
63
|
+
Misc.env_add('LD_LIBRARY_PATH',line.chomp)
|
64
|
+
end
|
65
|
+
|
66
|
+
Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
|
67
|
+
Misc.env_add('ACLOCAL_FLAGS', "-I#{File.join(opt_dir, line.chomp)}", ' ')
|
68
|
+
end
|
69
|
+
|
70
|
+
Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
|
71
|
+
Misc.env_add('CLASSPATH', "#{File.join(opt_dir,'java', 'lib', line.chomp)}")
|
72
|
+
end
|
73
|
+
|
74
|
+
Dir.glob(File.join opt_dir, 'jars', '*').each do |file|
|
75
|
+
Misc.env_add('CLASSPATH', "#{File.expand_path(file)}")
|
76
|
+
end
|
77
|
+
|
78
|
+
File.chmod 0774, File.join(opt_dir, '.post_install')
|
79
|
+
|
80
|
+
CMD.cmd(File.join(opt_dir, '.post_install'))
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
def rake_for(path)
|
85
|
+
entry = @rake_dirs.reject{|dir, content|
|
86
|
+
!Misc.common_path(dir, path)
|
87
|
+
}.sort_by{|dir, content|
|
88
|
+
dir.length
|
89
|
+
}.last
|
90
|
+
end
|
91
|
+
|
92
|
+
def has_rake(path)
|
93
|
+
!! rake_for(path)
|
94
|
+
end
|
95
|
+
|
96
|
+
def run_rake(path, rakefile, rake_dir)
|
97
|
+
task = Misc.path_relative_to rake_dir, path
|
98
|
+
rakefile = rakefile.produce if rakefile.respond_to? :produce
|
99
|
+
|
100
|
+
begin
|
101
|
+
Rake.run(rakefile, rake_dir, task)
|
102
|
+
rescue Rake::TaskNotFound
|
103
|
+
raise $! if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
|
104
|
+
task = File.join(File.basename(rake_dir), task)
|
105
|
+
rake_dir = File.dirname(rake_dir)
|
106
|
+
retry
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Resource
|
2
|
+
module WithKey
|
3
|
+
def self.extended(base)
|
4
|
+
class << base
|
5
|
+
attr_accessor :klass, :key
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
alias :old_method_missing :method_missing
|
10
|
+
def method_missing(name, *args)
|
11
|
+
return old_method_missing(name, *args) if name.to_s =~ /^to_/
|
12
|
+
if key
|
13
|
+
klass.send(name, key, *args)
|
14
|
+
else
|
15
|
+
klass.send(name, *args)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def with_key(key)
|
21
|
+
klass = self
|
22
|
+
o = Object.new
|
23
|
+
o.extend WithKey
|
24
|
+
o.klass = self
|
25
|
+
o.key = key
|
26
|
+
o
|
27
|
+
end
|
28
|
+
end
|
data/lib/rbbt/tsv.rb
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
require 'rbbt/util/misc'
|
4
|
+
require 'rbbt/util/log'
|
5
|
+
|
6
|
+
require 'rbbt/persist'
|
7
|
+
|
8
|
+
require 'rbbt/tsv/util'
|
9
|
+
require 'rbbt/tsv/serializers'
|
10
|
+
require 'rbbt/tsv/parser'
|
11
|
+
require 'rbbt/tsv/accessor'
|
12
|
+
require 'rbbt/tsv/manipulate'
|
13
|
+
require 'rbbt/tsv/index'
|
14
|
+
require 'rbbt/tsv/attach'
|
15
|
+
require 'rbbt/tsv/filter'
|
16
|
+
|
17
|
+
module TSV
|
18
|
+
def self.setup(hash, options = {})
|
19
|
+
hash = Misc.array2hash hash if Array === hash
|
20
|
+
hash.extend TSV
|
21
|
+
|
22
|
+
IndiferentHash.setup(options)
|
23
|
+
ENTRIES.each do |entry|
|
24
|
+
hash.send("#{ entry }=", options[entry]) if options.include? entry
|
25
|
+
hash.send("#{ entry }=", options[entry.to_sym]) if options.include? entry.to_sym
|
26
|
+
end
|
27
|
+
|
28
|
+
hash
|
29
|
+
end
|
30
|
+
|
31
|
+
# options shifts if type.nil?
|
32
|
+
def self.open(source, type = nil, options = nil)
|
33
|
+
type, options = nil, type if options.nil? and Hash === type
|
34
|
+
options ||= {}
|
35
|
+
options[:type] ||= type unless type.nil?
|
36
|
+
|
37
|
+
persist_options = Misc.pull_keys options, :persist
|
38
|
+
|
39
|
+
filename = get_filename source
|
40
|
+
serializer = Misc.process_options options, :serializer
|
41
|
+
|
42
|
+
Log.debug "TSV open: #{ filename } - #{options.inspect}"
|
43
|
+
|
44
|
+
Persist.persist_tsv source, filename, options, persist_options do |data|
|
45
|
+
if serializer
|
46
|
+
data.extend TSV unless TSV === data
|
47
|
+
data.serializer = serializer
|
48
|
+
end
|
49
|
+
|
50
|
+
stream = get_stream source
|
51
|
+
parse stream, data, options
|
52
|
+
|
53
|
+
data.filename = filename.to_s unless filename.nil?
|
54
|
+
if data.identifiers.nil? and Path === filename and filename.identifier_file_path
|
55
|
+
data.identifiers = filename.identifier_file_path.dup
|
56
|
+
end
|
57
|
+
|
58
|
+
data
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.parse_header(stream, options = {})
|
63
|
+
Parser.new stream, options
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.parse(stream, data, options = {})
|
67
|
+
monitor, grep = Misc.process_options options, :monitor, :grep
|
68
|
+
if grep
|
69
|
+
stream = Open.grep(stream, grep)
|
70
|
+
end
|
71
|
+
|
72
|
+
parser = Parser.new stream, options
|
73
|
+
line = parser.rescue_first_line
|
74
|
+
|
75
|
+
if TokyoCabinet::HDB === data and parser.straight
|
76
|
+
data.close
|
77
|
+
begin
|
78
|
+
CMD.cmd("tchmgr importtsv '#{data.persistence_path}'", :in => stream, :log => false)
|
79
|
+
rescue
|
80
|
+
Log.debug("tchmgr importtsv failed for: #{data.persistence_path}")
|
81
|
+
Log.debug($!.message)
|
82
|
+
end
|
83
|
+
data.write
|
84
|
+
end
|
85
|
+
|
86
|
+
data.extend TSV unless TSV === data
|
87
|
+
data.unnamed = true
|
88
|
+
|
89
|
+
data.serializer = parser.type if data.serializer == :type
|
90
|
+
|
91
|
+
if monitor and (stream.respond_to?(:size) or (stream.respond_to?(:stat) and stream.stat.respond_to? :size)) and stream.respond_to?(:pos)
|
92
|
+
size = case
|
93
|
+
when stream.respond_to?(:size)
|
94
|
+
stream.size
|
95
|
+
else
|
96
|
+
stream.stat.size
|
97
|
+
end
|
98
|
+
desc = "Parsing Stream"
|
99
|
+
step = 100
|
100
|
+
if Hash === monitor
|
101
|
+
desc = monitor[:desc] if monitor.include? :desc
|
102
|
+
step = monitor[:step] if monitor.include? :step
|
103
|
+
end
|
104
|
+
progress_monitor = Progress::Bar.new(size, 0, step, desc)
|
105
|
+
else
|
106
|
+
progress_monitor = nil
|
107
|
+
end
|
108
|
+
|
109
|
+
while not line.nil?
|
110
|
+
begin
|
111
|
+
|
112
|
+
progress_monitor.tick(stream.pos) if progress_monitor
|
113
|
+
|
114
|
+
line = parser.process line
|
115
|
+
parts = parser.chop_line line
|
116
|
+
key, values = parser.get_values parts
|
117
|
+
values = parser.cast_values values if parser.cast?
|
118
|
+
parser.add_to_data data, key, values
|
119
|
+
line = stream.gets
|
120
|
+
rescue Parser::SKIP_LINE
|
121
|
+
line = stream.gets
|
122
|
+
next
|
123
|
+
rescue IOError
|
124
|
+
break
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
parser.setup data
|
129
|
+
|
130
|
+
data.unnamed = false
|
131
|
+
|
132
|
+
data
|
133
|
+
end
|
134
|
+
end
|