scout-gear 1.2.0 → 5.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitmodules +4 -0
- data/.vimproject +663 -4
- data/Rakefile +1 -0
- data/VERSION +1 -1
- data/bin/scout +235 -0
- data/lib/scout/cmd.rb +344 -0
- data/lib/scout/concurrent_stream.rb +259 -0
- data/lib/scout/exceptions.rb +15 -8
- data/lib/scout/indiferent_hash/options.rb +8 -26
- data/lib/scout/indiferent_hash.rb +0 -30
- data/lib/scout/log/color.rb +2 -2
- data/lib/scout/log/fingerprint.rb +11 -1
- data/lib/scout/log/progress/report.rb +0 -1
- data/lib/scout/log/progress/util.rb +1 -1
- data/lib/scout/log/progress.rb +4 -4
- data/lib/scout/log.rb +10 -2
- data/lib/scout/meta_extension.rb +15 -1
- data/lib/scout/misc/digest.rb +56 -0
- data/lib/scout/misc/filesystem.rb +26 -0
- data/lib/scout/misc/format.rb +226 -0
- data/lib/scout/misc/insist.rb +56 -0
- data/lib/scout/misc.rb +5 -11
- data/lib/scout/open/lock.rb +61 -0
- data/lib/scout/open/remote.rb +120 -0
- data/lib/scout/open/stream.rb +372 -0
- data/lib/scout/open/util.rb +225 -0
- data/lib/scout/open.rb +169 -0
- data/lib/scout/path/find.rb +78 -26
- data/lib/scout/path/tmpfile.rb +8 -0
- data/lib/scout/path/util.rb +17 -5
- data/lib/scout/path.rb +13 -31
- data/lib/scout/persist/open.rb +17 -0
- data/lib/scout/persist/path.rb +15 -0
- data/lib/scout/persist/serialize.rb +140 -0
- data/lib/scout/persist.rb +54 -0
- data/lib/scout/resource/path.rb +15 -0
- data/lib/scout/resource/produce/rake.rb +69 -0
- data/lib/scout/resource/produce.rb +246 -0
- data/lib/scout/resource/scout.rb +3 -0
- data/lib/scout/resource.rb +37 -0
- data/lib/scout/simple_opt/accessor.rb +54 -0
- data/lib/scout/simple_opt/doc.rb +102 -0
- data/lib/scout/simple_opt/get.rb +57 -0
- data/lib/scout/simple_opt/parse.rb +67 -0
- data/lib/scout/simple_opt/setup.rb +26 -0
- data/lib/scout/simple_opt.rb +5 -0
- data/lib/scout/tmpfile.rb +39 -1
- data/lib/scout/workflow/definition.rb +72 -0
- data/lib/scout/workflow/documentation.rb +77 -0
- data/lib/scout/workflow/step/info.rb +77 -0
- data/lib/scout/workflow/step.rb +96 -0
- data/lib/scout/workflow/task/inputs.rb +112 -0
- data/lib/scout/workflow/task.rb +141 -0
- data/lib/scout/workflow/usage.rb +294 -0
- data/lib/scout/workflow/util.rb +11 -0
- data/lib/scout/workflow.rb +39 -0
- data/lib/scout-gear.rb +5 -0
- data/lib/scout.rb +1 -0
- data/lib/workflow-scout.rb +2 -0
- data/scout-gear.gemspec +78 -5
- data/scout_commands/alias +48 -0
- data/scout_commands/find +83 -0
- data/scout_commands/glob +0 -0
- data/scout_commands/rbbt +23 -0
- data/scout_commands/workflow/task +707 -0
- data/test/scout/indiferent_hash/test_case_insensitive.rb +16 -0
- data/test/scout/indiferent_hash/test_options.rb +11 -1
- data/test/scout/misc/test_digest.rb +30 -0
- data/test/scout/misc/test_filesystem.rb +30 -0
- data/test/scout/misc/test_insist.rb +13 -0
- data/test/scout/open/test_lock.rb +52 -0
- data/test/scout/open/test_remote.rb +25 -0
- data/test/scout/open/test_stream.rb +515 -0
- data/test/scout/open/test_util.rb +73 -0
- data/test/scout/path/test_find.rb +37 -1
- data/test/scout/persist/test_open.rb +37 -0
- data/test/scout/persist/test_path.rb +37 -0
- data/test/scout/persist/test_serialize.rb +114 -0
- data/test/scout/resource/test_path.rb +40 -0
- data/test/scout/resource/test_produce.rb +62 -0
- data/test/scout/simple_opt/test_get.rb +11 -0
- data/test/scout/simple_opt/test_parse.rb +10 -0
- data/test/scout/simple_opt/test_setup.rb +77 -0
- data/test/scout/test_cmd.rb +85 -0
- data/test/scout/test_concurrent_stream.rb +29 -0
- data/test/scout/test_misc.rb +0 -7
- data/test/scout/test_open.rb +146 -0
- data/test/scout/test_path.rb +3 -1
- data/test/scout/test_persist.rb +83 -0
- data/test/scout/test_resource.rb +26 -0
- data/test/scout/test_workflow.rb +87 -0
- data/test/scout/workflow/step/test_info.rb +28 -0
- data/test/scout/workflow/task/test_inputs.rb +182 -0
- data/test/scout/workflow/test_step.rb +36 -0
- data/test/scout/workflow/test_task.rb +178 -0
- data/test/scout/workflow/test_usage.rb +26 -0
- data/test/scout/workflow/test_util.rb +17 -0
- data/test/test_helper.rb +17 -0
- data/test/test_scout-gear.rb +0 -0
- metadata +76 -3
data/lib/scout/open.rb
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
require_relative 'tmpfile'
|
2
|
+
require_relative 'path'
|
3
|
+
require_relative 'cmd'
|
4
|
+
|
5
|
+
require_relative 'open/stream'
|
6
|
+
require_relative 'open/util'
|
7
|
+
require_relative 'open/remote'
|
8
|
+
require_relative 'open/lock'
|
9
|
+
|
10
|
+
module Open
|
11
|
+
module NamedStream
|
12
|
+
attr_accessor :filename
|
13
|
+
|
14
|
+
def digest_str
|
15
|
+
if Path === filename && ! filename.located?
|
16
|
+
filename
|
17
|
+
else
|
18
|
+
Misc.file_md5(filename)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.get_stream(file, mode = 'r')
|
24
|
+
file = file.find if Path === file
|
25
|
+
|
26
|
+
return Open.ssh(file) if Open.ssh?(file)
|
27
|
+
return Open.wget(file) if Open.remote?(file)
|
28
|
+
|
29
|
+
File.open(file, mode)
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.file_open(file, grep = false, mode = 'r', invert_grep = false)
|
33
|
+
Open.mkdir File.dirname(file) if mode.include? 'w'
|
34
|
+
|
35
|
+
stream = get_stream(file, mode)
|
36
|
+
|
37
|
+
if grep
|
38
|
+
grep(stream, grep, invert_grep)
|
39
|
+
else
|
40
|
+
stream
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.file_write(file, content, mode = 'w')
|
45
|
+
File.open(file, mode) do |f|
|
46
|
+
begin
|
47
|
+
f.flock(File::LOCK_EX)
|
48
|
+
f.write content
|
49
|
+
f.flock(File::LOCK_UN)
|
50
|
+
ensure
|
51
|
+
f.close unless f.closed?
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.open(file, options = {})
|
57
|
+
if IO === file || StringIO === file
|
58
|
+
if block_given?
|
59
|
+
res = yield file
|
60
|
+
file.close
|
61
|
+
return res
|
62
|
+
else
|
63
|
+
return file
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
options = IndiferentHash.add_defaults options, :noz => false, :mode => 'r'
|
68
|
+
|
69
|
+
mode = IndiferentHash.process_options options, :mode
|
70
|
+
|
71
|
+
options[:noz] = true if mode.include? "w"
|
72
|
+
|
73
|
+
io = file_open(file, options[:grep], mode, options[:invert_grep])
|
74
|
+
|
75
|
+
io = unzip(io) if ((String === file and zip?(file)) and not options[:noz]) or options[:zip]
|
76
|
+
io = gunzip(io) if ((String === file and gzip?(file)) and not options[:noz]) or options[:gzip]
|
77
|
+
io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
|
78
|
+
|
79
|
+
io.extend NamedStream
|
80
|
+
io.filename = file
|
81
|
+
|
82
|
+
if block_given?
|
83
|
+
res = nil
|
84
|
+
begin
|
85
|
+
res = yield(io)
|
86
|
+
rescue DontClose
|
87
|
+
res = $!.payload
|
88
|
+
rescue Exception
|
89
|
+
io.abort if io.respond_to? :abort
|
90
|
+
io.join if io.respond_to? :join
|
91
|
+
raise $!
|
92
|
+
ensure
|
93
|
+
io.close if io.respond_to? :close and not io.closed?
|
94
|
+
io.join if io.respond_to? :join
|
95
|
+
end
|
96
|
+
res
|
97
|
+
else
|
98
|
+
io
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def self.read(file, options = {}, &block)
|
103
|
+
open(file, options) do |f|
|
104
|
+
if block_given?
|
105
|
+
res = []
|
106
|
+
while not f.eof?
|
107
|
+
l = f.gets
|
108
|
+
l = Misc.fixutf8(l) unless options[:nofix]
|
109
|
+
res << yield(l)
|
110
|
+
end
|
111
|
+
res
|
112
|
+
else
|
113
|
+
if options[:nofix]
|
114
|
+
f.read
|
115
|
+
else
|
116
|
+
Misc.fixutf8(f.read)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def self.write(file, content = nil, options = {})
|
123
|
+
options = IndiferentHash.add_defaults options, :mode => 'w'
|
124
|
+
|
125
|
+
file = file.find(options[:where]) if Path === file
|
126
|
+
mode = IndiferentHash.process_options options, :mode
|
127
|
+
|
128
|
+
FileUtils.mkdir_p File.dirname(file)
|
129
|
+
|
130
|
+
case
|
131
|
+
when block_given?
|
132
|
+
begin
|
133
|
+
f = File.open(file, mode)
|
134
|
+
begin
|
135
|
+
yield f
|
136
|
+
ensure
|
137
|
+
f.close unless f.closed?
|
138
|
+
end
|
139
|
+
rescue Exception
|
140
|
+
FileUtils.rm file if File.exist? file
|
141
|
+
raise $!
|
142
|
+
end
|
143
|
+
when content.nil?
|
144
|
+
file_write(file, "", mode)
|
145
|
+
when String === content
|
146
|
+
file_write(file, content, mode)
|
147
|
+
when (IO === content || StringIO === content)
|
148
|
+
begin
|
149
|
+
File.open(file, mode) do |f|
|
150
|
+
f.flock(File::LOCK_EX)
|
151
|
+
while block = content.read(Open::BLOCK_SIZE)
|
152
|
+
f.write block
|
153
|
+
end
|
154
|
+
f.flock(File::LOCK_UN)
|
155
|
+
end
|
156
|
+
rescue Exception
|
157
|
+
FileUtils.rm_rf file if File.exist? file
|
158
|
+
raise $!
|
159
|
+
end
|
160
|
+
content.close unless content.closed?
|
161
|
+
content.join if content.respond_to? :join
|
162
|
+
else
|
163
|
+
raise "Content unknown #{Log.fingerprint content}"
|
164
|
+
end
|
165
|
+
|
166
|
+
notify_write(file)
|
167
|
+
end
|
168
|
+
|
169
|
+
end
|
data/lib/scout/path/find.rb
CHANGED
@@ -1,27 +1,57 @@
|
|
1
|
+
require_relative '../indiferent_hash'
|
1
2
|
module Path
|
2
|
-
def _parts
|
3
|
-
@_parts ||= self.split("/")
|
4
|
-
end
|
5
3
|
|
6
|
-
def
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
4
|
+
def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
|
5
|
+
|
6
|
+
if file.nil?
|
7
|
+
caller_dup = caller.dup
|
8
|
+
while file = caller_dup.shift
|
9
|
+
break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
|
10
|
+
file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
|
11
|
+
file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
|
12
|
+
file =~ /(?:scout|rbbt)\/persist.rb/
|
13
|
+
end
|
14
|
+
file = file.sub(/\.rb[^\w].*/,'.rb')
|
15
|
+
end
|
16
|
+
|
17
|
+
relative_to = [relative_to] unless Array === relative_to
|
18
|
+
file = File.expand_path(file)
|
19
|
+
return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
|
20
|
+
|
21
|
+
while file != '/'
|
22
|
+
dir = File.dirname file
|
23
|
+
|
24
|
+
return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
|
25
|
+
|
26
|
+
file = File.dirname file
|
27
|
+
end
|
28
|
+
|
29
|
+
return nil
|
12
30
|
end
|
13
31
|
|
14
|
-
def self.follow(path, map)
|
15
|
-
map.sub('{PKGDIR}', path.pkgdir || Path.default_pkgdir).
|
16
|
-
sub('{RESOURCE}', path.to_s).
|
32
|
+
def self.follow(path, map, map_name = nil)
|
33
|
+
file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
|
34
|
+
sub('{RESOURCE}', path.pkgdir.to_s).
|
17
35
|
sub('{PWD}', FileUtils.pwd).
|
18
36
|
sub('{TOPLEVEL}', path._toplevel).
|
19
37
|
sub('{SUBPATH}', path._subpath).
|
20
38
|
sub('{BASENAME}', File.basename(path)).
|
21
39
|
sub('{PATH}', path).
|
22
|
-
sub('{LIBDIR}', path.libdir || Path.caller_lib_dir).
|
40
|
+
sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
|
41
|
+
sub('{MAPNAME}', map_name.to_s).
|
23
42
|
sub('{REMOVE}/', '').
|
24
43
|
sub('{REMOVE}', '').gsub(/\/+/,'/')
|
44
|
+
|
45
|
+
while true
|
46
|
+
file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
|
47
|
+
key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
|
48
|
+
key_text = follow(path, "#{key}}", map_name)
|
49
|
+
key_text[orig] = replace[0..-2] if key_text.include?(orig)
|
50
|
+
key_text
|
51
|
+
end || break
|
52
|
+
end
|
53
|
+
|
54
|
+
file
|
25
55
|
end
|
26
56
|
|
27
57
|
def self.path_maps
|
@@ -40,19 +70,36 @@ module Path
|
|
40
70
|
})
|
41
71
|
end
|
42
72
|
|
43
|
-
def self.
|
44
|
-
@@
|
73
|
+
def self.basic_map_order
|
74
|
+
@@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
|
45
75
|
end
|
46
76
|
|
47
|
-
def self.
|
48
|
-
@@
|
77
|
+
def self.map_order
|
78
|
+
@@map_order ||= (path_maps.keys & basic_map_order) + (path_maps.keys - basic_map_order)
|
79
|
+
end
|
80
|
+
|
81
|
+
def self.add_path(name, map)
|
82
|
+
@@path_maps[name] = map
|
83
|
+
@@map_order = nil
|
84
|
+
end
|
85
|
+
|
86
|
+
def _parts
|
87
|
+
@_parts ||= self.split("/")
|
88
|
+
end
|
89
|
+
|
90
|
+
def _subpath
|
91
|
+
@subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
|
92
|
+
end
|
93
|
+
|
94
|
+
def _toplevel
|
95
|
+
@toplevel ||= _parts.length > 1 ? _parts[0] : ""
|
49
96
|
end
|
50
97
|
|
51
98
|
SLASH = "/"[0]
|
52
99
|
DOT = "."[0]
|
53
100
|
def located?
|
54
101
|
# OPEN RESOURCE
|
55
|
-
self.slice(0,1) == SLASH || (self.
|
102
|
+
self.slice(0,1) == SLASH || (self.slice(0,1) == DOT && self.slice(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
|
56
103
|
end
|
57
104
|
|
58
105
|
def annotate_found_where(found, where)
|
@@ -70,13 +117,19 @@ module Path
|
|
70
117
|
@original
|
71
118
|
end
|
72
119
|
|
120
|
+
def map_order
|
121
|
+
@map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
|
122
|
+
end
|
123
|
+
|
73
124
|
def follow(map_name = :default, annotate = true)
|
74
|
-
|
125
|
+
IndiferentHash.setup(path_maps)
|
126
|
+
map = path_maps[map_name]
|
127
|
+
raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
|
75
128
|
while Symbol === map
|
76
129
|
map_name = map
|
77
|
-
map =
|
130
|
+
map = path_maps[map_name]
|
78
131
|
end
|
79
|
-
found = Path.follow(self, map)
|
132
|
+
found = Path.follow(self, map, map_name)
|
80
133
|
|
81
134
|
annotate_found_where(found, map_name) if annotate
|
82
135
|
|
@@ -85,13 +138,13 @@ module Path
|
|
85
138
|
|
86
139
|
def find(where = nil)
|
87
140
|
return self if located?
|
141
|
+
return find_all if where == 'all' || where == :all
|
88
142
|
return follow(where) if where
|
89
143
|
|
90
|
-
|
91
|
-
Path.search_order.each do |map_name|
|
144
|
+
map_order.each do |map_name|
|
92
145
|
found = follow(map_name, false)
|
93
146
|
|
94
|
-
return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(
|
147
|
+
return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
|
95
148
|
end
|
96
149
|
|
97
150
|
return follow(:default)
|
@@ -106,9 +159,8 @@ module Path
|
|
106
159
|
alias exists? exist?
|
107
160
|
|
108
161
|
def find_all(caller_lib = nil, search_paths = nil)
|
109
|
-
|
162
|
+
map_order
|
110
163
|
.collect{|where| find(where) }
|
111
164
|
.select{|file| file.exist? }.uniq
|
112
165
|
end
|
113
|
-
|
114
166
|
end
|
data/lib/scout/path/util.rb
CHANGED
@@ -1,4 +1,12 @@
|
|
1
1
|
module Path
|
2
|
+
|
3
|
+
def self.is_filename?(string, need_to_exists = true)
|
4
|
+
return false if string.nil?
|
5
|
+
return true if Path === string
|
6
|
+
return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265 }.empty? and (! need_to_exists || File.exist?(string))
|
7
|
+
return false
|
8
|
+
end
|
9
|
+
|
2
10
|
def directory?
|
3
11
|
return nil unless self.exist?
|
4
12
|
File.directory?(self.find)
|
@@ -12,7 +20,6 @@ module Path
|
|
12
20
|
self.annotate(File.basename(self))
|
13
21
|
end
|
14
22
|
|
15
|
-
|
16
23
|
def glob(pattern = '*')
|
17
24
|
if self.include? "*"
|
18
25
|
self.glob_all
|
@@ -31,12 +38,11 @@ module Path
|
|
31
38
|
end
|
32
39
|
|
33
40
|
def glob_all(pattern = nil, caller_lib = nil, search_paths = nil)
|
34
|
-
search_paths ||= Path.
|
41
|
+
search_paths ||= Path.path_maps
|
35
42
|
search_paths = search_paths.dup
|
36
43
|
|
37
|
-
location_paths = {}
|
38
44
|
search_paths.keys.collect do |where|
|
39
|
-
found = find(where
|
45
|
+
found = find(where)
|
40
46
|
paths = pattern ? Dir.glob(File.join(found, pattern)) : Dir.glob(found)
|
41
47
|
|
42
48
|
paths = paths.collect{|p| self.annotate p }
|
@@ -46,7 +52,13 @@ module Path
|
|
46
52
|
p.where = where
|
47
53
|
end if found.original and pattern
|
48
54
|
|
49
|
-
|
55
|
+
paths
|
56
|
+
end.flatten.uniq
|
57
|
+
end
|
58
|
+
|
59
|
+
def no_method_missing
|
60
|
+
class << self
|
61
|
+
undef_method :method_missing
|
50
62
|
end
|
51
63
|
end
|
52
64
|
end
|
data/lib/scout/path.rb
CHANGED
@@ -1,40 +1,18 @@
|
|
1
1
|
require_relative 'meta_extension'
|
2
2
|
require_relative 'path/find'
|
3
3
|
require_relative 'path/util'
|
4
|
+
require_relative 'path/tmpfile'
|
4
5
|
|
5
6
|
module Path
|
6
7
|
extend MetaExtension
|
7
|
-
extension_attr :pkgdir, :libdir
|
8
|
-
|
9
|
-
def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
|
10
|
-
|
11
|
-
if file.nil?
|
12
|
-
caller_dup = caller.dup
|
13
|
-
while file = caller_dup.shift
|
14
|
-
break unless file =~ /scout\/(?:resource\.rb|workflow\.rb)/ or
|
15
|
-
file =~ /scout\/path\.rb/ or
|
16
|
-
file =~ /scout\/persist.rb/
|
17
|
-
end
|
18
|
-
file = file.sub(/\.rb[^\w].*/,'.rb')
|
19
|
-
end
|
20
|
-
|
21
|
-
relative_to = [relative_to] unless Array === relative_to
|
22
|
-
file = File.expand_path(file)
|
23
|
-
return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
|
24
|
-
|
25
|
-
while file != '/'
|
26
|
-
dir = File.dirname file
|
27
|
-
|
28
|
-
return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
|
29
|
-
|
30
|
-
file = File.dirname file
|
31
|
-
end
|
32
|
-
|
33
|
-
return nil
|
34
|
-
end
|
8
|
+
extension_attr :pkgdir, :libdir, :path_maps
|
35
9
|
|
36
10
|
def self.default_pkgdir
|
37
|
-
@@default_pkgdir
|
11
|
+
@@default_pkgdir ||= 'scout'
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.default_pkgdir=(pkgdir)
|
15
|
+
@@default_pkgdir = pkgdir
|
38
16
|
end
|
39
17
|
|
40
18
|
def pkgdir
|
@@ -42,7 +20,11 @@ module Path
|
|
42
20
|
end
|
43
21
|
|
44
22
|
def libdir
|
45
|
-
@libdir
|
23
|
+
@libdir || Path.caller_lib_dir
|
24
|
+
end
|
25
|
+
|
26
|
+
def path_maps
|
27
|
+
@path_maps ||= Path.path_maps
|
46
28
|
end
|
47
29
|
|
48
30
|
def join(subpath, prevpath = nil)
|
@@ -50,7 +32,7 @@ module Path
|
|
50
32
|
prevpath = prevpath.to_s if Symbol === prevpath
|
51
33
|
|
52
34
|
subpath = File.join(prevpath.to_s, subpath) if prevpath
|
53
|
-
new = File.join(self, subpath)
|
35
|
+
new = self.empty? ? subpath : File.join(self, subpath)
|
54
36
|
self.annotate(new)
|
55
37
|
new
|
56
38
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require_relative '../open'
|
2
|
+
require 'json'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module Open
|
6
|
+
def self.json(file)
|
7
|
+
Open.open(file){|f| JSON.load(f) }
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.yaml(file)
|
11
|
+
Open.open(file){|f| YAML.load(f) }
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.marshal(file)
|
15
|
+
Open.open(file){|f| Marshal.load(f) }
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require_relative 'open'
|
2
|
+
|
3
|
+
module Path
|
4
|
+
def yaml(*rest, &block)
|
5
|
+
Open.yaml(self, *rest, &block)
|
6
|
+
end
|
7
|
+
|
8
|
+
def json(*rest, &block)
|
9
|
+
Open.json(self, *rest, &block)
|
10
|
+
end
|
11
|
+
|
12
|
+
def marshal(*rest, &block)
|
13
|
+
Open.marshal(self, *rest, &block)
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,140 @@
|
|
1
|
+
require_relative '../open'
|
2
|
+
require_relative 'open'
|
3
|
+
|
4
|
+
module Persist
|
5
|
+
TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
|
6
|
+
|
7
|
+
class << self
|
8
|
+
attr_accessor :save_drivers, :load_drivers
|
9
|
+
def save_drivers
|
10
|
+
@save_drivers ||= {}
|
11
|
+
end
|
12
|
+
def load_drivers
|
13
|
+
@load_drivers ||= {}
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.serialize(content, type)
|
18
|
+
case type
|
19
|
+
when nil, :string, :integer, :float, :boolean, :file, :path
|
20
|
+
if IO === content || StringIO === content
|
21
|
+
content.read
|
22
|
+
else
|
23
|
+
content.to_s
|
24
|
+
end
|
25
|
+
when :array
|
26
|
+
content * "\n"
|
27
|
+
when :yaml
|
28
|
+
content.to_yaml
|
29
|
+
when :json
|
30
|
+
content.to_json
|
31
|
+
when :marshal, :serializer
|
32
|
+
Marshal.dump(content)
|
33
|
+
else
|
34
|
+
if m = type.to_s.match(/(.*)_array/)
|
35
|
+
type = m[1].to_sym
|
36
|
+
content.collect{|c| serialize(c, type) } * "\n"
|
37
|
+
else
|
38
|
+
raise "Persist does not know #{Log.fingerprint type}"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.deserialize(serialized, type)
|
44
|
+
case type
|
45
|
+
when nil, :string, :file, :stream
|
46
|
+
serialized
|
47
|
+
when :path
|
48
|
+
Path.setup(serialized)
|
49
|
+
when :integer
|
50
|
+
serialized.to_i
|
51
|
+
when :float
|
52
|
+
serialized.to_f
|
53
|
+
when :boolean
|
54
|
+
TRUE_STRINGS.include? serialized
|
55
|
+
when :array
|
56
|
+
serialized.split("\n")
|
57
|
+
when :yaml
|
58
|
+
YAML.parse(serialized)
|
59
|
+
when :json
|
60
|
+
JSON.parse(serialized)
|
61
|
+
when :marshal, :serializer
|
62
|
+
Marshal.load(serialized)
|
63
|
+
else
|
64
|
+
if m = type.to_s.match(/(.*)_array/)
|
65
|
+
type = m[1].to_sym
|
66
|
+
new_content = serialized.split("\n")
|
67
|
+
new_content.collect{|c| deserialize(c, type) }
|
68
|
+
else
|
69
|
+
raise "Persist does not know #{Log.fingerprint type}"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
MEMORY = {}
|
75
|
+
def self.save(content, file, type = :serializer)
|
76
|
+
return if content.nil?
|
77
|
+
type = MEMORY if type == :memory
|
78
|
+
type = :serializer if type.nil?
|
79
|
+
|
80
|
+
if Hash === type
|
81
|
+
type[file] = content
|
82
|
+
return
|
83
|
+
end
|
84
|
+
|
85
|
+
Log.debug "Save #{Log.fingerprint type} on #{file}"
|
86
|
+
if save_drivers[type]
|
87
|
+
Open.write(file, save_drivers[type].call(content))
|
88
|
+
return
|
89
|
+
end
|
90
|
+
|
91
|
+
if IO === content || StringIO === content
|
92
|
+
main, copy = Open.tee_stream_thread content
|
93
|
+
t = Thread.new do
|
94
|
+
Thread.current["name"] = "file saver: " + file
|
95
|
+
Open.sensible_write(file, main)
|
96
|
+
end
|
97
|
+
ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
|
98
|
+
else
|
99
|
+
serialized = serialize(content, type)
|
100
|
+
Open.sensible_write(file, serialized)
|
101
|
+
content
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.load(file, type = :serializer)
|
106
|
+
file = file.find if Path === file
|
107
|
+
type = MEMORY if type == :memory
|
108
|
+
return unless Hash === type || Open.exist?(file)
|
109
|
+
type = :serializer if type.nil?
|
110
|
+
|
111
|
+
Log.debug "Load #{Log.fingerprint type} on #{file}"
|
112
|
+
if load_drivers[type]
|
113
|
+
return load_drivers[type].call(file)
|
114
|
+
end
|
115
|
+
|
116
|
+
case type
|
117
|
+
when :yaml
|
118
|
+
Open.yaml(file)
|
119
|
+
when :json
|
120
|
+
Open.json(file)
|
121
|
+
when :marshal, :serializer
|
122
|
+
Open.marshal(file)
|
123
|
+
when :stream
|
124
|
+
Open.open(file)
|
125
|
+
when :file
|
126
|
+
value = Open.read(file)
|
127
|
+
value.sub!(/^\./, File.dirname(file)) if value.start_with?("./")
|
128
|
+
value
|
129
|
+
when :file_array
|
130
|
+
Open.read(file).split("\n").collect do |f|
|
131
|
+
f.sub!(/^\./, File.dirname(file)) if f.start_with?("./")
|
132
|
+
f
|
133
|
+
end
|
134
|
+
when Hash
|
135
|
+
type[file]
|
136
|
+
else
|
137
|
+
deserialize(Open.read(file), type)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require_relative 'persist/serialize'
|
2
|
+
require_relative 'persist/open'
|
3
|
+
require_relative 'persist/path'
|
4
|
+
|
5
|
+
module Persist
|
6
|
+
class << self
|
7
|
+
attr :cache_dir
|
8
|
+
def cache_dir=(cache_dir)
|
9
|
+
@cache_dir = Path === cache_dir ? cache_dir : Path.setup(cache_dir)
|
10
|
+
end
|
11
|
+
def cache_dir
|
12
|
+
@cache_dir ||= Path.setup("var/cache/persistence")
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_writer :lock_dir
|
16
|
+
def lock_dir
|
17
|
+
@lock_dir ||= Path.setup("var/cache/persist_locks")
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.persistence_path(name, options = {})
|
22
|
+
options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
|
23
|
+
other_options = IndiferentHash.pull_keys options, :other
|
24
|
+
TmpFile.tmp_for_file(name, options, other_options)
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.persist(name, type = :serializer, options = {}, &block)
|
28
|
+
persist_options = IndiferentHash.pull_keys options, :persist
|
29
|
+
file = persist_options[:path] || options[:path] || persistence_path(name, options)
|
30
|
+
|
31
|
+
update = options[:update] || persist_options[:update]
|
32
|
+
update = Open.mtime(update) if Path === update
|
33
|
+
update = Open.mtime(file) >= update ? false : true if Time === update
|
34
|
+
|
35
|
+
if Open.exist?(file) && ! update
|
36
|
+
Persist.load(file, type)
|
37
|
+
else
|
38
|
+
res = yield
|
39
|
+
begin
|
40
|
+
Open.rm(file)
|
41
|
+
res = Persist.save(res, file, type)
|
42
|
+
rescue
|
43
|
+
raise $! unless options[:canfail]
|
44
|
+
Log.debug "Could not persist #{type} on #{file}"
|
45
|
+
end
|
46
|
+
res
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.memory(name, *args, &block)
|
51
|
+
self.persist(name, :memory, *args, &block)
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|