scout-gear 1.2.0 → 5.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +4 -0
  3. data/.vimproject +663 -4
  4. data/Rakefile +1 -0
  5. data/VERSION +1 -1
  6. data/bin/scout +235 -0
  7. data/lib/scout/cmd.rb +344 -0
  8. data/lib/scout/concurrent_stream.rb +259 -0
  9. data/lib/scout/exceptions.rb +15 -8
  10. data/lib/scout/indiferent_hash/options.rb +8 -26
  11. data/lib/scout/indiferent_hash.rb +0 -30
  12. data/lib/scout/log/color.rb +2 -2
  13. data/lib/scout/log/fingerprint.rb +11 -1
  14. data/lib/scout/log/progress/report.rb +0 -1
  15. data/lib/scout/log/progress/util.rb +1 -1
  16. data/lib/scout/log/progress.rb +4 -4
  17. data/lib/scout/log.rb +10 -2
  18. data/lib/scout/meta_extension.rb +15 -1
  19. data/lib/scout/misc/digest.rb +56 -0
  20. data/lib/scout/misc/filesystem.rb +26 -0
  21. data/lib/scout/misc/format.rb +226 -0
  22. data/lib/scout/misc/insist.rb +56 -0
  23. data/lib/scout/misc.rb +5 -11
  24. data/lib/scout/open/lock.rb +61 -0
  25. data/lib/scout/open/remote.rb +120 -0
  26. data/lib/scout/open/stream.rb +372 -0
  27. data/lib/scout/open/util.rb +225 -0
  28. data/lib/scout/open.rb +169 -0
  29. data/lib/scout/path/find.rb +78 -26
  30. data/lib/scout/path/tmpfile.rb +8 -0
  31. data/lib/scout/path/util.rb +17 -5
  32. data/lib/scout/path.rb +13 -31
  33. data/lib/scout/persist/open.rb +17 -0
  34. data/lib/scout/persist/path.rb +15 -0
  35. data/lib/scout/persist/serialize.rb +140 -0
  36. data/lib/scout/persist.rb +54 -0
  37. data/lib/scout/resource/path.rb +15 -0
  38. data/lib/scout/resource/produce/rake.rb +69 -0
  39. data/lib/scout/resource/produce.rb +246 -0
  40. data/lib/scout/resource/scout.rb +3 -0
  41. data/lib/scout/resource.rb +37 -0
  42. data/lib/scout/simple_opt/accessor.rb +54 -0
  43. data/lib/scout/simple_opt/doc.rb +102 -0
  44. data/lib/scout/simple_opt/get.rb +57 -0
  45. data/lib/scout/simple_opt/parse.rb +67 -0
  46. data/lib/scout/simple_opt/setup.rb +26 -0
  47. data/lib/scout/simple_opt.rb +5 -0
  48. data/lib/scout/tmpfile.rb +39 -1
  49. data/lib/scout/workflow/definition.rb +72 -0
  50. data/lib/scout/workflow/documentation.rb +77 -0
  51. data/lib/scout/workflow/step/info.rb +77 -0
  52. data/lib/scout/workflow/step.rb +96 -0
  53. data/lib/scout/workflow/task/inputs.rb +112 -0
  54. data/lib/scout/workflow/task.rb +141 -0
  55. data/lib/scout/workflow/usage.rb +294 -0
  56. data/lib/scout/workflow/util.rb +11 -0
  57. data/lib/scout/workflow.rb +39 -0
  58. data/lib/scout-gear.rb +5 -0
  59. data/lib/scout.rb +1 -0
  60. data/lib/workflow-scout.rb +2 -0
  61. data/scout-gear.gemspec +78 -5
  62. data/scout_commands/alias +48 -0
  63. data/scout_commands/find +83 -0
  64. data/scout_commands/glob +0 -0
  65. data/scout_commands/rbbt +23 -0
  66. data/scout_commands/workflow/task +707 -0
  67. data/test/scout/indiferent_hash/test_case_insensitive.rb +16 -0
  68. data/test/scout/indiferent_hash/test_options.rb +11 -1
  69. data/test/scout/misc/test_digest.rb +30 -0
  70. data/test/scout/misc/test_filesystem.rb +30 -0
  71. data/test/scout/misc/test_insist.rb +13 -0
  72. data/test/scout/open/test_lock.rb +52 -0
  73. data/test/scout/open/test_remote.rb +25 -0
  74. data/test/scout/open/test_stream.rb +515 -0
  75. data/test/scout/open/test_util.rb +73 -0
  76. data/test/scout/path/test_find.rb +37 -1
  77. data/test/scout/persist/test_open.rb +37 -0
  78. data/test/scout/persist/test_path.rb +37 -0
  79. data/test/scout/persist/test_serialize.rb +114 -0
  80. data/test/scout/resource/test_path.rb +40 -0
  81. data/test/scout/resource/test_produce.rb +62 -0
  82. data/test/scout/simple_opt/test_get.rb +11 -0
  83. data/test/scout/simple_opt/test_parse.rb +10 -0
  84. data/test/scout/simple_opt/test_setup.rb +77 -0
  85. data/test/scout/test_cmd.rb +85 -0
  86. data/test/scout/test_concurrent_stream.rb +29 -0
  87. data/test/scout/test_misc.rb +0 -7
  88. data/test/scout/test_open.rb +146 -0
  89. data/test/scout/test_path.rb +3 -1
  90. data/test/scout/test_persist.rb +83 -0
  91. data/test/scout/test_resource.rb +26 -0
  92. data/test/scout/test_workflow.rb +87 -0
  93. data/test/scout/workflow/step/test_info.rb +28 -0
  94. data/test/scout/workflow/task/test_inputs.rb +182 -0
  95. data/test/scout/workflow/test_step.rb +36 -0
  96. data/test/scout/workflow/test_task.rb +178 -0
  97. data/test/scout/workflow/test_usage.rb +26 -0
  98. data/test/scout/workflow/test_util.rb +17 -0
  99. data/test/test_helper.rb +17 -0
  100. data/test/test_scout-gear.rb +0 -0
  101. metadata +76 -3
data/lib/scout/open.rb ADDED
@@ -0,0 +1,169 @@
1
+ require_relative 'tmpfile'
2
+ require_relative 'path'
3
+ require_relative 'cmd'
4
+
5
+ require_relative 'open/stream'
6
+ require_relative 'open/util'
7
+ require_relative 'open/remote'
8
+ require_relative 'open/lock'
9
+
10
+ module Open
11
+ module NamedStream
12
+ attr_accessor :filename
13
+
14
+ def digest_str
15
+ if Path === filename && ! filename.located?
16
+ filename
17
+ else
18
+ Misc.file_md5(filename)
19
+ end
20
+ end
21
+ end
22
+
23
+ def self.get_stream(file, mode = 'r')
24
+ file = file.find if Path === file
25
+
26
+ return Open.ssh(file) if Open.ssh?(file)
27
+ return Open.wget(file) if Open.remote?(file)
28
+
29
+ File.open(file, mode)
30
+ end
31
+
32
+ def self.file_open(file, grep = false, mode = 'r', invert_grep = false)
33
+ Open.mkdir File.dirname(file) if mode.include? 'w'
34
+
35
+ stream = get_stream(file, mode)
36
+
37
+ if grep
38
+ grep(stream, grep, invert_grep)
39
+ else
40
+ stream
41
+ end
42
+ end
43
+
44
+ def self.file_write(file, content, mode = 'w')
45
+ File.open(file, mode) do |f|
46
+ begin
47
+ f.flock(File::LOCK_EX)
48
+ f.write content
49
+ f.flock(File::LOCK_UN)
50
+ ensure
51
+ f.close unless f.closed?
52
+ end
53
+ end
54
+ end
55
+
56
+ def self.open(file, options = {})
57
+ if IO === file || StringIO === file
58
+ if block_given?
59
+ res = yield file
60
+ file.close
61
+ return res
62
+ else
63
+ return file
64
+ end
65
+ end
66
+
67
+ options = IndiferentHash.add_defaults options, :noz => false, :mode => 'r'
68
+
69
+ mode = IndiferentHash.process_options options, :mode
70
+
71
+ options[:noz] = true if mode.include? "w"
72
+
73
+ io = file_open(file, options[:grep], mode, options[:invert_grep])
74
+
75
+ io = unzip(io) if ((String === file and zip?(file)) and not options[:noz]) or options[:zip]
76
+ io = gunzip(io) if ((String === file and gzip?(file)) and not options[:noz]) or options[:gzip]
77
+ io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
78
+
79
+ io.extend NamedStream
80
+ io.filename = file
81
+
82
+ if block_given?
83
+ res = nil
84
+ begin
85
+ res = yield(io)
86
+ rescue DontClose
87
+ res = $!.payload
88
+ rescue Exception
89
+ io.abort if io.respond_to? :abort
90
+ io.join if io.respond_to? :join
91
+ raise $!
92
+ ensure
93
+ io.close if io.respond_to? :close and not io.closed?
94
+ io.join if io.respond_to? :join
95
+ end
96
+ res
97
+ else
98
+ io
99
+ end
100
+ end
101
+
102
+ def self.read(file, options = {}, &block)
103
+ open(file, options) do |f|
104
+ if block_given?
105
+ res = []
106
+ while not f.eof?
107
+ l = f.gets
108
+ l = Misc.fixutf8(l) unless options[:nofix]
109
+ res << yield(l)
110
+ end
111
+ res
112
+ else
113
+ if options[:nofix]
114
+ f.read
115
+ else
116
+ Misc.fixutf8(f.read)
117
+ end
118
+ end
119
+ end
120
+ end
121
+
122
+ def self.write(file, content = nil, options = {})
123
+ options = IndiferentHash.add_defaults options, :mode => 'w'
124
+
125
+ file = file.find(options[:where]) if Path === file
126
+ mode = IndiferentHash.process_options options, :mode
127
+
128
+ FileUtils.mkdir_p File.dirname(file)
129
+
130
+ case
131
+ when block_given?
132
+ begin
133
+ f = File.open(file, mode)
134
+ begin
135
+ yield f
136
+ ensure
137
+ f.close unless f.closed?
138
+ end
139
+ rescue Exception
140
+ FileUtils.rm file if File.exist? file
141
+ raise $!
142
+ end
143
+ when content.nil?
144
+ file_write(file, "", mode)
145
+ when String === content
146
+ file_write(file, content, mode)
147
+ when (IO === content || StringIO === content)
148
+ begin
149
+ File.open(file, mode) do |f|
150
+ f.flock(File::LOCK_EX)
151
+ while block = content.read(Open::BLOCK_SIZE)
152
+ f.write block
153
+ end
154
+ f.flock(File::LOCK_UN)
155
+ end
156
+ rescue Exception
157
+ FileUtils.rm_rf file if File.exist? file
158
+ raise $!
159
+ end
160
+ content.close unless content.closed?
161
+ content.join if content.respond_to? :join
162
+ else
163
+ raise "Content unknown #{Log.fingerprint content}"
164
+ end
165
+
166
+ notify_write(file)
167
+ end
168
+
169
+ end
@@ -1,27 +1,57 @@
1
+ require_relative '../indiferent_hash'
1
2
  module Path
2
- def _parts
3
- @_parts ||= self.split("/")
4
- end
5
3
 
6
- def _subpath
7
- @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0]
8
- end
9
-
10
- def _toplevel
11
- @toplevel ||= _parts.length > 1 ? _parts[0] : nil
4
+ def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
5
+
6
+ if file.nil?
7
+ caller_dup = caller.dup
8
+ while file = caller_dup.shift
9
+ break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
10
+ file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
11
+ file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
12
+ file =~ /(?:scout|rbbt)\/persist.rb/
13
+ end
14
+ file = file.sub(/\.rb[^\w].*/,'.rb')
15
+ end
16
+
17
+ relative_to = [relative_to] unless Array === relative_to
18
+ file = File.expand_path(file)
19
+ return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
20
+
21
+ while file != '/'
22
+ dir = File.dirname file
23
+
24
+ return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
25
+
26
+ file = File.dirname file
27
+ end
28
+
29
+ return nil
12
30
  end
13
31
 
14
- def self.follow(path, map)
15
- map.sub('{PKGDIR}', path.pkgdir || Path.default_pkgdir).
16
- sub('{RESOURCE}', path.to_s).
32
+ def self.follow(path, map, map_name = nil)
33
+ file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
34
+ sub('{RESOURCE}', path.pkgdir.to_s).
17
35
  sub('{PWD}', FileUtils.pwd).
18
36
  sub('{TOPLEVEL}', path._toplevel).
19
37
  sub('{SUBPATH}', path._subpath).
20
38
  sub('{BASENAME}', File.basename(path)).
21
39
  sub('{PATH}', path).
22
- sub('{LIBDIR}', path.libdir || Path.caller_lib_dir).
40
+ sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
41
+ sub('{MAPNAME}', map_name.to_s).
23
42
  sub('{REMOVE}/', '').
24
43
  sub('{REMOVE}', '').gsub(/\/+/,'/')
44
+
45
+ while true
46
+ file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
47
+ key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
48
+ key_text = follow(path, "#{key}}", map_name)
49
+ key_text[orig] = replace[0..-2] if key_text.include?(orig)
50
+ key_text
51
+ end || break
52
+ end
53
+
54
+ file
25
55
  end
26
56
 
27
57
  def self.path_maps
@@ -40,19 +70,36 @@ module Path
40
70
  })
41
71
  end
42
72
 
43
- def self.map_search
44
- @@map_search ||= %w(current workflow user local global lib fast cache bulk)
73
+ def self.basic_map_order
74
+ @@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
45
75
  end
46
76
 
47
- def self.search_order
48
- @@search_order ||= (path_maps.keys & map_search) + (path_maps.keys - map_search)
77
+ def self.map_order
78
+ @@map_order ||= (path_maps.keys & basic_map_order) + (path_maps.keys - basic_map_order)
79
+ end
80
+
81
+ def self.add_path(name, map)
82
+ @@path_maps[name] = map
83
+ @@map_order = nil
84
+ end
85
+
86
+ def _parts
87
+ @_parts ||= self.split("/")
88
+ end
89
+
90
+ def _subpath
91
+ @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
92
+ end
93
+
94
+ def _toplevel
95
+ @toplevel ||= _parts.length > 1 ? _parts[0] : ""
49
96
  end
50
97
 
51
98
  SLASH = "/"[0]
52
99
  DOT = "."[0]
53
100
  def located?
54
101
  # OPEN RESOURCE
55
- self.slice(0,1) == SLASH || (self.char(0,1) == DOT && self.char(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
102
+ self.slice(0,1) == SLASH || (self.slice(0,1) == DOT && self.slice(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
56
103
  end
57
104
 
58
105
  def annotate_found_where(found, where)
@@ -70,13 +117,19 @@ module Path
70
117
  @original
71
118
  end
72
119
 
120
+ def map_order
121
+ @map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
122
+ end
123
+
73
124
  def follow(map_name = :default, annotate = true)
74
- map = Path.path_maps[map_name]
125
+ IndiferentHash.setup(path_maps)
126
+ map = path_maps[map_name]
127
+ raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
75
128
  while Symbol === map
76
129
  map_name = map
77
- map = Path.path_maps[map_name]
130
+ map = path_maps[map_name]
78
131
  end
79
- found = Path.follow(self, map)
132
+ found = Path.follow(self, map, map_name)
80
133
 
81
134
  annotate_found_where(found, map_name) if annotate
82
135
 
@@ -85,13 +138,13 @@ module Path
85
138
 
86
139
  def find(where = nil)
87
140
  return self if located?
141
+ return find_all if where == 'all' || where == :all
88
142
  return follow(where) if where
89
143
 
90
-
91
- Path.search_order.each do |map_name|
144
+ map_order.each do |map_name|
92
145
  found = follow(map_name, false)
93
146
 
94
- return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(real_path)
147
+ return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
95
148
  end
96
149
 
97
150
  return follow(:default)
@@ -106,9 +159,8 @@ module Path
106
159
  alias exists? exist?
107
160
 
108
161
  def find_all(caller_lib = nil, search_paths = nil)
109
- Path.search_order
162
+ map_order
110
163
  .collect{|where| find(where) }
111
164
  .select{|file| file.exist? }.uniq
112
165
  end
113
-
114
166
  end
@@ -0,0 +1,8 @@
1
+ module TmpFile
2
+ def self.with_path(*args, &block)
3
+ TmpFile.with_file(*args) do |file|
4
+ Path.setup(file)
5
+ yield file
6
+ end
7
+ end
8
+ end
@@ -1,4 +1,12 @@
1
1
  module Path
2
+
3
+ def self.is_filename?(string, need_to_exists = true)
4
+ return false if string.nil?
5
+ return true if Path === string
6
+ return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265 }.empty? and (! need_to_exists || File.exist?(string))
7
+ return false
8
+ end
9
+
2
10
  def directory?
3
11
  return nil unless self.exist?
4
12
  File.directory?(self.find)
@@ -12,7 +20,6 @@ module Path
12
20
  self.annotate(File.basename(self))
13
21
  end
14
22
 
15
-
16
23
  def glob(pattern = '*')
17
24
  if self.include? "*"
18
25
  self.glob_all
@@ -31,12 +38,11 @@ module Path
31
38
  end
32
39
 
33
40
  def glob_all(pattern = nil, caller_lib = nil, search_paths = nil)
34
- search_paths ||= Path.search_paths
41
+ search_paths ||= Path.path_maps
35
42
  search_paths = search_paths.dup
36
43
 
37
- location_paths = {}
38
44
  search_paths.keys.collect do |where|
39
- found = find(where, Path.caller_lib_dir, search_paths)
45
+ found = find(where)
40
46
  paths = pattern ? Dir.glob(File.join(found, pattern)) : Dir.glob(found)
41
47
 
42
48
  paths = paths.collect{|p| self.annotate p }
@@ -46,7 +52,13 @@ module Path
46
52
  p.where = where
47
53
  end if found.original and pattern
48
54
 
49
- location_paths[where] = paths
55
+ paths
56
+ end.flatten.uniq
57
+ end
58
+
59
+ def no_method_missing
60
+ class << self
61
+ undef_method :method_missing
50
62
  end
51
63
  end
52
64
  end
data/lib/scout/path.rb CHANGED
@@ -1,40 +1,18 @@
1
1
  require_relative 'meta_extension'
2
2
  require_relative 'path/find'
3
3
  require_relative 'path/util'
4
+ require_relative 'path/tmpfile'
4
5
 
5
6
  module Path
6
7
  extend MetaExtension
7
- extension_attr :pkgdir, :libdir
8
-
9
- def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
10
-
11
- if file.nil?
12
- caller_dup = caller.dup
13
- while file = caller_dup.shift
14
- break unless file =~ /scout\/(?:resource\.rb|workflow\.rb)/ or
15
- file =~ /scout\/path\.rb/ or
16
- file =~ /scout\/persist.rb/
17
- end
18
- file = file.sub(/\.rb[^\w].*/,'.rb')
19
- end
20
-
21
- relative_to = [relative_to] unless Array === relative_to
22
- file = File.expand_path(file)
23
- return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
24
-
25
- while file != '/'
26
- dir = File.dirname file
27
-
28
- return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
29
-
30
- file = File.dirname file
31
- end
32
-
33
- return nil
34
- end
8
+ extension_attr :pkgdir, :libdir, :path_maps
35
9
 
36
10
  def self.default_pkgdir
37
- @@default_pkgdir = 'scout'
11
+ @@default_pkgdir ||= 'scout'
12
+ end
13
+
14
+ def self.default_pkgdir=(pkgdir)
15
+ @@default_pkgdir = pkgdir
38
16
  end
39
17
 
40
18
  def pkgdir
@@ -42,7 +20,11 @@ module Path
42
20
  end
43
21
 
44
22
  def libdir
45
- @libdir ||= Path.caller_lib_dir
23
+ @libdir || Path.caller_lib_dir
24
+ end
25
+
26
+ def path_maps
27
+ @path_maps ||= Path.path_maps
46
28
  end
47
29
 
48
30
  def join(subpath, prevpath = nil)
@@ -50,7 +32,7 @@ module Path
50
32
  prevpath = prevpath.to_s if Symbol === prevpath
51
33
 
52
34
  subpath = File.join(prevpath.to_s, subpath) if prevpath
53
- new = File.join(self, subpath)
35
+ new = self.empty? ? subpath : File.join(self, subpath)
54
36
  self.annotate(new)
55
37
  new
56
38
  end
@@ -0,0 +1,17 @@
1
+ require_relative '../open'
2
+ require 'json'
3
+ require 'yaml'
4
+
5
+ module Open
6
+ def self.json(file)
7
+ Open.open(file){|f| JSON.load(f) }
8
+ end
9
+
10
+ def self.yaml(file)
11
+ Open.open(file){|f| YAML.load(f) }
12
+ end
13
+
14
+ def self.marshal(file)
15
+ Open.open(file){|f| Marshal.load(f) }
16
+ end
17
+ end
@@ -0,0 +1,15 @@
1
+ require_relative 'open'
2
+
3
+ module Path
4
+ def yaml(*rest, &block)
5
+ Open.yaml(self, *rest, &block)
6
+ end
7
+
8
+ def json(*rest, &block)
9
+ Open.json(self, *rest, &block)
10
+ end
11
+
12
+ def marshal(*rest, &block)
13
+ Open.marshal(self, *rest, &block)
14
+ end
15
+ end
@@ -0,0 +1,140 @@
1
+ require_relative '../open'
2
+ require_relative 'open'
3
+
4
+ module Persist
5
+ TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
6
+
7
+ class << self
8
+ attr_accessor :save_drivers, :load_drivers
9
+ def save_drivers
10
+ @save_drivers ||= {}
11
+ end
12
+ def load_drivers
13
+ @load_drivers ||= {}
14
+ end
15
+ end
16
+
17
+ def self.serialize(content, type)
18
+ case type
19
+ when nil, :string, :integer, :float, :boolean, :file, :path
20
+ if IO === content || StringIO === content
21
+ content.read
22
+ else
23
+ content.to_s
24
+ end
25
+ when :array
26
+ content * "\n"
27
+ when :yaml
28
+ content.to_yaml
29
+ when :json
30
+ content.to_json
31
+ when :marshal, :serializer
32
+ Marshal.dump(content)
33
+ else
34
+ if m = type.to_s.match(/(.*)_array/)
35
+ type = m[1].to_sym
36
+ content.collect{|c| serialize(c, type) } * "\n"
37
+ else
38
+ raise "Persist does not know #{Log.fingerprint type}"
39
+ end
40
+ end
41
+ end
42
+
43
+ def self.deserialize(serialized, type)
44
+ case type
45
+ when nil, :string, :file, :stream
46
+ serialized
47
+ when :path
48
+ Path.setup(serialized)
49
+ when :integer
50
+ serialized.to_i
51
+ when :float
52
+ serialized.to_f
53
+ when :boolean
54
+ TRUE_STRINGS.include? serialized
55
+ when :array
56
+ serialized.split("\n")
57
+ when :yaml
58
+ YAML.parse(serialized)
59
+ when :json
60
+ JSON.parse(serialized)
61
+ when :marshal, :serializer
62
+ Marshal.load(serialized)
63
+ else
64
+ if m = type.to_s.match(/(.*)_array/)
65
+ type = m[1].to_sym
66
+ new_content = serialized.split("\n")
67
+ new_content.collect{|c| deserialize(c, type) }
68
+ else
69
+ raise "Persist does not know #{Log.fingerprint type}"
70
+ end
71
+ end
72
+ end
73
+
74
+ MEMORY = {}
75
+ def self.save(content, file, type = :serializer)
76
+ return if content.nil?
77
+ type = MEMORY if type == :memory
78
+ type = :serializer if type.nil?
79
+
80
+ if Hash === type
81
+ type[file] = content
82
+ return
83
+ end
84
+
85
+ Log.debug "Save #{Log.fingerprint type} on #{file}"
86
+ if save_drivers[type]
87
+ Open.write(file, save_drivers[type].call(content))
88
+ return
89
+ end
90
+
91
+ if IO === content || StringIO === content
92
+ main, copy = Open.tee_stream_thread content
93
+ t = Thread.new do
94
+ Thread.current["name"] = "file saver: " + file
95
+ Open.sensible_write(file, main)
96
+ end
97
+ ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
98
+ else
99
+ serialized = serialize(content, type)
100
+ Open.sensible_write(file, serialized)
101
+ content
102
+ end
103
+ end
104
+
105
+ def self.load(file, type = :serializer)
106
+ file = file.find if Path === file
107
+ type = MEMORY if type == :memory
108
+ return unless Hash === type || Open.exist?(file)
109
+ type = :serializer if type.nil?
110
+
111
+ Log.debug "Load #{Log.fingerprint type} on #{file}"
112
+ if load_drivers[type]
113
+ return load_drivers[type].call(file)
114
+ end
115
+
116
+ case type
117
+ when :yaml
118
+ Open.yaml(file)
119
+ when :json
120
+ Open.json(file)
121
+ when :marshal, :serializer
122
+ Open.marshal(file)
123
+ when :stream
124
+ Open.open(file)
125
+ when :file
126
+ value = Open.read(file)
127
+ value.sub!(/^\./, File.dirname(file)) if value.start_with?("./")
128
+ value
129
+ when :file_array
130
+ Open.read(file).split("\n").collect do |f|
131
+ f.sub!(/^\./, File.dirname(file)) if f.start_with?("./")
132
+ f
133
+ end
134
+ when Hash
135
+ type[file]
136
+ else
137
+ deserialize(Open.read(file), type)
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,54 @@
1
+ require_relative 'persist/serialize'
2
+ require_relative 'persist/open'
3
+ require_relative 'persist/path'
4
+
5
+ module Persist
6
+ class << self
7
+ attr :cache_dir
8
+ def cache_dir=(cache_dir)
9
+ @cache_dir = Path === cache_dir ? cache_dir : Path.setup(cache_dir)
10
+ end
11
+ def cache_dir
12
+ @cache_dir ||= Path.setup("var/cache/persistence")
13
+ end
14
+
15
+ attr_writer :lock_dir
16
+ def lock_dir
17
+ @lock_dir ||= Path.setup("var/cache/persist_locks")
18
+ end
19
+ end
20
+
21
+ def self.persistence_path(name, options = {})
22
+ options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
23
+ other_options = IndiferentHash.pull_keys options, :other
24
+ TmpFile.tmp_for_file(name, options, other_options)
25
+ end
26
+
27
+ def self.persist(name, type = :serializer, options = {}, &block)
28
+ persist_options = IndiferentHash.pull_keys options, :persist
29
+ file = persist_options[:path] || options[:path] || persistence_path(name, options)
30
+
31
+ update = options[:update] || persist_options[:update]
32
+ update = Open.mtime(update) if Path === update
33
+ update = Open.mtime(file) >= update ? false : true if Time === update
34
+
35
+ if Open.exist?(file) && ! update
36
+ Persist.load(file, type)
37
+ else
38
+ res = yield
39
+ begin
40
+ Open.rm(file)
41
+ res = Persist.save(res, file, type)
42
+ rescue
43
+ raise $! unless options[:canfail]
44
+ Log.debug "Could not persist #{type} on #{file}"
45
+ end
46
+ res
47
+ end
48
+ end
49
+
50
+ def self.memory(name, *args, &block)
51
+ self.persist(name, :memory, *args, &block)
52
+ end
53
+
54
+ end
@@ -0,0 +1,15 @@
1
+ module Path
2
+ def open(*args, &block)
3
+ produce
4
+ Open.open(self, *args, &block)
5
+ end
6
+
7
+ def read
8
+ produce
9
+ Open.read(self)
10
+ end
11
+
12
+ def write(*args, &block)
13
+ Open.write(self.find, *args, &block)
14
+ end
15
+ end