scout-gear 1.2.0 → 5.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +4 -0
  3. data/.vimproject +663 -4
  4. data/Rakefile +1 -0
  5. data/VERSION +1 -1
  6. data/bin/scout +235 -0
  7. data/lib/scout/cmd.rb +344 -0
  8. data/lib/scout/concurrent_stream.rb +259 -0
  9. data/lib/scout/exceptions.rb +15 -8
  10. data/lib/scout/indiferent_hash/options.rb +8 -26
  11. data/lib/scout/indiferent_hash.rb +0 -30
  12. data/lib/scout/log/color.rb +2 -2
  13. data/lib/scout/log/fingerprint.rb +11 -1
  14. data/lib/scout/log/progress/report.rb +0 -1
  15. data/lib/scout/log/progress/util.rb +1 -1
  16. data/lib/scout/log/progress.rb +4 -4
  17. data/lib/scout/log.rb +10 -2
  18. data/lib/scout/meta_extension.rb +15 -1
  19. data/lib/scout/misc/digest.rb +56 -0
  20. data/lib/scout/misc/filesystem.rb +26 -0
  21. data/lib/scout/misc/format.rb +226 -0
  22. data/lib/scout/misc/insist.rb +56 -0
  23. data/lib/scout/misc.rb +5 -11
  24. data/lib/scout/open/lock.rb +61 -0
  25. data/lib/scout/open/remote.rb +120 -0
  26. data/lib/scout/open/stream.rb +372 -0
  27. data/lib/scout/open/util.rb +225 -0
  28. data/lib/scout/open.rb +169 -0
  29. data/lib/scout/path/find.rb +78 -26
  30. data/lib/scout/path/tmpfile.rb +8 -0
  31. data/lib/scout/path/util.rb +17 -5
  32. data/lib/scout/path.rb +13 -31
  33. data/lib/scout/persist/open.rb +17 -0
  34. data/lib/scout/persist/path.rb +15 -0
  35. data/lib/scout/persist/serialize.rb +140 -0
  36. data/lib/scout/persist.rb +54 -0
  37. data/lib/scout/resource/path.rb +15 -0
  38. data/lib/scout/resource/produce/rake.rb +69 -0
  39. data/lib/scout/resource/produce.rb +246 -0
  40. data/lib/scout/resource/scout.rb +3 -0
  41. data/lib/scout/resource.rb +37 -0
  42. data/lib/scout/simple_opt/accessor.rb +54 -0
  43. data/lib/scout/simple_opt/doc.rb +102 -0
  44. data/lib/scout/simple_opt/get.rb +57 -0
  45. data/lib/scout/simple_opt/parse.rb +67 -0
  46. data/lib/scout/simple_opt/setup.rb +26 -0
  47. data/lib/scout/simple_opt.rb +5 -0
  48. data/lib/scout/tmpfile.rb +39 -1
  49. data/lib/scout/workflow/definition.rb +72 -0
  50. data/lib/scout/workflow/documentation.rb +77 -0
  51. data/lib/scout/workflow/step/info.rb +77 -0
  52. data/lib/scout/workflow/step.rb +96 -0
  53. data/lib/scout/workflow/task/inputs.rb +112 -0
  54. data/lib/scout/workflow/task.rb +141 -0
  55. data/lib/scout/workflow/usage.rb +294 -0
  56. data/lib/scout/workflow/util.rb +11 -0
  57. data/lib/scout/workflow.rb +39 -0
  58. data/lib/scout-gear.rb +5 -0
  59. data/lib/scout.rb +1 -0
  60. data/lib/workflow-scout.rb +2 -0
  61. data/scout-gear.gemspec +78 -5
  62. data/scout_commands/alias +48 -0
  63. data/scout_commands/find +83 -0
  64. data/scout_commands/glob +0 -0
  65. data/scout_commands/rbbt +23 -0
  66. data/scout_commands/workflow/task +707 -0
  67. data/test/scout/indiferent_hash/test_case_insensitive.rb +16 -0
  68. data/test/scout/indiferent_hash/test_options.rb +11 -1
  69. data/test/scout/misc/test_digest.rb +30 -0
  70. data/test/scout/misc/test_filesystem.rb +30 -0
  71. data/test/scout/misc/test_insist.rb +13 -0
  72. data/test/scout/open/test_lock.rb +52 -0
  73. data/test/scout/open/test_remote.rb +25 -0
  74. data/test/scout/open/test_stream.rb +515 -0
  75. data/test/scout/open/test_util.rb +73 -0
  76. data/test/scout/path/test_find.rb +37 -1
  77. data/test/scout/persist/test_open.rb +37 -0
  78. data/test/scout/persist/test_path.rb +37 -0
  79. data/test/scout/persist/test_serialize.rb +114 -0
  80. data/test/scout/resource/test_path.rb +40 -0
  81. data/test/scout/resource/test_produce.rb +62 -0
  82. data/test/scout/simple_opt/test_get.rb +11 -0
  83. data/test/scout/simple_opt/test_parse.rb +10 -0
  84. data/test/scout/simple_opt/test_setup.rb +77 -0
  85. data/test/scout/test_cmd.rb +85 -0
  86. data/test/scout/test_concurrent_stream.rb +29 -0
  87. data/test/scout/test_misc.rb +0 -7
  88. data/test/scout/test_open.rb +146 -0
  89. data/test/scout/test_path.rb +3 -1
  90. data/test/scout/test_persist.rb +83 -0
  91. data/test/scout/test_resource.rb +26 -0
  92. data/test/scout/test_workflow.rb +87 -0
  93. data/test/scout/workflow/step/test_info.rb +28 -0
  94. data/test/scout/workflow/task/test_inputs.rb +182 -0
  95. data/test/scout/workflow/test_step.rb +36 -0
  96. data/test/scout/workflow/test_task.rb +178 -0
  97. data/test/scout/workflow/test_usage.rb +26 -0
  98. data/test/scout/workflow/test_util.rb +17 -0
  99. data/test/test_helper.rb +17 -0
  100. data/test/test_scout-gear.rb +0 -0
  101. metadata +76 -3
data/lib/scout/open.rb ADDED
@@ -0,0 +1,169 @@
1
+ require_relative 'tmpfile'
2
+ require_relative 'path'
3
+ require_relative 'cmd'
4
+
5
+ require_relative 'open/stream'
6
+ require_relative 'open/util'
7
+ require_relative 'open/remote'
8
+ require_relative 'open/lock'
9
+
10
+ module Open
11
+ module NamedStream
12
+ attr_accessor :filename
13
+
14
+ def digest_str
15
+ if Path === filename && ! filename.located?
16
+ filename
17
+ else
18
+ Misc.file_md5(filename)
19
+ end
20
+ end
21
+ end
22
+
23
+ def self.get_stream(file, mode = 'r')
24
+ file = file.find if Path === file
25
+
26
+ return Open.ssh(file) if Open.ssh?(file)
27
+ return Open.wget(file) if Open.remote?(file)
28
+
29
+ File.open(file, mode)
30
+ end
31
+
32
+ def self.file_open(file, grep = false, mode = 'r', invert_grep = false)
33
+ Open.mkdir File.dirname(file) if mode.include? 'w'
34
+
35
+ stream = get_stream(file, mode)
36
+
37
+ if grep
38
+ grep(stream, grep, invert_grep)
39
+ else
40
+ stream
41
+ end
42
+ end
43
+
44
+ def self.file_write(file, content, mode = 'w')
45
+ File.open(file, mode) do |f|
46
+ begin
47
+ f.flock(File::LOCK_EX)
48
+ f.write content
49
+ f.flock(File::LOCK_UN)
50
+ ensure
51
+ f.close unless f.closed?
52
+ end
53
+ end
54
+ end
55
+
56
+ def self.open(file, options = {})
57
+ if IO === file || StringIO === file
58
+ if block_given?
59
+ res = yield file
60
+ file.close
61
+ return res
62
+ else
63
+ return file
64
+ end
65
+ end
66
+
67
+ options = IndiferentHash.add_defaults options, :noz => false, :mode => 'r'
68
+
69
+ mode = IndiferentHash.process_options options, :mode
70
+
71
+ options[:noz] = true if mode.include? "w"
72
+
73
+ io = file_open(file, options[:grep], mode, options[:invert_grep])
74
+
75
+ io = unzip(io) if ((String === file and zip?(file)) and not options[:noz]) or options[:zip]
76
+ io = gunzip(io) if ((String === file and gzip?(file)) and not options[:noz]) or options[:gzip]
77
+ io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
78
+
79
+ io.extend NamedStream
80
+ io.filename = file
81
+
82
+ if block_given?
83
+ res = nil
84
+ begin
85
+ res = yield(io)
86
+ rescue DontClose
87
+ res = $!.payload
88
+ rescue Exception
89
+ io.abort if io.respond_to? :abort
90
+ io.join if io.respond_to? :join
91
+ raise $!
92
+ ensure
93
+ io.close if io.respond_to? :close and not io.closed?
94
+ io.join if io.respond_to? :join
95
+ end
96
+ res
97
+ else
98
+ io
99
+ end
100
+ end
101
+
102
+ def self.read(file, options = {}, &block)
103
+ open(file, options) do |f|
104
+ if block_given?
105
+ res = []
106
+ while not f.eof?
107
+ l = f.gets
108
+ l = Misc.fixutf8(l) unless options[:nofix]
109
+ res << yield(l)
110
+ end
111
+ res
112
+ else
113
+ if options[:nofix]
114
+ f.read
115
+ else
116
+ Misc.fixutf8(f.read)
117
+ end
118
+ end
119
+ end
120
+ end
121
+
122
+ def self.write(file, content = nil, options = {})
123
+ options = IndiferentHash.add_defaults options, :mode => 'w'
124
+
125
+ file = file.find(options[:where]) if Path === file
126
+ mode = IndiferentHash.process_options options, :mode
127
+
128
+ FileUtils.mkdir_p File.dirname(file)
129
+
130
+ case
131
+ when block_given?
132
+ begin
133
+ f = File.open(file, mode)
134
+ begin
135
+ yield f
136
+ ensure
137
+ f.close unless f.closed?
138
+ end
139
+ rescue Exception
140
+ FileUtils.rm file if File.exist? file
141
+ raise $!
142
+ end
143
+ when content.nil?
144
+ file_write(file, "", mode)
145
+ when String === content
146
+ file_write(file, content, mode)
147
+ when (IO === content || StringIO === content)
148
+ begin
149
+ File.open(file, mode) do |f|
150
+ f.flock(File::LOCK_EX)
151
+ while block = content.read(Open::BLOCK_SIZE)
152
+ f.write block
153
+ end
154
+ f.flock(File::LOCK_UN)
155
+ end
156
+ rescue Exception
157
+ FileUtils.rm_rf file if File.exist? file
158
+ raise $!
159
+ end
160
+ content.close unless content.closed?
161
+ content.join if content.respond_to? :join
162
+ else
163
+ raise "Content unknown #{Log.fingerprint content}"
164
+ end
165
+
166
+ notify_write(file)
167
+ end
168
+
169
+ end
@@ -1,27 +1,57 @@
1
+ require_relative '../indiferent_hash'
1
2
  module Path
2
- def _parts
3
- @_parts ||= self.split("/")
4
- end
5
3
 
6
- def _subpath
7
- @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0]
8
- end
9
-
10
- def _toplevel
11
- @toplevel ||= _parts.length > 1 ? _parts[0] : nil
4
+ def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
5
+
6
+ if file.nil?
7
+ caller_dup = caller.dup
8
+ while file = caller_dup.shift
9
+ break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
10
+ file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
11
+ file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
12
+ file =~ /(?:scout|rbbt)\/persist.rb/
13
+ end
14
+ file = file.sub(/\.rb[^\w].*/,'.rb')
15
+ end
16
+
17
+ relative_to = [relative_to] unless Array === relative_to
18
+ file = File.expand_path(file)
19
+ return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
20
+
21
+ while file != '/'
22
+ dir = File.dirname file
23
+
24
+ return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
25
+
26
+ file = File.dirname file
27
+ end
28
+
29
+ return nil
12
30
  end
13
31
 
14
- def self.follow(path, map)
15
- map.sub('{PKGDIR}', path.pkgdir || Path.default_pkgdir).
16
- sub('{RESOURCE}', path.to_s).
32
+ def self.follow(path, map, map_name = nil)
33
+ file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
34
+ sub('{RESOURCE}', path.pkgdir.to_s).
17
35
  sub('{PWD}', FileUtils.pwd).
18
36
  sub('{TOPLEVEL}', path._toplevel).
19
37
  sub('{SUBPATH}', path._subpath).
20
38
  sub('{BASENAME}', File.basename(path)).
21
39
  sub('{PATH}', path).
22
- sub('{LIBDIR}', path.libdir || Path.caller_lib_dir).
40
+ sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
41
+ sub('{MAPNAME}', map_name.to_s).
23
42
  sub('{REMOVE}/', '').
24
43
  sub('{REMOVE}', '').gsub(/\/+/,'/')
44
+
45
+ while true
46
+ file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
47
+ key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
48
+ key_text = follow(path, "#{key}}", map_name)
49
+ key_text[orig] = replace[0..-2] if key_text.include?(orig)
50
+ key_text
51
+ end || break
52
+ end
53
+
54
+ file
25
55
  end
26
56
 
27
57
  def self.path_maps
@@ -40,19 +70,36 @@ module Path
40
70
  })
41
71
  end
42
72
 
43
- def self.map_search
44
- @@map_search ||= %w(current workflow user local global lib fast cache bulk)
73
+ def self.basic_map_order
74
+ @@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
45
75
  end
46
76
 
47
- def self.search_order
48
- @@search_order ||= (path_maps.keys & map_search) + (path_maps.keys - map_search)
77
+ def self.map_order
78
+ @@map_order ||= (path_maps.keys & basic_map_order) + (path_maps.keys - basic_map_order)
79
+ end
80
+
81
+ def self.add_path(name, map)
82
+ @@path_maps[name] = map
83
+ @@map_order = nil
84
+ end
85
+
86
+ def _parts
87
+ @_parts ||= self.split("/")
88
+ end
89
+
90
+ def _subpath
91
+ @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
92
+ end
93
+
94
+ def _toplevel
95
+ @toplevel ||= _parts.length > 1 ? _parts[0] : ""
49
96
  end
50
97
 
51
98
  SLASH = "/"[0]
52
99
  DOT = "."[0]
53
100
  def located?
54
101
  # OPEN RESOURCE
55
- self.slice(0,1) == SLASH || (self.char(0,1) == DOT && self.char(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
102
+ self.slice(0,1) == SLASH || (self.slice(0,1) == DOT && self.slice(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
56
103
  end
57
104
 
58
105
  def annotate_found_where(found, where)
@@ -70,13 +117,19 @@ module Path
70
117
  @original
71
118
  end
72
119
 
120
+ def map_order
121
+ @map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
122
+ end
123
+
73
124
  def follow(map_name = :default, annotate = true)
74
- map = Path.path_maps[map_name]
125
+ IndiferentHash.setup(path_maps)
126
+ map = path_maps[map_name]
127
+ raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
75
128
  while Symbol === map
76
129
  map_name = map
77
- map = Path.path_maps[map_name]
130
+ map = path_maps[map_name]
78
131
  end
79
- found = Path.follow(self, map)
132
+ found = Path.follow(self, map, map_name)
80
133
 
81
134
  annotate_found_where(found, map_name) if annotate
82
135
 
@@ -85,13 +138,13 @@ module Path
85
138
 
86
139
  def find(where = nil)
87
140
  return self if located?
141
+ return find_all if where == 'all' || where == :all
88
142
  return follow(where) if where
89
143
 
90
-
91
- Path.search_order.each do |map_name|
144
+ map_order.each do |map_name|
92
145
  found = follow(map_name, false)
93
146
 
94
- return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(real_path)
147
+ return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
95
148
  end
96
149
 
97
150
  return follow(:default)
@@ -106,9 +159,8 @@ module Path
106
159
  alias exists? exist?
107
160
 
108
161
  def find_all(caller_lib = nil, search_paths = nil)
109
- Path.search_order
162
+ map_order
110
163
  .collect{|where| find(where) }
111
164
  .select{|file| file.exist? }.uniq
112
165
  end
113
-
114
166
  end
@@ -0,0 +1,8 @@
1
+ module TmpFile
2
+ def self.with_path(*args, &block)
3
+ TmpFile.with_file(*args) do |file|
4
+ Path.setup(file)
5
+ yield file
6
+ end
7
+ end
8
+ end
@@ -1,4 +1,12 @@
1
1
  module Path
2
+
3
+ def self.is_filename?(string, need_to_exists = true)
4
+ return false if string.nil?
5
+ return true if Path === string
6
+ return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265 }.empty? and (! need_to_exists || File.exist?(string))
7
+ return false
8
+ end
9
+
2
10
  def directory?
3
11
  return nil unless self.exist?
4
12
  File.directory?(self.find)
@@ -12,7 +20,6 @@ module Path
12
20
  self.annotate(File.basename(self))
13
21
  end
14
22
 
15
-
16
23
  def glob(pattern = '*')
17
24
  if self.include? "*"
18
25
  self.glob_all
@@ -31,12 +38,11 @@ module Path
31
38
  end
32
39
 
33
40
  def glob_all(pattern = nil, caller_lib = nil, search_paths = nil)
34
- search_paths ||= Path.search_paths
41
+ search_paths ||= Path.path_maps
35
42
  search_paths = search_paths.dup
36
43
 
37
- location_paths = {}
38
44
  search_paths.keys.collect do |where|
39
- found = find(where, Path.caller_lib_dir, search_paths)
45
+ found = find(where)
40
46
  paths = pattern ? Dir.glob(File.join(found, pattern)) : Dir.glob(found)
41
47
 
42
48
  paths = paths.collect{|p| self.annotate p }
@@ -46,7 +52,13 @@ module Path
46
52
  p.where = where
47
53
  end if found.original and pattern
48
54
 
49
- location_paths[where] = paths
55
+ paths
56
+ end.flatten.uniq
57
+ end
58
+
59
+ def no_method_missing
60
+ class << self
61
+ undef_method :method_missing
50
62
  end
51
63
  end
52
64
  end
data/lib/scout/path.rb CHANGED
@@ -1,40 +1,18 @@
1
1
  require_relative 'meta_extension'
2
2
  require_relative 'path/find'
3
3
  require_relative 'path/util'
4
+ require_relative 'path/tmpfile'
4
5
 
5
6
  module Path
6
7
  extend MetaExtension
7
- extension_attr :pkgdir, :libdir
8
-
9
- def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
10
-
11
- if file.nil?
12
- caller_dup = caller.dup
13
- while file = caller_dup.shift
14
- break unless file =~ /scout\/(?:resource\.rb|workflow\.rb)/ or
15
- file =~ /scout\/path\.rb/ or
16
- file =~ /scout\/persist.rb/
17
- end
18
- file = file.sub(/\.rb[^\w].*/,'.rb')
19
- end
20
-
21
- relative_to = [relative_to] unless Array === relative_to
22
- file = File.expand_path(file)
23
- return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
24
-
25
- while file != '/'
26
- dir = File.dirname file
27
-
28
- return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
29
-
30
- file = File.dirname file
31
- end
32
-
33
- return nil
34
- end
8
+ extension_attr :pkgdir, :libdir, :path_maps
35
9
 
36
10
  def self.default_pkgdir
37
- @@default_pkgdir = 'scout'
11
+ @@default_pkgdir ||= 'scout'
12
+ end
13
+
14
+ def self.default_pkgdir=(pkgdir)
15
+ @@default_pkgdir = pkgdir
38
16
  end
39
17
 
40
18
  def pkgdir
@@ -42,7 +20,11 @@ module Path
42
20
  end
43
21
 
44
22
  def libdir
45
- @libdir ||= Path.caller_lib_dir
23
+ @libdir || Path.caller_lib_dir
24
+ end
25
+
26
+ def path_maps
27
+ @path_maps ||= Path.path_maps
46
28
  end
47
29
 
48
30
  def join(subpath, prevpath = nil)
@@ -50,7 +32,7 @@ module Path
50
32
  prevpath = prevpath.to_s if Symbol === prevpath
51
33
 
52
34
  subpath = File.join(prevpath.to_s, subpath) if prevpath
53
- new = File.join(self, subpath)
35
+ new = self.empty? ? subpath : File.join(self, subpath)
54
36
  self.annotate(new)
55
37
  new
56
38
  end
@@ -0,0 +1,17 @@
1
+ require_relative '../open'
2
+ require 'json'
3
+ require 'yaml'
4
+
5
+ module Open
6
+ def self.json(file)
7
+ Open.open(file){|f| JSON.load(f) }
8
+ end
9
+
10
+ def self.yaml(file)
11
+ Open.open(file){|f| YAML.load(f) }
12
+ end
13
+
14
+ def self.marshal(file)
15
+ Open.open(file){|f| Marshal.load(f) }
16
+ end
17
+ end
@@ -0,0 +1,15 @@
1
+ require_relative 'open'
2
+
3
+ module Path
4
+ def yaml(*rest, &block)
5
+ Open.yaml(self, *rest, &block)
6
+ end
7
+
8
+ def json(*rest, &block)
9
+ Open.json(self, *rest, &block)
10
+ end
11
+
12
+ def marshal(*rest, &block)
13
+ Open.marshal(self, *rest, &block)
14
+ end
15
+ end
@@ -0,0 +1,140 @@
1
+ require_relative '../open'
2
+ require_relative 'open'
3
+
4
+ module Persist
5
+ TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
6
+
7
+ class << self
8
+ attr_accessor :save_drivers, :load_drivers
9
+ def save_drivers
10
+ @save_drivers ||= {}
11
+ end
12
+ def load_drivers
13
+ @load_drivers ||= {}
14
+ end
15
+ end
16
+
17
+ def self.serialize(content, type)
18
+ case type
19
+ when nil, :string, :integer, :float, :boolean, :file, :path
20
+ if IO === content || StringIO === content
21
+ content.read
22
+ else
23
+ content.to_s
24
+ end
25
+ when :array
26
+ content * "\n"
27
+ when :yaml
28
+ content.to_yaml
29
+ when :json
30
+ content.to_json
31
+ when :marshal, :serializer
32
+ Marshal.dump(content)
33
+ else
34
+ if m = type.to_s.match(/(.*)_array/)
35
+ type = m[1].to_sym
36
+ content.collect{|c| serialize(c, type) } * "\n"
37
+ else
38
+ raise "Persist does not know #{Log.fingerprint type}"
39
+ end
40
+ end
41
+ end
42
+
43
+ def self.deserialize(serialized, type)
44
+ case type
45
+ when nil, :string, :file, :stream
46
+ serialized
47
+ when :path
48
+ Path.setup(serialized)
49
+ when :integer
50
+ serialized.to_i
51
+ when :float
52
+ serialized.to_f
53
+ when :boolean
54
+ TRUE_STRINGS.include? serialized
55
+ when :array
56
+ serialized.split("\n")
57
+ when :yaml
58
+ YAML.parse(serialized)
59
+ when :json
60
+ JSON.parse(serialized)
61
+ when :marshal, :serializer
62
+ Marshal.load(serialized)
63
+ else
64
+ if m = type.to_s.match(/(.*)_array/)
65
+ type = m[1].to_sym
66
+ new_content = serialized.split("\n")
67
+ new_content.collect{|c| deserialize(c, type) }
68
+ else
69
+ raise "Persist does not know #{Log.fingerprint type}"
70
+ end
71
+ end
72
+ end
73
+
74
+ MEMORY = {}
75
+ def self.save(content, file, type = :serializer)
76
+ return if content.nil?
77
+ type = MEMORY if type == :memory
78
+ type = :serializer if type.nil?
79
+
80
+ if Hash === type
81
+ type[file] = content
82
+ return
83
+ end
84
+
85
+ Log.debug "Save #{Log.fingerprint type} on #{file}"
86
+ if save_drivers[type]
87
+ Open.write(file, save_drivers[type].call(content))
88
+ return
89
+ end
90
+
91
+ if IO === content || StringIO === content
92
+ main, copy = Open.tee_stream_thread content
93
+ t = Thread.new do
94
+ Thread.current["name"] = "file saver: " + file
95
+ Open.sensible_write(file, main)
96
+ end
97
+ ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
98
+ else
99
+ serialized = serialize(content, type)
100
+ Open.sensible_write(file, serialized)
101
+ content
102
+ end
103
+ end
104
+
105
+ def self.load(file, type = :serializer)
106
+ file = file.find if Path === file
107
+ type = MEMORY if type == :memory
108
+ return unless Hash === type || Open.exist?(file)
109
+ type = :serializer if type.nil?
110
+
111
+ Log.debug "Load #{Log.fingerprint type} on #{file}"
112
+ if load_drivers[type]
113
+ return load_drivers[type].call(file)
114
+ end
115
+
116
+ case type
117
+ when :yaml
118
+ Open.yaml(file)
119
+ when :json
120
+ Open.json(file)
121
+ when :marshal, :serializer
122
+ Open.marshal(file)
123
+ when :stream
124
+ Open.open(file)
125
+ when :file
126
+ value = Open.read(file)
127
+ value.sub!(/^\./, File.dirname(file)) if value.start_with?("./")
128
+ value
129
+ when :file_array
130
+ Open.read(file).split("\n").collect do |f|
131
+ f.sub!(/^\./, File.dirname(file)) if f.start_with?("./")
132
+ f
133
+ end
134
+ when Hash
135
+ type[file]
136
+ else
137
+ deserialize(Open.read(file), type)
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,54 @@
1
+ require_relative 'persist/serialize'
2
+ require_relative 'persist/open'
3
+ require_relative 'persist/path'
4
+
5
+ module Persist
6
+ class << self
7
+ attr :cache_dir
8
+ def cache_dir=(cache_dir)
9
+ @cache_dir = Path === cache_dir ? cache_dir : Path.setup(cache_dir)
10
+ end
11
+ def cache_dir
12
+ @cache_dir ||= Path.setup("var/cache/persistence")
13
+ end
14
+
15
+ attr_writer :lock_dir
16
+ def lock_dir
17
+ @lock_dir ||= Path.setup("var/cache/persist_locks")
18
+ end
19
+ end
20
+
21
+ def self.persistence_path(name, options = {})
22
+ options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
23
+ other_options = IndiferentHash.pull_keys options, :other
24
+ TmpFile.tmp_for_file(name, options, other_options)
25
+ end
26
+
27
+ def self.persist(name, type = :serializer, options = {}, &block)
28
+ persist_options = IndiferentHash.pull_keys options, :persist
29
+ file = persist_options[:path] || options[:path] || persistence_path(name, options)
30
+
31
+ update = options[:update] || persist_options[:update]
32
+ update = Open.mtime(update) if Path === update
33
+ update = Open.mtime(file) >= update ? false : true if Time === update
34
+
35
+ if Open.exist?(file) && ! update
36
+ Persist.load(file, type)
37
+ else
38
+ res = yield
39
+ begin
40
+ Open.rm(file)
41
+ res = Persist.save(res, file, type)
42
+ rescue
43
+ raise $! unless options[:canfail]
44
+ Log.debug "Could not persist #{type} on #{file}"
45
+ end
46
+ res
47
+ end
48
+ end
49
+
50
+ def self.memory(name, *args, &block)
51
+ self.persist(name, :memory, *args, &block)
52
+ end
53
+
54
+ end
@@ -0,0 +1,15 @@
1
+ module Path
2
+ def open(*args, &block)
3
+ produce
4
+ Open.open(self, *args, &block)
5
+ end
6
+
7
+ def read
8
+ produce
9
+ Open.read(self)
10
+ end
11
+
12
+ def write(*args, &block)
13
+ Open.write(self.find, *args, &block)
14
+ end
15
+ end