scout-gear 2.0.0 → 5.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +65 -2
  3. data/Rakefile +2 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +233 -24
  6. data/lib/scout/cmd.rb +344 -0
  7. data/lib/scout/concurrent_stream.rb +259 -0
  8. data/lib/scout/exceptions.rb +15 -8
  9. data/lib/scout/indiferent_hash/options.rb +8 -26
  10. data/lib/scout/log/color.rb +2 -2
  11. data/lib/scout/log/fingerprint.rb +11 -1
  12. data/lib/scout/log/progress/report.rb +0 -1
  13. data/lib/scout/log/progress/util.rb +1 -1
  14. data/lib/scout/log/progress.rb +4 -4
  15. data/lib/scout/log.rb +10 -2
  16. data/lib/scout/meta_extension.rb +19 -3
  17. data/lib/scout/misc/digest.rb +56 -0
  18. data/lib/scout/misc/filesystem.rb +26 -0
  19. data/lib/scout/misc/format.rb +17 -6
  20. data/lib/scout/misc/insist.rb +56 -0
  21. data/lib/scout/misc/monitor.rb +23 -0
  22. data/lib/scout/misc.rb +5 -11
  23. data/lib/scout/open/lock.rb +61 -0
  24. data/lib/scout/open/remote.rb +120 -0
  25. data/lib/scout/open/stream.rb +373 -0
  26. data/lib/scout/open/util.rb +225 -0
  27. data/lib/scout/open.rb +169 -0
  28. data/lib/scout/path/find.rb +68 -21
  29. data/lib/scout/path/tmpfile.rb +8 -0
  30. data/lib/scout/path/util.rb +14 -1
  31. data/lib/scout/path.rb +6 -30
  32. data/lib/scout/persist/open.rb +17 -0
  33. data/lib/scout/persist/path.rb +15 -0
  34. data/lib/scout/persist/serialize.rb +151 -0
  35. data/lib/scout/persist.rb +54 -0
  36. data/lib/scout/resource/path.rb +20 -0
  37. data/lib/scout/resource/produce/rake.rb +69 -0
  38. data/lib/scout/resource/produce.rb +246 -0
  39. data/lib/scout/resource/scout.rb +3 -0
  40. data/lib/scout/resource/util.rb +48 -0
  41. data/lib/scout/resource.rb +39 -0
  42. data/lib/scout/simple_opt/accessor.rb +1 -1
  43. data/lib/scout/simple_opt/doc.rb +29 -23
  44. data/lib/scout/simple_opt/parse.rb +4 -3
  45. data/lib/scout/tmpfile.rb +39 -1
  46. data/lib/scout/workflow/definition.rb +78 -0
  47. data/lib/scout/workflow/documentation.rb +83 -0
  48. data/lib/scout/workflow/step/info.rb +77 -0
  49. data/lib/scout/workflow/step/load.rb +18 -0
  50. data/lib/scout/workflow/step.rb +132 -0
  51. data/lib/scout/workflow/task/inputs.rb +114 -0
  52. data/lib/scout/workflow/task.rb +155 -0
  53. data/lib/scout/workflow/usage.rb +314 -0
  54. data/lib/scout/workflow/util.rb +11 -0
  55. data/lib/scout/workflow.rb +40 -0
  56. data/lib/scout-gear.rb +4 -0
  57. data/lib/scout.rb +1 -0
  58. data/lib/workflow-scout.rb +2 -0
  59. data/scout-gear.gemspec +77 -5
  60. data/scout_commands/alias +48 -0
  61. data/scout_commands/find +83 -0
  62. data/scout_commands/glob +0 -0
  63. data/scout_commands/rbbt +23 -0
  64. data/scout_commands/workflow/info +29 -0
  65. data/scout_commands/workflow/list +27 -0
  66. data/scout_commands/workflow/task +58 -0
  67. data/scout_commands/workflow/task_old +706 -0
  68. data/test/scout/indiferent_hash/test_options.rb +11 -1
  69. data/test/scout/misc/test_digest.rb +30 -0
  70. data/test/scout/misc/test_filesystem.rb +30 -0
  71. data/test/scout/misc/test_insist.rb +13 -0
  72. data/test/scout/open/test_lock.rb +52 -0
  73. data/test/scout/open/test_remote.rb +25 -0
  74. data/test/scout/open/test_stream.rb +515 -0
  75. data/test/scout/open/test_util.rb +73 -0
  76. data/test/scout/path/test_find.rb +28 -0
  77. data/test/scout/persist/test_open.rb +37 -0
  78. data/test/scout/persist/test_path.rb +37 -0
  79. data/test/scout/persist/test_serialize.rb +114 -0
  80. data/test/scout/resource/test_path.rb +40 -0
  81. data/test/scout/resource/test_produce.rb +62 -0
  82. data/test/scout/resource/test_util.rb +27 -0
  83. data/test/scout/simple_opt/test_doc.rb +16 -0
  84. data/test/scout/test_cmd.rb +85 -0
  85. data/test/scout/test_concurrent_stream.rb +29 -0
  86. data/test/scout/test_meta_extension.rb +9 -0
  87. data/test/scout/test_misc.rb +0 -7
  88. data/test/scout/test_open.rb +146 -0
  89. data/test/scout/test_path.rb +3 -1
  90. data/test/scout/test_persist.rb +83 -0
  91. data/test/scout/test_resource.rb +26 -0
  92. data/test/scout/test_workflow.rb +87 -0
  93. data/test/scout/workflow/step/test_info.rb +30 -0
  94. data/test/scout/workflow/step/test_load.rb +65 -0
  95. data/test/scout/workflow/task/test_inputs.rb +182 -0
  96. data/test/scout/workflow/test_definition.rb +0 -0
  97. data/test/scout/workflow/test_documentation.rb +30 -0
  98. data/test/scout/workflow/test_step.rb +36 -0
  99. data/test/scout/workflow/test_task.rb +179 -0
  100. data/test/scout/workflow/test_usage.rb +35 -0
  101. data/test/scout/workflow/test_util.rb +17 -0
  102. data/test/test_helper.rb +17 -0
  103. data/test/test_scout-gear.rb +0 -0
  104. metadata +75 -3
data/lib/scout/open.rb ADDED
@@ -0,0 +1,169 @@
1
+ require_relative 'tmpfile'
2
+ require_relative 'path'
3
+ require_relative 'cmd'
4
+
5
+ require_relative 'open/stream'
6
+ require_relative 'open/util'
7
+ require_relative 'open/remote'
8
+ require_relative 'open/lock'
9
+
10
+ module Open
11
+ module NamedStream
12
+ attr_accessor :filename
13
+
14
+ def digest_str
15
+ if Path === filename && ! filename.located?
16
+ filename
17
+ else
18
+ Misc.file_md5(filename)
19
+ end
20
+ end
21
+ end
22
+
23
+ def self.get_stream(file, mode = 'r')
24
+ file = file.find if Path === file
25
+
26
+ return Open.ssh(file) if Open.ssh?(file)
27
+ return Open.wget(file) if Open.remote?(file)
28
+
29
+ File.open(file, mode)
30
+ end
31
+
32
+ def self.file_open(file, grep = false, mode = 'r', invert_grep = false)
33
+ Open.mkdir File.dirname(file) if mode.include? 'w'
34
+
35
+ stream = get_stream(file, mode)
36
+
37
+ if grep
38
+ grep(stream, grep, invert_grep)
39
+ else
40
+ stream
41
+ end
42
+ end
43
+
44
+ def self.file_write(file, content, mode = 'w')
45
+ File.open(file, mode) do |f|
46
+ begin
47
+ f.flock(File::LOCK_EX)
48
+ f.write content
49
+ f.flock(File::LOCK_UN)
50
+ ensure
51
+ f.close unless f.closed?
52
+ end
53
+ end
54
+ end
55
+
56
+ def self.open(file, options = {})
57
+ if IO === file || StringIO === file
58
+ if block_given?
59
+ res = yield file
60
+ file.close
61
+ return res
62
+ else
63
+ return file
64
+ end
65
+ end
66
+
67
+ options = IndiferentHash.add_defaults options, :noz => false, :mode => 'r'
68
+
69
+ mode = IndiferentHash.process_options options, :mode
70
+
71
+ options[:noz] = true if mode.include? "w"
72
+
73
+ io = file_open(file, options[:grep], mode, options[:invert_grep])
74
+
75
+ io = unzip(io) if ((String === file and zip?(file)) and not options[:noz]) or options[:zip]
76
+ io = gunzip(io) if ((String === file and gzip?(file)) and not options[:noz]) or options[:gzip]
77
+ io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
78
+
79
+ io.extend NamedStream
80
+ io.filename = file
81
+
82
+ if block_given?
83
+ res = nil
84
+ begin
85
+ res = yield(io)
86
+ rescue DontClose
87
+ res = $!.payload
88
+ rescue Exception
89
+ io.abort if io.respond_to? :abort
90
+ io.join if io.respond_to? :join
91
+ raise $!
92
+ ensure
93
+ io.close if io.respond_to? :close and not io.closed?
94
+ io.join if io.respond_to? :join
95
+ end
96
+ res
97
+ else
98
+ io
99
+ end
100
+ end
101
+
102
+ def self.read(file, options = {}, &block)
103
+ open(file, options) do |f|
104
+ if block_given?
105
+ res = []
106
+ while not f.eof?
107
+ l = f.gets
108
+ l = Misc.fixutf8(l) unless options[:nofix]
109
+ res << yield(l)
110
+ end
111
+ res
112
+ else
113
+ if options[:nofix]
114
+ f.read
115
+ else
116
+ Misc.fixutf8(f.read)
117
+ end
118
+ end
119
+ end
120
+ end
121
+
122
+ def self.write(file, content = nil, options = {})
123
+ options = IndiferentHash.add_defaults options, :mode => 'w'
124
+
125
+ file = file.find(options[:where]) if Path === file
126
+ mode = IndiferentHash.process_options options, :mode
127
+
128
+ FileUtils.mkdir_p File.dirname(file)
129
+
130
+ case
131
+ when block_given?
132
+ begin
133
+ f = File.open(file, mode)
134
+ begin
135
+ yield f
136
+ ensure
137
+ f.close unless f.closed?
138
+ end
139
+ rescue Exception
140
+ FileUtils.rm file if File.exist? file
141
+ raise $!
142
+ end
143
+ when content.nil?
144
+ file_write(file, "", mode)
145
+ when String === content
146
+ file_write(file, content, mode)
147
+ when (IO === content || StringIO === content)
148
+ begin
149
+ File.open(file, mode) do |f|
150
+ f.flock(File::LOCK_EX)
151
+ while block = content.read(Open::BLOCK_SIZE)
152
+ f.write block
153
+ end
154
+ f.flock(File::LOCK_UN)
155
+ end
156
+ rescue Exception
157
+ FileUtils.rm_rf file if File.exist? file
158
+ raise $!
159
+ end
160
+ content.close unless content.closed?
161
+ content.join if content.respond_to? :join
162
+ else
163
+ raise "Content unknown #{Log.fingerprint content}"
164
+ end
165
+
166
+ notify_write(file)
167
+ end
168
+
169
+ end
@@ -1,20 +1,36 @@
1
1
  require_relative '../indiferent_hash'
2
2
  module Path
3
3
 
4
- def _parts
5
- @_parts ||= self.split("/")
6
- end
4
+ def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
5
+
6
+ if file.nil?
7
+ caller_dup = caller.dup
8
+ while file = caller_dup.shift
9
+ break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
10
+ file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
11
+ file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
12
+ file =~ /(?:scout|rbbt)\/persist.rb/
13
+ end
14
+ file = file.sub(/\.rb[^\w].*/,'.rb')
15
+ end
7
16
 
8
- def _subpath
9
- @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
10
- end
11
-
12
- def _toplevel
13
- @toplevel ||= _parts.length > 1 ? _parts[0] : ""
17
+ relative_to = [relative_to] unless Array === relative_to
18
+ file = File.expand_path(file)
19
+ return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
20
+
21
+ while file != '/'
22
+ dir = File.dirname file
23
+
24
+ return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
25
+
26
+ file = File.dirname file
27
+ end
28
+
29
+ return nil
14
30
  end
15
31
 
16
- def self.follow(path, map)
17
- map.sub('{PKGDIR}', path.pkgdir.respond_to?(:subdir) ? path.pkgdir.pkgdir : path.pkgdir || Path.default_pkgdir).
32
+ def self.follow(path, map, map_name = nil)
33
+ file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
18
34
  sub('{RESOURCE}', path.pkgdir.to_s).
19
35
  sub('{PWD}', FileUtils.pwd).
20
36
  sub('{TOPLEVEL}', path._toplevel).
@@ -22,8 +38,20 @@ module Path
22
38
  sub('{BASENAME}', File.basename(path)).
23
39
  sub('{PATH}', path).
24
40
  sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
41
+ sub('{MAPNAME}', map_name.to_s).
25
42
  sub('{REMOVE}/', '').
26
43
  sub('{REMOVE}', '').gsub(/\/+/,'/')
44
+
45
+ while true
46
+ file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
47
+ key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
48
+ key_text = follow(path, "#{key}}", map_name)
49
+ key_text[orig] = replace[0..-2] if key_text.include?(orig)
50
+ key_text
51
+ end || break
52
+ end
53
+
54
+ file
27
55
  end
28
56
 
29
57
  def self.path_maps
@@ -42,17 +70,29 @@ module Path
42
70
  })
43
71
  end
44
72
 
45
- def self.map_search
46
- @@map_search ||= %w(current workflow user local global lib fast cache bulk)
73
+ def self.basic_map_order
74
+ @@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
47
75
  end
48
76
 
49
- def self.search_order
50
- @@search_order ||= (path_maps.keys & map_search) + (path_maps.keys - map_search)
77
+ def self.map_order
78
+ @@map_order ||= (path_maps.keys & basic_map_order) + (path_maps.keys - basic_map_order)
51
79
  end
52
80
 
53
81
  def self.add_path(name, map)
54
82
  @@path_maps[name] = map
55
- @@search_order = nil
83
+ @@map_order = nil
84
+ end
85
+
86
+ def _parts
87
+ @_parts ||= self.split("/")
88
+ end
89
+
90
+ def _subpath
91
+ @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
92
+ end
93
+
94
+ def _toplevel
95
+ @toplevel ||= _parts.length > 1 ? _parts[0] : ""
56
96
  end
57
97
 
58
98
  SLASH = "/"[0]
@@ -77,13 +117,19 @@ module Path
77
117
  @original
78
118
  end
79
119
 
120
+ def map_order
121
+ @map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
122
+ end
123
+
80
124
  def follow(map_name = :default, annotate = true)
81
- map = Path.path_maps[map_name]
125
+ IndiferentHash.setup(path_maps)
126
+ map = path_maps[map_name] || Path.path_maps[map_name]
127
+ raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
82
128
  while Symbol === map
83
129
  map_name = map
84
- map = Path.path_maps[map_name]
130
+ map = path_maps[map_name]
85
131
  end
86
- found = Path.follow(self, map)
132
+ found = Path.follow(self, map, map_name)
87
133
 
88
134
  annotate_found_where(found, map_name) if annotate
89
135
 
@@ -95,7 +141,7 @@ module Path
95
141
  return find_all if where == 'all' || where == :all
96
142
  return follow(where) if where
97
143
 
98
- Path.search_order.each do |map_name|
144
+ map_order.each do |map_name|
99
145
  found = follow(map_name, false)
100
146
 
101
147
  return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
@@ -113,8 +159,9 @@ module Path
113
159
  alias exists? exist?
114
160
 
115
161
  def find_all(caller_lib = nil, search_paths = nil)
116
- Path.search_order
162
+ map_order
117
163
  .collect{|where| find(where) }
118
164
  .select{|file| file.exist? }.uniq
119
165
  end
166
+
120
167
  end
@@ -0,0 +1,8 @@
1
+ module TmpFile
2
+ def self.with_path(*args, &block)
3
+ TmpFile.with_file(*args) do |file|
4
+ Path.setup(file)
5
+ yield file
6
+ end
7
+ end
8
+ end
@@ -1,4 +1,12 @@
1
1
  module Path
2
+
3
+ def self.is_filename?(string, need_to_exists = true)
4
+ return false if string.nil?
5
+ return true if Path === string
6
+ return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265 }.empty? and (! need_to_exists || File.exist?(string))
7
+ return false
8
+ end
9
+
2
10
  def directory?
3
11
  return nil unless self.exist?
4
12
  File.directory?(self.find)
@@ -12,7 +20,6 @@ module Path
12
20
  self.annotate(File.basename(self))
13
21
  end
14
22
 
15
-
16
23
  def glob(pattern = '*')
17
24
  if self.include? "*"
18
25
  self.glob_all
@@ -48,4 +55,10 @@ module Path
48
55
  paths
49
56
  end.flatten.uniq
50
57
  end
58
+
59
+ def no_method_missing
60
+ class << self
61
+ undef_method :method_missing
62
+ end
63
+ end
51
64
  end
data/lib/scout/path.rb CHANGED
@@ -1,39 +1,12 @@
1
1
  require_relative 'meta_extension'
2
2
  require_relative 'path/find'
3
3
  require_relative 'path/util'
4
+ require_relative 'path/tmpfile'
4
5
 
5
6
  module Path
6
7
  extend MetaExtension
7
8
  extension_attr :pkgdir, :libdir, :path_maps
8
9
 
9
- def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
10
-
11
- if file.nil?
12
- caller_dup = caller.dup
13
- while file = caller_dup.shift
14
- break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
15
- file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
16
- file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
17
- file =~ /(?:scout|rbbt)\/persist.rb/
18
- end
19
- file = file.sub(/\.rb[^\w].*/,'.rb')
20
- end
21
-
22
- relative_to = [relative_to] unless Array === relative_to
23
- file = File.expand_path(file)
24
- return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
25
-
26
- while file != '/'
27
- dir = File.dirname file
28
-
29
- return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
30
-
31
- file = File.dirname file
32
- end
33
-
34
- return nil
35
- end
36
-
37
10
  def self.default_pkgdir
38
11
  @@default_pkgdir ||= 'scout'
39
12
  end
@@ -42,13 +15,16 @@ module Path
42
15
  @@default_pkgdir = pkgdir
43
16
  end
44
17
 
45
-
46
18
  def pkgdir
47
19
  @pkgdir ||= Path.default_pkgdir
48
20
  end
49
21
 
50
22
  def libdir
51
- @libdir
23
+ @libdir || Path.caller_lib_dir
24
+ end
25
+
26
+ def path_maps
27
+ @path_maps ||= Path.path_maps.dup
52
28
  end
53
29
 
54
30
  def join(subpath, prevpath = nil)
@@ -0,0 +1,17 @@
1
+ require_relative '../open'
2
+ require 'json'
3
+ require 'yaml'
4
+
5
+ module Open
6
+ def self.json(file)
7
+ Open.open(file){|f| JSON.load(f) }
8
+ end
9
+
10
+ def self.yaml(file)
11
+ Open.open(file){|f| YAML.load(f) }
12
+ end
13
+
14
+ def self.marshal(file)
15
+ Open.open(file){|f| Marshal.load(f) }
16
+ end
17
+ end
@@ -0,0 +1,15 @@
1
+ require_relative 'open'
2
+
3
+ module Path
4
+ def yaml(*rest, &block)
5
+ Open.yaml(self, *rest, &block)
6
+ end
7
+
8
+ def json(*rest, &block)
9
+ Open.json(self, *rest, &block)
10
+ end
11
+
12
+ def marshal(*rest, &block)
13
+ Open.marshal(self, *rest, &block)
14
+ end
15
+ end
@@ -0,0 +1,151 @@
1
+ require_relative '../open'
2
+ require_relative 'open'
3
+
4
+ module Persist
5
+ TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
6
+ SERIALIZER = :json
7
+
8
+ class << self
9
+ attr_accessor :save_drivers, :load_drivers
10
+ def save_drivers
11
+ @save_drivers ||= {}
12
+ end
13
+ def load_drivers
14
+ @load_drivers ||= {}
15
+ end
16
+ end
17
+
18
+ def self.serialize(content, type)
19
+ type = type.to_sym if String === type
20
+ type = SERIALIZER if type == :serializer
21
+ case type
22
+ when nil, :string, :integer, :float, :boolean, :file, :path
23
+ if IO === content || StringIO === content
24
+ content.read
25
+ else
26
+ content.to_s
27
+ end
28
+ when :array
29
+ content * "\n"
30
+ when :yaml
31
+ content.to_yaml
32
+ when :json
33
+ content.to_json
34
+ when :marshal
35
+ Marshal.dump(content)
36
+ else
37
+ if m = type.to_s.match(/(.*)_array/)
38
+ type = m[1].to_sym
39
+ content.collect{|c| serialize(c, type) } * "\n"
40
+ else
41
+ raise "Persist does not know #{Log.fingerprint type}"
42
+ end
43
+ end
44
+ end
45
+
46
+ def self.deserialize(serialized, type)
47
+ type = type.to_sym if String === type
48
+ type = SERIALIZER if type == :serializer
49
+ case type
50
+ when nil, :string, :file, :stream
51
+ serialized
52
+ when :path
53
+ Path.setup(serialized)
54
+ when :integer
55
+ serialized.to_i
56
+ when :float
57
+ serialized.to_f
58
+ when :boolean
59
+ TRUE_STRINGS.include? serialized
60
+ when :array
61
+ serialized.split("\n")
62
+ when :yaml
63
+ YAML.parse(serialized)
64
+ when :json
65
+ JSON.parse(serialized)
66
+ when :marshal
67
+ Marshal.load(serialized)
68
+ else
69
+ if m = type.to_s.match(/(.*)_array/)
70
+ type = m[1].to_sym
71
+ new_content = serialized.split("\n")
72
+ new_content.collect{|c| deserialize(c, type) }
73
+ else
74
+ raise "Persist does not know #{Log.fingerprint type}"
75
+ end
76
+ end
77
+ end
78
+
79
+ MEMORY = {}
80
+ def self.save(content, file, type = :serializer)
81
+ type = :serializer if type.nil?
82
+ type = type.to_sym if String === type
83
+ type = SERIALIZER if type == :serializer
84
+ type = MEMORY if type == :memory
85
+ return if content.nil?
86
+ type = MEMORY if type == :memory
87
+ type = :serializer if type.nil?
88
+
89
+ if Hash === type
90
+ type[file] = content
91
+ return
92
+ end
93
+
94
+ Log.debug "Save #{Log.fingerprint type} on #{file}"
95
+ if save_drivers[type]
96
+ Open.write(file, save_drivers[type].call(content))
97
+ return
98
+ end
99
+
100
+ if IO === content || StringIO === content
101
+ main, copy = Open.tee_stream_thread content
102
+ t = Thread.new do
103
+ Thread.current["name"] = "file saver: " + file
104
+ Open.sensible_write(file, main)
105
+ end
106
+ ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
107
+ else
108
+ serialized = serialize(content, type)
109
+ Open.sensible_write(file, serialized, :force => true)
110
+ content
111
+ end
112
+ end
113
+
114
+ def self.load(file, type = :serializer)
115
+ file = file.find if Path === file
116
+ type = :serializer if type.nil?
117
+ type = type.to_sym if String === type
118
+ type = SERIALIZER if type == :serializer
119
+ type = MEMORY if type == :memory
120
+ return unless Hash === type || Open.exist?(file)
121
+
122
+ Log.debug "Load #{Log.fingerprint type} on #{file}"
123
+ if load_drivers[type]
124
+ return load_drivers[type].call(file)
125
+ end
126
+
127
+ case type
128
+ when :yaml
129
+ Open.yaml(file)
130
+ when :json
131
+ Open.json(file)
132
+ when :marshal, :serializer
133
+ Open.marshal(file)
134
+ when :stream
135
+ Open.open(file)
136
+ when :file
137
+ value = Open.read(file)
138
+ value.sub!(/^\./, File.dirname(file)) if value.start_with?("./")
139
+ value
140
+ when :file_array
141
+ Open.read(file).split("\n").collect do |f|
142
+ f.sub!(/^\./, File.dirname(file)) if f.start_with?("./")
143
+ f
144
+ end
145
+ when Hash
146
+ type[file]
147
+ else
148
+ deserialize(Open.read(file), type)
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,54 @@
1
+ require_relative 'persist/serialize'
2
+ require_relative 'persist/open'
3
+ require_relative 'persist/path'
4
+
5
+ module Persist
6
+ class << self
7
+ attr :cache_dir
8
+ def cache_dir=(cache_dir)
9
+ @cache_dir = Path === cache_dir ? cache_dir : Path.setup(cache_dir)
10
+ end
11
+ def cache_dir
12
+ @cache_dir ||= Path.setup("var/cache/persistence")
13
+ end
14
+
15
+ attr_writer :lock_dir
16
+ def lock_dir
17
+ @lock_dir ||= Path.setup("var/cache/persist_locks")
18
+ end
19
+ end
20
+
21
+ def self.persistence_path(name, options = {})
22
+ options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
23
+ other_options = IndiferentHash.pull_keys options, :other
24
+ TmpFile.tmp_for_file(name, options, other_options)
25
+ end
26
+
27
+ def self.persist(name, type = :serializer, options = {}, &block)
28
+ persist_options = IndiferentHash.pull_keys options, :persist
29
+ file = persist_options[:path] || options[:path] || persistence_path(name, options)
30
+
31
+ update = options[:update] || persist_options[:update]
32
+ update = Open.mtime(update) if Path === update
33
+ update = Open.mtime(file) >= update ? false : true if Time === update
34
+
35
+ if Open.exist?(file) && ! update
36
+ Persist.load(file, type)
37
+ else
38
+ res = yield
39
+ begin
40
+ Open.rm(file)
41
+ res = Persist.save(res, file, type)
42
+ rescue
43
+ raise $! unless options[:canfail]
44
+ Log.debug "Could not persist #{type} on #{file}"
45
+ end
46
+ res
47
+ end
48
+ end
49
+
50
+ def self.memory(name, *args, &block)
51
+ self.persist(name, :memory, *args, &block)
52
+ end
53
+
54
+ end
@@ -0,0 +1,20 @@
1
+ module Path
2
+ def relocate
3
+ return self if Open.exists?(self)
4
+ Resource.relocate(self)
5
+ end
6
+
7
+ def open(*args, &block)
8
+ produce
9
+ Open.open(self, *args, &block)
10
+ end
11
+
12
+ def read
13
+ produce
14
+ Open.read(self)
15
+ end
16
+
17
+ def write(*args, &block)
18
+ Open.write(self.find, *args, &block)
19
+ end
20
+ end