scout-gear 10.3.0 → 10.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (146) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +100 -657
  3. data/Rakefile +1 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +1 -3
  6. data/lib/scout/association/fields.rb +170 -0
  7. data/lib/scout/association/index.rb +229 -0
  8. data/lib/scout/association/item.rb +227 -0
  9. data/lib/scout/association/util.rb +7 -0
  10. data/lib/scout/association.rb +100 -0
  11. data/lib/scout/entity/format.rb +62 -0
  12. data/lib/scout/entity/identifiers.rb +111 -0
  13. data/lib/scout/entity/object.rb +20 -0
  14. data/lib/scout/entity/property.rb +165 -0
  15. data/lib/scout/entity.rb +40 -0
  16. data/lib/scout/offsite/step.rb +2 -2
  17. data/lib/scout/{tsv/persist → persist/engine}/fix_width_table.rb +25 -33
  18. data/lib/scout/persist/engine/packed_index.rb +100 -0
  19. data/lib/scout/persist/engine/sharder.rb +219 -0
  20. data/lib/scout/{tsv/persist → persist/engine}/tkrzw.rb +0 -17
  21. data/lib/scout/{tsv/persist → persist/engine}/tokyocabinet.rb +55 -31
  22. data/lib/scout/persist/engine.rb +4 -0
  23. data/lib/scout/{tsv/persist/adapter.rb → persist/tsv/adapter/base.rb} +80 -51
  24. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +106 -0
  25. data/lib/scout/persist/tsv/adapter/packed_index.rb +95 -0
  26. data/lib/scout/persist/tsv/adapter/sharder.rb +54 -0
  27. data/lib/scout/persist/tsv/adapter/tkrzw.rb +18 -0
  28. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +65 -0
  29. data/lib/scout/persist/tsv/adapter.rb +6 -0
  30. data/lib/scout/{tsv/persist → persist/tsv}/serialize.rb +5 -0
  31. data/lib/scout/persist/tsv.rb +107 -0
  32. data/lib/scout/tsv/annotation/repo.rb +83 -0
  33. data/lib/scout/tsv/annotation.rb +169 -0
  34. data/lib/scout/tsv/attach.rb +104 -20
  35. data/lib/scout/tsv/change_id/translate.rb +148 -0
  36. data/lib/scout/tsv/change_id.rb +6 -3
  37. data/lib/scout/tsv/csv.rb +85 -0
  38. data/lib/scout/tsv/dumper.rb +113 -25
  39. data/lib/scout/tsv/entity.rb +5 -0
  40. data/lib/scout/tsv/index.rb +89 -37
  41. data/lib/scout/tsv/open.rb +21 -8
  42. data/lib/scout/tsv/parser.rb +156 -91
  43. data/lib/scout/tsv/path.rb +7 -2
  44. data/lib/scout/tsv/stream.rb +48 -6
  45. data/lib/scout/tsv/transformer.rb +25 -3
  46. data/lib/scout/tsv/traverse.rb +26 -18
  47. data/lib/scout/tsv/util/process.rb +8 -1
  48. data/lib/scout/tsv/util/reorder.rb +25 -15
  49. data/lib/scout/tsv/util/select.rb +9 -1
  50. data/lib/scout/tsv/util/sort.rb +90 -2
  51. data/lib/scout/tsv/util/unzip.rb +56 -0
  52. data/lib/scout/tsv/util.rb +52 -5
  53. data/lib/scout/tsv.rb +85 -19
  54. data/lib/scout/work_queue/socket.rb +8 -0
  55. data/lib/scout/work_queue/worker.rb +22 -5
  56. data/lib/scout/work_queue.rb +38 -24
  57. data/lib/scout/workflow/definition.rb +19 -11
  58. data/lib/scout/workflow/deployment/orchestrator.rb +20 -3
  59. data/lib/scout/workflow/deployment/trace.rb +205 -0
  60. data/lib/scout/workflow/deployment.rb +1 -0
  61. data/lib/scout/workflow/documentation.rb +1 -1
  62. data/lib/scout/workflow/step/archive.rb +42 -0
  63. data/lib/scout/workflow/step/children.rb +51 -0
  64. data/lib/scout/workflow/step/config.rb +1 -1
  65. data/lib/scout/workflow/step/dependencies.rb +24 -7
  66. data/lib/scout/workflow/step/file.rb +19 -0
  67. data/lib/scout/workflow/step/info.rb +37 -9
  68. data/lib/scout/workflow/step/progress.rb +11 -2
  69. data/lib/scout/workflow/step/status.rb +8 -1
  70. data/lib/scout/workflow/step.rb +80 -25
  71. data/lib/scout/workflow/task/dependencies.rb +4 -1
  72. data/lib/scout/workflow/task/inputs.rb +91 -41
  73. data/lib/scout/workflow/task.rb +54 -57
  74. data/lib/scout/workflow/usage.rb +1 -1
  75. data/lib/scout/workflow/util.rb +4 -0
  76. data/lib/scout/workflow.rb +110 -13
  77. data/lib/scout-gear.rb +2 -0
  78. data/lib/scout.rb +0 -1
  79. data/scout-gear.gemspec +80 -23
  80. data/scout_commands/rbbt +2 -0
  81. data/test/data/person/brothers +4 -0
  82. data/test/data/person/identifiers +10 -0
  83. data/test/data/person/marriages +3 -0
  84. data/test/data/person/parents +6 -0
  85. data/test/scout/association/test_fields.rb +105 -0
  86. data/test/scout/association/test_index.rb +70 -0
  87. data/test/scout/association/test_item.rb +21 -0
  88. data/test/scout/entity/test_format.rb +19 -0
  89. data/test/scout/entity/test_identifiers.rb +58 -0
  90. data/test/scout/entity/test_object.rb +0 -0
  91. data/test/scout/entity/test_property.rb +345 -0
  92. data/test/scout/{tsv/persist → persist/engine}/test_fix_width_table.rb +0 -1
  93. data/test/scout/persist/engine/test_packed_index.rb +99 -0
  94. data/test/scout/persist/engine/test_sharder.rb +31 -0
  95. data/test/scout/persist/engine/test_tkrzw.rb +0 -0
  96. data/test/scout/persist/engine/test_tokyocabinet.rb +17 -0
  97. data/test/scout/persist/test_tsv.rb +146 -0
  98. data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb} +3 -4
  99. data/test/scout/persist/tsv/adapter/test_fix_width_table.rb +46 -0
  100. data/test/scout/persist/tsv/adapter/test_packed_index.rb +37 -0
  101. data/test/scout/persist/tsv/adapter/test_serialize.rb +0 -0
  102. data/test/scout/persist/tsv/adapter/test_sharder.rb +290 -0
  103. data/test/scout/persist/tsv/adapter/test_tkrzw.rb +126 -0
  104. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +282 -0
  105. data/test/scout/persist/tsv/test_serialize.rb +12 -0
  106. data/test/scout/test_association.rb +51 -0
  107. data/test/scout/test_entity.rb +40 -0
  108. data/test/scout/test_tsv.rb +63 -4
  109. data/test/scout/test_work_queue.rb +3 -2
  110. data/test/scout/test_workflow.rb +16 -15
  111. data/test/scout/tsv/annotation/test_repo.rb +150 -0
  112. data/test/scout/tsv/change_id/test_translate.rb +178 -0
  113. data/test/scout/tsv/test_annotation.rb +52 -0
  114. data/test/scout/tsv/test_attach.rb +226 -1
  115. data/test/scout/tsv/test_change_id.rb +25 -0
  116. data/test/scout/tsv/test_csv.rb +50 -0
  117. data/test/scout/tsv/test_dumper.rb +38 -0
  118. data/test/scout/tsv/test_entity.rb +0 -0
  119. data/test/scout/tsv/test_index.rb +82 -0
  120. data/test/scout/tsv/test_open.rb +44 -0
  121. data/test/scout/tsv/test_parser.rb +70 -0
  122. data/test/scout/tsv/test_stream.rb +22 -0
  123. data/test/scout/tsv/test_transformer.rb +39 -3
  124. data/test/scout/tsv/test_traverse.rb +78 -0
  125. data/test/scout/tsv/util/test_process.rb +36 -0
  126. data/test/scout/tsv/util/test_reorder.rb +67 -0
  127. data/test/scout/tsv/util/test_sort.rb +28 -1
  128. data/test/scout/tsv/util/test_unzip.rb +32 -0
  129. data/test/scout/work_queue/test_socket.rb +4 -1
  130. data/test/scout/workflow/deployment/test_orchestrator.rb +17 -26
  131. data/test/scout/workflow/deployment/test_trace.rb +25 -0
  132. data/test/scout/workflow/step/test_archive.rb +28 -0
  133. data/test/scout/workflow/step/test_children.rb +25 -0
  134. data/test/scout/workflow/step/test_info.rb +16 -0
  135. data/test/scout/workflow/task/test_dependencies.rb +16 -16
  136. data/test/scout/workflow/task/test_inputs.rb +45 -1
  137. data/test/scout/workflow/test_definition.rb +52 -0
  138. data/test/scout/workflow/test_step.rb +57 -0
  139. data/test/scout/workflow/test_task.rb +26 -1
  140. data/test/scout/workflow/test_usage.rb +4 -4
  141. data/test/test_helper.rb +23 -1
  142. metadata +71 -14
  143. data/lib/scout/tsv/persist.rb +0 -27
  144. data/test/scout/tsv/persist/test_tkrzw.rb +0 -123
  145. data/test/scout/tsv/persist/test_tokyocabinet.rb +0 -120
  146. data/test/scout/tsv/test_persist.rb +0 -45
@@ -0,0 +1,106 @@
1
+ require_relative 'base'
2
+ require_relative '../../engine/fix_width_table'
3
+
4
+ module FWTAdapter
5
+ include TSVAdapter
6
+
7
+ def self.extended(base)
8
+ base.extend TSVAdapter
9
+ base
10
+ end
11
+
12
+ attr_accessor :pos_function
13
+
14
+ def persistence_path=(value)
15
+ @persistence_path = value
16
+ @filename = value
17
+ end
18
+
19
+ def metadata_file
20
+ @metadata_file ||= self.persistence_path + '.metadata'
21
+ end
22
+
23
+ def load_annotation_hash
24
+ ANNOTATION_ATTR_HASH_SERIALIZER.load(Open.read(metadata_file, mode: 'b'))
25
+ end
26
+
27
+ def save_annotation_hash
28
+ Open.write(metadata_file, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash), mode: 'wb')
29
+ end
30
+
31
+ def add(key, value)
32
+ key = pos_function.call(key) if pos_function and not (@range and Array === key)
33
+ value = save_value(value)
34
+ super(key, value)
35
+ end
36
+
37
+ def add_range_point(key, value)
38
+ key = pos_function.call(key) if pos_function
39
+ value = save_value(value)
40
+ super(key, value)
41
+ end
42
+
43
+ def [](key, clean = false)
44
+ key = pos_function.call(key) if pos_function and not clean
45
+ res = super(key)
46
+ return nil if res.nil?
47
+ res = res.collect{|r| load_value(r) }
48
+ res.extend MultipleResult
49
+ res
50
+ end
51
+
52
+ def []=(key, value)
53
+ self.add(key, value)
54
+ end
55
+
56
+ def include?(i)
57
+ return true if Numeric === i and i < pos(@size)
58
+ @annotations
59
+ false
60
+ end
61
+
62
+ def size
63
+ @size
64
+ end
65
+
66
+ def each
67
+ read
68
+ @size.times do |i|
69
+ v = idx_value(i)
70
+ yield i, v
71
+ end
72
+ end
73
+
74
+ def keys
75
+ []
76
+ end
77
+ end
78
+
79
+ module Persist
80
+
81
+ def self.open_fwt(path, value_size, range = false, serializer = nil, update = false, in_memory = false, &pos_function)
82
+ FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
83
+
84
+ database = FixWidthTable.new(path, value_size, range, update, in_memory, &pos_function)
85
+
86
+ database.extend FWTAdapter
87
+
88
+ database.pos_function = pos_function
89
+
90
+ unless serializer == :clean
91
+ TSV.setup database
92
+ database.serializer ||= TSVAdapter.serializer_module(serializer)
93
+ end
94
+
95
+ database
96
+ end
97
+ end
98
+
99
+ Persist.save_drivers[:fwt] = proc do |file, content|
100
+ content.file.seek 0
101
+ Misc.sensiblewrite(file, content.file.read)
102
+ end
103
+
104
+ Persist.load_drivers[:fwt] = proc do |file|
105
+ FixWidthTable.new file
106
+ end
@@ -0,0 +1,95 @@
1
+ require_relative 'base'
2
+ require_relative '../../engine/packed_index'
3
+
4
+ module PKIAdapter
5
+ include TSVAdapter
6
+
7
+ attr_accessor :pos_function
8
+
9
+ def self.extended(base)
10
+ base.extend TSVAdapter
11
+ base
12
+ end
13
+
14
+ def persistence_path=(value)
15
+ @persistence_path = value
16
+ @file = value
17
+ end
18
+
19
+ def metadata_file
20
+ @metadata_file ||= self.persistence_path + '.metadata'
21
+ end
22
+
23
+ def load_annotation_hash
24
+ ANNOTATION_ATTR_HASH_SERIALIZER.load(Open.read(metadata_file, mode: 'rb'))
25
+ end
26
+
27
+ def save_annotation_hash
28
+ Open.write(metadata_file, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash), mode: 'wb')
29
+ end
30
+
31
+ def [](key, clean = false)
32
+ key = pos_function.call(key) if pos_function and not clean
33
+ res = super(key)
34
+ res.extend MultipleResult unless res.nil?
35
+ res
36
+ end
37
+
38
+ def value(pos)
39
+ self.send(:[], pos, true)
40
+ end
41
+
42
+ def []=(key, value)
43
+ add key, value
44
+ end
45
+
46
+ def add(key, value)
47
+ key = pos_function.call(key) if pos_function
48
+ if Numeric === key
49
+ @_last ||= -1
50
+ skipped = key - @_last - 1
51
+ skipped.times do
52
+ self.send(:<<, nil)
53
+ end
54
+ @_last = key
55
+ end
56
+ self.send(:<<, value)
57
+ end
58
+
59
+ def add_range_point(key, value)
60
+ key = pos_function.call(key) if pos_function
61
+ super(key, value)
62
+ end
63
+
64
+ def include?(i)
65
+ return true if Numeric === i and i < size
66
+ false
67
+ end
68
+
69
+ def each
70
+ size.times do |i|
71
+ yield i, value(i)
72
+ end
73
+ end
74
+
75
+ def keys
76
+ []
77
+ end
78
+
79
+ def size
80
+ orig_size
81
+ end
82
+ end
83
+
84
+ module Persist
85
+ def self.open_pki(path, write, pattern, &pos_function)
86
+ FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
87
+
88
+ database = PackedIndex.new(path, write, pattern, &pos_function)
89
+ database.extend PKIAdapter
90
+
91
+ database.pos_function = pos_function
92
+
93
+ database
94
+ end
95
+ end
@@ -0,0 +1,54 @@
1
+ require_relative 'base'
2
+ require_relative '../../engine/sharder'
3
+
4
+ module ShardAdapter
5
+ include TSVAdapter
6
+
7
+ def self.extended(obj)
8
+ obj.extend TSVAdapter
9
+ class << obj
10
+ alias keys orig_keys
11
+ alias each orig_each
12
+ end
13
+
14
+ obj
15
+ end
16
+
17
+ def metadata_file
18
+ @metadata_file ||= File.join(self.persistence_path, 'metadata')
19
+ end
20
+
21
+ def load_annotation_hash
22
+ ANNOTATION_ATTR_HASH_SERIALIZER.load(Open.read(metadata_file, mode: 'rb'))
23
+ end
24
+
25
+ def save_annotation_hash
26
+ Open.write(metadata_file, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash), mode: 'wb')
27
+ end
28
+
29
+ def size
30
+ databases.values.inject(0){|acc,d| acc += d.size }
31
+ end
32
+
33
+ def keys
34
+ databases.values.inject([]){|acc,d| acc.concat(d.keys); acc }
35
+ end
36
+
37
+ def prefix(...)
38
+ databases.values.inject([]){|acc,d| acc.concat(d.prefix(...)); acc }
39
+ end
40
+
41
+ def include?(key)
42
+ self[key] != nil
43
+ end
44
+ end
45
+
46
+ module Persist
47
+ def self.open_sharder(persistence_path, write=false, db_type=nil, persist_options={}, &block)
48
+ database = Sharder.new(persistence_path, write, db_type, persist_options, &block)
49
+ database.extend ShardAdapter
50
+ database.serializer = TSVAdapter.serializer_module(persist_options[:serializer]) if persist_options[:serializer]
51
+ database.save_annotation_hash
52
+ database
53
+ end
54
+ end
@@ -0,0 +1,18 @@
1
+ require_relative 'base'
2
+ require_relative '../../engine/tkrzw'
3
+
4
+ Persist.save_drivers[:tkh] = proc do |file, content|
5
+ data = ScoutTKRZW.open(file, true, "tkh")
6
+ content.annotate(data)
7
+ data.extend TSVAdapter
8
+ data.merge!(content)
9
+ data.close
10
+ data.read
11
+ data
12
+ end
13
+
14
+ Persist.load_drivers[:tkh] = proc do |file|
15
+ data = ScoutTKRZW.open(file, false, "tkh")
16
+ data.extend TSVAdapter unless TSVAdapter === data
17
+ data
18
+ end
@@ -0,0 +1,65 @@
1
+ require_relative 'base'
2
+ require_relative '../../engine/tokyocabinet'
3
+
4
+ module TKAdapter
5
+ include TSVAdapter
6
+ def self.extended(obj)
7
+ obj.extend TSVAdapter
8
+ obj
9
+ end
10
+ end
11
+
12
+ module Persist
13
+ def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
14
+ write = true unless File.exist? path
15
+
16
+ FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
17
+
18
+ database = ScoutCabinet.open(path, write, tokyocabinet_class)
19
+
20
+ database.extend TKAdapter
21
+ database.serializer ||= TSVAdapter.serializer_module(serializer)
22
+
23
+ database
24
+ end
25
+ end
26
+
27
+ Persist.save_drivers[:HDB] = proc do |file, content|
28
+ if ScoutCabinet === content
29
+ Open.mv(content.persistence_path, file)
30
+ content.persistence_path = file
31
+ content
32
+ else
33
+ data = ScoutCabinet.open(file, true, "HDB")
34
+ content.annotate(data)
35
+ data.extend TKAdapter
36
+ data.merge!(content)
37
+ data
38
+ end
39
+ end
40
+
41
+ Persist.load_drivers[:HDB] = proc do |file|
42
+ data = ScoutCabinet.open(file, false, "HDB")
43
+ data.extend TKAdapter unless TKAdapter === data
44
+ data
45
+ end
46
+
47
+ Persist.save_drivers[:BDB] = proc do |file, content|
48
+ if ScoutCabinet === content
49
+ Open.mv(content.persistence_path, file)
50
+ content.persistence_path = file
51
+ content
52
+ else
53
+ data = ScoutCabinet.open(file, true, "BDB")
54
+ content.annotate(data)
55
+ data.extend TKAdapter
56
+ data.merge!(content)
57
+ data
58
+ end
59
+ end
60
+
61
+ Persist.load_drivers[:BDB] = proc do |file|
62
+ data = ScoutCabinet.open(file, false, "BDB")
63
+ data.extend TKAdapter unless TKAdapter === data
64
+ data
65
+ end
@@ -0,0 +1,6 @@
1
+ require_relative 'serialize'
2
+ require_relative 'adapter/base'
3
+ require_relative 'adapter/fix_width_table'
4
+ require_relative 'adapter/packed_index'
5
+ require_relative 'adapter/tokyocabinet'
6
+ require_relative 'adapter/sharder'
@@ -108,10 +108,15 @@ module TSVAdapter
108
108
  :strict_integer_array => StrictIntegerArraySerializer,
109
109
  :strict_float_array => StrictFloatArraySerializer,
110
110
  :marshal => Marshal,
111
+ :json => JSON,
111
112
  :string => StringSerializer,
112
113
  :binary => BinarySerializer,
113
114
  :tsv => TSVSerializer,
114
115
  :marshal_tsv => TSVMarshalSerializer
115
116
  }
116
117
 
118
+ def self.serializer_module(serializer)
119
+ Symbol === serializer ? SERIALIZER_ALIAS[serializer] : serializer
120
+ end
121
+
117
122
  end
@@ -0,0 +1,107 @@
1
+ require 'scout/persist'
2
+ require_relative 'engine'
3
+ require_relative 'tsv/adapter'
4
+
5
+ Persist.save_drivers[:tsv] = proc do |file,content|
6
+ case content
7
+ when IO
8
+ Open.sensible_write(file, content)
9
+ when TSV
10
+ Open.sensible_write(file, content)
11
+ else
12
+ stream = if content.respond_to?(:stream)
13
+ content.stream
14
+ elsif content.respond_to?(:dumper_stream)
15
+ content.dumper_stream
16
+ else
17
+ content
18
+ end
19
+ Open.sensible_write(file, stream)
20
+ end
21
+ end
22
+
23
+ Persist.load_drivers[:tsv] = proc do |file| TSV.open file end
24
+
25
+ module Persist
26
+
27
+ def self.open_database(path, write, serializer = nil, type = "HDB", options = {})
28
+ db = case type
29
+ when 'fwt'
30
+ value_size, range, update, in_memory, pos_function = IndiferentHash.process_options options.dup, :value_size, :range, :update, :in_memory, :pos_function
31
+ if pos_function
32
+ Persist.open_fwt(path, value_size, range, serializer, update, in_memory, &pos_function)
33
+ else
34
+ Persist.open_fwt(path, value_size, range, serializer, update, in_memory)
35
+ end
36
+ when 'pki'
37
+ pattern, pos_function = IndiferentHash.process_options options.dup, :pattern, :pos_function
38
+ if pos_function
39
+ Persist.open_pki(path, write, pattern, &pos_function)
40
+ else
41
+ Persist.open_pki(path, write, pattern)
42
+ end
43
+ else
44
+ Persist.open_tokyocabinet(path, write, serializer, type)
45
+ end
46
+ db
47
+ end
48
+
49
+ def self.tsv(id, options = {}, engine: nil, persist_options: {})
50
+ engine ||= persist_options[:engine] || :HDB
51
+ Persist.persist(id, engine, persist_options.merge(:other_options => options)) do |filename|
52
+ if filename.nil?
53
+ yield(persist_options[:data] || {})
54
+ else
55
+ if persist_options.include?(:shard_function)
56
+ data = persist_options[:data] ||= Persist.open_sharder(filename, true, engine, options.merge(persist_options), &persist_options[:shard_function])
57
+ else
58
+ data = persist_options[:data] ||= Persist.open_database(filename, true, persist_options[:serializer], engine, options)
59
+ end
60
+
61
+ data.serializer = TSVAdapter.serializer_module(persist_options[:serializer]) if persist_options[:serializer]
62
+
63
+ yield(data)
64
+ data.save_annotation_hash if Annotation.is_annotated?(data)
65
+ data
66
+ end
67
+ end
68
+ end
69
+
70
+ def self.persist_tsv(file, filename = nil, options = {}, persist_options = {}, &block)
71
+ persist_options = IndiferentHash.add_defaults persist_options,
72
+ IndiferentHash.pull_keys(options, :persist)
73
+
74
+ persist_options[:data] ||= options.delete(:data)
75
+ engine = IndiferentHash.process_options persist_options, :engine, engine: "HDB"
76
+ other_options = IndiferentHash.pull_keys persist_options, :other
77
+ other_options[:original_options] = options
78
+
79
+ Persist.tsv(file, engine: engine, persist_options: persist_options.merge(other: other_options), &block)
80
+ end
81
+
82
+ #def self.persist_tsv(file, filename = nil, options = {}, persist_options = {})
83
+ # persist_options = IndiferentHash.add_defaults persist_options,
84
+ # IndiferentHash.pull_keys(options, :persist)
85
+
86
+ # persist_options[:data] ||= options.delete(:data)
87
+ # engine = IndiferentHash.process_options persist_options, :engine, engine: "HDB"
88
+ # other_options = IndiferentHash.pull_keys persist_options, :other
89
+ # other_options[:original_options] = options
90
+ # Persist.persist(file, engine, persist_options.merge(:other => other_options)) do |filename|
91
+ # if filename
92
+ # if persist_options.include?(:shard_function)
93
+ # data = persist_options[:data] ||= Persist.open_sharder(filename, true, engine, options.merge(persist_options), &persist_options[:shard_function])
94
+ # else
95
+ # data = persist_options[:data] ||= Persist.open_database(filename, true, persist_options[:serializer], engine, options)
96
+ # end
97
+
98
+
99
+ # yield(data)
100
+ # data.save_annotation_hash if Annotation.is_annotated?(data)
101
+ # data
102
+ # else
103
+ # yield(persist_options[:data] || {})
104
+ # end
105
+ # end
106
+ #end
107
+ end
@@ -0,0 +1,83 @@
1
+ module Persist
2
+ def self.annotation_repo_persist(repo, name, &block)
3
+
4
+ if String === repo
5
+ repo = repo.find if Path === repo
6
+ repo = Persist.open_tokyocabinet(repo, false, :list, :BDB)
7
+ repo_fields = ["literal", "annotation_types", "JSON"]
8
+ TSV.setup(repo, :fields => repo_fields, :key_field => "Annotation ID")
9
+ repo.save_annotation_hash
10
+ repo.close
11
+ else
12
+ repo_fields = repo.fields
13
+ end
14
+
15
+ subkey = name + ":"
16
+
17
+ keys = repo.read_and_close do
18
+ repo.range subkey + 0.chr, true, subkey + 254.chr, true
19
+ end
20
+
21
+ case
22
+ when (keys.length == 1 and keys.first == subkey + 'NIL')
23
+ nil
24
+ when (keys.length == 1 and keys.first == subkey + 'EMPTY')
25
+ []
26
+ when (keys.length == 1 && keys.first =~ /:SINGLE$/)
27
+ key = keys.first
28
+ values = repo.read_and_close do
29
+ repo[key]
30
+ end
31
+ Annotation.load_tsv_values(key, values, *repo_fields)
32
+ when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
33
+ repo.read_and_close do
34
+ keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
35
+ v = repo[key]
36
+ Annotation.load_tsv_values(key, v, *repo_fields)
37
+ }
38
+ end
39
+ when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
40
+ repo.read_and_close do
41
+
42
+ res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
43
+ v = repo[key]
44
+ Annotation.load_tsv_values(key, v, *repo_fields)
45
+ }
46
+
47
+ res.first.annotate res
48
+ res.extend AnnotatedArray
49
+
50
+ res
51
+ end
52
+ else
53
+ annotations = yield
54
+
55
+ repo.write_and_close do
56
+ case
57
+ when annotations.nil?
58
+ repo[subkey + "NIL"] = nil
59
+ when annotations.empty?
60
+ repo[subkey + "EMPTY"] = nil
61
+ when (not Array === annotations or (AnnotatedArray === annotations and not Array === annotations.first))
62
+ tsv_values = Annotation.obj_tsv_values(annotations, repo_fields)
63
+ repo[subkey + annotations.id << ":" << "SINGLE"] = tsv_values
64
+ when (not Array === annotations or (AnnotatedArray === annotations and AnnotatedArray === annotations.first))
65
+ annotations.each_with_index do |e,i|
66
+ next if e.nil?
67
+ tsv_values = Annotation.obj_tsv_values(e, repo_fields)
68
+ repo[subkey + "ANNOTATED_DOUBLE_ARRAY:" << i.to_s] = tsv_values
69
+ end
70
+ else
71
+ annotations.each_with_index do |e,i|
72
+ next if e.nil?
73
+ tsv_values = Annotation.obj_tsv_values(e, repo_fields)
74
+ repo[subkey + i.to_s] = tsv_values
75
+ end
76
+ end
77
+ end
78
+
79
+ annotations
80
+ end
81
+
82
+ end
83
+ end