scout-gear 10.4.0 → 10.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +100 -656
  3. data/Rakefile +1 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +1 -3
  6. data/lib/scout/association/fields.rb +170 -0
  7. data/lib/scout/association/index.rb +229 -0
  8. data/lib/scout/association/item.rb +227 -0
  9. data/lib/scout/association/util.rb +7 -0
  10. data/lib/scout/association.rb +100 -0
  11. data/lib/scout/entity/format.rb +62 -0
  12. data/lib/scout/entity/identifiers.rb +111 -0
  13. data/lib/scout/entity/object.rb +20 -0
  14. data/lib/scout/entity/property.rb +165 -0
  15. data/lib/scout/entity.rb +41 -0
  16. data/lib/scout/offsite/step.rb +2 -2
  17. data/lib/scout/{tsv/persist → persist/engine}/fix_width_table.rb +25 -33
  18. data/lib/scout/persist/engine/packed_index.rb +100 -0
  19. data/lib/scout/persist/engine/sharder.rb +219 -0
  20. data/lib/scout/{tsv/persist → persist/engine}/tkrzw.rb +0 -17
  21. data/lib/scout/{tsv/persist → persist/engine}/tokyocabinet.rb +55 -31
  22. data/lib/scout/persist/engine.rb +4 -0
  23. data/lib/scout/{tsv/persist/adapter.rb → persist/tsv/adapter/base.rb} +80 -51
  24. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +106 -0
  25. data/lib/scout/persist/tsv/adapter/packed_index.rb +95 -0
  26. data/lib/scout/persist/tsv/adapter/sharder.rb +54 -0
  27. data/lib/scout/persist/tsv/adapter/tkrzw.rb +18 -0
  28. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +65 -0
  29. data/lib/scout/persist/tsv/adapter.rb +6 -0
  30. data/lib/scout/{tsv/persist → persist/tsv}/serialize.rb +5 -0
  31. data/lib/scout/persist/tsv.rb +107 -0
  32. data/lib/scout/tsv/annotation/repo.rb +87 -0
  33. data/lib/scout/tsv/annotation.rb +169 -0
  34. data/lib/scout/tsv/attach.rb +97 -21
  35. data/lib/scout/tsv/change_id/translate.rb +148 -0
  36. data/lib/scout/tsv/change_id.rb +3 -0
  37. data/lib/scout/tsv/csv.rb +85 -0
  38. data/lib/scout/tsv/dumper.rb +113 -25
  39. data/lib/scout/tsv/index.rb +88 -36
  40. data/lib/scout/tsv/open.rb +21 -8
  41. data/lib/scout/tsv/parser.rb +153 -90
  42. data/lib/scout/tsv/path.rb +7 -2
  43. data/lib/scout/tsv/stream.rb +48 -6
  44. data/lib/scout/tsv/transformer.rb +5 -3
  45. data/lib/scout/tsv/traverse.rb +28 -19
  46. data/lib/scout/tsv/util/process.rb +7 -0
  47. data/lib/scout/tsv/util/reorder.rb +25 -15
  48. data/lib/scout/tsv/util/select.rb +9 -1
  49. data/lib/scout/tsv/util/sort.rb +90 -2
  50. data/lib/scout/tsv/util/unzip.rb +56 -0
  51. data/lib/scout/tsv/util.rb +52 -5
  52. data/lib/scout/tsv.rb +42 -27
  53. data/lib/scout/work_queue/socket.rb +8 -0
  54. data/lib/scout/work_queue/worker.rb +22 -5
  55. data/lib/scout/work_queue.rb +41 -24
  56. data/lib/scout/workflow/definition.rb +15 -12
  57. data/lib/scout/workflow/deployment/orchestrator.rb +21 -3
  58. data/lib/scout/workflow/deployment/trace.rb +205 -0
  59. data/lib/scout/workflow/deployment.rb +1 -0
  60. data/lib/scout/workflow/documentation.rb +1 -1
  61. data/lib/scout/workflow/step/archive.rb +42 -0
  62. data/lib/scout/workflow/step/children.rb +51 -0
  63. data/lib/scout/workflow/step/config.rb +1 -1
  64. data/lib/scout/workflow/step/dependencies.rb +25 -8
  65. data/lib/scout/workflow/step/file.rb +19 -0
  66. data/lib/scout/workflow/step/info.rb +37 -9
  67. data/lib/scout/workflow/step/progress.rb +11 -2
  68. data/lib/scout/workflow/step/status.rb +9 -1
  69. data/lib/scout/workflow/step.rb +80 -25
  70. data/lib/scout/workflow/task/dependencies.rb +5 -2
  71. data/lib/scout/workflow/task/inputs.rb +91 -41
  72. data/lib/scout/workflow/task.rb +54 -57
  73. data/lib/scout/workflow/usage.rb +1 -1
  74. data/lib/scout/workflow/util.rb +4 -0
  75. data/lib/scout/workflow.rb +110 -13
  76. data/lib/scout-gear.rb +2 -0
  77. data/lib/scout.rb +0 -1
  78. data/scout-gear.gemspec +78 -23
  79. data/scout_commands/rbbt +2 -0
  80. data/test/data/person/brothers +4 -0
  81. data/test/data/person/identifiers +10 -0
  82. data/test/data/person/marriages +3 -0
  83. data/test/data/person/parents +6 -0
  84. data/test/scout/association/test_fields.rb +105 -0
  85. data/test/scout/association/test_index.rb +70 -0
  86. data/test/scout/association/test_item.rb +21 -0
  87. data/test/scout/entity/test_format.rb +19 -0
  88. data/test/scout/entity/test_identifiers.rb +58 -0
  89. data/test/scout/entity/test_object.rb +0 -0
  90. data/test/scout/entity/test_property.rb +345 -0
  91. data/test/scout/{tsv/persist → persist/engine}/test_fix_width_table.rb +0 -1
  92. data/test/scout/persist/engine/test_packed_index.rb +99 -0
  93. data/test/scout/persist/engine/test_sharder.rb +31 -0
  94. data/test/scout/persist/engine/test_tkrzw.rb +0 -0
  95. data/test/scout/persist/engine/test_tokyocabinet.rb +17 -0
  96. data/test/scout/persist/test_tsv.rb +146 -0
  97. data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb} +3 -4
  98. data/test/scout/persist/tsv/adapter/test_fix_width_table.rb +46 -0
  99. data/test/scout/persist/tsv/adapter/test_packed_index.rb +37 -0
  100. data/test/scout/persist/tsv/adapter/test_serialize.rb +0 -0
  101. data/test/scout/persist/tsv/adapter/test_sharder.rb +290 -0
  102. data/test/scout/{tsv/persist → persist/tsv/adapter}/test_tkrzw.rb +3 -6
  103. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +282 -0
  104. data/test/scout/persist/tsv/test_serialize.rb +12 -0
  105. data/test/scout/test_association.rb +51 -0
  106. data/test/scout/test_entity.rb +40 -0
  107. data/test/scout/test_tsv.rb +33 -4
  108. data/test/scout/test_work_queue.rb +5 -2
  109. data/test/scout/test_workflow.rb +31 -14
  110. data/test/scout/tsv/annotation/test_repo.rb +150 -0
  111. data/test/scout/tsv/change_id/test_translate.rb +178 -0
  112. data/test/scout/tsv/test_annotation.rb +52 -0
  113. data/test/scout/tsv/test_attach.rb +255 -1
  114. data/test/scout/tsv/test_change_id.rb +25 -0
  115. data/test/scout/tsv/test_csv.rb +50 -0
  116. data/test/scout/tsv/test_dumper.rb +38 -0
  117. data/test/scout/tsv/test_index.rb +82 -0
  118. data/test/scout/tsv/test_open.rb +44 -0
  119. data/test/scout/tsv/test_parser.rb +70 -0
  120. data/test/scout/tsv/test_stream.rb +22 -0
  121. data/test/scout/tsv/test_transformer.rb +27 -3
  122. data/test/scout/tsv/test_traverse.rb +78 -0
  123. data/test/scout/tsv/util/test_process.rb +16 -0
  124. data/test/scout/tsv/util/test_reorder.rb +67 -0
  125. data/test/scout/tsv/util/test_sort.rb +28 -1
  126. data/test/scout/tsv/util/test_unzip.rb +32 -0
  127. data/test/scout/work_queue/test_socket.rb +4 -1
  128. data/test/scout/workflow/deployment/test_orchestrator.rb +17 -26
  129. data/test/scout/workflow/deployment/test_trace.rb +25 -0
  130. data/test/scout/workflow/step/test_archive.rb +28 -0
  131. data/test/scout/workflow/step/test_children.rb +25 -0
  132. data/test/scout/workflow/step/test_info.rb +16 -0
  133. data/test/scout/workflow/task/test_dependencies.rb +16 -16
  134. data/test/scout/workflow/task/test_inputs.rb +45 -1
  135. data/test/scout/workflow/test_definition.rb +52 -0
  136. data/test/scout/workflow/test_step.rb +57 -0
  137. data/test/scout/workflow/test_task.rb +26 -1
  138. data/test/scout/workflow/test_usage.rb +4 -4
  139. data/test/test_helper.rb +23 -1
  140. metadata +69 -14
  141. data/lib/scout/tsv/persist.rb +0 -27
  142. data/test/scout/tsv/persist/test_tokyocabinet.rb +0 -120
  143. data/test/scout/tsv/test_persist.rb +0 -45
@@ -0,0 +1,100 @@
1
+ require_relative 'tsv'
2
+ require_relative 'association/fields'
3
+ require_relative 'association/util'
4
+ require_relative 'association/index'
5
+ require_relative 'association/item'
6
+
7
+ module Association
8
+ def self.open(obj, source: nil, target: nil, fields: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
9
+ all_fields = TSV.all_fields(obj)
10
+ source_pos, field_pos, source_header, field_headers, source_format, target_format = headers(all_fields, fields, kwargs.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format))
11
+
12
+ original_source_header = all_fields[source_pos]
13
+ original_field_headers = all_fields.values_at(*field_pos)
14
+ original_target_header = all_fields[field_pos.first]
15
+
16
+ type, identifiers = IndiferentHash.process_options kwargs, :type, :identifiers
17
+
18
+ if source_format
19
+ translation_files = [TSV.identifier_files(obj), Entity.identifier_files(source_format), identifiers].flatten.compact
20
+ source_index = begin
21
+ TSV.translation_index(translation_files, source_header, source_format)
22
+ rescue
23
+ TSV.translation_index(translation_files, original_source_header, source_format)
24
+ end
25
+ end
26
+
27
+ if target_format
28
+ translation_files = [TSV.identifier_files(obj), Entity.identifier_files(target_format), identifiers].flatten.compact
29
+ target_index = begin
30
+ TSV.translation_index(translation_files, field_headers.first, target_format)
31
+ rescue
32
+ TSV.translation_index(translation_files, original_target_header, target_format)
33
+ end
34
+ end
35
+
36
+ final_key_field = if source_format
37
+ if m = original_source_header.match(/(.*) \(.*\)/)
38
+ m[1] + " (#{source_format})"
39
+ elsif m = source_header.match(/(.*) \(.*\)/)
40
+ m[1] + " (#{source_format})"
41
+ else
42
+ source_format
43
+ end
44
+ else
45
+ if source_header
46
+ original_source_header.include?(source_header) ? original_source_header : source_header
47
+ else
48
+ original_source_header
49
+ end
50
+ end
51
+
52
+ fields = original_field_headers
53
+ final_target_field = if target_format
54
+ if m = original_target_header.match(/(.*) \(.*\)/)
55
+ m[1] + " (#{target_format})"
56
+ elsif m = field_headers.first.match(/(.*) \(.*\)/)
57
+ m[1] + " (#{target_format})"
58
+ else
59
+ target_format
60
+ end
61
+ else
62
+ target_header = field_headers.first
63
+ original_target_header.include?(target_header) ? original_target_header : target_header
64
+ end
65
+ final_fields = [final_target_field] + original_field_headers[1..-1]
66
+
67
+ if source_index.nil? && target_index.nil?
68
+ if TSV === obj
69
+ IndiferentHash.pull_keys kwargs, :persist
70
+ type = kwargs[:type] || obj.type
71
+ res = obj.reorder original_source_header, all_fields.values_at(*field_pos), **kwargs.merge(type: type, merge: true)
72
+ else
73
+ res = TSV.open(obj, key_field: original_source_header, fields: all_fields.values_at(*field_pos), **kwargs.merge(type: type))
74
+ end
75
+ res.key_field = final_key_field
76
+ res.fields = final_fields
77
+
78
+ return res
79
+ end
80
+
81
+ transformer = TSV::Transformer.new obj
82
+ transformer.key_field = final_key_field
83
+ transformer.fields = final_fields
84
+ transformer.type = type if type
85
+
86
+ transformer.traverse key_field: original_source_header, fields: all_fields.values_at(*field_pos) do |k,v|
87
+ v = v.dup if TSV === obj
88
+ k = source_index[k] if source_index
89
+ v[0] = Array === v[0] ? target_index.values_at(*v[0]) : target_index[v[0]] if target_index
90
+ [k, v]
91
+ end
92
+
93
+ transformer
94
+ end
95
+
96
+ def self.database(*args, **kwargs)
97
+ tsv = open(*args, **kwargs)
98
+ TSV::Transformer === tsv ? tsv.tsv(merge: true) : tsv
99
+ end
100
+ end
@@ -0,0 +1,62 @@
1
+ module Entity
2
+ def format=(formats)
3
+ formats = [formats] unless Array === formats
4
+ formats.each do |format|
5
+ Entity.formats[format] ||= self
6
+ end
7
+ end
8
+
9
+ class FormatIndex < Hash
10
+
11
+ alias orig_include? include?
12
+
13
+ def initialize
14
+ @find_cache = {}
15
+ end
16
+
17
+ def find(value)
18
+ @find_cache ||= {}
19
+
20
+ @find_cache[value] ||= begin
21
+ if orig_include? value
22
+ @find_cache[value] = value
23
+ else
24
+ found = nil
25
+ each do |k,v|
26
+ if value.to_s == k.to_s
27
+ found = k
28
+ break
29
+ elsif value.to_s =~ /\(#{Regexp.quote k}\)/
30
+ found = k
31
+ break
32
+ end
33
+ end
34
+ found
35
+ end
36
+ end
37
+ end
38
+
39
+ def [](value)
40
+ res = super
41
+ return res if res
42
+ key = find(value)
43
+ key ? super(key) : nil
44
+ end
45
+
46
+ def []=(key,value)
47
+ @find_cache = {}
48
+ super(key, value)
49
+ end
50
+
51
+ def include?(value)
52
+ find(value) != nil
53
+ end
54
+ end
55
+
56
+ FORMATS ||= FormatIndex.new
57
+
58
+ def self.formats
59
+ FORMATS
60
+ end
61
+
62
+ end
@@ -0,0 +1,111 @@
1
+ module Entity
2
+ def self.identifier_files(field)
3
+ entity_type = Entity.formats[field]
4
+ return [] unless entity_type and entity_type.include? Entity::Identified
5
+ entity_type.identifier_files
6
+ end
7
+
8
+ module Identified
9
+ NAMESPACE_TAG = 'NAMESPACE'
10
+
11
+ def self.included(base)
12
+ base.annotation :format
13
+ base.annotation :namespace
14
+
15
+ class << base
16
+ attr_accessor :identifier_files, :formats, :default_format, :name_format, :description_format
17
+ end
18
+
19
+ base.property :to => :both do |target_format|
20
+
21
+ target_format = case target_format
22
+ when :name
23
+ identity_type.name_format
24
+ when :default
25
+ identity_type.default_format
26
+ else
27
+ target_format
28
+ end
29
+
30
+ if target_format == format
31
+ self
32
+ elsif Array === self
33
+ self.annotate(identifier_index(target_format, self.format).values_at(*self))
34
+ else
35
+ self.annotate(identifier_index(target_format, self.format)[self])
36
+ end.tap{|o| o.format = target_format unless o.nil? }
37
+ end
38
+
39
+ base.property :name => :both do
40
+ to(:name)
41
+ end
42
+
43
+ base.property :default => :both do
44
+ to(:default)
45
+ end
46
+ end
47
+
48
+ def identifier_files
49
+ files = identity_type.identifier_files.dup
50
+ return [] if files.nil?
51
+ files.collect!{|f| f.annotate f.gsub(/\b#{NAMESPACE_TAG}\b/, namespace.to_s) } if annotations.include? :namespace and self.namespace
52
+ if files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ }.any?
53
+ Log.warn "Rejecting some identifier files for lack of 'namespace': " << files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ } * ", "
54
+ end
55
+ files.reject!{|f| f =~ /\b#{NAMESPACE_TAG}\b/ }
56
+ files
57
+ end
58
+
59
+ def identity_type
60
+ self.annotation_types.select{|m| m.include? Entity::Identified }.last
61
+ end
62
+
63
+ def identifier_index(format = nil, source = nil)
64
+ Persist.memory("Entity index #{identity_type}: #{format} (from #{source || "All"})", :persist => true, :format => format, :source => source) do
65
+ source ||= self.respond_to?(:format)? self.format : nil
66
+
67
+ begin
68
+ index = TSV.translation_index(identifier_files, source, format, :persist => true)
69
+ raise "No index from #{ Misc.fingerprint source } to #{ Misc.fingerprint format }: #{Misc.fingerprint identifier_files}" if index.nil?
70
+ index.unnamed = true
71
+ index
72
+ rescue
73
+ raise $! if source.nil?
74
+ source = nil
75
+ retry
76
+ end
77
+ end
78
+ end
79
+ end
80
+
81
+ def add_identifiers(file, default = nil, name = nil, description = nil)
82
+ if TSV === file
83
+ all_fields = file.all_fields
84
+ else
85
+ if file =~ /#{Identified::NAMESPACE_TAG}/
86
+ all_fields = file.sub(/#{Identified::NAMESPACE_TAG}/,'**').glob.collect do |f|
87
+ TSV.parse_header(f)["all_fields"]
88
+ end.flatten.compact.uniq
89
+ else
90
+ all_fields = TSV.parse_header(file)["all_fields"]
91
+ end
92
+ end
93
+
94
+ self.send(:include, Entity::Identified) unless Entity::Identified === self
95
+
96
+ self.format = all_fields
97
+ @formats ||= []
98
+ @formats.concat all_fields
99
+ @formats.uniq!
100
+
101
+ @default_format = default if default
102
+ @name_format = name if name
103
+ @description_format = description if description
104
+
105
+ @identifier_files ||= []
106
+ @identifier_files << file
107
+ @identifier_files.uniq!
108
+ end
109
+
110
+
111
+ end
@@ -0,0 +1,20 @@
1
+ module Entity
2
+ module Object
3
+
4
+ def entity_classes
5
+ annotation_types.select{|t| Entity === t}
6
+ end
7
+
8
+ def base_entity
9
+ entity_classes.last
10
+ end
11
+
12
+ def _ary_property_cache
13
+ @_ary_property_cache ||= {}
14
+ end
15
+
16
+ def all_properties
17
+ entity_classes.inject([]){|acc,e| acc.concat(e.properties) }
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,165 @@
1
+ module Entity
2
+ class << self
3
+ attr_accessor :entity_property_cache
4
+
5
+ def entity_property_cache
6
+ @entity_property_cache ||= Path.setup('var/entity_property')
7
+ end
8
+ end
9
+
10
+ module Property
11
+
12
+ class MultipleEntityProperty < DontPersist; end
13
+
14
+ attr_accessor :persisted_methods, :properties
15
+
16
+ def self.persist(name, obj, type, options, &block)
17
+ return yield unless options[:persist]
18
+ if (type == :annotation || type == :annotations) && options[:annotation_repo]
19
+ repo = options[:annotation_repo]
20
+ Persist.annotation_repo_persist(repo, [name, obj.id] * ":", &block)
21
+ else
22
+
23
+ Persist.persist([name, obj.id] * ":", type, options.dup, &block)
24
+ end
25
+ end
26
+
27
+ def self.single_method(name)
28
+ ("_single_" + name.to_s).to_sym
29
+ end
30
+
31
+ def self.array_method(name)
32
+ ("_ary_" + name.to_s).to_sym
33
+ end
34
+
35
+ def self.multi_method(name)
36
+ ("_multi_" + name.to_s).to_sym
37
+ end
38
+
39
+ DEFAULT_PROPERTY_TYPE = :both
40
+ def property(name_and_type, &block)
41
+ name, type = case name_and_type
42
+ when Symbol, String
43
+ [name_and_type.to_sym, DEFAULT_PROPERTY_TYPE]
44
+ else
45
+ name_and_type.collect.first
46
+ end
47
+
48
+ real_method = case type
49
+ when :single, :single2array
50
+ Entity::Property.single_method(name)
51
+ when :array, :array2single
52
+ Entity::Property.array_method(name)
53
+ when :multiple
54
+ Entity::Property.multi_method(name)
55
+ when :both
56
+ name
57
+ else
58
+ raise "Type of property unknown #{type}"
59
+ end
60
+
61
+ properties.push name
62
+
63
+
64
+ entity_class = self
65
+ if type == :multiple
66
+ self.define_method(real_method) do |*args,**kwargs|
67
+ if entity_class.persisted_methods && entity_class.persisted_methods[name]
68
+ type, options = entity_class.persisted_methods[name]
69
+ else
70
+ type, options = nil, {persist: false}
71
+ end
72
+
73
+ missing = []
74
+ responses = {}
75
+ self.each do |item|
76
+ begin
77
+ responses[item] = Entity::Property.persist(name, item, type, options) do
78
+ raise MultipleEntityProperty
79
+ end
80
+ rescue MultipleEntityProperty
81
+ missing << item
82
+ end
83
+ end
84
+
85
+ if missing.any?
86
+ self.annotate(missing)
87
+
88
+ new_responses = missing.instance_exec(*args, **kwargs, &block)
89
+
90
+ missing.each do |item,i|
91
+ responses[item] = Entity::Property.persist(name, item, type, options) do
92
+ Array === new_responses ? new_responses[item.container_index] : new_responses[item]
93
+ end
94
+ end
95
+ end
96
+
97
+ responses.values_at(*self)
98
+ end
99
+ else
100
+ self.define_method(real_method) do |*args,**kwargs|
101
+ if entity_class.persisted_methods && entity_class.persisted_methods[name]
102
+ type, options = entity_class.persisted_methods[name]
103
+ else
104
+ type, options = nil, {persist: false}
105
+ end
106
+
107
+ Entity::Property.persist(name, self, type, options) do
108
+ self.instance_exec(*args, **kwargs, &block)
109
+ end
110
+ end
111
+ end
112
+
113
+ return if type == :both
114
+
115
+ self.define_method(name) do |*args,**kwargs|
116
+
117
+ method_type = %w(single_method array_method multi_method).select do |method_type|
118
+ self.methods.include?(Entity::Property.send(method_type, name))
119
+ end.first
120
+
121
+ real_method = Entity::Property.send(method_type, name)
122
+
123
+ if Array === self
124
+ case method_type
125
+ when 'single_method'
126
+ self.collect{|item| item.send(real_method, *args, **kwargs) }
127
+ when 'array_method', 'multi_method'
128
+ self.send(real_method, *args, **kwargs)
129
+ end
130
+ else
131
+ case method_type
132
+ when 'single_method'
133
+ self.send(real_method, *args, **kwargs)
134
+ when 'array_method', 'multi_method'
135
+ if AnnotatedArray.is_contained?(self)
136
+ cache_code = Misc.digest({:name => name, :args => args})
137
+ res = (self.container._ary_property_cache[cache_code] ||= self.container.send(real_method, *args, **kwargs))
138
+ Array === res ? res[self.container_index] : res[self]
139
+ else
140
+ res = self.make_array.send(real_method)
141
+ Array === res ? res[0] : res[self]
142
+ end
143
+ end
144
+ end
145
+ end
146
+ end
147
+
148
+ def persist(name, type = :marshal, options = {})
149
+ options = IndiferentHash.add_defaults options, persist: true,
150
+ dir: File.join(Entity.entity_property_cache, self.to_s, name.to_s)
151
+ @persisted_methods ||= {}
152
+ @persisted_methods[name] = [type, options]
153
+ end
154
+
155
+ def persisted?(name)
156
+ @persisted_methods ||= {}
157
+ @persisted_methods.include?(name)
158
+ end
159
+
160
+ def unpersist(name)
161
+ @persisted_methods ||= {}
162
+ @persisted_methods.delete(name)
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,41 @@
1
+ require 'scout/annotation'
2
+ require_relative 'entity/format'
3
+ require_relative 'entity/property'
4
+ require_relative 'entity/object'
5
+ require_relative 'entity/identifiers'
6
+
7
+ module Entity
8
+ def self.extended(base)
9
+ base.extend Annotation
10
+ base.extend Entity::Property
11
+ base.instance_variable_set(:@properties, [])
12
+ base.instance_variable_set(:@persisted_methods, {})
13
+ base.include Entity::Object
14
+ base.include AnnotatedArray
15
+ base
16
+ end
17
+
18
+ def self.prepare_entity(entity, field, options = {})
19
+ return entity unless defined? Entity
20
+ return entity unless String === entity or Array === entity
21
+ options ||= {}
22
+
23
+ dup_array = options.delete :dup_array
24
+
25
+ if Entity === field or (Entity.respond_to?(:formats) and (_format = Entity.formats.find(field)))
26
+ params = options.dup
27
+
28
+ params[:format] ||= params.delete "format"
29
+ params.merge!(:format => _format) unless _format.nil? or (params.include?(:format) and not ((f = params[:format]).nil? or (String === f and f.empty?)))
30
+
31
+ mod = Entity === field ? field : Entity.formats[field]
32
+
33
+ entity = entity.dup
34
+ entity = (entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity)
35
+
36
+ entity = mod.setup(entity, params)
37
+ end
38
+
39
+ entity
40
+ end
41
+ end
@@ -4,8 +4,8 @@ require_relative 'sync'
4
4
 
5
5
  module OffsiteStep
6
6
 
7
- extend MetaExtension
8
- extension_attr :server, :workflow_name, :clean_id, :slurm
7
+ extend Annotation
8
+ annotation :server, :workflow_name, :clean_id, :slurm
9
9
 
10
10
  def inputs_directory
11
11
  @inputs_directory ||= begin
@@ -1,24 +1,24 @@
1
1
  class FixWidthTable
2
2
 
3
- attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask, :write
4
- def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
5
- filename = filename.find if Path === filename
6
- @filename = filename
3
+ attr_accessor :persistence_path, :file, :value_size, :record_size, :range, :size, :mask, :write
4
+ def initialize(persistence_path, value_size = nil, range = nil, update = false, in_memory = true)
5
+ persistence_path = persistence_path.find if Path === persistence_path
6
+ @persistence_path = persistence_path
7
7
 
8
- if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
9
- Log.debug "FixWidthTable create: #{ filename }"
8
+ if update || %w(memory stringio).include?(persistence_path.to_s.downcase) || ! File.exist?(persistence_path)
9
+ Log.debug "FixWidthTable create: #{ persistence_path }"
10
10
  @value_size = value_size
11
11
  @range = range
12
12
  @record_size = @value_size + (@range ? 16 : 8)
13
13
  @write = true
14
14
 
15
- if %w(memory stringio).include?(filename.to_s.downcase)
16
- @filename = :memory
15
+ if %w(memory stringio).include?(persistence_path.to_s.downcase)
16
+ @persistence_path = :memory
17
17
  @file = StringIO.new
18
18
  else
19
- FileUtils.rm @filename if File.exist? @filename
20
- FileUtils.mkdir_p File.dirname(@filename) unless File.exist? @filename
21
- @file = File.open(@filename, 'w:ASCII-8BIT')
19
+ FileUtils.rm @persistence_path if File.exist? @persistence_path
20
+ FileUtils.mkdir_p File.dirname(@persistence_path) unless File.exist? @persistence_path
21
+ @file = File.open(@persistence_path, 'w:ASCII-8BIT')
22
22
  end
23
23
 
24
24
  @file.write [value_size].pack("L")
@@ -26,18 +26,18 @@ class FixWidthTable
26
26
 
27
27
  @size = 0
28
28
  else
29
- Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
29
+ Log.debug "FixWidthTable up-to-date: #{ persistence_path } - (in_memory:#{in_memory})"
30
30
  if in_memory
31
- @file = Open.open(@filename, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
31
+ @file = Open.open(@persistence_path, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
32
32
  else
33
- @file = File.open(@filename, 'r:ASCII-8BIT')
33
+ @file = File.open(@persistence_path, 'r:ASCII-8BIT')
34
34
  end
35
35
  @value_size = @file.read(4).unpack("L").first
36
36
  @range = @file.read(1).unpack("C").first == 1
37
37
  @record_size = @value_size + (@range ? 16 : 8)
38
38
  @write = false
39
39
 
40
- @size = (File.size(@filename) - 5) / (@record_size)
40
+ @size = (File.size(@persistence_path) - 5) / (@record_size)
41
41
  end
42
42
 
43
43
  @mask = "a#{@value_size}"
@@ -47,22 +47,22 @@ class FixWidthTable
47
47
  @write
48
48
  end
49
49
 
50
- def persistence_path
51
- @filename
50
+ def filename
51
+ @persistence_path
52
52
  end
53
53
 
54
54
  def persistence_path=(value)
55
- @filename=value
55
+ @persistence_path=value
56
56
  end
57
57
 
58
- def self.get(filename, value_size = nil, range = nil, update = false)
59
- return self.new(filename, value_size, range, update) if filename == :memory
58
+ def self.get(persistence_path, value_size = nil, range = nil, update = false)
59
+ return self.new(persistence_path, value_size, range, update) if persistence_path == :memory
60
60
  case
61
- when (!File.exist?(filename) or update or not Persist::CONNECTIONS.include?(filename))
62
- Persist::CONNECTIONS[filename] = self.new(filename, value_size, range, update)
61
+ when (!File.exist?(persistence_path) or update or not Persist::CONNECTIONS.include?(persistence_path))
62
+ Persist::CONNECTIONS[persistence_path] = self.new(persistence_path, value_size, range, update)
63
63
  end
64
64
 
65
- Persist::CONNECTIONS[filename]
65
+ Persist::CONNECTIONS[persistence_path]
66
66
  end
67
67
 
68
68
  def format(pos, value)
@@ -113,10 +113,10 @@ class FixWidthTable
113
113
  end
114
114
 
115
115
  def read(force = false)
116
- return if @filename == :memory
116
+ return if @persistence_path == :memory
117
117
  @write = false
118
118
  @file.close unless @file.closed?
119
- @file = File.open(filename, 'r:ASCII-8BIT')
119
+ @file = File.open(persistence_path, 'r:ASCII-8BIT')
120
120
  end
121
121
 
122
122
  def close
@@ -317,11 +317,3 @@ class FixWidthTable
317
317
  alias length size
318
318
  end
319
319
 
320
- Persist.save_drivers[:fwt] = proc do |file, content|
321
- content.file.seek 0
322
- Misc.sensiblewrite(file, content.file.read)
323
- end
324
-
325
- Persist.load_drivers[:fwt] = proc do |file|
326
- FixWidthTable.new file
327
- end