scout-gear 10.4.0 → 10.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +100 -656
  3. data/Rakefile +1 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +1 -3
  6. data/lib/scout/association/fields.rb +170 -0
  7. data/lib/scout/association/index.rb +229 -0
  8. data/lib/scout/association/item.rb +227 -0
  9. data/lib/scout/association/util.rb +7 -0
  10. data/lib/scout/association.rb +100 -0
  11. data/lib/scout/entity/format.rb +62 -0
  12. data/lib/scout/entity/identifiers.rb +111 -0
  13. data/lib/scout/entity/object.rb +20 -0
  14. data/lib/scout/entity/property.rb +165 -0
  15. data/lib/scout/entity.rb +41 -0
  16. data/lib/scout/offsite/step.rb +2 -2
  17. data/lib/scout/{tsv/persist → persist/engine}/fix_width_table.rb +25 -33
  18. data/lib/scout/persist/engine/packed_index.rb +100 -0
  19. data/lib/scout/persist/engine/sharder.rb +219 -0
  20. data/lib/scout/{tsv/persist → persist/engine}/tkrzw.rb +0 -17
  21. data/lib/scout/{tsv/persist → persist/engine}/tokyocabinet.rb +55 -31
  22. data/lib/scout/persist/engine.rb +4 -0
  23. data/lib/scout/{tsv/persist/adapter.rb → persist/tsv/adapter/base.rb} +80 -51
  24. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +106 -0
  25. data/lib/scout/persist/tsv/adapter/packed_index.rb +95 -0
  26. data/lib/scout/persist/tsv/adapter/sharder.rb +54 -0
  27. data/lib/scout/persist/tsv/adapter/tkrzw.rb +18 -0
  28. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +65 -0
  29. data/lib/scout/persist/tsv/adapter.rb +6 -0
  30. data/lib/scout/{tsv/persist → persist/tsv}/serialize.rb +5 -0
  31. data/lib/scout/persist/tsv.rb +107 -0
  32. data/lib/scout/tsv/annotation/repo.rb +87 -0
  33. data/lib/scout/tsv/annotation.rb +169 -0
  34. data/lib/scout/tsv/attach.rb +97 -21
  35. data/lib/scout/tsv/change_id/translate.rb +148 -0
  36. data/lib/scout/tsv/change_id.rb +3 -0
  37. data/lib/scout/tsv/csv.rb +85 -0
  38. data/lib/scout/tsv/dumper.rb +113 -25
  39. data/lib/scout/tsv/index.rb +88 -36
  40. data/lib/scout/tsv/open.rb +21 -8
  41. data/lib/scout/tsv/parser.rb +153 -90
  42. data/lib/scout/tsv/path.rb +7 -2
  43. data/lib/scout/tsv/stream.rb +48 -6
  44. data/lib/scout/tsv/transformer.rb +5 -3
  45. data/lib/scout/tsv/traverse.rb +28 -19
  46. data/lib/scout/tsv/util/process.rb +7 -0
  47. data/lib/scout/tsv/util/reorder.rb +25 -15
  48. data/lib/scout/tsv/util/select.rb +9 -1
  49. data/lib/scout/tsv/util/sort.rb +90 -2
  50. data/lib/scout/tsv/util/unzip.rb +56 -0
  51. data/lib/scout/tsv/util.rb +52 -5
  52. data/lib/scout/tsv.rb +42 -27
  53. data/lib/scout/work_queue/socket.rb +8 -0
  54. data/lib/scout/work_queue/worker.rb +22 -5
  55. data/lib/scout/work_queue.rb +41 -24
  56. data/lib/scout/workflow/definition.rb +15 -12
  57. data/lib/scout/workflow/deployment/orchestrator.rb +21 -3
  58. data/lib/scout/workflow/deployment/trace.rb +205 -0
  59. data/lib/scout/workflow/deployment.rb +1 -0
  60. data/lib/scout/workflow/documentation.rb +1 -1
  61. data/lib/scout/workflow/step/archive.rb +42 -0
  62. data/lib/scout/workflow/step/children.rb +51 -0
  63. data/lib/scout/workflow/step/config.rb +1 -1
  64. data/lib/scout/workflow/step/dependencies.rb +25 -8
  65. data/lib/scout/workflow/step/file.rb +19 -0
  66. data/lib/scout/workflow/step/info.rb +37 -9
  67. data/lib/scout/workflow/step/progress.rb +11 -2
  68. data/lib/scout/workflow/step/status.rb +9 -1
  69. data/lib/scout/workflow/step.rb +80 -25
  70. data/lib/scout/workflow/task/dependencies.rb +5 -2
  71. data/lib/scout/workflow/task/inputs.rb +91 -41
  72. data/lib/scout/workflow/task.rb +54 -57
  73. data/lib/scout/workflow/usage.rb +1 -1
  74. data/lib/scout/workflow/util.rb +4 -0
  75. data/lib/scout/workflow.rb +110 -13
  76. data/lib/scout-gear.rb +2 -0
  77. data/lib/scout.rb +0 -1
  78. data/scout-gear.gemspec +78 -23
  79. data/scout_commands/rbbt +2 -0
  80. data/test/data/person/brothers +4 -0
  81. data/test/data/person/identifiers +10 -0
  82. data/test/data/person/marriages +3 -0
  83. data/test/data/person/parents +6 -0
  84. data/test/scout/association/test_fields.rb +105 -0
  85. data/test/scout/association/test_index.rb +70 -0
  86. data/test/scout/association/test_item.rb +21 -0
  87. data/test/scout/entity/test_format.rb +19 -0
  88. data/test/scout/entity/test_identifiers.rb +58 -0
  89. data/test/scout/entity/test_object.rb +0 -0
  90. data/test/scout/entity/test_property.rb +345 -0
  91. data/test/scout/{tsv/persist → persist/engine}/test_fix_width_table.rb +0 -1
  92. data/test/scout/persist/engine/test_packed_index.rb +99 -0
  93. data/test/scout/persist/engine/test_sharder.rb +31 -0
  94. data/test/scout/persist/engine/test_tkrzw.rb +0 -0
  95. data/test/scout/persist/engine/test_tokyocabinet.rb +17 -0
  96. data/test/scout/persist/test_tsv.rb +146 -0
  97. data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb} +3 -4
  98. data/test/scout/persist/tsv/adapter/test_fix_width_table.rb +46 -0
  99. data/test/scout/persist/tsv/adapter/test_packed_index.rb +37 -0
  100. data/test/scout/persist/tsv/adapter/test_serialize.rb +0 -0
  101. data/test/scout/persist/tsv/adapter/test_sharder.rb +290 -0
  102. data/test/scout/{tsv/persist → persist/tsv/adapter}/test_tkrzw.rb +3 -6
  103. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +282 -0
  104. data/test/scout/persist/tsv/test_serialize.rb +12 -0
  105. data/test/scout/test_association.rb +51 -0
  106. data/test/scout/test_entity.rb +40 -0
  107. data/test/scout/test_tsv.rb +33 -4
  108. data/test/scout/test_work_queue.rb +5 -2
  109. data/test/scout/test_workflow.rb +31 -14
  110. data/test/scout/tsv/annotation/test_repo.rb +150 -0
  111. data/test/scout/tsv/change_id/test_translate.rb +178 -0
  112. data/test/scout/tsv/test_annotation.rb +52 -0
  113. data/test/scout/tsv/test_attach.rb +255 -1
  114. data/test/scout/tsv/test_change_id.rb +25 -0
  115. data/test/scout/tsv/test_csv.rb +50 -0
  116. data/test/scout/tsv/test_dumper.rb +38 -0
  117. data/test/scout/tsv/test_index.rb +82 -0
  118. data/test/scout/tsv/test_open.rb +44 -0
  119. data/test/scout/tsv/test_parser.rb +70 -0
  120. data/test/scout/tsv/test_stream.rb +22 -0
  121. data/test/scout/tsv/test_transformer.rb +27 -3
  122. data/test/scout/tsv/test_traverse.rb +78 -0
  123. data/test/scout/tsv/util/test_process.rb +16 -0
  124. data/test/scout/tsv/util/test_reorder.rb +67 -0
  125. data/test/scout/tsv/util/test_sort.rb +28 -1
  126. data/test/scout/tsv/util/test_unzip.rb +32 -0
  127. data/test/scout/work_queue/test_socket.rb +4 -1
  128. data/test/scout/workflow/deployment/test_orchestrator.rb +17 -26
  129. data/test/scout/workflow/deployment/test_trace.rb +25 -0
  130. data/test/scout/workflow/step/test_archive.rb +28 -0
  131. data/test/scout/workflow/step/test_children.rb +25 -0
  132. data/test/scout/workflow/step/test_info.rb +16 -0
  133. data/test/scout/workflow/task/test_dependencies.rb +16 -16
  134. data/test/scout/workflow/task/test_inputs.rb +45 -1
  135. data/test/scout/workflow/test_definition.rb +52 -0
  136. data/test/scout/workflow/test_step.rb +57 -0
  137. data/test/scout/workflow/test_task.rb +26 -1
  138. data/test/scout/workflow/test_usage.rb +4 -4
  139. data/test/test_helper.rb +23 -1
  140. metadata +69 -14
  141. data/lib/scout/tsv/persist.rb +0 -27
  142. data/test/scout/tsv/persist/test_tokyocabinet.rb +0 -120
  143. data/test/scout/tsv/test_persist.rb +0 -45
data/Rakefile CHANGED
@@ -39,6 +39,7 @@ Rake::TestTask.new(:test) do |test|
39
39
  test.libs << 'lib' << 'test'
40
40
  test.pattern = 'test/**/test_*.rb'
41
41
  test.verbose = true
42
+ test.warning = false
42
43
  end
43
44
 
44
45
  desc "Code coverage detail"
data/VERSION CHANGED
@@ -1 +1 @@
1
- 10.4.0
1
+ 10.7.0
data/bin/scout CHANGED
@@ -6,8 +6,6 @@ ENV["SCOUT_NOCOLOR"] = "true" if ARGV.include? "--nocolor"
6
6
 
7
7
  ENV["SCOUT_NO_PROGRESS"] = "true" if ARGV.include? "--nobar"
8
8
 
9
- require 'scout-gear'
10
-
11
9
  class CmdStop < Exception
12
10
  attr_accessor :exit_status
13
11
  def initialize(exit_status = 0)
@@ -54,7 +52,7 @@ if dev_dir
54
52
  end
55
53
  end
56
54
 
57
- require 'scout'
55
+ require 'scout-gear'
58
56
 
59
57
  require 'scout/simple_opt'
60
58
 
@@ -0,0 +1,170 @@
1
+ require 'scout/entity'
2
+
3
+ module Association
4
+
5
+ def self.identify_entity_format(format, fields)
6
+ entity_type = Entity.formats[format]
7
+ raise "Field #{ format } could not be resolved: #{fields}" if entity_type.nil?
8
+ main_field = fields.select{|f| Entity.formats[f] == entity_type}.first
9
+ raise "Field #{ format } not present, options: #{Log.fingerprint fields}" if main_field.nil?
10
+ [main_field, nil, format]
11
+ end
12
+
13
+ def self.parse_field_specification(spec)
14
+ return [spec,nil,nil] if Numeric === spec
15
+ spec = spec.split "=>" unless Array === spec
16
+ field_part, final_format = spec
17
+
18
+ field, format = field_part.split "=~", -1
19
+
20
+ field = nil if field.nil? or field.empty?
21
+
22
+ [field, format, final_format]
23
+ end
24
+
25
+ def self.normalize_specs(spec, all_fields = nil)
26
+ return nil if spec.nil?
27
+ field, header, format = parse_field_specification spec
28
+
29
+ specs = if all_fields.nil? or all_fields.include? field
30
+ [field, header, format]
31
+ else
32
+ if all_fields.nil?
33
+ begin
34
+ identify_entity_format field, all_fields
35
+ rescue
36
+ [field, header, format]
37
+ end
38
+ else
39
+ [field, header, format]
40
+ end
41
+ end
42
+ specs
43
+ end
44
+
45
+ def self.extract_specs(all_fields=nil, options = {})
46
+ source, source_format, target, target_format, format = IndiferentHash.process_options options, :source, :source_format, :target, :target_format, :format
47
+
48
+ key_field, *fields = all_fields.nil? ? [nil] : all_fields
49
+
50
+ source_specs = normalize_specs source, all_fields
51
+ target_specs = normalize_specs target, all_fields
52
+
53
+ source_specs = [nil, nil, nil] if source_specs.nil?
54
+ target_specs = [nil, nil, nil] if target_specs.nil?
55
+
56
+ source_specs[2] = source_format if source_format
57
+ target_specs[2] = target_format if target_format
58
+
59
+ if source_specs.first and not all_fields.include? source_specs.first and defined? Entity and (_format = Entity.formats[source_specs.first.to_s])
60
+ _source = all_fields.select{|f| Entity.formats[f.to_s] == _format }.first
61
+ raise "Source not found #{source_specs}. Options: #{Log.fingerprint all_fields}" if _source.nil?
62
+ source_specs[0] = _source
63
+ end
64
+
65
+ if target_specs.first and not all_fields.include? target_specs.first and defined? Entity and (_format = Entity.formats[target_specs.first.to_s])
66
+ _target = all_fields.select{|f| Entity.formats[f.to_s].to_s == _format.to_s }.first
67
+ raise "Target not found #{target_specs}. Options: #{Log.fingerprint all_fields}" if _target.nil?
68
+ target_specs[0] = _target
69
+ end
70
+
71
+ if source_specs[0].nil? and target_specs[0].nil?
72
+ source_specs[0] = key_field
73
+ target_specs[0] = fields[0]
74
+ elsif source_specs[0].nil?
75
+ if target_specs[0] == :key or target_specs[0] == key_field
76
+ source_specs[0] = fields[0]
77
+ else
78
+ source_specs[0] = key_field
79
+ end
80
+ elsif target_specs[0].nil?
81
+ if source_specs[0] == fields.first
82
+ target_specs[0] = key_field
83
+ else
84
+ target_specs[0] = fields.first
85
+ end
86
+ end
87
+
88
+ # If format is specified, then perhaps we need to change the
89
+ if target_specs[2].nil?
90
+ target_type = Entity.formats[target_specs[1] || target_specs[0]]
91
+ target_specs[2] = format[target_type.to_s] if format
92
+ target_specs[2] = nil if target_specs[2] == target_specs[0] or target_specs[2] == target_specs[1]
93
+ end
94
+
95
+ if source_specs[2].nil?
96
+ source_type = Entity.formats[source_specs[1] || source_specs[0]]
97
+ source_specs[2] = format[source_type.to_s] if format
98
+ source_specs[2] = nil if source_specs[2] == source_specs[0] or source_specs[2] == source_specs[1]
99
+ end
100
+
101
+ {:source => source_specs, :target => target_specs}
102
+ end
103
+
104
+ def self.process_formats(field, default_format = {})
105
+ return nil if default_format.nil? or default_format.empty?
106
+ default_format.each do |type, format|
107
+ entity_type = Entity.formats[field] || format
108
+ return format if entity_type.to_s === type
109
+ end
110
+ return nil
111
+ end
112
+
113
+ def self.headers(all_fields, info_fields = nil, options = {})
114
+ specs = extract_specs all_fields, options
115
+
116
+ source_field = specs[:source][0]
117
+ target_field = specs[:target][0]
118
+
119
+ #source_pos = all_fields.index source_field
120
+ #target_pos = all_fields.index target_field
121
+
122
+ source_pos = TSV.identify_field all_fields.first, all_fields[1..-1], source_field
123
+ target_pos = TSV.identify_field all_fields.first, all_fields[1..-1], target_field
124
+
125
+ source_pos = source_pos == :key ? 0 : source_pos + 1
126
+ target_pos = target_pos == :key ? 0 : target_pos + 1
127
+
128
+ source_header = specs[:source][1] || specs[:source][0]
129
+ target_header = specs[:target][1] || specs[:target][0]
130
+
131
+ info_fields = all_fields.dup if info_fields.nil?
132
+ info_fields.delete_at NamedArray.identify_name(info_fields, source_field) if NamedArray.identify_name(info_fields, source_field)
133
+ info_fields.delete_at NamedArray.identify_name(info_fields, target_field) if NamedArray.identify_name(info_fields, target_field)
134
+ info_fields.unshift target_field
135
+
136
+ field_headers = [target_header]
137
+ info_fields[1..-1].each do |field|
138
+ header = case field
139
+ when String
140
+ field
141
+ when Numeric
142
+ all_fields[field]
143
+ when :key
144
+ all_fields.first
145
+ end
146
+
147
+ field_headers << header
148
+ end
149
+
150
+ field_pos = info_fields.collect do |f|
151
+ p = TSV.identify_field all_fields.first, all_fields[1..-1], f
152
+ p == :key ? 0 : p + 1
153
+ end
154
+
155
+ field_pos.delete source_pos
156
+
157
+ source_format = specs[:source][2]
158
+ target_format = specs[:target][2]
159
+
160
+
161
+ if format = options[:format]
162
+ source_format = process_formats(specs[:source][1] || specs[:source][0], format) || source_format unless source_format
163
+ target_format = process_formats(specs[:target][1] || specs[:target][0], format) || target_format unless target_format
164
+ end
165
+
166
+ res = [source_pos, field_pos, source_header, field_headers, source_format, target_format]
167
+ Log.low "Headers -- #{res}"
168
+ res
169
+ end
170
+ end
@@ -0,0 +1,229 @@
1
+ require 'scout/annotation'
2
+ module Association
3
+
4
+ def self.index(file, source: nil, target: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
5
+ persist_options = IndiferentHash.pull_keys kwargs, :persist
6
+ index_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true,
7
+ prefix: "Association::Index",
8
+ other_options: kwargs.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format)
9
+
10
+ index = Persist.tsv(file, kwargs, engine: "BDB", persist_options: index_persist_options) do |data|
11
+ recycle, undirected = IndiferentHash.process_options kwargs, :recycle, :undirected
12
+
13
+ database = Association.open(file, source: source, target: target, source_format: source_format, target_format: target_format, **kwargs.merge(persist_prefix: "Association::Database"))
14
+
15
+ source_field = database.key_field
16
+ target_field, *fields = database.fields
17
+
18
+ undirected = true if undirected.nil? and source_field == target_field
19
+
20
+ key_field = [source_field, target_field, undirected ? "undirected" : nil].compact * "~"
21
+
22
+ dumper = TSV::Dumper.new database.options.merge(key_field: key_field, fields: fields, type: :list)
23
+ transformer = TSV::Transformer.new database, dumper
24
+
25
+ if database.type == :double
26
+ transformer.traverse do |source,value_list|
27
+ res = []
28
+ NamedArray.zip_fields(value_list).collect do |values|
29
+ target, *info = values
30
+ key = [source, target] * "~"
31
+ res << [key, info]
32
+ if undirected
33
+ key = [target, source] * "~"
34
+ res << [key, info]
35
+ end
36
+ end
37
+ res.extend MultipleResult
38
+ end
39
+ elsif database.type == :flat
40
+ transformer.traverse do |source,targets|
41
+ res = []
42
+ res.extend MultipleResult
43
+ targets.each do |target|
44
+ key = [source, target] * "~"
45
+ res << [key, []]
46
+ if undirected
47
+ key = [target, source] * "~"
48
+ res << [key, []]
49
+ end
50
+ end
51
+ res
52
+ end
53
+ else
54
+ transformer.traverse do |source,values|
55
+ res = []
56
+ res.extend MultipleResult
57
+ target, *info = values
58
+ key = [source, target] * "~"
59
+ res << [key, info]
60
+ if undirected
61
+ key = [target, source] * "~"
62
+ res << [key, info]
63
+ end
64
+ res
65
+ end
66
+ end
67
+
68
+ tsv = transformer.tsv **kwargs.merge(data: data, fields: fields)
69
+ end
70
+ index.extend Index
71
+ index.parse_key_field
72
+ index
73
+ end
74
+
75
+ module Index
76
+ extend Annotation
77
+
78
+ annotation :source_field, :target_field, :undirected
79
+
80
+ def parse_key_field
81
+ @source_field, @target_field, @undirected = key_field.split("~")
82
+ end
83
+
84
+ def match(entity)
85
+ return entity.inject([]){|acc,e| acc.concat match(e); acc } if Array === entity
86
+ return [] if entity.nil?
87
+ prefix(entity + "~")
88
+ end
89
+
90
+ def subset(source, target)
91
+ return [] if source.nil? or target.nil? or source.empty? or target.empty?
92
+
93
+ if source == :all or source == "all"
94
+ if target == :all or target == "all"
95
+ return keys
96
+ else
97
+ matches = reverse.subset(target, source)
98
+ return matches.collect{|m| r = m.partition "~"; r.reverse*"" }
99
+ end
100
+ end
101
+
102
+ matches = source.uniq.inject([]){|acc,e|
103
+ if block_given?
104
+ acc.concat(match(e))
105
+ else
106
+ acc.concat(match(e))
107
+ end
108
+ }
109
+
110
+ return matches if target == :all or target == "all"
111
+
112
+ target_matches = {}
113
+
114
+ matches.each{|code|
115
+ s,sep,t = code.partition "~"
116
+ next if undirected and t > s and source.include? t
117
+ target_matches[t] ||= []
118
+ target_matches[t] << code
119
+ }
120
+
121
+ target_matches.values_at(*target.uniq).flatten.compact
122
+ end
123
+
124
+ def reverse
125
+ @reverse ||= begin
126
+ if self.respond_to? :persistence_path
127
+ persistence_path = self.persistence_path
128
+ persistence_path = persistence_path.find if Path === persistence_path
129
+ reverse_filename = persistence_path + '.reverse'
130
+ else
131
+ raise "Can only reverse a TokyoCabinet::BDB dataset at the time"
132
+ end
133
+
134
+ if Open.exist?(reverse_filename)
135
+ new = Persist.open_tokyocabinet(reverse_filename, false, serializer, TokyoCabinet::BDB)
136
+ raise "Index has no info: #{reverse_filename}" if new.key_field.nil?
137
+ new.extend Index
138
+ new
139
+ else
140
+ FileUtils.mkdir_p File.dirname(reverse_filename) unless File.exist?(File.dirname(reverse_filename))
141
+
142
+ new = Persist.open_tokyocabinet(reverse_filename, true, serializer, TokyoCabinet::BDB)
143
+
144
+ self.with_unnamed do
145
+ self.traverse do |key, value|
146
+ new_key = key.split("~").reverse.join("~")
147
+ new[new_key] = value
148
+ end
149
+ end
150
+ annotate(new)
151
+ new.key_field = key_field.split("~").values_at(1,0,2).compact * "~"
152
+ new.save_annotation_hash
153
+ new.read_and_close do
154
+ Association::Index.setup new
155
+ end
156
+ new.parse_key_field
157
+ new.read
158
+ end
159
+
160
+ new.unnamed = true
161
+
162
+ new.undirected = undirected
163
+
164
+ new
165
+ rescue Exception
166
+ Log.error "Deleting after error reversing database: #{ reverse_filename }"
167
+ FileUtils.rm reverse_filename if File.exist? reverse_filename
168
+ raise $!
169
+ end
170
+ end
171
+
172
+ def filter(value_field = nil, target_value = nil, &block)
173
+ if block_given?
174
+ matches = []
175
+ if value_field
176
+ through :key, value_field do |key,values|
177
+ pass = block.call values
178
+ matches << key if pass
179
+ end
180
+ else
181
+ through do |key,values|
182
+ pass = block.call [key, values]
183
+ matches << key if pass
184
+ end
185
+ end
186
+ matches
187
+
188
+ else
189
+ matches = []
190
+ if target_value
191
+ target_value = [target_value] unless Array === target_value
192
+ through :key, value_field do |key,values|
193
+ pass = (values & target_value).any?
194
+ matches << key if pass
195
+ end
196
+ else
197
+ through :key, value_field do |key,values|
198
+ pass = false
199
+ values.each do |value|
200
+ pass = true unless value.nil? or value.empty? or value.downcase == 'false'
201
+ end
202
+ matches << key if pass
203
+ end
204
+ end
205
+ matches
206
+ end
207
+ end
208
+
209
+ def to_matrix(value_field = nil, &block)
210
+ value_field = fields.first if value_field.nil? and fields.length == 1
211
+ value_pos = identify_field value_field if value_field and String === value_field
212
+ key_field = source_field
213
+
214
+ tsv = if value_pos
215
+ AssociationItem.incidence self.keys, key_field do |key|
216
+ if block_given?
217
+ yield self[key][value_pos]
218
+ else
219
+ self[key][value_pos]
220
+ end
221
+ end
222
+ elsif block_given?
223
+ AssociationItem.incidence self.keys, key_field, &block
224
+ else
225
+ AssociationItem.incidence self.keys, key_field
226
+ end
227
+ end
228
+ end
229
+ end
@@ -0,0 +1,227 @@
1
+ require_relative '../entity'
2
+
3
+ module AssociationItem
4
+ extend Entity
5
+
6
+ annotation :knowledge_base
7
+ annotation :database
8
+ annotation :reverse
9
+
10
+ property :name => :single do
11
+ [source_entity, target_entity].collect{|e| e.respond_to?(:name)? e.name || e : e } * "~"
12
+ end
13
+
14
+ property :full_name => :single do
15
+ database ? [database, name] * ":" : name
16
+ end
17
+
18
+ property :invert => :both do
19
+ if Array === self
20
+ inverted = self.collect do |item|
21
+ s,_sep,t= item.partition "~"
22
+ new = [t,s] * _sep
23
+ end
24
+ self.annotate inverted
25
+ inverted.reverse = ! reverse
26
+ inverted
27
+ else
28
+ s,_sep,t= self.partition "~"
29
+ inverted = self.annotate([t,s] * _sep)
30
+ inverted.reverse = ! reverse
31
+ inverted
32
+ end
33
+ end
34
+
35
+ property :namespace => :both do
36
+ knowledge_base.namespace
37
+ end
38
+
39
+ property :part => :array2single do
40
+ self.purge.collect{|p| p.partition("~") }
41
+ end
42
+
43
+ property :target => :array2single do
44
+ self.part.collect{|p| p[2]}
45
+ end
46
+
47
+ property :source => :array2single do
48
+ self.purge.collect{|p| p[/[^~]+/] }
49
+ end
50
+
51
+ property :target_entity_type => :both do
52
+ Entity.formats[target_type].to_s
53
+ end
54
+
55
+ property :source_entity_type => :both do
56
+ Entity.formats[source_type].to_s
57
+ end
58
+
59
+ property :target_type => :both do
60
+ if reverse
61
+ knowledge_base.source(database)
62
+ else
63
+ knowledge_base.target(database)
64
+ end
65
+ end
66
+
67
+ property :source_type => :both do
68
+ if reverse
69
+ knowledge_base.target(database)
70
+ else
71
+ knowledge_base.source(database)
72
+ end
73
+ end
74
+
75
+ property :undirected => :both do
76
+ knowledge_base.undirected(database)
77
+ end
78
+
79
+ property :target_entity => :array2single do
80
+ type = target_type
81
+ knowledge_base.annotate self.target, type, database #if self.target.any?
82
+ end
83
+
84
+ property :source_entity => :array2single do
85
+ type = source_type
86
+ knowledge_base.annotate self.source, type, database #if self.source.any?
87
+ end
88
+
89
+ property :index => :both do |database|
90
+ @index ||= knowledge_base.get_index(database)
91
+ end
92
+ property :value => :array2single do
93
+ index = index(database)
94
+ value = self.reverse ? index.chunked_values_at(self.invert) : index.chunked_values_at(self)
95
+ value.collect{|v| NamedArray.setup(v, index.fields)}
96
+ end
97
+
98
+ property :info_fields => :both do
99
+ knowledge_base.index_fields(database)
100
+ end
101
+
102
+ property :info => :array2single do
103
+ fields = self.info_fields
104
+
105
+ return [{}] * self.length if fields.nil? or fields.empty?
106
+
107
+ value = self.value
108
+ value.collect{|v|
109
+ raise "No info for pair; not registered in index" if v.nil?
110
+ Hash[*fields.zip(v).flatten]
111
+ }
112
+ end
113
+
114
+ property :tsv => :array do
115
+ info_fields = self.info_fields
116
+ fields = [self.source_type, self.target_type].concat info_fields
117
+ type = [self.source_type, self.target_type] * "~"
118
+ tsv = TSV.setup({}, :key_field => type, :fields => fields, :type => :list, :namespace => self.namespace)
119
+ index = index(database)
120
+ index.with_unnamed do
121
+ index.chunked_values_at(self).each_with_index do |v,i|
122
+ p = self[i]
123
+ source, _sep, target = p.partition("~")
124
+ if info_fields.empty?
125
+ tsv[p] = [source, target]
126
+ else
127
+ tsv[p] = [source, target].concat v
128
+ end
129
+ end
130
+ end
131
+ tsv.entity_options = {:namespace => namespace}
132
+ knowledge_base.entity_options.each do |type,options|
133
+ tsv.entity_options.merge! options
134
+ end
135
+ tsv
136
+ end
137
+
138
+ property :filter => :array do |*args,&block|
139
+ block = args.pop if Proc === args.last
140
+ keys = tsv.with_unnamed do tsv.select(*args, &block).keys end
141
+ keys = self.annotate keys
142
+ keys
143
+ end
144
+
145
+ def self.incidence(pairs, key_field = nil, &block)
146
+ matrix = {}
147
+ targets = []
148
+ sources = []
149
+ matches = {}
150
+
151
+ pairs.inject([]){|acc,m| acc << m; acc << m.invert if m.respond_to?(:undirected) and m.undirected; acc }.each do |p|
152
+ s, sep, t = p.partition "~"
153
+
154
+ sources << s
155
+ targets << t
156
+ if block_given?
157
+ matches[s] ||= Hash.new{nil}
158
+ value = block.call p
159
+ matches[s][t] = value unless value.nil? or (mv = matches[s][t] and value > mv)
160
+ else
161
+ matches[s] ||= Hash.new{false}
162
+ matches[s][t] ||= true
163
+ end
164
+ end
165
+
166
+ sources.uniq!
167
+ targets = targets.uniq.sort
168
+
169
+ matches.each do |s,hash|
170
+ matrix[s] = hash.values_at(*targets)
171
+ end
172
+
173
+ defined?(TSV)? TSV.setup(matrix, :key_field => (key_field || "Source") , :fields => targets, :type => :list) : matrix
174
+ end
175
+
176
+ def self.adjacency(pairs, key_field = nil, &block)
177
+ incidence = incidence(pairs, key_field, &block)
178
+
179
+ targets = incidence.fields
180
+ adjacency = TSV.setup({}, :key_field => incidence.key_field, :fields => ["Target"], :type => :double)
181
+ TSV.traverse incidence, :into => adjacency, :unnamed => true do |k,values|
182
+ target_values = targets.zip(values).reject{|t,v| v.nil? }.collect{|t,v| [t,v]}
183
+ next if target_values.empty?
184
+ [k, Misc.zip_fields(target_values)]
185
+ end
186
+ end
187
+
188
+ def self._select_match(orig, elem)
189
+ if Array === orig and Array === elem
190
+ (orig & elem).any?
191
+ elsif Array === orig
192
+ orig.include? elem
193
+ elsif Array === elem
194
+ elem.include? orig
195
+ else
196
+ elem === orif
197
+ end
198
+ false
199
+ end
200
+
201
+ def self.select(list, method = nil, &block)
202
+ if method and method.any?
203
+ list.select do |item|
204
+ method.collect do |key,value|
205
+ case key
206
+ when :target
207
+ _select_match item.target, value
208
+ when :source
209
+ _select_match item.source, value
210
+ else
211
+ orig = item.info[key]
212
+ orig = orig.split(";;") if String orig
213
+ _select_match orig, value
214
+ end
215
+ end
216
+ end
217
+ else
218
+ list.select(&block)
219
+ end
220
+ end
221
+ end
222
+
223
+ module TSV
224
+ def self.incidence(tsv, **kwargs)
225
+ AssociationItem.incidence Association.index(tsv, **kwargs).keys
226
+ end
227
+ end
@@ -0,0 +1,7 @@
1
+ module Association
2
+ def self.version_file(file, namespace)
3
+ old_file, file = file, file.sub(Entity::Identified::NAMESPACE_TAG, namespace) if namespace and String === file
4
+ old_file.annotate file if Path === old_file
5
+ file
6
+ end
7
+ end