gd_es 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +6 -0
- data/bin/es +375 -0
- data/es.rdoc +5 -0
- data/lib/commands.rb +406 -0
- data/lib/es.rb +1077 -0
- data/lib/es_version.rb +3 -0
- data/lib/templates/deleted_records.jsonify +12 -0
- data/lib/templates/extract_map.jsonify +41 -0
- data/lib/templates/extract_task.jsonify +13 -0
- data/lib/templates/load.jsonify +8 -0
- metadata +271 -0
data/lib/es.rb
ADDED
@@ -0,0 +1,1077 @@
|
|
1
|
+
require 'pry'
|
2
|
+
require 'chronic'
|
3
|
+
require 'jsonify'
|
4
|
+
require 'json'
|
5
|
+
require 'rainbow'
|
6
|
+
require 'yajl'
|
7
|
+
require 'active_support/time'
|
8
|
+
require 'active_support/ordered_hash'
|
9
|
+
require 'terminal-table'
|
10
|
+
require 'pathname'
|
11
|
+
require 'tempfile'
|
12
|
+
require 'commands'
|
13
|
+
|
14
|
+
module Es
|
15
|
+
|
16
|
+
class InsufficientSpecificationError < RuntimeError
|
17
|
+
end
|
18
|
+
|
19
|
+
class IncorrectSpecificationError < RuntimeError
|
20
|
+
end
|
21
|
+
|
22
|
+
class UnableToMerge < RuntimeError
|
23
|
+
end
|
24
|
+
|
25
|
+
class Timeframe
|
26
|
+
INTERVAL_UNITS = [:day, :week, :month, :year]
|
27
|
+
DAY_WITHIN_PERIOD = [:first, :last]
|
28
|
+
attr_accessor :to, :from, :interval_unit, :interval, :day_within_period, :spec_from, :spec_to
|
29
|
+
|
30
|
+
def self.parse(spec, options={})
|
31
|
+
if spec == 'latest' then
|
32
|
+
Timeframe.new({
|
33
|
+
:to => 'tomorrow',
|
34
|
+
:from => 'yesterday'
|
35
|
+
}, options)
|
36
|
+
else
|
37
|
+
Timeframe.new(spec, options)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.parseOldFormat(spec, options={})
|
42
|
+
if spec == 'latest' then
|
43
|
+
Timeframe.new({
|
44
|
+
:to => 'today',
|
45
|
+
:from => 'yesterday'
|
46
|
+
})
|
47
|
+
else
|
48
|
+
Timeframe.new({
|
49
|
+
:to => spec[:endDate],
|
50
|
+
:from => spec[:startDate],
|
51
|
+
:interval_unit => spec[:intervalUnit],
|
52
|
+
:interval => spec[:interval],
|
53
|
+
:day_within_period => spec[:dayWithinPeriod].downcase
|
54
|
+
})
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def initialize(spec,options = {})
|
59
|
+
validate_spec(spec)
|
60
|
+
@now = options[:now] || Time.now
|
61
|
+
@spec = spec
|
62
|
+
@to = Chronic.parse(spec[:to], :now => @now)
|
63
|
+
@from = spec[:from] ? Chronic.parse(spec[:from], :now => @now) : to.advance(:days => -1)
|
64
|
+
@spec_from = spec[:from]
|
65
|
+
@spec_to = spec[:to]
|
66
|
+
@interval_unit = spec[:interval_unit] || :day
|
67
|
+
@interval = spec[:interval] || 1
|
68
|
+
@day_within_period = spec[:day_within_period] || :last
|
69
|
+
end
|
70
|
+
|
71
|
+
|
72
|
+
def validate_spec(spec)
|
73
|
+
fail IncorrectSpecificationError.new("Timeframe should have a specification") if spec.nil?
|
74
|
+
fail InsufficientSpecificationError.new("To key was not specified during the Timeframe creation") unless spec.has_key?(:to)
|
75
|
+
fail InsufficientSpecificationError.new("From key was not specified during the Timeframe creation") unless spec.has_key?(:from)
|
76
|
+
fail IncorrectSpecificationError.new("Interval key should be a number") if spec[:interval] && !spec[:interval].is_a?(Fixnum)
|
77
|
+
fail IncorrectSpecificationError.new("Interval_unit key should be one of :day, :week, :month, :year") if spec[:interval_unit] && !INTERVAL_UNITS.include?(spec[:interval_unit].to_sym)
|
78
|
+
fail IncorrectSpecificationError.new("Day within period should be one of #{DAY_WITHIN_PERIOD.join(', ')}") if spec[:day_within_period] && !DAY_WITHIN_PERIOD.include?(spec[:day_within_period].to_sym)
|
79
|
+
end
|
80
|
+
|
81
|
+
def to_extract_fragment(pid, options = {})
|
82
|
+
{
|
83
|
+
:endDate => to.strftime('%Y-%m-%d'),
|
84
|
+
:startDate => from.strftime('%Y-%m-%d'),
|
85
|
+
:intervalUnit => interval_unit,
|
86
|
+
:dayWithinPeriod => day_within_period.to_s.upcase,
|
87
|
+
:interval => interval
|
88
|
+
}
|
89
|
+
end
|
90
|
+
|
91
|
+
|
92
|
+
def to_config_generator_extract
|
93
|
+
{
|
94
|
+
:from => from.strftime('%Y-%m-%d'),
|
95
|
+
:to => to.strftime('%Y-%m-%d'),
|
96
|
+
:interval_unit => interval_unit,
|
97
|
+
:interval => day_within_period.to_s.upcase,
|
98
|
+
:day_within_period => interval
|
99
|
+
}
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
|
104
|
+
class Extract
|
105
|
+
|
106
|
+
attr_accessor :entities, :timeframe, :timezone, :partial
|
107
|
+
|
108
|
+
def self.parse(spec, a_load, options={})
|
109
|
+
global_timeframe = parse_timeframes(spec[:timeframes], options) || parse_timeframes("latest", options)
|
110
|
+
timezone = spec[:timezone]
|
111
|
+
parsed_entities = spec[:entities].map do |entity_spec|
|
112
|
+
entity_name = entity_spec[:entity]
|
113
|
+
partial = entity_spec[:partial] || "false"
|
114
|
+
load_entity = a_load.get_merged_entity_for(entity_name)
|
115
|
+
fields = entity_spec[:fields].map do |field|
|
116
|
+
if load_entity.has_field?(field)
|
117
|
+
if (load_entity.get_field(field).is_recordid?)
|
118
|
+
Es::RecordIdField.new(load_entity.get_field(field).name, load_entity.get_field(field).type, partial)
|
119
|
+
else
|
120
|
+
load_entity.get_field(field)
|
121
|
+
end
|
122
|
+
elsif field == "DeletedAt"
|
123
|
+
Es::Field.new("DeletedAt", "time")
|
124
|
+
elsif field == "IsDeleted"
|
125
|
+
Es::Field.new("IsDeleted", "attribute")
|
126
|
+
elsif field == "snapshot"
|
127
|
+
Es::SnapshotField.new("snapshot", "snapshot")
|
128
|
+
elsif field == "autoincrement"
|
129
|
+
Es::AutoincrementField.new("generate", "autoincrement")
|
130
|
+
elsif field == "duration" || (field.respond_to?(:keys) && field.keys.first == :duration )
|
131
|
+
if (field == "duration") then
|
132
|
+
Es::DurationField.new("duration", "duration")
|
133
|
+
else
|
134
|
+
Es::DurationField.new("duration", "duration",{
|
135
|
+
:attribute => field[:duration][:attribute],
|
136
|
+
:value => field[:duration][:value],
|
137
|
+
:control_attribute => field[:duration][:name]
|
138
|
+
})
|
139
|
+
end
|
140
|
+
elsif field == "velocity" || (field.respond_to?(:keys) && field.keys.first == :velocity )
|
141
|
+
if (field == "velocity") then
|
142
|
+
Es::VelocityField.new("velocity", "velocity")
|
143
|
+
else
|
144
|
+
Es::VelocityField.new("velocity", "velocity",{
|
145
|
+
:control_attribute => field[:velocity][:name]
|
146
|
+
})
|
147
|
+
end
|
148
|
+
elsif field.respond_to?(:keys) && field.keys.first == :hid
|
149
|
+
Es::HIDField.new('hid', "historicid", {
|
150
|
+
:entity => field[:hid][:from_entity],
|
151
|
+
:fields => field[:hid][:from_fields],
|
152
|
+
:through => field[:hid][:connected_through]
|
153
|
+
})
|
154
|
+
else
|
155
|
+
fail InsufficientSpecificationError.new("The field #{field.to_s.bright} was not found in either the loading specification nor was recognized as a special column")
|
156
|
+
end
|
157
|
+
end
|
158
|
+
parsed_timeframe = parse_timeframes(entity_spec[:timeframes], options)
|
159
|
+
Entity.new(entity_name, {
|
160
|
+
:fields => fields,
|
161
|
+
:file => Pathname.new(entity_spec[:file]).expand_path.to_s,
|
162
|
+
:timeframe => parsed_timeframe || global_timeframe || (fail "Timeframe has to be defined"),
|
163
|
+
:timezone => timezone
|
164
|
+
})
|
165
|
+
end
|
166
|
+
|
167
|
+
Extract.new(parsed_entities)
|
168
|
+
end
|
169
|
+
|
170
|
+
|
171
|
+
def self.parseOldFormat(spec,a_load)
|
172
|
+
global_timeframe = parseOldFormat_timeframes(spec[:timeFrames])
|
173
|
+
timezone = spec[:timezone]
|
174
|
+
entity_name = spec[:entity]
|
175
|
+
load_entity = a_load.get_merged_entity_for(entity_name)
|
176
|
+
parser = Yajl::Parser.new(:symbolize_keys => true)
|
177
|
+
i=0
|
178
|
+
begin
|
179
|
+
doc = parser.parse(spec[:readMap])
|
180
|
+
doc.map do |internal|
|
181
|
+
fields = internal[:columns].map do |definition|
|
182
|
+
if load_entity.has_field?(definition[:name])
|
183
|
+
load_entity.get_field(definition[:name])
|
184
|
+
elsif definition[:name] == "DeletedAt"
|
185
|
+
Es::Field.new("DeletedAt", "time")
|
186
|
+
elsif definition[:name] == "IsDeleted"
|
187
|
+
Es::Field.new("IsDeleted", "attribute")
|
188
|
+
elsif definition[:name] == "snapshot" || definition[:definition][:type] == "snapshot"
|
189
|
+
Es::SnapshotField.new("snapshot", "snapshot")
|
190
|
+
elsif definition[:name] == "autoincrement"
|
191
|
+
Es::AutoincrementField.new("generate", "autoincrement")
|
192
|
+
elsif definition[:name] == "duration"
|
193
|
+
Es::DurationField.new("duration", "duration")
|
194
|
+
elsif definition[:name] == "velocity"
|
195
|
+
Es::DurationField.new("velocity", "velocity")
|
196
|
+
elsif definition[:definition][:type] == "historicid"
|
197
|
+
Es::HIDField.new('hid', "historicid",Es::Helpers.get_historyid_settings(definition[:definition][:ops]))
|
198
|
+
elsif definition[:name].downcase == "iswon" || definition[:name].downcase == "isclosed" || definition[:name].downcase == "stagename" || definition[:name].downcase == "daytoclose" || definition[:name] == "dayssincelastactivity"
|
199
|
+
Es::Field.new(definition[:name], "attribute")
|
200
|
+
else
|
201
|
+
puts "WARNING! Transformer has found out field #{definition[:name]} which is not in load script, puting to extract as attribute"
|
202
|
+
Es::Field.new("#{definition[:name]}", "attribute")
|
203
|
+
end
|
204
|
+
end
|
205
|
+
parsed_timeframe = parseOldFormat_timeframes(internal[:timeframes])
|
206
|
+
entity = Entity.new(entity_name, {
|
207
|
+
:fields => fields,
|
208
|
+
:file => internal[:file],
|
209
|
+
:timeframe => parsed_timeframe || global_timeframe || (fail "Timeframe has to be defined"),
|
210
|
+
:timezone => timezone
|
211
|
+
})
|
212
|
+
entity
|
213
|
+
end
|
214
|
+
rescue Yajl::ParseError => e
|
215
|
+
fail Yajl::ParseError.new("Failed during parsing internal JSON. Error message: " + e.message)
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
|
220
|
+
def self.parse_timeframes(timeframe_spec, options={})
|
221
|
+
return nil if timeframe_spec.nil?
|
222
|
+
if timeframe_spec.is_a?(Array) then
|
223
|
+
timeframe_spec.map {|t_spec| Es::Timeframe.parse(t_spec, options)}
|
224
|
+
else
|
225
|
+
Es::Timeframe.parse(timeframe_spec, options)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def self.parseOldFormat_timeframes(timeframe_spec)
|
230
|
+
return nil if timeframe_spec.nil?
|
231
|
+
return Timeframe.parse("latest") if timeframe_spec == "latest"
|
232
|
+
if timeframe_spec.is_a?(Array) then
|
233
|
+
timeframe_spec.map {|t_spec| Es::Timeframe.parseOldFormat(t_spec)}
|
234
|
+
else
|
235
|
+
Es::Timeframe.parseOldFormat(timeframe_spec)
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
|
240
|
+
def initialize(entities, options = {})
|
241
|
+
@entities = entities
|
242
|
+
@timeframe = options[:timeframe]
|
243
|
+
@timezone = options[:timezone] || 'UTC'
|
244
|
+
end
|
245
|
+
|
246
|
+
def get_entity(name)
|
247
|
+
entities.detect {|e| e.name == name}
|
248
|
+
end
|
249
|
+
|
250
|
+
def to_extract_fragment(pid, options = {})
|
251
|
+
entities.map do |entity|
|
252
|
+
entity.to_extract_fragment(pid, options)
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
end
|
257
|
+
|
258
|
+
class Load
|
259
|
+
attr_accessor :entities
|
260
|
+
|
261
|
+
def self.parse(spec)
|
262
|
+
Load.new(spec.map do |entity_spec|
|
263
|
+
Entity.parse(entity_spec)
|
264
|
+
end)
|
265
|
+
end
|
266
|
+
|
267
|
+
def self.parseOldFormat(spec)
|
268
|
+
Load.new(spec.map do |entity_spec|
|
269
|
+
Entity.parseOldFormat(entity_spec[1])
|
270
|
+
end)
|
271
|
+
end
|
272
|
+
|
273
|
+
def initialize(entities)
|
274
|
+
@entities = entities
|
275
|
+
validate
|
276
|
+
end
|
277
|
+
|
278
|
+
def get_merged_entity_for(name)
|
279
|
+
entities_to_merge = entities.find_all {|e| e.name == name}
|
280
|
+
fail UnableToMerge.new("There is no entity #{name.bright} in current load object.") if entities_to_merge.empty?
|
281
|
+
merged_fields = entities_to_merge.inject([]) {|all, e| all.concat e.fields}
|
282
|
+
merged_fields = merged_fields.uniq_by {|obj| [obj.name, obj.type]}
|
283
|
+
Entity.new(name, {
|
284
|
+
:file => "MERGED",
|
285
|
+
:fields => merged_fields
|
286
|
+
})
|
287
|
+
end
|
288
|
+
|
289
|
+
def get_entity(name)
|
290
|
+
entities.detect {|e| e.name == name}
|
291
|
+
end
|
292
|
+
|
293
|
+
def validate
|
294
|
+
names = entities.map {|e| e.name}.uniq
|
295
|
+
names.each do |name|
|
296
|
+
merged_entity = get_merged_entity_for(name)
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
def to_config
|
301
|
+
entities.map {|e| e.to_load_config}
|
302
|
+
end
|
303
|
+
|
304
|
+
def to_config_generator
|
305
|
+
entities.map do |e|
|
306
|
+
d = ActiveSupport::OrderedHash.new
|
307
|
+
d['entity'] = e.to_config_generator[:entity]
|
308
|
+
d['file'] = "data/estore-in/#{e.to_config_generator[:file].match(/[^\/]*.csv/)[0]}"
|
309
|
+
d['fields'] = e.to_config_generator[:fields]
|
310
|
+
d
|
311
|
+
end
|
312
|
+
end
|
313
|
+
|
314
|
+
def to_config_file(filename)
|
315
|
+
File.open(filename, 'w') do |f|
|
316
|
+
f.write(JSON.pretty_generate(to_config))
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
end
|
321
|
+
|
322
|
+
class Entity
|
323
|
+
attr_accessor :name, :fields, :file, :timeframes, :timezone
|
324
|
+
|
325
|
+
def self.create_deleted_entity(name, options = {})
|
326
|
+
compatibility_mode = options[:compatibility_mode]
|
327
|
+
deleted_type = compatibility_mode ? "isDeleted" : "attribute"
|
328
|
+
file = options[:file]
|
329
|
+
|
330
|
+
e = Es::Entity.new(name, {
|
331
|
+
:file => file,
|
332
|
+
:fields => [
|
333
|
+
Es::Field.new('Timestamp', 'timestamp'),
|
334
|
+
Es::Field.new('Id', 'recordid'),
|
335
|
+
Es::Field.new('IsDeleted', deleted_type)
|
336
|
+
]
|
337
|
+
})
|
338
|
+
end
|
339
|
+
|
340
|
+
def self.parse(spec)
|
341
|
+
entity = Entity.new(spec[:entity], {
|
342
|
+
:file => Pathname.new(spec[:file]).expand_path.to_s,
|
343
|
+
:fields => spec[:fields] && spec[:fields].map {|field_spec| Field.parse(field_spec)}
|
344
|
+
})
|
345
|
+
end
|
346
|
+
|
347
|
+
def self.parseOldFormat(spec)
|
348
|
+
entity = Entity.new(spec[:entity], {
|
349
|
+
:file => spec[:file],
|
350
|
+
:fields => spec[:attributes] && spec[:attributes].map {|field_spec| Field.parse(field_spec)}
|
351
|
+
})
|
352
|
+
end
|
353
|
+
|
354
|
+
|
355
|
+
def initialize(name, options)
|
356
|
+
fail Es::IncorrectSpecificationError.new("Entity name is not specified.") if name.nil?
|
357
|
+
fail Es::IncorrectSpecificationError.new("Entity name should be a string.") unless name.is_a?(String)
|
358
|
+
fail Es::IncorrectSpecificationError.new("Entity name should not be empty.") if name.strip.empty?
|
359
|
+
fail Es::IncorrectSpecificationError.new("File is not specified.") if options[:file].nil?
|
360
|
+
fail Es::IncorrectSpecificationError.new("File should be a string.") unless options[:file].is_a?(String)
|
361
|
+
fail Es::IncorrectSpecificationError.new("Fields are not specified.") if options[:fields].nil?
|
362
|
+
fail Es::IncorrectSpecificationError.new("Entity should contain at least one field.") if options[:fields].empty?
|
363
|
+
# fail Es::IncorrectSpecificationError.new("Entity should contain at least one recordid field.") if !options[:fields].any? {|f| f.is_recordid?}
|
364
|
+
|
365
|
+
@name = name
|
366
|
+
@fields = options[:fields]
|
367
|
+
@file = options[:file]
|
368
|
+
if options[:timeframe] && !options[:timeframe].is_a?(Array)
|
369
|
+
@timeframes = [options[:timeframe]]
|
370
|
+
else
|
371
|
+
@timeframes = options[:timeframe]
|
372
|
+
end
|
373
|
+
@timezone = options[:timezone] || 'UTC'
|
374
|
+
fail Es::IncorrectSpecificationError.new("Entity #{name} should not contain multiple fields with the same name.") if has_multiple_same_fields?
|
375
|
+
end
|
376
|
+
|
377
|
+
def has_multiple_same_fields?
|
378
|
+
fields.uniq_by {|s| s.name}.count != fields.count
|
379
|
+
end
|
380
|
+
|
381
|
+
def to_extract_fragment(pid, options = {})
|
382
|
+
populates_element = (fields.find {|f| f.is_hid?} || fields.find {|f| f.is_recordid?} || fields.find {|f| f.is_autoincrement?})
|
383
|
+
fail "Needs to have at least on ID element. Use Id, HID, autoincrement" if populates_element.nil?
|
384
|
+
pretty = options[:pretty].nil? ? true : options[:pretty]
|
385
|
+
read_map = [{
|
386
|
+
:file => Es::Helpers.web_dav_extract_destination_dir(pid, self) + '/' + Es::Helpers.destination_file(self),
|
387
|
+
:populates => populates_element.name,
|
388
|
+
:columns => (fields.map do |field|
|
389
|
+
field.to_extract_fragment(pid, fields, options)
|
390
|
+
end)
|
391
|
+
}]
|
392
|
+
|
393
|
+
|
394
|
+
d = ActiveSupport::OrderedHash.new
|
395
|
+
d['entity'] = name
|
396
|
+
d['timezone'] = timezone
|
397
|
+
d['readMap'] = (pretty ? read_map : read_map.to_json)
|
398
|
+
d['computedStreams'] = '[{"type":"computed","ops":[]}]'
|
399
|
+
d['timeFrames'] = (timeframes.map{|t| t.to_extract_fragment(pid, options)})
|
400
|
+
|
401
|
+
task = ActiveSupport::OrderedHash.new
|
402
|
+
task['readTask'] = d
|
403
|
+
task
|
404
|
+
|
405
|
+
end
|
406
|
+
|
407
|
+
def to_extract_configuration
|
408
|
+
d = ActiveSupport::OrderedHash.new
|
409
|
+
d['entity'] = name
|
410
|
+
d['file'] = "data/estore-out/#{file.match(/[^\/]*.csv/)[0]}"
|
411
|
+
d['fields'] = (fields.map do |field|
|
412
|
+
field.to_config_generator_extract
|
413
|
+
end)
|
414
|
+
d['timeframes'] = (timeframes.map{|t| t.to_config_generator_extract})
|
415
|
+
final = ActiveSupport::OrderedHash.new
|
416
|
+
final['entities'] = [ d ]
|
417
|
+
final
|
418
|
+
end
|
419
|
+
|
420
|
+
|
421
|
+
def to_load_fragment(pid)
|
422
|
+
{
|
423
|
+
:uploadTask => {
|
424
|
+
:entity => name,
|
425
|
+
:file => Es::Helpers.web_dav_load_destination_dir(pid, self) + '/' + Es::Helpers.destination_file(self),
|
426
|
+
:attributes => fields.map {|f| f.to_load_fragment(pid)}
|
427
|
+
}
|
428
|
+
}
|
429
|
+
end
|
430
|
+
|
431
|
+
def to_load_config
|
432
|
+
{
|
433
|
+
:entity => name,
|
434
|
+
:file => file,
|
435
|
+
:fields => fields.map {|f| f.to_load_config}
|
436
|
+
}
|
437
|
+
end
|
438
|
+
|
439
|
+
def to_config_generator
|
440
|
+
{
|
441
|
+
:entity => name,
|
442
|
+
:file => file,
|
443
|
+
:fields => fields.map {|f| f.to_config_generator}
|
444
|
+
}
|
445
|
+
end
|
446
|
+
|
447
|
+
|
448
|
+
def to_extract_config
|
449
|
+
{
|
450
|
+
:timezone => timezone,
|
451
|
+
:entities => [{
|
452
|
+
:entity => name,
|
453
|
+
:file => file,
|
454
|
+
:fields => fields.map {|f| f.name}
|
455
|
+
}]
|
456
|
+
}
|
457
|
+
end
|
458
|
+
|
459
|
+
def to_table
|
460
|
+
t = Terminal::Table.new :headings => [name]
|
461
|
+
fields.map {|f| t << [f.name]}
|
462
|
+
t
|
463
|
+
end
|
464
|
+
|
465
|
+
def has_field?(name)
|
466
|
+
!!fields.detect {|f| f.name == name}
|
467
|
+
end
|
468
|
+
|
469
|
+
def get_field(name)
|
470
|
+
fields.detect {|f| f.name == name}
|
471
|
+
end
|
472
|
+
|
473
|
+
def add_field(field)
|
474
|
+
fail Es::IncorrectSpecificationError.new("There already is a field with name #{field.name} in entity #{name}") if fields.detect {|f| f.name == field.name}
|
475
|
+
fields << field
|
476
|
+
end
|
477
|
+
|
478
|
+
def load(pid, es_name)
|
479
|
+
begin
|
480
|
+
GoodData.connection.upload file, Es::Helpers.load_destination_dir(pid, self)
|
481
|
+
data = GoodData.post "/gdc/projects/#{pid}/eventStore/stores/#{es_name}/uploadTasks", to_load_fragment(pid).to_json
|
482
|
+
link = data["asyncTask"]["link"]["poll"]
|
483
|
+
response = GoodData.get(link, :process => false)
|
484
|
+
while response.code != 204
|
485
|
+
sleep 5
|
486
|
+
GoodData.connection.retryable(:tries => 3, :on => RestClient::InternalServerError) do
|
487
|
+
sleep 5
|
488
|
+
response = GoodData.get(link, :process => false)
|
489
|
+
end
|
490
|
+
end
|
491
|
+
rescue RestClient::RequestFailed => error
|
492
|
+
begin
|
493
|
+
doc = Yajl::Parser.parse(error.response, :symbolize_keys => true)
|
494
|
+
rescue Yajl::ParseError => e
|
495
|
+
puts "Error parsing \"#{error.response}\""
|
496
|
+
end
|
497
|
+
pp doc
|
498
|
+
raise error
|
499
|
+
end
|
500
|
+
end
|
501
|
+
|
502
|
+
def extract(pid, es_name)
|
503
|
+
begin
|
504
|
+
data = GoodData.post "/gdc/projects/#{pid}/eventStore/stores/#{es_name}/readTasks", to_extract_fragment(pid, {:pretty => false}).to_json
|
505
|
+
link = data["asyncTask"]["link"]["poll"]
|
506
|
+
response = GoodData.get(link, :process => false)
|
507
|
+
while response.code != 204
|
508
|
+
GoodData.connection.retryable(:tries => 3, :on => RestClient::InternalServerError) do
|
509
|
+
sleep 5
|
510
|
+
response = GoodData.get(link, :process => false)
|
511
|
+
end
|
512
|
+
response = GoodData.get(link, :process => false)
|
513
|
+
end
|
514
|
+
puts "Done downloading"
|
515
|
+
web_dav_file = Es::Helpers.extract_destination_dir(pid, self) + '/' + Es::Helpers.destination_file(self)
|
516
|
+
puts "Grabbing from web dav"
|
517
|
+
GoodData.connection.download web_dav_file, file
|
518
|
+
puts "Done"
|
519
|
+
rescue RestClient::RequestFailed => error
|
520
|
+
begin
|
521
|
+
doc = Yajl::Parser.parse(error.response, :symbolize_keys => true)
|
522
|
+
rescue Yajl::ParseError => e
|
523
|
+
puts "Error parsing \"#{error.response}\""
|
524
|
+
end
|
525
|
+
pp doc
|
526
|
+
raise error
|
527
|
+
end
|
528
|
+
end
|
529
|
+
|
530
|
+
def truncate(pid, es_name, timestamp)
|
531
|
+
begin
|
532
|
+
data = GoodData.post "/gdc/projects/#{pid}/eventStore/stores/#{es_name}/truncateTasks", {
|
533
|
+
:truncateTask => {
|
534
|
+
:entity => @name,
|
535
|
+
:timestamp => timestamp.to_i
|
536
|
+
}
|
537
|
+
}
|
538
|
+
rescue RestClient::BadRequest => error
|
539
|
+
puts error.inspect
|
540
|
+
raise error
|
541
|
+
end
|
542
|
+
link = data["asyncTask"]["link"]["poll"]
|
543
|
+
response = GoodData.get(link, :process => false)
|
544
|
+
while response.code != 204
|
545
|
+
sleep 10
|
546
|
+
response = GoodData.get(link, :process => false)
|
547
|
+
end
|
548
|
+
end
|
549
|
+
|
550
|
+
end
|
551
|
+
|
552
|
+
# Fields
|
553
|
+
|
554
|
+
class Field
|
555
|
+
|
556
|
+
ATTRIBUTE_TYPE = "attribute"
|
557
|
+
RECORDID_TYPE = "recordid"
|
558
|
+
DATE_TYPE = "date"
|
559
|
+
TIME_TYPE = "time"
|
560
|
+
FACT_TYPE = "fact"
|
561
|
+
TIMESTAMP_TYPE = "timestamp"
|
562
|
+
AUTOINCREMENT_TYPE = "autoincrement"
|
563
|
+
SNAPSHOT_TYPE = "snapshot"
|
564
|
+
HID_TYPE = "hid"
|
565
|
+
HISTORIC_TYPE = "historicid"
|
566
|
+
DURATION_TYPE = "duration"
|
567
|
+
VELOCITY_TYPE = "velocity"
|
568
|
+
IS_DELETED_TYPE = "isDeleted"
|
569
|
+
TIMEATTRIBUTE_TYPE = "timeAttribute"
|
570
|
+
|
571
|
+
FIELD_TYPES = [ATTRIBUTE_TYPE, RECORDID_TYPE, DATE_TYPE, TIME_TYPE, FACT_TYPE, TIMESTAMP_TYPE, AUTOINCREMENT_TYPE, SNAPSHOT_TYPE, HID_TYPE, HISTORIC_TYPE, DURATION_TYPE, VELOCITY_TYPE, IS_DELETED_TYPE,TIMEATTRIBUTE_TYPE]
|
572
|
+
|
573
|
+
def self.parse(spec)
|
574
|
+
fail InsufficientSpecificationError.new("Field specification is empty") if spec.nil?
|
575
|
+
fail InsufficientSpecificationError.new("Field specification is should be an object") unless spec.is_a?(Hash)
|
576
|
+
Field.new(spec[:name], spec[:type])
|
577
|
+
end
|
578
|
+
|
579
|
+
attr_accessor :type, :name
|
580
|
+
|
581
|
+
def is_recordid?
|
582
|
+
type == RECORDID_TYPE
|
583
|
+
end
|
584
|
+
|
585
|
+
def is_timestamp?
|
586
|
+
type == TIMESTAMP_TYPE
|
587
|
+
end
|
588
|
+
|
589
|
+
def is_attribute?
|
590
|
+
type == ATTRIBUTE_TYPE
|
591
|
+
end
|
592
|
+
|
593
|
+
def is_fact?
|
594
|
+
type == FACT_TYPE
|
595
|
+
end
|
596
|
+
|
597
|
+
def is_date?
|
598
|
+
type == DATE_TYPE
|
599
|
+
end
|
600
|
+
|
601
|
+
def is_snapshot?
|
602
|
+
false
|
603
|
+
end
|
604
|
+
|
605
|
+
def is_duration?
|
606
|
+
false
|
607
|
+
end
|
608
|
+
|
609
|
+
def is_autoincrement?
|
610
|
+
false
|
611
|
+
end
|
612
|
+
|
613
|
+
def is_hid?
|
614
|
+
false
|
615
|
+
end
|
616
|
+
|
617
|
+
def is_velocity?
|
618
|
+
false
|
619
|
+
end
|
620
|
+
|
621
|
+
def initialize(name, type)
|
622
|
+
fail Es::IncorrectSpecificationError.new("The field name \"#{name.bright}\" does not have type specified. Type should be one of [#{FIELD_TYPES.join(', ')}]") if type.nil?
|
623
|
+
fail Es::IncorrectSpecificationError.new("The type of field name \"#{name.bright}\" should be a string.") unless type.is_a?(String)
|
624
|
+
fail Es::IncorrectSpecificationError.new("The field name \"#{name.bright}\" does have wrong type specified. Specified \"#{type.bright}\" should be one of [#{FIELD_TYPES.join(', ')}]") unless FIELD_TYPES.include?(type) || type == "none"
|
625
|
+
@name = name
|
626
|
+
@type = type
|
627
|
+
end
|
628
|
+
|
629
|
+
def to_extract_fragment(pid,fields, options = {})
|
630
|
+
{
|
631
|
+
:name => name,
|
632
|
+
:preferred => name,
|
633
|
+
:definition => {
|
634
|
+
:ops => [{
|
635
|
+
:type => Es::Helpers.type_to_type(type),
|
636
|
+
:data => name
|
637
|
+
}],
|
638
|
+
:type => Es::Helpers.type_to_operation(type)
|
639
|
+
}
|
640
|
+
}
|
641
|
+
end
|
642
|
+
|
643
|
+
def to_load_fragment(pid)
|
644
|
+
{
|
645
|
+
:name => name,
|
646
|
+
:type => Es::Helpers.type_to_load_type(type)
|
647
|
+
}
|
648
|
+
end
|
649
|
+
|
650
|
+
def to_load_config
|
651
|
+
{
|
652
|
+
:name => name,
|
653
|
+
:type => (type == 'none' ? '' : type)
|
654
|
+
}
|
655
|
+
end
|
656
|
+
|
657
|
+
def to_config_generator
|
658
|
+
d = ActiveSupport::OrderedHash.new
|
659
|
+
d['name'] = name
|
660
|
+
d['type'] = Es::Helpers.type_to_generator_load_type(type)
|
661
|
+
d
|
662
|
+
end
|
663
|
+
|
664
|
+
|
665
|
+
def to_config_generator_extract
|
666
|
+
name
|
667
|
+
end
|
668
|
+
|
669
|
+
def ==(other)
|
670
|
+
other.name == name
|
671
|
+
end
|
672
|
+
|
673
|
+
end
|
674
|
+
|
675
|
+
class RecordIdField < Field
|
676
|
+
|
677
|
+
attr_accessor :type, :name, :partial
|
678
|
+
|
679
|
+
def initialize(name, type, partial)
|
680
|
+
fail Es::IncorrectSpecificationError.new("The field name \"#{name.bright}\" does not have type specified. Type should be one of [#{FIELD_TYPES.join(', ')}]") if type.nil?
|
681
|
+
fail Es::IncorrectSpecificationError.new("The type of field name \"#{name.bright}\" should be a string.") unless type.is_a?(String)
|
682
|
+
fail Es::IncorrectSpecificationError.new("The field name \"#{name.bright}\" does have wrong type specified. Specified \"#{type.bright}\" should be one of [#{FIELD_TYPES.join(', ')}]") unless FIELD_TYPES.include?(type) || type == "none"
|
683
|
+
@name = name
|
684
|
+
@type = type
|
685
|
+
@partial = partial
|
686
|
+
end
|
687
|
+
|
688
|
+
def is_recordid?
|
689
|
+
true
|
690
|
+
end
|
691
|
+
|
692
|
+
def to_extract_fragment(pid, fields, options = {})
|
693
|
+
if (partial == "true") then
|
694
|
+
{
|
695
|
+
:name => name,
|
696
|
+
:preferred => name,
|
697
|
+
:definition => {
|
698
|
+
:ops => [{
|
699
|
+
:type => Es::Helpers.type_to_type(type),
|
700
|
+
:data => name,
|
701
|
+
:ops => fields.select{|x| Es::Helpers.type_to_type(x.type) == "stream"}.map do |f|
|
702
|
+
{
|
703
|
+
:type => "stream",
|
704
|
+
:data => f.name
|
705
|
+
}
|
706
|
+
end
|
707
|
+
}],
|
708
|
+
:type => Es::Helpers.type_to_operation(type)
|
709
|
+
}
|
710
|
+
}
|
711
|
+
else
|
712
|
+
{
|
713
|
+
:name => name,
|
714
|
+
:preferred => name,
|
715
|
+
:definition => {
|
716
|
+
:ops => [{
|
717
|
+
:type => Es::Helpers.type_to_type(type),
|
718
|
+
:data => name
|
719
|
+
}],
|
720
|
+
:type => Es::Helpers.type_to_operation(type)
|
721
|
+
}
|
722
|
+
}
|
723
|
+
end
|
724
|
+
end
|
725
|
+
end
|
726
|
+
|
727
|
+
|
728
|
+
class SnapshotField < Field
|
729
|
+
|
730
|
+
attr_accessor :type, :name
|
731
|
+
|
732
|
+
def is_snapshot?
|
733
|
+
true
|
734
|
+
end
|
735
|
+
|
736
|
+
def to_extract_fragment(pid, fields, options = {})
|
737
|
+
{
|
738
|
+
:name => name,
|
739
|
+
:preferred => name,
|
740
|
+
:definition => {
|
741
|
+
:type => "snapshot",
|
742
|
+
:data => "date"
|
743
|
+
}
|
744
|
+
}
|
745
|
+
end
|
746
|
+
|
747
|
+
end
|
748
|
+
|
749
|
+
class HIDField < Field
|
750
|
+
|
751
|
+
attr_accessor :type, :name, :entity, :fieldsInner, :through
|
752
|
+
|
753
|
+
def is_hid?
|
754
|
+
true
|
755
|
+
end
|
756
|
+
|
757
|
+
def initialize(name, type, options)
|
758
|
+
name = "#{name}-#{options[:entity]}"
|
759
|
+
super(name, type)
|
760
|
+
@entity = options[:entity] || fail("Entity has to be scpecified for a HID Field")
|
761
|
+
@fieldsInner = options[:fields] || fail("Fields has to be scpecified for a HID Field")
|
762
|
+
@through = options[:through]
|
763
|
+
end
|
764
|
+
|
765
|
+
def to_extract_fragment(pid, fields, options = {})
|
766
|
+
{
|
767
|
+
:name => name,
|
768
|
+
:preferred => name,
|
769
|
+
:definition => {
|
770
|
+
:ops => [
|
771
|
+
through.nil? ? {:type => RECORDID_TYPE} : {:type => "stream", :data => through},
|
772
|
+
{
|
773
|
+
:type => "entity",
|
774
|
+
:data => entity,
|
775
|
+
:ops => fieldsInner.map do |f|
|
776
|
+
{
|
777
|
+
:type => "stream",
|
778
|
+
:data => f
|
779
|
+
}
|
780
|
+
end
|
781
|
+
}
|
782
|
+
],
|
783
|
+
:type => "historicid"
|
784
|
+
}
|
785
|
+
}
|
786
|
+
end
|
787
|
+
|
788
|
+
|
789
|
+
def to_config_generator_extract
|
790
|
+
if through.empty? then
|
791
|
+
{
|
792
|
+
:hid =>
|
793
|
+
{
|
794
|
+
:from_entity => entity,
|
795
|
+
:from_fields => fields.map{|f| f}
|
796
|
+
}
|
797
|
+
}
|
798
|
+
else
|
799
|
+
{
|
800
|
+
:hid =>
|
801
|
+
{
|
802
|
+
:from_entity => entity,
|
803
|
+
:from_fields => fields.map{|f| f},
|
804
|
+
:connected_through => through
|
805
|
+
}
|
806
|
+
|
807
|
+
}
|
808
|
+
end
|
809
|
+
end
|
810
|
+
|
811
|
+
|
812
|
+
end
|
813
|
+
|
814
|
+
class DurationField < Field
|
815
|
+
|
816
|
+
attr_accessor :type, :name, :attribute, :value, :control_attribute
|
817
|
+
|
818
|
+
def initialize(name, type, options = {})
|
819
|
+
super(name, type)
|
820
|
+
@attribute = options[:attribute] || "IsClosed"
|
821
|
+
@value = options[:value] || "false"
|
822
|
+
@control_attribute = options[:control_attribute] || "StageName"
|
823
|
+
end
|
824
|
+
|
825
|
+
def is_duration?
|
826
|
+
true
|
827
|
+
end
|
828
|
+
|
829
|
+
def to_extract_fragment(pid, fields, options = {})
|
830
|
+
{
|
831
|
+
:name => "StageDuration",
|
832
|
+
:preferred => "stageduration",
|
833
|
+
:definition => {
|
834
|
+
:type => "case",
|
835
|
+
:ops => [{
|
836
|
+
:type => "option",
|
837
|
+
:ops => [{
|
838
|
+
:type => "=",
|
839
|
+
:ops => [{
|
840
|
+
:type => "stream",
|
841
|
+
:data => attribute
|
842
|
+
},
|
843
|
+
{
|
844
|
+
:type => "match",
|
845
|
+
:data => value
|
846
|
+
}]
|
847
|
+
},
|
848
|
+
{
|
849
|
+
:type => "duration",
|
850
|
+
:ops => [{
|
851
|
+
:type => "stream",
|
852
|
+
:data => control_attribute
|
853
|
+
}]
|
854
|
+
}]
|
855
|
+
},
|
856
|
+
{
|
857
|
+
:type => "option",
|
858
|
+
:ops => [{
|
859
|
+
:type => "const",
|
860
|
+
:data => 1
|
861
|
+
},
|
862
|
+
{
|
863
|
+
:type => "const",
|
864
|
+
:data => 0
|
865
|
+
}]
|
866
|
+
}]
|
867
|
+
}
|
868
|
+
}
|
869
|
+
end
|
870
|
+
end
|
871
|
+
|
872
|
+
class VelocityField < Field
|
873
|
+
|
874
|
+
attr_accessor :type, :name, :control_attribute
|
875
|
+
|
876
|
+
def initialize(name, type, options = {})
|
877
|
+
super(name, type)
|
878
|
+
@control_attribute = options[:control_attribute] || "StageName"
|
879
|
+
end
|
880
|
+
|
881
|
+
def is_velocity?
|
882
|
+
true
|
883
|
+
end
|
884
|
+
|
885
|
+
def to_extract_fragment(pid, fields, options = {})
|
886
|
+
{
|
887
|
+
:name => "StageVelocity",
|
888
|
+
:preferred => "stagevelocity",
|
889
|
+
:definition => {
|
890
|
+
:type => "velocity",
|
891
|
+
:ops => [{
|
892
|
+
:type => "stream",
|
893
|
+
:data => control_attribute
|
894
|
+
}]
|
895
|
+
}
|
896
|
+
}
|
897
|
+
end
|
898
|
+
end
|
899
|
+
|
900
|
+
|
901
|
+
class AutoincrementField < Field
|
902
|
+
|
903
|
+
attr_accessor :type, :name
|
904
|
+
|
905
|
+
def is_autoincrement?
|
906
|
+
true
|
907
|
+
end
|
908
|
+
|
909
|
+
def to_extract_fragment(pid, fields, options = {})
|
910
|
+
{
|
911
|
+
:name => name,
|
912
|
+
:preferred => name,
|
913
|
+
:definition => {
|
914
|
+
:type => "generate",
|
915
|
+
:data => "autoincrement"
|
916
|
+
}
|
917
|
+
}
|
918
|
+
end
|
919
|
+
end
|
920
|
+
|
921
|
+
module Helpers
|
922
|
+
TEMPLATE_DIR = "./lib/templates"
|
923
|
+
|
924
|
+
def self.has_more_lines?(path)
|
925
|
+
counter = 0
|
926
|
+
File.open(path, "r") do |infile|
|
927
|
+
while (line = infile.gets)
|
928
|
+
counter += 1
|
929
|
+
break if counter > 2
|
930
|
+
end
|
931
|
+
end
|
932
|
+
counter > 1
|
933
|
+
end
|
934
|
+
|
935
|
+
def self.load_config(filename, validate=true)
|
936
|
+
json = File.new(filename, 'r')
|
937
|
+
parser = Yajl::Parser.new(:symbolize_keys => true)
|
938
|
+
begin
|
939
|
+
doc = parser.parse(json)
|
940
|
+
rescue Yajl::ParseError => e
|
941
|
+
fail Yajl::ParseError.new("Failed during parsing file #{filename}\n" + e.message)
|
942
|
+
end
|
943
|
+
end
|
944
|
+
|
945
|
+
def self.web_dav_load_destination_dir(pid, entity)
|
946
|
+
"/uploads/#{pid}"
|
947
|
+
end
|
948
|
+
|
949
|
+
def self.web_dav_extract_destination_dir(pid, entity)
|
950
|
+
"/out_#{pid}_#{entity.name}"
|
951
|
+
end
|
952
|
+
|
953
|
+
def self.load_destination_dir(pid, entity)
|
954
|
+
"#{pid}"
|
955
|
+
end
|
956
|
+
|
957
|
+
def self.extract_destination_dir(pid, entity)
|
958
|
+
"out_#{pid}_#{entity.name}"
|
959
|
+
end
|
960
|
+
|
961
|
+
def self.destination_file(entity, options={})
|
962
|
+
with_date = options[:with_date]
|
963
|
+
deleted = options[:deleted]
|
964
|
+
source = entity.file
|
965
|
+
filename = File.basename(source)
|
966
|
+
base = File.basename(source, '.*')
|
967
|
+
ext = File.extname(filename)
|
968
|
+
base = deleted ? "#{base}_deleted" : base
|
969
|
+
with_date ? base + '_' + DateTime.now.strftime("%Y-%M-%d_%H:%M:%S") + ext : base + ext
|
970
|
+
end
|
971
|
+
|
972
|
+
def self.type_to_load_type(type)
|
973
|
+
types = {
|
974
|
+
Es::Field::RECORDID_TYPE => "recordid",
|
975
|
+
Es::Field::TIMESTAMP_TYPE => "timestamp",
|
976
|
+
Es::Field::ATTRIBUTE_TYPE => "attribute",
|
977
|
+
Es::Field::FACT_TYPE => "fact",
|
978
|
+
Es::Field::TIME_TYPE => "timeAttribute",
|
979
|
+
Es::Field::DATE_TYPE => "timeAttribute",
|
980
|
+
Es::Field::IS_DELETED_TYPE => "isDeleted",
|
981
|
+
Es::Field::TIMEATTRIBUTE_TYPE => "timeAttribute"
|
982
|
+
}
|
983
|
+
if types.has_key?(type) then
|
984
|
+
types[type]
|
985
|
+
else
|
986
|
+
fail "Type #{type} not found."
|
987
|
+
end
|
988
|
+
end
|
989
|
+
|
990
|
+
|
991
|
+
def self.type_to_type(type)
|
992
|
+
types = {
|
993
|
+
Es::Field::RECORDID_TYPE => "recordid",
|
994
|
+
Es::Field::ATTRIBUTE_TYPE => "stream",
|
995
|
+
Es::Field::FACT_TYPE => "stream",
|
996
|
+
Es::Field::SNAPSHOT_TYPE => "snapshot",
|
997
|
+
Es::Field::TIME_TYPE => "stream",
|
998
|
+
Es::Field::DATE_TYPE => "stream",
|
999
|
+
Es::Field::TIMEATTRIBUTE_TYPE => "stream"
|
1000
|
+
}
|
1001
|
+
if types.has_key?(type) then
|
1002
|
+
types[type]
|
1003
|
+
else
|
1004
|
+
fail "Type #{type} not found."
|
1005
|
+
end
|
1006
|
+
end
|
1007
|
+
|
1008
|
+
def self.type_to_operation(type)
|
1009
|
+
types = {
|
1010
|
+
Es::Field::RECORDID_TYPE => "value",
|
1011
|
+
Es::Field::ATTRIBUTE_TYPE => "value",
|
1012
|
+
Es::Field::FACT_TYPE => "number",
|
1013
|
+
Es::Field::SNAPSHOT_TYPE => "snapshot",
|
1014
|
+
Es::Field::TIME_TYPE => "key",
|
1015
|
+
Es::Field::DATE_TYPE => "date",
|
1016
|
+
Es::Field::TIMEATTRIBUTE_TYPE => "key"
|
1017
|
+
}
|
1018
|
+
if types.has_key?(type) then
|
1019
|
+
types[type]
|
1020
|
+
else
|
1021
|
+
fail "Type #{type} not found."
|
1022
|
+
end
|
1023
|
+
end
|
1024
|
+
|
1025
|
+
def self.type_to_generator_load_type(type)
|
1026
|
+
types = {
|
1027
|
+
Es::Field::RECORDID_TYPE => "recordid",
|
1028
|
+
Es::Field::TIMESTAMP_TYPE => "timestamp",
|
1029
|
+
Es::Field::ATTRIBUTE_TYPE => "attribute",
|
1030
|
+
Es::Field::FACT_TYPE => "fact",
|
1031
|
+
Es::Field::TIME_TYPE => "time",
|
1032
|
+
Es::Field::DATE_TYPE => "date",
|
1033
|
+
Es::Field::IS_DELETED_TYPE => "isDeleted",
|
1034
|
+
Es::Field::TIMEATTRIBUTE_TYPE => "time"
|
1035
|
+
}
|
1036
|
+
if types.has_key?(type) then
|
1037
|
+
types[type]
|
1038
|
+
else
|
1039
|
+
fail "Type #{type} not found."
|
1040
|
+
end
|
1041
|
+
end
|
1042
|
+
|
1043
|
+
|
1044
|
+
def self.get_historyid_settings(json)
|
1045
|
+
entity_fields = Array.new
|
1046
|
+
entity_name = ""
|
1047
|
+
connected_through = ""
|
1048
|
+
json.map do |inner_part|
|
1049
|
+
if (inner_part[:type] == "entity")
|
1050
|
+
entity_name = inner_part[:data]
|
1051
|
+
inner_part[:ops].map do |fields|
|
1052
|
+
entity_fields << fields[:data]
|
1053
|
+
end
|
1054
|
+
elsif (inner_part[:type] == "stream")
|
1055
|
+
connected_through = inner_part[:data]
|
1056
|
+
end
|
1057
|
+
end
|
1058
|
+
{
|
1059
|
+
:entity => entity_name,
|
1060
|
+
:fields => entity_fields,
|
1061
|
+
:through => connected_through
|
1062
|
+
}
|
1063
|
+
end
|
1064
|
+
|
1065
|
+
|
1066
|
+
end
|
1067
|
+
|
1068
|
+
end
|
1069
|
+
|
1070
|
+
# Hack for 1.8.7
|
1071
|
+
# uniq on array does not take block
|
1072
|
+
module Enumerable
|
1073
|
+
def uniq_by
|
1074
|
+
seen = Hash.new { |h,k| h[k] = true; false }
|
1075
|
+
reject { |v| seen[yield(v)] }
|
1076
|
+
end
|
1077
|
+
end
|