gd_bam 0.1.3 → 0.1.5
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/bam +29 -9
- data/bin/bam.orig +375 -0
- data/lib/bam/version.rb +1 -1
- data/lib/base/project.rb +8 -7
- data/lib/base/tap.rb +4 -0
- data/lib/cloud_connect/dsl/es_helpers.rb +1 -1
- data/lib/cloud_connect/dsl/sf_helpers.rb +33 -2
- data/lib/commands/commands.rb +16 -8
- data/lib/commands/docs_commands.rb +1 -1
- data/lib/commands/gd_commands.rb +2 -2
- data/lib/commands/sf_commands.rb +1 -1
- data/lib/compiler/etl_visitor.rb +1 -1
- data/lib/generators/downloaders.rb +209 -0
- data/lib/generators/downloaders.rb.orig +663 -0
- data/lib/generators/etl.rb +5 -1
- data/lib/graphs/goodsales_v2/process_opportunity_line_item.grf +4 -2
- data/lib/graphs/goodsales_v2/process_velocity_duration.grf +1 -1
- metadata +4 -2
@@ -313,6 +313,202 @@ HEREDOC
|
|
313
313
|
end
|
314
314
|
end
|
315
315
|
|
316
|
+
|
317
|
+
|
318
|
+
def self.create_incremental_sf_history_init_graph(file, taps, options={})
|
319
|
+
metadata = options[:metadata]
|
320
|
+
store = options[:store] || "${GDC_EVENTSTORE}"
|
321
|
+
s3_backup = options[:s3_backup]
|
322
|
+
|
323
|
+
|
324
|
+
client = GoodData::Bam::Commands.get_sf_client(options)
|
325
|
+
|
326
|
+
File.open(file, "w") do |file|
|
327
|
+
builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
|
328
|
+
builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
|
329
|
+
builder.Graph({:name => "Goodsales incremental Downloader"}) do
|
330
|
+
builder.Global do
|
331
|
+
Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
|
332
|
+
Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
|
333
|
+
|
334
|
+
Helpers::create_lookup_meta(builder)
|
335
|
+
builder.Metadata({:id => "data_metadata"}) do |builder|
|
336
|
+
Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => "Value"}]})
|
337
|
+
end
|
338
|
+
|
339
|
+
builder.Metadata({:id => "es_metadata"}) do |builder|
|
340
|
+
Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp", :type => "date"}, {:name => "Value"}]})
|
341
|
+
end
|
342
|
+
|
343
|
+
Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
|
344
|
+
|
345
|
+
Helpers::sf_connection(builder, {})
|
346
|
+
Helpers::property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
|
347
|
+
Helpers::property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
|
348
|
+
Helpers::property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
|
349
|
+
Helpers::property(builder, {:id => "SFDC_PASSWORD", :value => options[:sf_password]})
|
350
|
+
Helpers::property(builder, {:id => "SFDC_TOKEN", :value => options[:sf_token]})
|
351
|
+
Helpers::property(builder, {:id => "SFDC_USERNAME", :value => options[:sf_login]})
|
352
|
+
Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
|
353
|
+
end
|
354
|
+
|
355
|
+
phase = 1
|
356
|
+
|
357
|
+
taps_with_history = []
|
358
|
+
taps.each do |tap|
|
359
|
+
module_name = tap[:object]
|
360
|
+
dataset = tap[:id]
|
361
|
+
source_file = tap[:source]
|
362
|
+
|
363
|
+
has_timestamp = Tap.has_output_field?(tap, "Timestamp")
|
364
|
+
timestamp_field = Tap.find_output_field(tap, "Timestamp")
|
365
|
+
id_field = Tap.find_output_field(tap, "Id")
|
366
|
+
tap = Tap.prepare_for_sf_downloader(tap)
|
367
|
+
builder.Phase(:number => phase += 1) do
|
368
|
+
# Core::build_node2(builder, Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
|
369
|
+
end
|
370
|
+
|
371
|
+
fields = tap[:fields]
|
372
|
+
|
373
|
+
puts tap[:object]
|
374
|
+
sf_object = tap[:object]
|
375
|
+
id_field = Tap.find_output_field(tap, "Id")
|
376
|
+
timestamp_field = Tap.find_output_field(tap, "Timestamp")
|
377
|
+
|
378
|
+
|
379
|
+
objects_to_get = Helpers.objects_for_history(client, tap)
|
380
|
+
|
381
|
+
(tap[:fields] - [id_field, timestamp_field]).each_with_index do |field, i|
|
382
|
+
builder.Phase(:number => phase += 1) do
|
383
|
+
|
384
|
+
f = field[:name]
|
385
|
+
|
386
|
+
selects = objects_to_get.reduce([]) do |memo, o|
|
387
|
+
fields = client.fields(o)
|
388
|
+
generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|f| fields.include?(f)}
|
389
|
+
specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|f| fields.include?(f)}
|
390
|
+
specific_history = ["SystemModstamp", "#{sf_object}Id"].all? {|f| fields.include?(f)}
|
391
|
+
|
392
|
+
select, mapping, mandatory = if generic_field_history
|
393
|
+
[
|
394
|
+
"SELECT NewValue, CreatedDate, ParentId FROM #{o} WHERE Field = '#{f}'",
|
395
|
+
"{\"xmlFieldsMapping\":{\"xmlFields\":[
|
396
|
+
{\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
|
397
|
+
{\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
|
398
|
+
{\"xmlFieldMapping\":{\"name\":\"ParentId\",\"label\":\"ParentId\",\"xmlPath\":\"ParentId\",\"metadataField\":\"Id\"}}
|
399
|
+
]}}",
|
400
|
+
["ParentId", "NewValue", "CreatedDate"]
|
401
|
+
]
|
402
|
+
elsif specific_field_history
|
403
|
+
[
|
404
|
+
|
405
|
+
"SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
|
406
|
+
"{\"xmlFieldsMapping\":{\"xmlFields\":[
|
407
|
+
{\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
|
408
|
+
{\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
|
409
|
+
{\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
|
410
|
+
]}}",
|
411
|
+
["#{sf_object}Id", "NewValue", "CreatedDate"]
|
412
|
+
]
|
413
|
+
elsif specific_history
|
414
|
+
if fields.include?(f)
|
415
|
+
[
|
416
|
+
"SELECT #{f}, SystemModstamp, #{sf_object}Id FROM #{o}",
|
417
|
+
"{\"xmlFieldsMapping\":{\"xmlFields\":[
|
418
|
+
{\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
|
419
|
+
{\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
|
420
|
+
{\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{o}Id\",\"xmlPath\":\"#{o}Id\",\"metadataField\":\"Id\"}}
|
421
|
+
]}}",
|
422
|
+
["#{sf_object}Id", f, "SystemModstamp"]
|
423
|
+
]
|
424
|
+
else
|
425
|
+
[nil, nil]
|
426
|
+
end
|
427
|
+
else
|
428
|
+
fail "Unrecognized fields configuration for historization in SF."
|
429
|
+
end
|
430
|
+
if select.nil?
|
431
|
+
memo
|
432
|
+
else
|
433
|
+
memo.concat([{
|
434
|
+
:object => o,
|
435
|
+
:query => select,
|
436
|
+
:mapping => mapping,
|
437
|
+
:mandatory => mandatory
|
438
|
+
}])
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
unless selects.empty?
|
443
|
+
taps_with_history.concat([tap])
|
444
|
+
selects = selects.concat([{
|
445
|
+
:object => sf_object,
|
446
|
+
:query => "SELECT Id, #{f}, SystemModstamp FROM #{sf_object}",
|
447
|
+
:mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
|
448
|
+
{\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
|
449
|
+
{\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
|
450
|
+
{\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
|
451
|
+
]}}",
|
452
|
+
:mandatory => ["Id", f, "SystemModstamp"]
|
453
|
+
}])
|
454
|
+
selects.each_with_index do |obj, i|
|
455
|
+
|
456
|
+
o = obj[:object]
|
457
|
+
mapping = obj[:mapping]
|
458
|
+
select = obj[:query]
|
459
|
+
mandatory = obj[:mandatory]
|
460
|
+
|
461
|
+
Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{o} SF Reader #{i}", :id => "#{f}_#{o}_reader", :soql => select, :sfdcConnection => "SFDC", :fieldsMapping => mapping, :mandatoryFields => mandatory.join(';')}))
|
462
|
+
puts "#{f}_#{o}_reader:0 -> #{f}_#{sf_object}_gather:#{i}"
|
463
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_gather:#{i}", :fromNode => "#{f}_#{o}_reader:0", :metadata => "data_metadata"}))
|
464
|
+
end
|
465
|
+
Core::build_node2(builder, Nodes.gather2({:name => "#{f} Gather", :id => "#{f}_#{sf_object}_gather"}))
|
466
|
+
Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{f}_#{sf_object}_sort", :id => "#{f}_#{sf_object}_sort"}))
|
467
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_sort:0", :fromNode => "#{f}_#{sf_object}_gather:0", :metadata => "data_metadata"}))
|
468
|
+
|
469
|
+
Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{f}_#{sf_object}_copy"}))
|
470
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_copy:0", :fromNode => "#{f}_#{sf_object}_sort:0", :metadata => "data_metadata"}))
|
471
|
+
|
472
|
+
if s3_backup
|
473
|
+
Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{f}_#{sf_object} s3 Writer", :id => "#{f}_#{sf_object}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{sf_object}/#{f}_#{sf_object}_\`date2long(today())\`", :outputFieldNames => true}))
|
474
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_s3:0", :fromNode => "#{f}_#{sf_object}_copy:1", :metadata => "data_metadata"}))
|
475
|
+
end
|
476
|
+
|
477
|
+
transform = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = str2date($in.0.Timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\nreturn OK;\n}"
|
478
|
+
|
479
|
+
Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "#{f}_#{sf_object}_reformat", :transformation => transform}))
|
480
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_reformat:0", :fromNode => "#{f}_#{sf_object}_copy:0", :metadata => "data_metadata"}))
|
481
|
+
|
482
|
+
Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{f}_#{sf_object}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json({:type => :tap, :id => tap[:id], :fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => f, :meta => "Value"}]}).to_json}))
|
483
|
+
|
484
|
+
# Core::build_node2(builder, Nodes.trash2({:name => "#{f}_#{sf_object}_es", :id => "#{f}_#{sf_object}_es", :debugPrint => true}))
|
485
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_es:0", :fromNode => "#{f}_#{sf_object}_reformat:0", :metadata => "es_metadata"}))
|
486
|
+
end
|
487
|
+
end
|
488
|
+
end
|
489
|
+
end
|
490
|
+
taps_with_history.each do |tap|
|
491
|
+
puts "#{tap[:id]}_LAST_RUN"
|
492
|
+
builder.Phase(:number => phase += 1) do
|
493
|
+
generate_func = <<HEREDOC
|
494
|
+
function integer generate() {
|
495
|
+
date time_end = jodaStr2date("${#{tap[:id]}_END}",["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC');
|
496
|
+
$out.0.key = "#{tap[:id]}_LAST_RUN";
|
497
|
+
$out.0.value = jodaDate2str(time_end,"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", 'en_US', 'UTC');
|
498
|
+
return OK;
|
499
|
+
}
|
500
|
+
HEREDOC
|
501
|
+
|
502
|
+
Core::build_node2(builder, Nodes.data_generator2({:guiName => tap[:id], :name => "generator_#{phase}", :id => "generator_#{phase}", :generate => generate_func}))
|
503
|
+
Core::build_node2(builder, Nodes.lookup_reader_writer2({:guiName => tap[:id], :lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{phase}" }))
|
504
|
+
Core::build_node2(builder, Nodes.edge2({:toNode => "gd_lookup_reader_#{phase}:0", :fromNode => "generator_#{phase}:0", :metadata => "lookup_metadata"}))
|
505
|
+
|
506
|
+
end
|
507
|
+
end
|
508
|
+
end
|
509
|
+
end
|
510
|
+
end
|
511
|
+
|
316
512
|
def self.generate_downloaders(home, project, params)
|
317
513
|
home = Pathname(home)
|
318
514
|
|
@@ -336,6 +532,19 @@ HEREDOC
|
|
336
532
|
end
|
337
533
|
end
|
338
534
|
|
535
|
+
|
536
|
+
def self.generate_history_downloaders(home, project, params)
|
537
|
+
home = Pathname(home)
|
538
|
+
|
539
|
+
incremental_taps = Taps.get_incremental(project[:taps])
|
540
|
+
|
541
|
+
sf_taps = Taps.get_salesforce(incremental_taps)
|
542
|
+
file_taps = Taps.get_file(incremental_taps)
|
543
|
+
|
544
|
+
create_incremental_downloader_run_graph(home + "main.grf", incremental_taps)
|
545
|
+
create_incremental_sf_history_init_graph(home + "incremental.grf", sf_taps, params)
|
546
|
+
end
|
547
|
+
|
339
548
|
def self.create_incremental_file_downloading_graph(file, taps, options={})
|
340
549
|
metadata = options[:metadata]
|
341
550
|
store = options[:store] || "${GDC_EVENTSTORE}"
|