gd_bam 0.1.3 → 0.1.5

Sign up to get free protection for your applications and to get access to all the features.
@@ -313,6 +313,202 @@ HEREDOC
313
313
  end
314
314
  end
315
315
 
316
+
317
+
318
+ def self.create_incremental_sf_history_init_graph(file, taps, options={})
319
+ metadata = options[:metadata]
320
+ store = options[:store] || "${GDC_EVENTSTORE}"
321
+ s3_backup = options[:s3_backup]
322
+
323
+
324
+ client = GoodData::Bam::Commands.get_sf_client(options)
325
+
326
+ File.open(file, "w") do |file|
327
+ builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
328
+ builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
329
+ builder.Graph({:name => "Goodsales incremental Downloader"}) do
330
+ builder.Global do
331
+ Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
332
+ Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
333
+
334
+ Helpers::create_lookup_meta(builder)
335
+ builder.Metadata({:id => "data_metadata"}) do |builder|
336
+ Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => "Value"}]})
337
+ end
338
+
339
+ builder.Metadata({:id => "es_metadata"}) do |builder|
340
+ Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp", :type => "date"}, {:name => "Value"}]})
341
+ end
342
+
343
+ Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
344
+
345
+ Helpers::sf_connection(builder, {})
346
+ Helpers::property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
347
+ Helpers::property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
348
+ Helpers::property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
349
+ Helpers::property(builder, {:id => "SFDC_PASSWORD", :value => options[:sf_password]})
350
+ Helpers::property(builder, {:id => "SFDC_TOKEN", :value => options[:sf_token]})
351
+ Helpers::property(builder, {:id => "SFDC_USERNAME", :value => options[:sf_login]})
352
+ Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
353
+ end
354
+
355
+ phase = 1
356
+
357
+ taps_with_history = []
358
+ taps.each do |tap|
359
+ module_name = tap[:object]
360
+ dataset = tap[:id]
361
+ source_file = tap[:source]
362
+
363
+ has_timestamp = Tap.has_output_field?(tap, "Timestamp")
364
+ timestamp_field = Tap.find_output_field(tap, "Timestamp")
365
+ id_field = Tap.find_output_field(tap, "Id")
366
+ tap = Tap.prepare_for_sf_downloader(tap)
367
+ builder.Phase(:number => phase += 1) do
368
+ # Core::build_node2(builder, Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
369
+ end
370
+
371
+ fields = tap[:fields]
372
+
373
+ puts tap[:object]
374
+ sf_object = tap[:object]
375
+ id_field = Tap.find_output_field(tap, "Id")
376
+ timestamp_field = Tap.find_output_field(tap, "Timestamp")
377
+
378
+
379
+ objects_to_get = Helpers.objects_for_history(client, tap)
380
+
381
+ (tap[:fields] - [id_field, timestamp_field]).each_with_index do |field, i|
382
+ builder.Phase(:number => phase += 1) do
383
+
384
+ f = field[:name]
385
+
386
+ selects = objects_to_get.reduce([]) do |memo, o|
387
+ fields = client.fields(o)
388
+ generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|f| fields.include?(f)}
389
+ specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|f| fields.include?(f)}
390
+ specific_history = ["SystemModstamp", "#{sf_object}Id"].all? {|f| fields.include?(f)}
391
+
392
+ select, mapping, mandatory = if generic_field_history
393
+ [
394
+ "SELECT NewValue, CreatedDate, ParentId FROM #{o} WHERE Field = '#{f}'",
395
+ "{\"xmlFieldsMapping\":{\"xmlFields\":[
396
+ {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
397
+ {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
398
+ {\"xmlFieldMapping\":{\"name\":\"ParentId\",\"label\":\"ParentId\",\"xmlPath\":\"ParentId\",\"metadataField\":\"Id\"}}
399
+ ]}}",
400
+ ["ParentId", "NewValue", "CreatedDate"]
401
+ ]
402
+ elsif specific_field_history
403
+ [
404
+
405
+ "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
406
+ "{\"xmlFieldsMapping\":{\"xmlFields\":[
407
+ {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
408
+ {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
409
+ {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
410
+ ]}}",
411
+ ["#{sf_object}Id", "NewValue", "CreatedDate"]
412
+ ]
413
+ elsif specific_history
414
+ if fields.include?(f)
415
+ [
416
+ "SELECT #{f}, SystemModstamp, #{sf_object}Id FROM #{o}",
417
+ "{\"xmlFieldsMapping\":{\"xmlFields\":[
418
+ {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
419
+ {\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
420
+ {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{o}Id\",\"xmlPath\":\"#{o}Id\",\"metadataField\":\"Id\"}}
421
+ ]}}",
422
+ ["#{sf_object}Id", f, "SystemModstamp"]
423
+ ]
424
+ else
425
+ [nil, nil]
426
+ end
427
+ else
428
+ fail "Unrecognized fields configuration for historization in SF."
429
+ end
430
+ if select.nil?
431
+ memo
432
+ else
433
+ memo.concat([{
434
+ :object => o,
435
+ :query => select,
436
+ :mapping => mapping,
437
+ :mandatory => mandatory
438
+ }])
439
+ end
440
+ end
441
+
442
+ unless selects.empty?
443
+ taps_with_history.concat([tap])
444
+ selects = selects.concat([{
445
+ :object => sf_object,
446
+ :query => "SELECT Id, #{f}, SystemModstamp FROM #{sf_object}",
447
+ :mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
448
+ {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
449
+ {\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
450
+ {\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
451
+ ]}}",
452
+ :mandatory => ["Id", f, "SystemModstamp"]
453
+ }])
454
+ selects.each_with_index do |obj, i|
455
+
456
+ o = obj[:object]
457
+ mapping = obj[:mapping]
458
+ select = obj[:query]
459
+ mandatory = obj[:mandatory]
460
+
461
+ Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{o} SF Reader #{i}", :id => "#{f}_#{o}_reader", :soql => select, :sfdcConnection => "SFDC", :fieldsMapping => mapping, :mandatoryFields => mandatory.join(';')}))
462
+ puts "#{f}_#{o}_reader:0 -> #{f}_#{sf_object}_gather:#{i}"
463
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_gather:#{i}", :fromNode => "#{f}_#{o}_reader:0", :metadata => "data_metadata"}))
464
+ end
465
+ Core::build_node2(builder, Nodes.gather2({:name => "#{f} Gather", :id => "#{f}_#{sf_object}_gather"}))
466
+ Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{f}_#{sf_object}_sort", :id => "#{f}_#{sf_object}_sort"}))
467
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_sort:0", :fromNode => "#{f}_#{sf_object}_gather:0", :metadata => "data_metadata"}))
468
+
469
+ Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{f}_#{sf_object}_copy"}))
470
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_copy:0", :fromNode => "#{f}_#{sf_object}_sort:0", :metadata => "data_metadata"}))
471
+
472
+ if s3_backup
473
+ Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{f}_#{sf_object} s3 Writer", :id => "#{f}_#{sf_object}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{sf_object}/#{f}_#{sf_object}_\`date2long(today())\`", :outputFieldNames => true}))
474
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_s3:0", :fromNode => "#{f}_#{sf_object}_copy:1", :metadata => "data_metadata"}))
475
+ end
476
+
477
+ transform = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = str2date($in.0.Timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\nreturn OK;\n}"
478
+
479
+ Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "#{f}_#{sf_object}_reformat", :transformation => transform}))
480
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_reformat:0", :fromNode => "#{f}_#{sf_object}_copy:0", :metadata => "data_metadata"}))
481
+
482
+ Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{f}_#{sf_object}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json({:type => :tap, :id => tap[:id], :fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => f, :meta => "Value"}]}).to_json}))
483
+
484
+ # Core::build_node2(builder, Nodes.trash2({:name => "#{f}_#{sf_object}_es", :id => "#{f}_#{sf_object}_es", :debugPrint => true}))
485
+ Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_es:0", :fromNode => "#{f}_#{sf_object}_reformat:0", :metadata => "es_metadata"}))
486
+ end
487
+ end
488
+ end
489
+ end
490
+ taps_with_history.each do |tap|
491
+ puts "#{tap[:id]}_LAST_RUN"
492
+ builder.Phase(:number => phase += 1) do
493
+ generate_func = <<HEREDOC
494
+ function integer generate() {
495
+ date time_end = jodaStr2date("${#{tap[:id]}_END}",["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC');
496
+ $out.0.key = "#{tap[:id]}_LAST_RUN";
497
+ $out.0.value = jodaDate2str(time_end,"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", 'en_US', 'UTC');
498
+ return OK;
499
+ }
500
+ HEREDOC
501
+
502
+ Core::build_node2(builder, Nodes.data_generator2({:guiName => tap[:id], :name => "generator_#{phase}", :id => "generator_#{phase}", :generate => generate_func}))
503
+ Core::build_node2(builder, Nodes.lookup_reader_writer2({:guiName => tap[:id], :lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{phase}" }))
504
+ Core::build_node2(builder, Nodes.edge2({:toNode => "gd_lookup_reader_#{phase}:0", :fromNode => "generator_#{phase}:0", :metadata => "lookup_metadata"}))
505
+
506
+ end
507
+ end
508
+ end
509
+ end
510
+ end
511
+
316
512
  def self.generate_downloaders(home, project, params)
317
513
  home = Pathname(home)
318
514
 
@@ -336,6 +532,19 @@ HEREDOC
336
532
  end
337
533
  end
338
534
 
535
+
536
+ def self.generate_history_downloaders(home, project, params)
537
+ home = Pathname(home)
538
+
539
+ incremental_taps = Taps.get_incremental(project[:taps])
540
+
541
+ sf_taps = Taps.get_salesforce(incremental_taps)
542
+ file_taps = Taps.get_file(incremental_taps)
543
+
544
+ create_incremental_downloader_run_graph(home + "main.grf", incremental_taps)
545
+ create_incremental_sf_history_init_graph(home + "incremental.grf", sf_taps, params)
546
+ end
547
+
339
548
  def self.create_incremental_file_downloading_graph(file, taps, options={})
340
549
  metadata = options[:metadata]
341
550
  store = options[:store] || "${GDC_EVENTSTORE}"