gd_bam 0.1.3 → 0.1.5

data/lib/generators/downloaders.rb

@@ -313,6 +313,202 @@ HEREDOC
           end
         end
 
+
+
+        def self.create_incremental_sf_history_init_graph(file, taps, options={})
+          metadata = options[:metadata]
+          store = options[:store] || "${GDC_EVENTSTORE}"
+          s3_backup = options[:s3_backup]
+
+
+          client = GoodData::Bam::Commands.get_sf_client(options)
+
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({:name => "Goodsales incremental Downloader"}) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+
+                Helpers::create_lookup_meta(builder)
+                builder.Metadata({:id   => "data_metadata"}) do |builder|
+                  Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => "Value"}]})
+                end
+
+                builder.Metadata({:id   => "es_metadata"}) do |builder|
+                  Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp", :type => "date"}, {:name => "Value"}]})
+                end
+
+                Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
+
+                Helpers::sf_connection(builder, {})
+                Helpers::property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
+                Helpers::property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
+                Helpers::property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
+                Helpers::property(builder, {:id => "SFDC_PASSWORD", :value => options[:sf_password]})
+                Helpers::property(builder, {:id => "SFDC_TOKEN", :value => options[:sf_token]})
+                Helpers::property(builder, {:id => "SFDC_USERNAME", :value => options[:sf_login]})
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+              end
+
+              phase = 1
+
+              taps_with_history = []
+              taps.each do |tap|
+                module_name = tap[:object]
+                dataset = tap[:id]
+                source_file  = tap[:source]
+
+                has_timestamp = Tap.has_output_field?(tap, "Timestamp")
+                timestamp_field = Tap.find_output_field(tap, "Timestamp")
+                id_field = Tap.find_output_field(tap, "Id")
+                tap = Tap.prepare_for_sf_downloader(tap)
+                builder.Phase(:number => phase += 1) do
+                  # Core::build_node2(builder, Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
+                end
+
+                fields = tap[:fields]
+                
+                puts tap[:object]
+                sf_object = tap[:object]
+                id_field = Tap.find_output_field(tap, "Id")
+                timestamp_field = Tap.find_output_field(tap, "Timestamp")
+                
+                
+                objects_to_get = Helpers.objects_for_history(client, tap)
+                
+                (tap[:fields] - [id_field, timestamp_field]).each_with_index do |field, i|
+                  builder.Phase(:number => phase += 1) do
+                    
+                    f = field[:name]
+
+                    selects = objects_to_get.reduce([]) do |memo, o|
+                      fields = client.fields(o)
+                      generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|f| fields.include?(f)}
+                      specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|f| fields.include?(f)}
+                      specific_history = ["SystemModstamp", "#{sf_object}Id"].all? {|f| fields.include?(f)}
+
+                      select, mapping, mandatory = if generic_field_history
+                        [
+                          "SELECT NewValue, CreatedDate, ParentId FROM #{o} WHERE Field = '#{f}'",
+                          "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                          {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
+                          {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
+                          {\"xmlFieldMapping\":{\"name\":\"ParentId\",\"label\":\"ParentId\",\"xmlPath\":\"ParentId\",\"metadataField\":\"Id\"}}
+                          ]}}",
+                          ["ParentId", "NewValue", "CreatedDate"]
+                        ]
+                      elsif specific_field_history
+                        [
+                          
+                          "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
+                          "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                          {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
+                          {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
+                          {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
+                          ]}}",
+                          ["#{sf_object}Id", "NewValue", "CreatedDate"]
+                        ]
+                      elsif specific_history
+                        if fields.include?(f)
+                          [
+                            "SELECT #{f}, SystemModstamp, #{sf_object}Id FROM #{o}",
+                            "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                            {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
+                            {\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
+                            {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{o}Id\",\"xmlPath\":\"#{o}Id\",\"metadataField\":\"Id\"}}
+                            ]}}",
+                            ["#{sf_object}Id", f, "SystemModstamp"]
+                          ]
+                        else
+                          [nil, nil]
+                        end
+                      else
+                        fail "Unrecognized fields configuration for historization in SF."
+                      end
+                      if select.nil?
+                        memo
+                      else
+                        memo.concat([{
+                          :object => o,
+                          :query => select,
+                          :mapping => mapping,
+                          :mandatory => mandatory
+                        }])
+                      end
+                    end
+
+                    unless selects.empty?
+                      taps_with_history.concat([tap])
+                      selects = selects.concat([{
+                        :object => sf_object,
+                        :query => "SELECT Id, #{f}, SystemModstamp FROM #{sf_object}",
+                        :mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                        {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
+                        {\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
+                        {\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
+                        ]}}",
+                        :mandatory => ["Id", f, "SystemModstamp"]
+                      }])
+                      selects.each_with_index do |obj, i|
+                      
+                        o = obj[:object]
+                        mapping = obj[:mapping]
+                        select = obj[:query]
+                        mandatory = obj[:mandatory]
+
+                        Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{o} SF Reader #{i}", :id => "#{f}_#{o}_reader", :soql => select, :sfdcConnection => "SFDC", :fieldsMapping => mapping, :mandatoryFields => mandatory.join(';')}))
+                        puts "#{f}_#{o}_reader:0 -> #{f}_#{sf_object}_gather:#{i}"
+                        Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_gather:#{i}", :fromNode => "#{f}_#{o}_reader:0", :metadata => "data_metadata"}))
+                      end
+                      Core::build_node2(builder, Nodes.gather2({:name => "#{f} Gather", :id => "#{f}_#{sf_object}_gather"}))
+                      Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{f}_#{sf_object}_sort", :id => "#{f}_#{sf_object}_sort"}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_sort:0", :fromNode => "#{f}_#{sf_object}_gather:0", :metadata => "data_metadata"}))
+
+                      Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{f}_#{sf_object}_copy"}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_copy:0", :fromNode => "#{f}_#{sf_object}_sort:0", :metadata => "data_metadata"}))
+
+                      if s3_backup
+                        Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{f}_#{sf_object} s3 Writer", :id => "#{f}_#{sf_object}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{sf_object}/#{f}_#{sf_object}_\`date2long(today())\`", :outputFieldNames => true}))
+                        Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_s3:0", :fromNode => "#{f}_#{sf_object}_copy:1", :metadata => "data_metadata"}))
+                      end
+
+                      transform = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = str2date($in.0.Timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\nreturn OK;\n}"
+
+                      Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "#{f}_#{sf_object}_reformat", :transformation => transform}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_reformat:0", :fromNode => "#{f}_#{sf_object}_copy:0", :metadata => "data_metadata"}))
+
+                      Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{f}_#{sf_object}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json({:type => :tap, :id => tap[:id], :fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => f, :meta => "Value"}]}).to_json}))
+
+                      # Core::build_node2(builder, Nodes.trash2({:name => "#{f}_#{sf_object}_es", :id => "#{f}_#{sf_object}_es", :debugPrint => true}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_es:0", :fromNode => "#{f}_#{sf_object}_reformat:0", :metadata => "es_metadata"}))
+                    end
+                  end
+                end
+              end
+              taps_with_history.each do |tap|
+                puts "#{tap[:id]}_LAST_RUN"
+                builder.Phase(:number => phase += 1) do
+                              generate_func = <<HEREDOC
+  function integer generate() {
+date time_end = jodaStr2date("${#{tap[:id]}_END}",["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC');
+$out.0.key = "#{tap[:id]}_LAST_RUN";
+$out.0.value = jodaDate2str(time_end,"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", 'en_US', 'UTC');
+return OK;
+}
+HEREDOC
+
+                  Core::build_node2(builder, Nodes.data_generator2({:guiName => tap[:id], :name => "generator_#{phase}", :id => "generator_#{phase}", :generate => generate_func}))
+                  Core::build_node2(builder, Nodes.lookup_reader_writer2({:guiName => tap[:id], :lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{phase}" }))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "gd_lookup_reader_#{phase}:0", :fromNode => "generator_#{phase}:0", :metadata => "lookup_metadata"}))
+
+                end
+              end
+            end
+          end
+        end
+
         def self.generate_downloaders(home, project, params)
           home = Pathname(home)
 
@@ -336,6 +532,19 @@ HEREDOC
           end
         end
 
+
+        def self.generate_history_downloaders(home, project, params)
+          home = Pathname(home)
+
+          incremental_taps = Taps.get_incremental(project[:taps])
+
+          sf_taps = Taps.get_salesforce(incremental_taps)
+          file_taps = Taps.get_file(incremental_taps)
+
+          create_incremental_downloader_run_graph(home + "main.grf", incremental_taps)
+          create_incremental_sf_history_init_graph(home + "incremental.grf", sf_taps, params)
+        end
+
         def self.create_incremental_file_downloading_graph(file, taps, options={})
           metadata = options[:metadata]
           store = options[:store] || "${GDC_EVENTSTORE}"