gd_bam 0.1.30 → 0.1.31

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.30'
+  VERSION = '0.1.31'
 end

data/lib/base/taps.rb.orig

@@ -0,0 +1,35 @@
+module GoodData 
+  module Bam
+    module Taps
+
+      def self.get_incremental(taps)
+        taps.find_all {|t| t[:direct] == false || t[:direct].nil? }
+      end
+
+      def self.get_salesforce(taps)
+        taps.find_all {|t| t[:source_type] == :salesforce}
+      end
+
+      def self.get_file(taps)
+        taps.find_all {|t| [:web, :file].include?(t[:source_type]) }
+      end
+
+<<<<<<< HEAD
+      def self.get_deleted_records_info(taps, client)
+        deleted = client.modules.map {|i| {:name => i[:name], :flag => i[:replicateable]}}.reduce({}) {|memo, x| memo.merge({x[:name] => x[:flag]})}
+        taps.map do |t|
+          if t[:source_type] == :salesforce
+            t.merge({:has_deleted_records => deleted[t[:object]]})
+          else
+            t.merge({:has_deleted_records => false})
+          end
+        end
+=======
+      def self.get_box(taps)
+        taps.find_all {|t| [:box].include?(t[:source_type]) }
+>>>>>>> Box init
+      end
+
+    end
+  end
+end

data/lib/cloud_connect/dsl/helpers.rb.orig

@@ -0,0 +1,229 @@
+module GoodData
+  module CloudConnect
+    module Helpers
+
+      def self.property(builder, data)
+        builder.Property({
+          :id => data[:id],
+          :name => data[:name] || data[:id],
+          :value => data[:value]
+        })
+      end
+
+      def self.property_file(builder, data)
+        builder.Property({
+          :id => data[:id],
+          :fileURL => data[:fileURL]
+        })
+      end
+
+<<<<<<< HEAD
+      def self.create_deleted_records_meta(builder)
+        builder.Metadata({:id => "deleted_records_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "deleted_records_metadata",
+            :fields => [
+              {:type => "string", :name => "Id"},
+              {:type => "string", :name => "Timestamp"}
+=======
+      def self.create_http_request_meta(builder)
+        builder.Metadata({:id => "http_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "http_metadata",
+            :fields => [
+              {:type => "string", :name => "response"},
+              {:type => "string", :name => "all"}
+>>>>>>> Box init
+            ]
+          })
+        end
+      end
+
+      def self.create_trash_meta(builder)
+        builder.Metadata({:id => "trash_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "trash_metadata",
+            :fields => [{:type => "string", :name => "all"}]
+          })
+        end
+      end
+
+      def self.save_metadata(filename, data)
+        dirname = File.dirname(filename)
+        FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
+ 
+        File.open(filename, "w") do |file|
+          builder = Builder::XmlMarkup.new(:target => file, :indent=>2)
+          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"  
+          csv_metadata(builder, data)
+        end
+      end
+
+      def self.create_moving_graph(file, options={})
+        source = options[:source]
+        target = options[:target]
+        operation = options[:operation]
+        force = options[:force] || false
+
+        File.open(file, "w") do |file|
+          builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+          builder.Graph({
+            :name => "File Copy"
+          }) do
+            builder.Global do
+              builder.Metadata({:id   => "list_metadata"}) do |builder|
+                Helpers::csv_metadata(builder, {
+                  :name => "list_metadata",
+                  :fields => [{:name=>"filePath", :type=>"string"}]
+                })
+              end
+              Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+            end
+            builder.Phase(:number => 0) do
+
+              transformation_source = "function integer transform() {\n" + ([["filePath", "filePath"]].map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+              Core::build_node2(builder, Nodes.file_list2(:baseURL => target, :id => "file_list", :output_mapping => transformation_source))
+
+              Core::build_node2(builder, Nodes.file_delete2(:baseURL => "${filePath}", :id => "file_delete"))
+              Core::build_node2(builder, Nodes.edge2({:toNode => "file_delete:0", :fromNode => "file_list:0", :metadata => "list_metadata"}))
+            end  
+            builder.Phase(:number => 1) do
+              Core::build_node2(builder, Nodes.file_copy2({:sourcePath => source, :targetPath => target, :operation => operation, :id => "file_copy"}))
+            end
+          end
+        end
+      end
+
+      def self.csv_metadata(builder, description)
+        sf_description = description.merge({
+          :fieldDelimiter => ",",
+          :recordDelimiter => "\\n",
+          :type => "delimited",
+        })
+        metadata(builder, sf_description)
+      end
+
+      # prepares a sink for sinking into GD. This means dealing with stuff like scientific notation which is handled bu uploading as a decimal and not string
+      def self.sink_out_gd_metadata(metadata, sink)
+        metadata_fields = metadata[:fields]
+        sink_facts = GoodData::Bam::Sink.get_fact_fields(sink).map {|f| f[:meta]}
+        x = metadata_fields.map {|f| sink_facts.include?(f[:name]) ? f.merge({:type => "decimal"}) : f}
+        GoodData::Bam::Metadata.create(metadata.merge({:fields => x}))
+      end
+
+      def self.metadata(builder, description)
+        builder.Record({
+            :fieldDelimiter => description[:fieldDelimiter],
+            :name => description[:name],
+            :recordDelimiter => description[:recordDelimiter],
+            :type => description[:type]
+          }) do |record|
+          description[:fields].each do |field|
+            builder.Field :name => field[:name], :type => field[:type] || "string", :nullable => "true"
+          end
+        end
+      end
+
+      def self.create_lookup_meta(builder)
+        builder.Metadata({:id   => "lookup_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "lookup_metadata",
+            :fields => [{:type => "string", :name => "key"}, {:type => "string", :name => "value"}]
+          })
+        end
+      end
+
+      def self.create_file_list_meta(builder)
+        builder.Metadata({:id   => "file_list"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "file_list",
+            :fields => [
+              {:name => "filePath", :type => "string"},
+              {:name => "fileName", :type => "string"}
+            ]
+          })
+        end
+      end
+
+      def self.create_read_error_meta(builder)
+         builder.Metadata({:id   => "reader_error_metadata"}) do |builder|
+           csv_metadata(builder, {
+             :name   => "reader_error_metadata",
+             :fields => [
+               {:name => "line_number", :type => "integer"},
+               {:name => "field_number", :type => "integer"},
+               {:name => "record", :type => "string"},
+               {:name => "message", :type => "string"},
+               {:name => "file", :type => "string"}
+             ]
+           })
+         end
+      end
+
+      def self.create_run_graph_failure_metadata(builder)
+        builder.Metadata({:id   => "run_graph_failure_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "run_graph_failure_metadata",
+            :fields => [
+              {:type => "string", :name => "graph"},
+              {:type => "string", :name => "result"},
+              {:type => "string", :name => "description"},
+              {:type => "string", :name => "message"},
+              {:type => "decimal", :name => "duration"}
+            ]
+          })
+        end
+      end
+
+      def self.create_run_graph(file, options={})
+        subgraphs = options[:subgraphs]
+        flow = options[:flow]
+        File.open(file, "w") do |file|
+          builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+          builder.Graph({
+            :name => "Run graph"
+          }) do
+            builder.Global do
+              property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+              property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
+              create_trash_meta(builder)
+              create_lookup_meta(builder)
+
+            end
+            phase = 0
+            subgraphs.each do |subgraph|
+              name = GoodData::Bam::Step.step_name(GoodData::Bam::Graph.create(:path => subgraph))
+              # Pathname(subgraph).basename.to_s.chomp(Pathname(subgraph).extname)
+              builder.Phase(:number => phase+1) do
+                id1 = GoodData::CloudConnect::Nodes.get_id
+                id2 = GoodData::CloudConnect::Nodes.get_id
+                ctl = "function integer generate() {$out.0.all = \"FLOW=#{flow}\";return OK;}"
+                Core::build_node2(builder, Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
+                Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
+              end
+              builder.Phase(:number => phase+2) do
+
+                id1 = GoodData::CloudConnect::Nodes.get_id
+                id2 = GoodData::CloudConnect::Nodes.get_id
+                ctl = "function integer generate() {$out.0.all = \"NAME=#{name}\";return OK;}"
+                Core::build_node2(builder, Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
+                Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
+              end  
+
+              builder.Phase(:number => phase+3) do
+                Core::build_node2(builder, Nodes.run_graph2({:guiName => name, :name => name, :id => flow, :graphName => subgraph}))
+              end
+              phase += 4
+            end
+          end
+        end
+      end
+    
+
+    end
+  end
+end

data/lib/generators/downloaders.rb

@@ -427,9 +427,8 @@ HEREDOC
                 Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
               end
 
-              phase = 1
+              taps_memo = {}
 
-              taps_with_history = []
               taps.each do |tap|
                 module_name = tap[:object]
                 dataset = tap[:id]
@@ -450,134 +449,154 @@ HEREDOC
                 objects_to_get = Helpers.objects_for_history(client, tap)
 
                 (tap[:fields] - [id_field, timestamp_field]).each_with_index do |field, i|
-                  builder.Phase(:number => phase += 1) do
-
-                    f = field[:name]
-                    selects = if (f == "CreatedDate") then
-                      [{
-                        :object => sf_object,
-                        :query => "SELECT Id, CreatedDate FROM #{sf_object}",
-                        :mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                        {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Value\"}},
-                        {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
-                        {\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
-                        ]}}",
-                        :mandatory => ["Id", "CreatedDate"]
-                      }]
-                      # binding.pry
-                      # temp
-                    else
-                      objects_to_get.reduce([]) do |memo, o|
-                        fields = client.fields(o)
-                        # how the history is stored does not rely on table naming convention. It is a couple of conventionsmangled together.This piece of code is trying to recognize which it is and produce the right downloader
-                        generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|fl| fields.include?(fl)}
-                        specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
-                        specific_history = ["CreatedDate", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
-
-                        select, mapping, mandatory = if generic_field_history && !field[:multi_currency]
-                          # This kicks in when you have Field History type of table. There are several versions in SF. This one deals with the one that references ID of the record as Pernt Id. This is not true for opportunity, account.
-                          [
-                            "SELECT NewValue, CreatedDate, ParentId FROM #{o} WHERE Field = '#{f}'",
-                            "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                            {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
-                            {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
-                            {\"xmlFieldMapping\":{\"name\":\"ParentId\",\"label\":\"ParentId\",\"xmlPath\":\"ParentId\",\"metadataField\":\"Id\"}}
-                            ]}}",
-                            ["ParentId", "NewValue", "CreatedDate"]
-                          ]
-                        elsif (specific_field_history && !field[:multi_currency])
-                          # This is like the previous with the difference that it specifically names the id of the record. Like OpportunityId, AccountId, etc.
+                  f = field[:name]
+
+                  selects = if (f == "CreatedDate") then
+                    [{
+                      :object => sf_object,
+                      :query => "SELECT Id, CreatedDate FROM #{sf_object}",
+                      :mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                      {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Value\"}},
+                      {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
+                      {\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
+                      ]}}",
+                      :mandatory => ["Id", "CreatedDate"]
+                    }]
+                  else
+                    objects_to_get.reduce([]) do |memo, o|
+                      fields = client.fields(o)
+                      # how the history is stored does not rely on table naming convention. It is a couple of conventionsmangled together.This piece of code is trying to recognize which it is and produce the right downloader
+                      generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|fl| fields.include?(fl)}
+                      specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
+                      specific_history = ["CreatedDate", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
+
+                      select, mapping, mandatory = if generic_field_history && !field[:multi_currency]
+                        # This kicks in when you have Field History type of table. There are several versions in SF. This one deals with the one that references ID of the record as Pernt Id. This is not true for opportunity, account.
+                        [
+                          "SELECT NewValue, CreatedDate, ParentId FROM #{o} WHERE Field = '#{f}'",
+                          "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                          {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
+                          {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
+                          {\"xmlFieldMapping\":{\"name\":\"ParentId\",\"label\":\"ParentId\",\"xmlPath\":\"ParentId\",\"metadataField\":\"Id\"}}
+                          ]}}",
+                          ["ParentId", "NewValue", "CreatedDate"]
+                        ]
+                      elsif (specific_field_history && !field[:multi_currency])
+                        # This is like the previous with the difference that it specifically names the id of the record. Like OpportunityId, AccountId, etc.
+                        [
+                          "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
+                          "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                          {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
+                          {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
+                          {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
+                          ]}}",
+                          ["#{sf_object}Id", "NewValue", "CreatedDate"]
+                        ]
+                      elsif specific_history
+                        # Specific history is the row like history. The only instance I know of is OpportunityHistory.
+                        if fields.include?(f)
                           [
-                            "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
+                            (field[:multi_currency] ? "SELECT convertCurrency(#{f}), CreatedDate, #{sf_object}Id FROM #{o}" : "SELECT #{f}, CreatedDate, #{sf_object}Id FROM #{o}"),
                             "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                            {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
+                            {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
                             {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
                             {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
                             ]}}",
-                            ["#{sf_object}Id", "NewValue", "CreatedDate"]
+                            ["#{sf_object}Id", f, "CreatedDate"]
                           ]
-                        elsif specific_history
-                          # Specific history is the row like history. The only instance I know of is OpportunityHistory.
-                          if fields.include?(f)
-                            [
-                              (field[:multi_currency] ? "SELECT convertCurrency(#{f}), CreatedDate, #{sf_object}Id FROM #{o}" : "SELECT #{f}, CreatedDate, #{sf_object}Id FROM #{o}"),
-                              "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                              {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
-                              {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
-                              {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
-                              ]}}",
-                              ["#{sf_object}Id", f, "CreatedDate"]
-                            ]
-                          else
-                            [nil, nil, nil]
-                          end
-                        else
-                          puts "Warning: Unrecognized fields configuration for historization in SF for #{o}"
-                        end
-                        if select.nil?
-                          memo
                         else
-                          memo.concat([{
-                            :object => o,
-                            :query => select,
-                            :mapping => mapping,
-                            :mandatory => mandatory
-                          }])
+                          [nil, nil, nil]
                         end
+                      else
+                        puts "Warning: Unrecognized fields configuration for historization in SF for #{o}"
                       end
-                    end
-                    unless selects.empty?
-                      # binding.pry if f== "CreatedDate"
-                      taps_with_history.concat([tap])
-                      selects = selects.concat([{
-                        :object => sf_object,
-                        :query => (field[:multi_currency] ? "SELECT Id, convertCurrency(#{f}), SystemModstamp FROM #{sf_object}" : "SELECT Id, #{f}, SystemModstamp FROM #{sf_object}"),
-                        :mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                        {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
-                        {\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
-                        {\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
-                        ]}}",
-                        :mandatory => ["Id", f, "SystemModstamp"]
-                      }])
-                      selects.each_with_index do |obj, i|
-                        # binding.pry if f== "CreatedDate"
-                        o = obj[:object]
-                        mapping = obj[:mapping]
-                        select = obj[:query]
-                        mandatory = obj[:mandatory]
-                        # puts "#{f}_#{o}_reader:0 -> #{f}_#{sf_object}_gather:#{i}"
-
-                        Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{o} SF Reader #{i}", :id => "#{f}_#{o}_#{i}_reader", :soql => select, :sfdcConnection => "SFDC", :fieldsMapping => mapping, :mandatoryFields => mandatory.join(';')}))
-                        Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_gather:#{i}", :fromNode => "#{f}_#{o}_#{i}_reader:0", :metadata => "data_metadata"}))
+                      if select.nil?
+                        memo
+                      else
+                        memo.concat([{
+                          :object => o,
+                          :query => select,
+                          :mapping => mapping,
+                          :mandatory => mandatory
+                        }])
                       end
-                      Core::build_node2(builder, Nodes.gather2({:name => "#{f} Gather", :id => "#{f}_#{sf_object}_gather"}))
-                      Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{f}_#{sf_object}_sort", :id => "#{f}_#{sf_object}_sort"}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_sort:0", :fromNode => "#{f}_#{sf_object}_gather:0", :metadata => "data_metadata"}))
+                    end
+                  end
+                  unless selects.empty?
+                    # binding.pry if f== "CreatedDate"
+                    # taps_with_history.concat([tap])
+                    selects = selects.concat([{
+                      :object => sf_object,
+                      :query => (field[:multi_currency] ? "SELECT Id, convertCurrency(#{f}), SystemModstamp FROM #{sf_object}" : "SELECT Id, #{f}, SystemModstamp FROM #{sf_object}"),
+                      :mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                      {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
+                      {\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
+                      {\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
+                      ]}}",
+                      :mandatory => ["Id", f, "SystemModstamp"]
+                    }])
+
+                    taps_memo["#{tap[:id]}:#{f}"] = {
+                      :tap => tap,
+                      :field => f,
+                      :selects => selects
+                    }
+                  end
+                end
+              end
 
-                      Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{f}_#{sf_object}_copy"}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_copy:0", :fromNode => "#{f}_#{sf_object}_sort:0", :metadata => "data_metadata"}))
+              phase = 1
 
-                      if s3_backup
-                        Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{f}_#{sf_object} s3 Writer", :id => "#{f}_#{sf_object}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{sf_object}/#{f}_#{sf_object}_\`date2long(today())\`", :outputFieldNames => true}))
-                        Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_s3:0", :fromNode => "#{f}_#{sf_object}_copy:1", :metadata => "data_metadata"}))
-                      end
+              binding.pry
+              taps_memo.values.each_slice(6).to_a.each do |slice|
+                builder.Phase(:number => phase +=1) do
+                  slice.each do |field|
+                # taps_memo.each do |key, field|
+                    f = field[:field]
+                    sf_object = field[:tap][:object]
+                    selects = field[:selects]
+                    selects.each_with_index do |obj, i|
+                      o = obj[:object]
+                      mapping = obj[:mapping]
+                      select = obj[:query]
+                      mandatory = obj[:mandatory]
+                      Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{o} SF Reader #{i}", :id => "#{f}_#{o}_#{i}_reader", :soql => select, :sfdcConnection => "SFDC", :fieldsMapping => mapping, :mandatoryFields => mandatory.join(';')}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_gather:#{i}", :fromNode => "#{f}_#{o}_#{i}_reader:0", :metadata => "data_metadata"}))
+                    end
+                    Core::build_node2(builder, Nodes.gather2({:name => "#{f} Gather", :id => "#{f}_#{sf_object}_gather"}))
+                    Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{f}_#{sf_object}_sort", :id => "#{f}_#{sf_object}_sort"}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_sort:0", :fromNode => "#{f}_#{sf_object}_gather:0", :metadata => "data_metadata"}))
 
-                      transform = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = str2date($in.0.Timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\nreturn OK;\n}"
+                    Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{f}_#{sf_object}_copy"}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_copy:0", :fromNode => "#{f}_#{sf_object}_sort:0", :metadata => "data_metadata"}))
 
-                      Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "#{f}_#{sf_object}_reformat", :transformation => transform}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_reformat:0", :fromNode => "#{f}_#{sf_object}_copy:0", :metadata => "data_metadata"}))
+                    if s3_backup
+                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{f}_#{sf_object} s3 Writer", :id => "#{f}_#{sf_object}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{sf_object}/#{f}_#{sf_object}_\`date2long(today())\`", :outputFieldNames => true}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_s3:0", :fromNode => "#{f}_#{sf_object}_copy:1", :metadata => "data_metadata"}))
+                    end
 
-                      Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{f}_#{sf_object}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json({:type => :tap, :id => tap[:id], :fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => f, :meta => "Value"}]}).to_json}))
+                    transform = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = str2date($in.0.Timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\nreturn OK;\n}"
+
+                    Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "#{f}_#{sf_object}_reformat", :transformation => transform}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_reformat:0", :fromNode => "#{f}_#{sf_object}_copy:0", :metadata => "data_metadata"}))
 
-                      # Core::build_node2(builder, Nodes.trash2({:name => "#{f}_#{sf_object}_es", :id => "#{f}_#{sf_object}_es", :debugPrint => true}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_es:0", :fromNode => "#{f}_#{sf_object}_reformat:0", :metadata => "es_metadata"}))
-                    end
                   end
                 end
               end
 
-              taps_with_history.uniq_by {|x| x[:id]}.each do |tap|
+              taps_memo.each do |key, field|
+                builder.Phase(:number => phase +=1) do 
+                  tap = field[:tap]
+                  file = field[:tap][:id]
+                  sf_object = field[:tap][:object]
+                  f = field[:field]
+                  Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{f}_#{sf_object}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json({:type => :tap, :id => tap[:id], :fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => f, :meta => "Value"}]}).to_json}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_es:0", :fromNode => "#{f}_#{sf_object}_reformat:0", :metadata => "es_metadata"}))
+                end
+              end
 
+              taps_memo.values.uniq_by {|x| x[:tap][:id]}.each do |field_tap|
+                tap = field_tap[:tap]
                 # Download the Deleted records for these taps. since we will not have a chance to download them later
 
                 if tap[:has_deleted_records]
@@ -652,7 +671,7 @@ HEREDOC
               :graph_to_run => "graphs/#{id}_download.grf"
             })
             create_incremental_file_downloading_graph(home + "#{tap[:id]}_download.grf", [tap], params)
-          
+
           end
         end
 
@@ -681,6 +700,7 @@ HEREDOC
                 Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
                 Helpers::create_lookup_meta(builder)
                 taps.each do |tap|
+
                   module_name = tap[:object]
                   file    = tap[:id] || module_name
 
@@ -710,7 +730,6 @@ HEREDOC
                 dataset = file || module_name
                 source_file  = tap[:source]
 
-                
                 has_timestamp = GoodData::Bam::Tap.has_output_field?(tap, "Timestamp")
                 timestamp_field = GoodData::Bam::Tap.find_output_field(tap, "Timestamp")
                 id_field = GoodData::Bam::Tap.find_output_field(tap, "Id")
@@ -725,17 +744,13 @@ HEREDOC
                   end
                   stuff = mapping + add.join(",") + "]}}"
 
-                  mandatory_fields = fields.reject {|f| f[:is_mandatory] == false }.map {|f| f[:name] + ";"}.join
-
-                  Core::build_node2(builder, Nodes.reader2({:name => "#{file} File Reader", :id => "#{file}_reader", :fileURL =>  "${FILE}"}))
+                  Core::build_node2(builder, Nodes.reader2(Utils.merge_only_values({:name => "#{file} File Reader", :id => "#{file}_reader", :fileURL =>  "${FILE}"}, tap, [:charset])))
                   es_transformation_source = if has_timestamp
-                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = str2date($in.0.#{timestamp_field[:name]},\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");;\nreturn OK;\n}"
+                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = str2date($in.0.#{timestamp_field[:name]},\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\nreturn OK;\n}"
                   else
                     "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = long2date(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"
                   end
 
-                  # build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_reformat", :transformation => transformation_source}))
-
                   Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reader:0", :metadata => "#{file}_source_metadata"}))
                   Core::build_node2(builder, Nodes.writer2({:enabled => "disabled", :name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))

data/lib/generators/etl.rb

@@ -191,7 +191,7 @@ HEREDOC
               end
 
               builder.Phase(:number => 0) do
-                Core::build_node2(builder, Nodes.reader2({:name => "reader", :id => "reader", :fileURL => tap[:source] }))
+                Core::build_node2(builder, Nodes.reader2(Utils.merge_only_values({:name => "reader", :id => "reader", :fileURL => tap[:source]}, tap, [:charset])))
 
                 Core::build_node2(builder, Nodes.copy2({:name => "copy", :id => "copy"}))
                 Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "reader:0", :metadata => "file_metadata"}))

data/lib/utils/utils.rb

@@ -61,6 +61,15 @@ module GoodData
         keys.all? {|key| !params[:additional_params][key].blank?}
       end
 
-		end
-	end
+      def self.merge_only_values(merge_onto, data, keys=[])
+        filtered_data = data.inject({}) {|memo, k| memo[k.first] = k[1] unless k[0].blank?; memo}
+        if keys.empty?
+          merge_onto.merge(filtered_data)
+        else
+          merge_onto.merge(filtered_data.select {|key, val| keys.include?(key)})
+        end
+      end
+
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.1.30
+  version: 0.1.31
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-12 00:00:00.000000000 Z
+date: 2013-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -320,9 +320,11 @@ files:
 - lib/base/step.rb
 - lib/base/tap.rb
 - lib/base/taps.rb
+- lib/base/taps.rb.orig
 - lib/cloud_connect/dsl/cc.rb
 - lib/cloud_connect/dsl/es_helpers.rb
 - lib/cloud_connect/dsl/helpers.rb
+- lib/cloud_connect/dsl/helpers.rb.orig
 - lib/cloud_connect/dsl/nodes.rb
 - lib/cloud_connect/dsl/sf_helpers.rb
 - lib/cloud_connect/dsl/structure_helpers.rb