gd_bam 0.1.14 → 0.1.15

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.14'
+  VERSION = '0.1.15'
 end

data/lib/generators/downloaders.rb

@@ -378,72 +378,81 @@ HEREDOC
                   builder.Phase(:number => phase += 1) do
 
                     f = field[:name]
-                    selects = objects_to_get.reduce([]) do |memo, o|
-                      fields = client.fields(o)
-                      generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|fl| fields.include?(fl)}
-                      specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
-                      specific_history = ["CreatedDate", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
-
-                      select, mapping, mandatory = if generic_field_history
-                        [
-                          "SELECT NewValue, CreatedDate, ParentId FROM #{o} WHERE Field = '#{f}'",
-                          "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                          {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
-                          {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
-                          {\"xmlFieldMapping\":{\"name\":\"ParentId\",\"label\":\"ParentId\",\"xmlPath\":\"ParentId\",\"metadataField\":\"Id\"}}
-                          ]}}",
-                          ["ParentId", "NewValue", "CreatedDate"]
-                        ]
-                      elsif specific_field_history
-                        [
-                          "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
-                          "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                          {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
-                          {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
-                          {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
-                          ]}}",
-                          ["#{sf_object}Id", "NewValue", "CreatedDate"]
-                        ]
-                      elsif specific_history
-                        if f == "CreatedDate"
+                    selects = if (f == "CreatedDate") then
+                      [{
+                        :object => sf_object,
+                        :query => "SELECT Id, CreatedDate FROM #{sf_object}",
+                        :mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                        {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Value\"}},
+                        {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
+                        {\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
+                        ]}}",
+                        :mandatory => ["Id", "CreatedDate"]
+                      }]
+                      # binding.pry
+                      # temp
+                    else
+                      objects_to_get.reduce([]) do |memo, o|
+                        fields = client.fields(o)
+                        # how the history is stored does not rely on table naming convention. It is a couple of conventionsmangled together.This piece of code is trying to recognize which it is and produce the right downloader
+                        generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|fl| fields.include?(fl)}
+                        specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
+                        specific_history = ["CreatedDate", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
+
+                        select, mapping, mandatory = if generic_field_history
+                          # This kicks in when you have Field History type of table. There are several versions in SF. This one deals with the one that references ID of the record as Pernt Id. This is not true for opportunity, account.
                           [
-                            "SELECT CreatedDate, #{sf_object}Id FROM #{o}",
+                            "SELECT NewValue, CreatedDate, ParentId FROM #{o} WHERE Field = '#{f}'",
                             "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                            {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Value\"}},
+                            {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
                             {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
-                            {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
+                            {\"xmlFieldMapping\":{\"name\":\"ParentId\",\"label\":\"ParentId\",\"xmlPath\":\"ParentId\",\"metadataField\":\"Id\"}}
                             ]}}",
-                            ["#{sf_object}Id", "CreatedDate"]
+                            ["ParentId", "NewValue", "CreatedDate"]
                           ]
-                        elsif fields.include?(f)
+                        elsif specific_field_history
+                          # This is like the previous with the difference that it specifically names the id of the record. Like OpportunityId, AccountId, etc.
                           [
-                            "SELECT #{f}, CreatedDate, #{sf_object}Id FROM #{o}",
+                            "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
                             "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                            {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
+                            {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
                             {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
                             {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
                             ]}}",
-                            ["#{sf_object}Id", f, "CreatedDate"]
+                            ["#{sf_object}Id", "NewValue", "CreatedDate"]
                           ]
+                        elsif specific_history
+                          # Specific history is the row like history. The only instance I know of is OpportunityHistory.
+                          if fields.include?(f)
+                            [
+                              "SELECT #{f}, CreatedDate, #{sf_object}Id FROM #{o}",
+                              "{\"xmlFieldsMapping\":{\"xmlFields\":[
+                              {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
+                              {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
+                              {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
+                              ]}}",
+                              ["#{sf_object}Id", f, "CreatedDate"]
+                            ]
+                          else
+                            [nil, nil, nil]
+                          end
                         else
-                          [nil, nil]
+                          fail "Unrecognized fields configuration for historization in SF."
+                        end
+                        if select.nil?
+                          memo
+                        else
+                          memo.concat([{
+                            :object => o,
+                            :query => select,
+                            :mapping => mapping,
+                            :mandatory => mandatory
+                          }])
                         end
-                      else
-                        fail "Unrecognized fields configuration for historization in SF."
-                      end
-                      if select.nil?
-                        memo
-                      else
-                        memo.concat([{
-                          :object => o,
-                          :query => select,
-                          :mapping => mapping,
-                          :mandatory => mandatory
-                        }])
                       end
                     end
-
                     unless selects.empty?
+                      # binding.pry if f== "CreatedDate"
                       taps_with_history.concat([tap])
                       selects = selects.concat([{
                         :object => sf_object,
@@ -456,15 +465,15 @@ HEREDOC
                         :mandatory => ["Id", f, "SystemModstamp"]
                       }])
                       selects.each_with_index do |obj, i|
-
+                        # binding.pry if f== "CreatedDate"
                         o = obj[:object]
                         mapping = obj[:mapping]
                         select = obj[:query]
                         mandatory = obj[:mandatory]
                         # puts "#{f}_#{o}_reader:0 -> #{f}_#{sf_object}_gather:#{i}"
 
-                        Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{o} SF Reader #{i}", :id => "#{f}_#{o}_reader", :soql => select, :sfdcConnection => "SFDC", :fieldsMapping => mapping, :mandatoryFields => mandatory.join(';')}))
-                        Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_gather:#{i}", :fromNode => "#{f}_#{o}_reader:0", :metadata => "data_metadata"}))
+                        Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{o} SF Reader #{i}", :id => "#{f}_#{o}_#{i}_reader", :soql => select, :sfdcConnection => "SFDC", :fieldsMapping => mapping, :mandatoryFields => mandatory.join(';')}))
+                        Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_gather:#{i}", :fromNode => "#{f}_#{o}_#{i}_reader:0", :metadata => "data_metadata"}))
                       end
                       Core::build_node2(builder, Nodes.gather2({:name => "#{f} Gather", :id => "#{f}_#{sf_object}_gather"}))
                       Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{f}_#{sf_object}_sort", :id => "#{f}_#{sf_object}_sort"}))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.15
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-17 00:00:00.000000000 Z
+date: 2013-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -308,7 +308,6 @@ extra_rdoc_files:
 - README.md
 files:
 - bin/bam
-- bin/bam.orig
 - lib/bam/version.rb
 - lib/bam.rb
 - lib/base/errors.rb
@@ -339,7 +338,6 @@ files:
 - lib/compiler/etl_visitor.rb
 - lib/dsl/dsl.rb
 - lib/generators/downloaders.rb
-- lib/generators/downloaders.rb.orig
 - lib/generators/etl.rb
 - lib/generators/validators.rb
 - lib/graphs/docentize.grf

data/bin/bam.orig

@@ -1,375 +0,0 @@
-#!/usr/bin/env ruby
-require 'gli'
-require 'bam'
-
-include GLI::App
-
-program_desc 'Describe your application here'
-
-version Bam::VERSION
-
-# desc 'Describe some switch here'
-# switch [:s,:switch]
-# 
-desc 'Verbose'
-default_value false
-arg_name 'verbose'
-switch [:v,:verbose]
-
-desc 'Http logger'
-default_value false
-arg_name 'logger'
-switch [:l,:logger]
-
-desc 'Generates clover project based on information in current directory. The default ouptut is the directory ./clover-project'
-# arg_name 'Describe arguments to new here'
-command :generate do |c|
-  
-  c.desc 'generate only specified flow'
-  c.arg_name 'only'
-  c.flag :only
-  
-  c.action do |global_options,options,args|
-    params = PARAMS.merge(options).merge({
-      :project_name => "etl-#{PARAMS[:project_name]}",
-      :graph_repos => [
-        GoodData::Bam::Repository.create(:type => :file, :base => Pathname('./local_graphs').expand_path),
-        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)
-      ]})
-    GoodData::Bam::Commands::generate('.', params)
-  end
-end
-
-desc 'Jacks into SF.'
-# arg_name 'Describe arguments to new here'
-command :sf_jack_in do |c|
-
-  c.action do |global_options,options,args|
-    GoodData::Bam::Commands::sf_jack_in(PARAMS)
-  end
-end
-
-# desc 'Jacks into project.'
-# # arg_name 'Describe arguments to new here'
-# command :project_jack_in do |c|
-# 
-#   c.action do |global_options,options,args|
-#     GoodData::Bam::Commands::project_jack_in(PARAMS)
-#   end
-# end
-
-
-desc 'Validatates connection to SalesForce.'
-# arg_name 'Describe arguments to new here'
-command :sf_validate_connection do |c|
-
-  c.action do |global_options,options,args|
-    GoodData::Bam::Commands::sf_validate_connection(PARAMS)
-  end
-end
-
-
-desc 'Generate data validator.'
-# arg_name 'Describe arguments to new here'
-command :generate_validator do |c|
-  c.action do |global_options,options,args|
-    GoodData::Bam::Commands::generate_validators('.', PARAMS.merge(options).merge({
-      :project_name => "validator-#{PARAMS[:project_name]}",
-      :graph_repos => [
-        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
-        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)
-      ]
-      }))
-  end
-end
-
-desc 'Run data validator.'
-# arg_name 'Describe arguments to new here'
-command :run_validator do |c|
-
-  c.desc 'Checker process ID'
-  c.arg_name 'process'
-  c.flag :process
-
-  c.action do |global_options,options,args|
-    process = options[:process]
-    files = args.map {|f| Pathname(f)}
-    files.each do |f|
-      fail "Provded file \"#{f}\" does not exist." unless File.exist?(f)
-    end
-    fail "You need to specify process" if process.blank?
-    
-    GoodData::Bam::Commands::connect_to_gd()
-    GoodData::Bam::Commands::run_validator(process, files, PARAMS)
-  end
-end
-
-
-desc 'Generates clover project for downloaders.'
-# arg_name 'Describe arguments to new here'
-command :generate_downloaders do |c|
-
-  c.desc 's3 backup'
-  c.arg_name 'backup'
-  c.flag :backup
-
-  c.action do |global_options,options,args|
-    params = global_options[:user_params]
-    params = PARAMS.merge({
-      :project_name => "downloaders-#{PARAMS[:project_name]}",
-      :graph_repos => [
-        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
-        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)]
-    })
-    GoodData::Bam::Commands::generate_downloaders(".", params)
-  end
-end
-
-command :generate_history_downloaders do |c|
-
-  c.desc 's3 backup'
-  c.arg_name 'backup'
-  c.flag :backup
-
-  c.action do |global_options,options,args|
-    params = global_options[:user_params]
-    params = PARAMS.merge({
-      :project_name => "downloaders-#{PARAMS[:project_name]}",
-      :graph_repos => [
-        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
-        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)]
-    })
-<<<<<<< HEAD
-    GoodData::Bam::Commands::generate_downloaders(".", params)
-=======
-    GoodData::Bam::Commands::generate_history_downloaders(".", params)
->>>>>>> Added historical SF downloader
-  end
-end
-
-
-
-desc 'Validates that the tap has the fields it is claimed it should have. This is supposed to make the mitigate errors during deploy.'
-# arg_name 'Describe arguments to new here'
-command :sf_taps_validate do |c|
-  c.action do |global_options,options,args|
-    verbose = global_options[:v]
-    params = PARAMS.merge({
-      :graph_repos => [
-        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
-        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)]})
-    result = GoodData::Bam::Commands::validate_sf_taps(params)
-
-    error = false
-    result.each_pair do |obj, fields|
-      if fields.empty?
-        puts HighLine::color("GOOD", :green) + " #{obj}"
-      else
-        error = true
-        puts HighLine::color("BAD", :red) + " #{obj} [" + fields.join(', ') + "]"
-      end
-    end
-    exit_now!("Errors found",exit_code=1) if error
-  end
-end
-
-desc "Prepares params.prm file for debugging that particular graph"
-command :debug do |c|
-  c.action do |global_options,options,args|
-    fail "Arguments should be passed as \"project\" \"flow\" \"graph\"" if args.length < 3
-    GoodData::Bam::Commands::set_up_debug(args.first, args[1], args[2])
-  end
-end
-
-desc 'Validates that the tap has the fields it is claimed it should have. This is supposed to make the mitigate errors during deploy.'
-# arg_name 'Describe arguments to new here'
-command :docs do |c|
-  c.action do |global_options,options,args|
-    GoodData::Bam::Commands::generate_docs
-  end
-end
-
-desc 'Lists processes for the project.'
-# arg_name 'Describe arguments to new here'
-command :procs do |c|
-
-  c.desc 'procs for all projects'
-  c.arg_name 'all'
-  c.switch :all
-
-  c.action do |global_options,options,args|
-    out = GoodData::Bam::Commands::procs_list(options)
-    out.each do |proc|
-      puts proc.join(',')
-    end
-  end
-end
-
-
-desc 'Creates project'
-command :project do |c|
-
-  c.desc 'blueprint name. Currently support goodsales'
-  c.arg_name 'blueprint'
-  c.flag :blueprint
-
-  c.desc 'token'
-  c.arg_name 'token'
-  c.flag :token
-
-  c.action do |global_options,options,args|
-    fail "You need to specify token to create a project" if options[:token].nil?
-    
-    GoodData::Bam::Commands::connect_to_gd()
-    pid = case options[:blueprint]
-      when "goodsales"
-        "nt935rwzls50zfqwy6dh62tabu8h0ocy"
-    end
-
-    params = PARAMS.merge({:token => options[:token]})
-
-    new_project = if pid
-      GoodData::Bam::Commands.clone_project(pid, params)
-    else
-      GoodData::Bam::Commands.create_project(params)
-    end
-
-    puts "Your project pid is #{new_project}"
-  end
-
-end
-
-
-desc 'Generates structures'
-arg_name 'what you want to generate project, tap, flow, sink'
-command :scaffold do |c|
-
-  c.desc 'blueprint name. Currently support goodsales'
-  c.arg_name 'blueprint'
-  c.flag :blueprint
-
-  c.action do |global_options,options,args|
-    command = args.first
-    fail "You did not provide what I should scaffold. I can generate project, tap, flow, sink, graph_template nothing else" unless ["project", "tap", "flow", "sink", "graph_template"].include?(command) 
-    case command
-    when "project"
-      directory = args[1]
-      fail "Directory has to be provided as an argument. See help" if directory.nil?
-      if options[:blueprint].nil?
-        GoodData::Bam::Commands::setup_bash_structure(directory)  
-      else
-        case options[:blueprint]
-        when "goodsales"
-          system "git clone git://github.com/gooddata/goodsales_base.git #{directory}"
-        end
-      end
-    when "flow"
-      name = args[1]
-      fail "Name of the flow has to be provided as an argument. See help" if name.nil?
-      GoodData::Bam::Commands::setup_flow(name)
-    when "tap"
-      name = args[1]
-      fail "Name of the tap has to be provided as an argument. See help" if name.nil?
-      GoodData::Bam::Commands::setup_tap(name)
-    when "sink"
-      name = args[1]
-      fail "Name of the sink has to be provided as an argument. See help" if name.nil?
-      GoodData::Bam::Commands::setup_sink(name)
-    when "graph_template"
-      name = args[1]
-      target = args[2]
-      fail "Name of the template has to be provided as an argument. See help" if name.nil?
-      fail "Name of the target has to be provided as an argument. See help" if target.nil?
-      GoodData::Bam::Commands::generate_graph_template(name, target)
-    end
-  end
-end
-
-desc 'Runs the project on server'
-command :model_sync do |c|
-
-  c.desc 'do not execute'
-  c.arg_name 'dry'
-  c.switch :dry
-
-  c.action do |global_options,options,args|
-    GoodData::Bam::Commands::model_sync('.', options)
-  end
-end
-
-
-desc 'Deploys the project on server and schedules it'
-command :deploy do |c|
-  
-  c.desc 'existing process id under which it is going to be redeployed'
-  c.arg_name 'process'
-  c.flag :process
-
-  c.desc 'name of the process'
-  c.arg_name 'name'
-  c.flag :name
-
-  c.action do |global_options,options,args|
-    dir = args.first
-    fail "You have to specify directory to deploy as an argument" if dir.nil?
-    fail "Specified directory does not exist" unless File.exist?(dir)
-    
-    GoodData::Bam::Commands::connect_to_gd()
-    params = PARAMS.merge(global_options.merge(options))
-    response = GoodData::Bam::Commands::deploy(dir, params)
-  end
-end
-
-desc 'Runs the project on server'
-command :run do |c|
-
-  c.desc 'email to addresses when the run is finished'
-  c.arg_name 'email'
-  c.flag :email
-
-  c.action do |global_options,options,args|
-
-    dir = args.first
-    fail "You have to specify directory to deploy as an argument" if dir.nil?
-    fail "Specified directory does not exist" unless File.exist?(dir)
-
-    params = PARAMS.merge(global_options.merge(options))
-    GoodData::Bam::Commands::connect_to_gd()
-    GoodData::Bam::Commands::run(dir, params)
-  end
-end
-
-
-pre do |global_options,command,options,args|
-
-  logger = Logger.new(STDOUT) if global_options[:l]
-  GoodData.logger = logger
-  params = GoodData::Bam::Utils::get_user_params('.')
-  s3_backup = GoodData::Bam::Utils::should_backup_to_s3?(params)
-  PARAMS = params.merge({:s3_backup => s3_backup})
-
-  # Pre logic here
-  # Return true to proceed; false to abort and not call the
-  # chosen command
-  # Use skips_pre before a command to skip this block
-  # on that command only
-
-  true
-end
-
-post do |global_options,command,options,args|
-  # Post logic here
-  # Use skips_post before a command to skip this
-  # block on that command only
-  verbose = global_options[:v]
-  puts HighLine::color("DONE", :green) if verbose
-end
-
-on_error do |exception|
-  pp exception.backtrace
-  # Error logic here
-  # return false to skip default error handling
-  true
-end
-
-exit run(ARGV)

data/lib/generators/downloaders.rb.orig

@@ -1,663 +0,0 @@
-module GoodData
-  module Bam
-    module Generators
-      module Downloaders
-
-        include GoodData::Bam
-        include GoodData::CloudConnect
-
-        def self.create_incremental_downloader_run_graph(file, taps, options={})
-
-          File.open(file, "w") do |file|
-            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-            builder.Graph({
-              :name => "Run graph"
-            }) do
-              builder.Global do
-                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-                Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
-                Helpers::create_trash_meta(builder)
-                Helpers::create_lookup_meta(builder)
-                Helpers::create_file_list_meta(builder)
-                Helpers::create_run_graph_failure_metadata(builder)
-
-                Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
-
-              end
-              phase = 0
-              sf_taps = Taps.get_salesforce(taps)
-
-              sf_taps.each do |tap|
-                module_name = tap[:object]
-                file    = tap[:id] || module_name
-                dataset = file || module_name
-
-
-            normalize_code = <<HEREDOC
-boolean done = false;
-function integer count() {
-
-  if (indexOf($in.0.key, "#{dataset}_LAST_RUN") != -1) {
-    return 4;
-  }
-  else {
-    return 0;
-  }
-}
-
-string last_run = null;
-string end_date = null;
-
-function integer transform(integer idx) {
-  if (last_run == null) {
-    last_run = $in.0.value;
-  }
-  if (end_date == null) {
-    end_date = jodaDate2str(today(), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
-  }
-
-
-  if (idx == 1) {
-    $out.0.all = "#{dataset}_TRUNCATE_DATE=" + jodaDate2str(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'), "yyyy-MM-dd HH:mm:ss", 'en_US', 'UTC');
-  } else if (idx == 2) {
-    $out.0.all = "#{dataset}_START=" + last_run;
-  } else if (idx == 3) {
-    $out.0.all = "#{dataset}_END=" + end_date;
-  } else {
-    $out.0.all = "#{dataset}_LAST_RUN=" + end_date;
-  }
-
-  return OK;
-}
-
-HEREDOC
-
-
-
-
-                builder.Phase(:number => phase += 1) do
-            generate_func = <<HEREDOC
-function integer generate() {
-$out.0.key = "#{dataset}_LAST_RUN";
-$out.0.value = "1970-01-01T00:00:00.000+00:00";
-return OK;
-}
-HEREDOC
-
-              join_func = <<HEREDOC
-function integer transform() {
-  $out.0.key = nvl2($in.1.value, $in.1.key, $in.0.key);
-  $out.0.value = nvl2($in.1.value, $in.1.value, $in.0.value);
-return OK;
-}
-HEREDOC
-
-                  Core::build_node2(builder, Nodes.data_generator2({:name => "generator_#{dataset}", :id => "generator_#{dataset}", :generate => generate_func}))
-                  Core::build_node2(builder, Nodes.lookup_reader_writer2({:lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{dataset}" }))
-                  Core::build_node2(builder, Nodes.hash_join2({:id => "join_#{dataset}", :joinType => "leftOuter", :joinKey => "$key=$key", :transformation => join_func}))
-
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "join_#{dataset}:0", :fromNode => "generator_#{dataset}:0", :metadata => "lookup_metadata"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "join_#{dataset}:1", :fromNode => "gd_lookup_reader_#{dataset}:0", :metadata => "lookup_metadata"}))
-
-                  Core::build_node2(builder, Nodes.normalizer2({:name => "normalizer_#{dataset}", :id => "normalizer_#{dataset}", :normalize => normalize_code }))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "normalizer_#{dataset}:0", :fromNode => "join_#{dataset}:0", :metadata => "lookup_metadata"}))
-
-                  Core::build_node2(builder, Nodes.writer2({:quotedStrings => "false", :name => "params_writer_#{dataset}", :id => "params_writer_#{dataset}", :fileURL => "params.prm", :outputFieldNames => "false", :append => "true"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "params_writer_#{dataset}:0", :fromNode => "normalizer_#{dataset}:0", :metadata => "trash_metadata"}))
-                end
-              end
-              builder.Phase(:number => phase += 1) do
-                if !sf_taps.empty?
-                  Core::build_node2(builder, Nodes.run_graph2({:guiName => "incremental", :name => "incremental_downloaders", :id => "downlaoders", :graphName => "graphs/incremental.grf"}))
-                end
-              end
-
-              file_taps = Taps.get_file(taps)
-
-              file_taps.each do |tap|
-                source = tap[:source]
-                id =  tap[:id]
-
-
-            reformat_func = <<HEREDOC
-
-function integer transform() {
-    $out.0.filePath = replace($in.0.filePath, "${GDC_WEBDAV_HOST}", replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + ":${GD_PASSWORD}@${GDC_WEBDAV_HOST}");
-    $out.0.fileName = $in.0.fileName;
-    return ALL;
-}
-HEREDOC
-
-                builder.Phase(:number => phase += 1) do
-                  Core::build_node2(builder, Nodes.file_list2({:id => "#{id}_file_list", :name => "#{id}_file_list", :dataPolicy => "Strict", :baseURL => "#{tap[:source]}", :output_mapping => Nodes::MAP_ALL}))
-                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_file_list:0", :metadata => "file_list"}))
-                  Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/#{id}_files_to_read.csv"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list"}))
-                end
-
-                builder.Phase(:number => phase += 1) do
-                  ctl = "function integer generate() {$out.0.all = \"#{id}_SKIP_LINES=0\";return OK;}"
-                  Core::build_node2(builder, Nodes.data_generator2({:name => "#{id}_generator", :id => "#{id}_generator", :generate => ctl}))
-                  Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_csv_writer", :fileURL => "#{id}_counter.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_csv_writer:0", :fromNode => "#{id}_generator:0", :metadata => "trash_metadata"}))
-                end
-
-               subgraph_reformat_func = <<HEREDOC
-
-function integer transform() {
-    $out.0.all = "graphs/#{id}_loop.grf";  
-    return ALL;
-}
-HEREDOC
-
-            fail_reformat = <<HEREDOC
-function integer transform() {
-  raiseError("Loop failed");
-}
-HEREDOC
-
-
-                builder.Phase(:number => phase += 1) do
-                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat graph", :id => "#{id}_reformat_graph", :transformation => subgraph_reformat_func}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_graph:0", :fromNode => "#{id}_reformat:1", :metadata => "file_list"}))
-                  Core::build_node2(builder, Nodes.run_graph2({
-                    :guiName => id,
-                    :name => id,
-                    :id => "#{id}_run_graph"
-                  }))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_run_graph:0", :fromNode => "#{id}_reformat_graph:0", :metadata => "trash_metadata"}))
-                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat fail", :id => "#{id}_reformat_fail", :transformation => fail_reformat}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_fail:0", :fromNode => "#{id}_run_graph:1", :metadata => "run_graph_failure_metadata"}))
-                  Core::build_node2(builder, Nodes.trash2({:name => "#{id}_trash", :id => "#{id}_trash", :debugPrint => true}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat_fail:0", :metadata => "run_graph_failure_metadata"}))
-                end
-              end
-
-
-            end
-          end
-        end
-
-        def self.create_incremental_sf_downloading_graph(file, taps, options={})
-          metadata = options[:metadata]
-          store = options[:store] || "${GDC_EVENTSTORE}"
-          s3_backup = options[:s3_backup]
-
-          File.open(file, "w") do |file|
-            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-            builder.Graph({:name => "Goodsales incremental Downloader"}) do
-              builder.Global do
-                Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
-                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-
-                Helpers::create_lookup_meta(builder)
-                taps.each do |tap|
-                  module_name = tap[:object]
-                  file = tap[:id] || module_name
-
-                  builder.Metadata({:id   => "#{file}_s3_metadata"}) do |builder|
-                    Helpers::csv_metadata(builder, GoodData::Bam::Metadata.add_field(Metadata.get_target_metadata(Tap.prepare_for_s3_backup(tap)), {:name => "Timestamp"}))
-                  end
-
-                  tap = Tap.prepare_for_sf_downloader(tap)
-                  builder.Metadata({:id   => "#{file}_source_metadata"}) do |builder|
-                    Helpers::csv_metadata(builder, Metadata.get_source_metadata(tap))
-                  end
-                  
-                  builder.Metadata({:id   => "#{file}_es_metadata"}) do |builder|
-                    Helpers::csv_metadata(builder, Metadata.add_timestamp_as_date(Metadata.get_target_metadata(tap)))
-                  end
-
-                  Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
-
-                end
-
-                Helpers::sf_connection(builder, {})
-                Helpers::property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
-                Helpers::property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
-                Helpers::property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
-                Helpers::property(builder, {:id => "SFDC_PASSWORD", :value => options[:sf_password]})
-                Helpers::property(builder, {:id => "SFDC_TOKEN", :value => options[:sf_token]})
-                Helpers::property(builder, {:id => "SFDC_USERNAME", :value => options[:sf_login]})
-                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-              end
-
-              phase = 1
-
-              taps.each do |tap|
-                module_name = tap[:object]
-                file    = tap[:id]
-                dataset = file || module_name
-                source_file  = tap[:source]
-                
-                has_timestamp = Tap.has_output_field?(tap, "Timestamp")
-                timestamp_field = Tap.find_output_field(tap, "Timestamp")
-                id_field = Tap.find_output_field(tap, "Id")
-                tap = Tap.prepare_for_sf_downloader(tap)
-                builder.Phase(:number => phase += 1) do
-                  Core::build_node2(builder, Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
-                end
-
-                builder.Phase(:number => phase += 1) do
-                  fields = tap[:fields]
-                  mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
-                  add = fields.map do |f|
-                    "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
-                  end
-                  stuff = mapping + add.join(",") + "]}}"
-
-                  mandatory_fields = fields.reject {|f| f[:is_mandatory] == false }.map {|f| f[:name] + ";"}.join
-
-                  Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{file} SF Reader", :id => "#{file}_reader", :soql => Helpers::generate_incremental_select(tap), :sfdcConnection => "SFDC", :fieldsMapping => stuff, :mandatoryFields => mandatory_fields}))
-
-
-                  es_transformation_source = if has_timestamp
-                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = str2date($in.0.#{timestamp_field[:name]},\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");;\nreturn OK;\n}"
-                  else
-                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = long2date(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"
-                  end
-
-                  Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reader:0", :metadata => "#{file}_source_metadata"}))
-                  Core::build_node2(builder, Nodes.writer2({:enabled => "disabled", :name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))
-
-                  Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_es_reformat", :transformation => es_transformation_source}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_reformat:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_source_metadata"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_source_metadata"}))
-                  
-                  if s3_backup then
-                    if has_timestamp
-
-                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
-
-                    else
-                      Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_s3_reformat", :transformation => "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = toString(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3_reformat:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
-
-                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_s3_reformat:0", :metadata => "#{file}_s3_metadata"}))
-                      
-                    end
-                  end
-
-
-                  Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_es_sort"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_sort:0", :fromNode => "#{file}_es_reformat:0", :metadata => "#{file}_es_metadata"}))
-
-                  Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json(Tap.add_timestamp_field(tap)).to_json}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es:0", :fromNode => "#{file}_es_sort:0", :metadata => "#{file}_es_metadata"}))
-
-                end
-
-                builder.Phase(:number => phase += 1) do
-                              generate_func = <<HEREDOC
-    function integer generate() {
-  date time_end = jodaStr2date("${#{dataset}_END}",["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC');
-  $out.0.key = "#{dataset}_LAST_RUN";
-  $out.0.value = jodaDate2str(time_end,"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", 'en_US', 'UTC');
-  return OK;
-}
-HEREDOC
-
-                  Core::build_node2(builder, Nodes.data_generator2({:guiName => dataset, :name => "generator_#{phase}", :id => "generator_#{phase}", :generate => generate_func}))
-                  Core::build_node2(builder, Nodes.lookup_reader_writer2({:guiName => dataset, :lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{phase}" }))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "gd_lookup_reader_#{phase}:0", :fromNode => "generator_#{phase}:0", :metadata => "lookup_metadata"}))
-
-                end
-              end
-            end
-          end
-        end
-
-
-
-        def self.create_incremental_sf_history_init_graph(file, taps, options={})
-          metadata = options[:metadata]
-          store = options[:store] || "${GDC_EVENTSTORE}"
-          s3_backup = options[:s3_backup]
-
-
-          client = GoodData::Bam::Commands.get_sf_client(options)
-
-          File.open(file, "w") do |file|
-            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-            builder.Graph({:name => "Goodsales incremental Downloader"}) do
-              builder.Global do
-                Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
-                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-
-                Helpers::create_lookup_meta(builder)
-                builder.Metadata({:id   => "data_metadata"}) do |builder|
-                  Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => "Value"}]})
-                end
-
-                builder.Metadata({:id   => "es_metadata"}) do |builder|
-                  Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp", :type => "date"}, {:name => "Value"}]})
-                end
-
-                Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
-
-                Helpers::sf_connection(builder, {})
-                Helpers::property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
-                Helpers::property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
-                Helpers::property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
-                Helpers::property(builder, {:id => "SFDC_PASSWORD", :value => options[:sf_password]})
-                Helpers::property(builder, {:id => "SFDC_TOKEN", :value => options[:sf_token]})
-                Helpers::property(builder, {:id => "SFDC_USERNAME", :value => options[:sf_login]})
-                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-              end
-
-              phase = 1
-
-              taps_with_history = []
-              taps.each do |tap|
-                module_name = tap[:object]
-                dataset = tap[:id]
-                source_file  = tap[:source]
-
-                has_timestamp = Tap.has_output_field?(tap, "Timestamp")
-                timestamp_field = Tap.find_output_field(tap, "Timestamp")
-                id_field = Tap.find_output_field(tap, "Id")
-                tap = Tap.prepare_for_sf_downloader(tap)
-                builder.Phase(:number => phase += 1) do
-                  # Core::build_node2(builder, Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
-                end
-
-                fields = tap[:fields]
-                
-                puts tap[:object]
-                sf_object = tap[:object]
-                id_field = Tap.find_output_field(tap, "Id")
-                timestamp_field = Tap.find_output_field(tap, "Timestamp")
-                
-                
-                objects_to_get = Helpers.objects_for_history(client, tap)
-                
-                (tap[:fields] - [id_field, timestamp_field]).each_with_index do |field, i|
-                  builder.Phase(:number => phase += 1) do
-                    
-                    f = field[:name]
-
-                    selects = objects_to_get.reduce([]) do |memo, o|
-                      fields = client.fields(o)
-                      generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|f| fields.include?(f)}
-                      specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|f| fields.include?(f)}
-                      specific_history = ["SystemModstamp", "#{sf_object}Id"].all? {|f| fields.include?(f)}
-
-                      select, mapping, mandatory = if generic_field_history
-                        [
-                          "SELECT NewValue, CreatedDate, ParentId FROM #{o} WHERE Field = '#{f}'",
-                          "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                          {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
-                          {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
-                          {\"xmlFieldMapping\":{\"name\":\"ParentId\",\"label\":\"ParentId\",\"xmlPath\":\"ParentId\",\"metadataField\":\"Id\"}}
-                          ]}}",
-                          ["ParentId", "NewValue", "CreatedDate"]
-                        ]
-                      elsif specific_field_history
-                        [
-                          
-                          "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
-                          "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                          {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
-                          {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
-                          {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{sf_object}Id\",\"xmlPath\":\"#{sf_object}Id\",\"metadataField\":\"Id\"}}
-                          ]}}",
-                          ["#{sf_object}Id", "NewValue", "CreatedDate"]
-                        ]
-                      elsif specific_history
-                        if fields.include?(f)
-                          [
-                            "SELECT #{f}, SystemModstamp, #{sf_object}Id FROM #{o}",
-                            "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                            {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
-                            {\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
-                            {\"xmlFieldMapping\":{\"name\":\"#{sf_object}Id\",\"label\":\"#{o}Id\",\"xmlPath\":\"#{o}Id\",\"metadataField\":\"Id\"}}
-                            ]}}",
-                            ["#{sf_object}Id", f, "SystemModstamp"]
-                          ]
-                        else
-                          [nil, nil]
-                        end
-                      else
-                        fail "Unrecognized fields configuration for historization in SF."
-                      end
-                      if select.nil?
-                        memo
-                      else
-                        memo.concat([{
-                          :object => o,
-                          :query => select,
-<<<<<<< HEAD
-                          :mapping => mapping
-=======
-                          :mapping => mapping,
-                          :mandatory => mandatory
->>>>>>> Added historical SF downloader
-                        }])
-                      end
-                    end
-
-                    unless selects.empty?
-                      taps_with_history.concat([tap])
-                      selects = selects.concat([{
-                        :object => sf_object,
-                        :query => "SELECT Id, #{f}, SystemModstamp FROM #{sf_object}",
-                        :mapping => "{\"xmlFieldsMapping\":{\"xmlFields\":[
-                        {\"xmlFieldMapping\":{\"name\":\"#{f}\",\"label\":\"#{f}\",\"xmlPath\":\"#{f}\",\"metadataField\":\"Value\"}},
-                        {\"xmlFieldMapping\":{\"name\":\"SystemModstamp\",\"label\":\"SystemModstamp\",\"xmlPath\":\"SystemModstamp\",\"metadataField\":\"Timestamp\"}},
-                        {\"xmlFieldMapping\":{\"name\":\"Id\",\"label\":\"Id\",\"xmlPath\":\"Id\",\"metadataField\":\"Id\"}}
-                        ]}}",
-                        :mandatory => ["Id", f, "SystemModstamp"]
-                      }])
-                      selects.each_with_index do |obj, i|
-                      
-                        o = obj[:object]
-                        mapping = obj[:mapping]
-                        select = obj[:query]
-                        mandatory = obj[:mandatory]
-
-                        Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{o} SF Reader #{i}", :id => "#{f}_#{o}_reader", :soql => select, :sfdcConnection => "SFDC", :fieldsMapping => mapping, :mandatoryFields => mandatory.join(';')}))
-                        puts "#{f}_#{o}_reader:0 -> #{f}_#{sf_object}_gather:#{i}"
-                        Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_gather:#{i}", :fromNode => "#{f}_#{o}_reader:0", :metadata => "data_metadata"}))
-                      end
-                      Core::build_node2(builder, Nodes.gather2({:name => "#{f} Gather", :id => "#{f}_#{sf_object}_gather"}))
-                      Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{f}_#{sf_object}_sort", :id => "#{f}_#{sf_object}_sort"}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_sort:0", :fromNode => "#{f}_#{sf_object}_gather:0", :metadata => "data_metadata"}))
-
-                      Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{f}_#{sf_object}_copy"}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_copy:0", :fromNode => "#{f}_#{sf_object}_sort:0", :metadata => "data_metadata"}))
-
-                      if s3_backup
-                        Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{f}_#{sf_object} s3 Writer", :id => "#{f}_#{sf_object}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{sf_object}/#{f}_#{sf_object}_\`date2long(today())\`", :outputFieldNames => true}))
-                        Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_s3:0", :fromNode => "#{f}_#{sf_object}_copy:1", :metadata => "data_metadata"}))
-                      end
-
-                      transform = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = str2date($in.0.Timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\nreturn OK;\n}"
-
-                      Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "#{f}_#{sf_object}_reformat", :transformation => transform}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_reformat:0", :fromNode => "#{f}_#{sf_object}_copy:0", :metadata => "data_metadata"}))
-
-                      Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{f}_#{sf_object}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json({:type => :tap, :id => tap[:id], :fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => f, :meta => "Value"}]}).to_json}))
-
-                      # Core::build_node2(builder, Nodes.trash2({:name => "#{f}_#{sf_object}_es", :id => "#{f}_#{sf_object}_es", :debugPrint => true}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{f}_#{sf_object}_es:0", :fromNode => "#{f}_#{sf_object}_reformat:0", :metadata => "es_metadata"}))
-                    end
-                  end
-                end
-              end
-              taps_with_history.each do |tap|
-                puts "#{tap[:id]}_LAST_RUN"
-                builder.Phase(:number => phase += 1) do
-                              generate_func = <<HEREDOC
-  function integer generate() {
-date time_end = jodaStr2date("${#{tap[:id]}_END}",["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC');
-$out.0.key = "#{tap[:id]}_LAST_RUN";
-$out.0.value = jodaDate2str(time_end,"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", 'en_US', 'UTC');
-return OK;
-}
-HEREDOC
-
-                  Core::build_node2(builder, Nodes.data_generator2({:guiName => tap[:id], :name => "generator_#{phase}", :id => "generator_#{phase}", :generate => generate_func}))
-                  Core::build_node2(builder, Nodes.lookup_reader_writer2({:guiName => tap[:id], :lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{phase}" }))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "gd_lookup_reader_#{phase}:0", :fromNode => "generator_#{phase}:0", :metadata => "lookup_metadata"}))
-
-                end
-              end
-            end
-          end
-        end
-
-        def self.generate_downloaders(home, project, params)
-          home = Pathname(home)
-
-          incremental_taps = Taps.get_incremental(project[:taps])
-
-          sf_taps = Taps.get_salesforce(incremental_taps)
-          file_taps = Taps.get_file(incremental_taps)
-
-          create_incremental_downloader_run_graph(home + "main.grf", incremental_taps)
-          # create_incremental_sf_downloading_graph(home + "incremental.grf", sf_taps, params)
-          create_incremental_sf_history_init_graph(home + "incremental.grf", sf_taps, params)
-
-          file_taps.each do |tap|
-            id = tap[:id]
-            Helpers::loop_over_file(home + "#{tap[:id]}_loop.grf", {
-              :token => id,
-              :file_to_loop => "data/#{id}_files_to_read.csv",
-              :graph_to_run => "graphs/#{id}_download.grf"
-            })
-            create_incremental_file_downloading_graph(home + "#{tap[:id]}_download.grf", [tap], params)
-          
-          end
-        end
-
-
-        def self.generate_history_downloaders(home, project, params)
-          home = Pathname(home)
-
-          incremental_taps = Taps.get_incremental(project[:taps])
-
-          sf_taps = Taps.get_salesforce(incremental_taps)
-          file_taps = Taps.get_file(incremental_taps)
-
-          create_incremental_downloader_run_graph(home + "main.grf", incremental_taps)
-          create_incremental_sf_history_init_graph(home + "incremental.grf", sf_taps, params)
-        end
-
-        def self.create_incremental_file_downloading_graph(file, taps, options={})
-          metadata = options[:metadata]
-          store = options[:store] || "${GDC_EVENTSTORE}"
-          s3_backup = options[:s3_backup]
-          File.open(file, "w") do |file|
-            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-            builder.Graph({:name => "Goodsales incremental Downloader"}) do
-              builder.Global do
-                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-                Helpers::create_lookup_meta(builder)
-                taps.each do |tap|
-                  module_name = tap[:object]
-                  file    = tap[:id] || module_name
-
-                  Helpers::property_file(builder, {:id => "params_params", :fileURL => "#{file}_item.prm"})
-
-                  builder.Metadata({:id   => "#{file}_s3_metadata"}) do |builder|
-                    Helpers::csv_metadata(builder, GoodData::Bam::Metadata.add_field(Metadata.get_target_metadata(Tap.prepare_for_s3_backup(tap)), {:name => "Timestamp"}))
-                  end
-
-                  tap = Tap.prepare_for_sf_downloader(tap)
-                  builder.Metadata({:id   => "#{file}_source_metadata"}) do |builder|
-                    Helpers::csv_metadata(builder, Metadata.get_source_metadata(tap))
-                  end
-
-                  builder.Metadata({:id   => "#{file}_es_metadata"}) do |builder|
-                    Helpers::csv_metadata(builder, GoodData::Bam::Metadata.add_timestamp_as_date(Metadata.get_target_metadata(tap)))
-                  end
-                  Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
-                end
-              end
-
-              phase = 1
-
-              taps.each do |tap|
-                module_name = tap[:object]
-                file    = tap[:id]
-                dataset = file || module_name
-                source_file  = tap[:source]
-
-                
-                has_timestamp = GoodData::Bam::Tap.has_output_field?(tap, "Timestamp")
-                timestamp_field = GoodData::Bam::Tap.find_output_field(tap, "Timestamp")
-                id_field = GoodData::Bam::Tap.find_output_field(tap, "Id")
-
-                tap = Tap.prepare_for_sf_downloader(tap)
-
-                builder.Phase(:number => phase += 1) do
-                  fields = tap[:fields]
-                  mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
-                  add = fields.map do |f|
-                    "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
-                  end
-                  stuff = mapping + add.join(",") + "]}}"
-
-                  mandatory_fields = fields.reject {|f| f[:is_mandatory] == false }.map {|f| f[:name] + ";"}.join
-
-                  Core::build_node2(builder, Nodes.reader2({:name => "#{file} File Reader", :id => "#{file}_reader", :fileURL =>  "${FILE}"}))
-                  es_transformation_source = if has_timestamp
-                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = str2date($in.0.#{timestamp_field[:name]},\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");;\nreturn OK;\n}"
-                  else
-                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = long2date(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"
-                  end
-
-                  # build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_reformat", :transformation => transformation_source}))
-
-                  Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reader:0", :metadata => "#{file}_source_metadata"}))
-                  Core::build_node2(builder, Nodes.writer2({:enabled => "disabled", :name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))
-
-                  Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_es_reformat", :transformation => es_transformation_source}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_reformat:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_source_metadata"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_source_metadata"}))
-                  
-                  if s3_backup 
-                    if has_timestamp
-                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
-                    else
-                      Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_s3_reformat", :transformation => "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = toString(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3_reformat:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
-                
-                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
-                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_s3_reformat:0", :metadata => "#{file}_s3_metadata"}))
-                    end
-                    # Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
-                    # Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
-                  end
-                  
-
-                  Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_es_sort"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_sort:0", :fromNode => "#{file}_es_reformat:0", :metadata => "#{file}_es_metadata"}))
-                  Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json(Tap.add_timestamp_field(tap)).to_json}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es:0", :fromNode => "#{file}_es_sort:0", :metadata => "#{file}_es_metadata"}))
-                end
-
-                builder.Phase(:number => phase += 1) do
-                  Core::build_node2(builder, Nodes.file_delete2({:guiName => "#{file}_file_delete", :name => "#{file}_file_delete", :id => "#{file}_file_delete", :baseURL => "${FILE}"}))
-                end
-
-              end
-
-            end
-          end
-        end
-
-      end
-    end
-  end
-end