gd_bam 0.0.1

data/lib/nodes/clover_gen.rb

@@ -0,0 +1,1283 @@
+module Enumerable
+  def uniq_by
+    seen = Hash.new { |h,k| h[k] = true; false }
+    reject { |v| seen[yield(v)] }
+  end
+end
+
+class Hash
+  #pass single or array of keys, which will be removed, returning the remaining hash
+  def remove!(*keys)
+    keys.each{|key| self.delete(key) }
+    self
+  end
+
+  #non-destructive version
+  def remove(*keys)
+    self.dup.remove!(*keys)
+  end
+end
+
+
+module GoodData 
+  module CloverGenerator
+
+    ADD_NA_TRANSFORMATION = <<-trasnf
+    function integer generate() {
+    	integer number_of_fields = length($out.0);
+    	integer i = 0;
+    	string type = "";
+    	for (i; i< number_of_fields; ++i) {
+    		type = getFieldType($out.0, i);
+    		switch(type) {
+    			case "string": setStringValue($out.0, i, 'N/A'); break;
+    			case "number": setNumValue($out.0, i, 0); break;
+    			case "date": setDateValue($out.0, i, null); break;
+    		}
+    	};
+
+    	return OK;
+    }
+    trasnf
+
+    def self.parse_json(file)
+      begin
+        JSON.parse(File.read(file), :symbolize_names => true)
+      rescue RuntimeError => e
+        puts "Error parsing \"#{file}\": #{e.inspect}"
+      end
+    end
+
+
+    def self.metadata(builder, description)
+      builder.Record({
+          :fieldDelimiter => description[:fieldDelimiter],
+          :name => description[:name],
+          :recordDelimiter => description[:recordDelimiter],
+          :type => description[:type]
+        }) do |record|
+        description[:fields].each do |field|
+          builder.Field :name => field[:name], :type => field[:type], :nullable => "true"
+        end
+      end
+    end
+
+    def self.csv_metadata(builder, description)
+
+      sf_description = description.merge({
+        :fieldDelimiter => ",",
+        :recordDelimiter => "\\n",
+        :type => "delimited",
+      })
+      metadata(builder, sf_description)
+    end
+
+
+    def self.sf_connection(builder, data)
+      builder.Connection({
+        :clientId => "${SFDC_CLIENT_ID}",
+        :id => "SFDC",
+        :loginHostname => "${SFDC_LOGIN_HOSTNAME}",
+        :name => "${SFDC_NAME}",
+        :password => "${SFDC_PASSWORD}",
+        :passwordEncrypted => "true",
+        :token => "${SFDC_TOKEN}",
+        :type => "SFDC",
+        :username => "${SFDC_USERNAME}",
+        :passwordEncrypted => "false"
+      })
+    end
+
+    def self.external_metadata_link(builder, data)
+      builder.Metadata({
+        :fileURL => data[:fileURL],
+        :id => data[:id]
+      })
+    end
+
+    def self.property(builder, data)
+      builder.Property({
+        :id => data[:id],
+        :name => data[:name] || data[:id],
+        :value => data[:value]
+      })
+    end
+
+
+    def self.property_file(builder, data)
+      builder.Property({
+        :id => data[:id],
+        :fileURL => data[:fileURL]
+      })
+    end
+
+    def graph2(file, stuff)
+      metadata = stuff[:metadata]
+      nodes = stuff[:nodes]
+      connections = stuff[:connections]
+      File.open(file, "w") do |file|
+        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+        builder.Graph({
+          :name => "Goodsales Downloader"
+        }) do
+          builder.Global do
+            metadata.each do |m|
+              build_metadata2(builder, m)
+            end
+            connections.each do |conn|
+              build_node2(builder, conn)
+            end
+        
+          end
+          builder.Phase(:number => 0) do
+            nodes.each do |node|
+              build_node2(builder, node)
+            end
+          end  
+        end
+      end
+    end
+
+    def self.build_node2(builder, node)
+      if node[:type] == GoodData::CloverGenerator::Nodes::EDGE
+        builder.tag!("Edge", node)
+      elsif node[:type] == GoodData::CloverGenerator::Nodes::SF_CONNECTION
+        builder.tag!("Connection", node)
+      elsif node[:type] == GoodData::CloverGenerator::Nodes::FILE_LIST
+        builder.tag!("Node", node.remove(:transformation)) do |xml|
+          xml.attr({:name => "outputMapping"}) do |attr|
+            transformation = node[:transformation]
+            attr.cdata! transformation
+          end
+        end
+      elsif node[:type] == GoodData::CloverGenerator::Nodes::REFORMAT || node[:type] == GoodData::CloverGenerator::Nodes::EXT_HASH_JOIN
+        builder.tag!("Node", node.remove(:transformation)) do |xml|
+          xml.attr({:name => "transform"}) do |attr|
+            transformation = node[:transformation]
+            attr.cdata! transformation
+          end
+        end
+      elsif node[:type] == GoodData::CloverGenerator::Nodes::DATA_GENERATOR
+        builder.tag!("Node", node.remove(:transformation)) do |xml|
+          xml.attr({:name => "generate"}) do |attr|
+            transformation = node[:generate]
+            attr.cdata! transformation
+          end
+        end
+      elsif node[:type] == GoodData::CloverGenerator::Nodes::PERSISTENT_LOOKUP || node[:type] == GoodData::CloverGenerator::Nodes::GD_LOOKUP
+        builder.tag!("LookupTable", node)
+      else
+        builder.tag!("Node", node)
+      end
+    end
+
+    def self.run_ctl(builder, ctl)
+      id1 = rand(10000000000)
+      id2 = rand(10000000000)
+      id3 = rand(10000000000)
+      
+      build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => "#{id1}", :id => "#{id1}", :generate => ctl}))
+      build_node2(builder, GoodData::CloverGenerator::Nodes.trash2({:name => "#{id2}", :id => "#{id2}"}))
+      build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata", :id => "#{id3}"}))
+    end
+
+    def build_metadata2(builder, node)
+      builder.Metadata({:id => node[:id]}) do    
+        csv_metadata(builder, {
+          :name   => "account_sf_reformat",
+          :fields => translate(node[:fields])
+        })
+      end
+    end
+
+    def sf_field_mapping(stuff)
+      fields = stuff["fields"]
+      mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
+      add = fields.map do |f|
+        "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:name]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
+      end
+      stuff = mapping + add.join(",") + "]}}"
+      stuff
+    end
+
+    def self.validate_gd_datasets(datasets, gd_datasets)
+      dataset_names = datasets.map {|d| d[:gd_name] || d[:id]}
+      gd_dataset_names = gd_datasets.map {|d| d["dataSet"]["meta"]["identifier"].gsub("dataset.", "")}.find_all {|d| d.index(".dt").nil?}
+      {
+        :not_in_gd  => dataset_names - gd_dataset_names,
+        :not_loaded => gd_dataset_names - dataset_names
+      }
+    end
+
+    def self.validate_gd_datasets_metadata(datasets, gd_datasets)
+
+      init = {
+        :datasets => {}
+      }
+
+      datasets.reduce(init) do |memo, dataset|
+        gd_name = dataset[:gd_name] || dataset[:id]
+        gd_dataset = gd_datasets.find {|d| d["dataSet"]["meta"]["identifier"].gsub("dataset.", "") == gd_name}
+        gd_dataset_names = gd_datasets.map {|d| d["dataSet"]["meta"]["identifier"].gsub("dataset.", "")}.find_all {|d| d.index(".dt").nil?}
+
+        next(memo) if gd_dataset.nil?
+
+        refs = dataset[:fields].find_all {|f| f[:type] == "reference"}
+        refs.each do |ref|
+          (memo[:datasets][gd_name] ||= []) << ref unless gd_dataset_names.include?(ref[:schema])
+          ref_attr_indentifier = "attr.#{ref[:schema]}.#{ref[:ref]}"
+          (memo[:datasets][gd_name] ||= []) << ref if GoodData::MdObject[ref_attr_indentifier].nil?  
+        end
+
+        labels = dataset[:fields].find_all {|f| f[:type] == "label"}
+        gd_labels = gd_dataset['dataSet']['content']['dataLoadingColumns'].map {|label| GoodData.get(label)}
+        labels.each do |label|
+          label_name = label[:name]
+          label_identifier = "label.#{gd_name}.#{label[:for]}.#{label_name}"
+          (memo[:datasets][gd_name] ||= []) << label if GoodData::MdObject[label_identifier].nil?
+        end
+
+        facts = dataset[:fields].find_all {|f| f[:type] == "fact"}
+        gd_facts = gd_dataset['dataSet']['content']['facts'].map {|fact| GoodData.get(fact)}
+        facts.each do |fact|
+          fact_name = fact[:name]
+          unless gd_facts.any? {|a| a['fact']['meta']['identifier'] == "fact.#{gd_name}.#{fact_name}"}
+            (memo[:datasets][gd_name] ||= []) << fact
+          end
+        end
+
+        attributes = dataset[:fields].find_all {|f| f[:type] == "attribute" || f[:type] == "connection_point"}
+        gd_attributes = gd_dataset['dataSet']['content']['attributes'].map {|attr| GoodData.get(attr)}
+        attributes.each do |attr|
+          attr_name = attr[:name]
+          unless gd_attributes.any? {|a| a['attribute']['meta']['identifier'] == "attr.#{gd_name}.#{attr_name}"}
+            (memo[:datasets][gd_name] ||= []) << attr
+          end
+        end
+        memo
+      end
+    end
+
+
+    def self.validate_sf_metadata(sf_client, sources)
+      sources.reduce({}) do |memo, source|
+        puts "Checking #{source[:object]}"
+        sf_object = source[:object]
+        u = sf_client.describe(sf_object)
+        sf_fields = u[:describeSObjectResponse][:result][:fields].map {|field| field[:name]}
+        fields_to_validate = source[:fields].map {|field| field[:name]}
+        memo[sf_object] = (fields_to_validate - sf_fields)
+        pp fields_to_validate - sf_fields
+        memo
+      end
+    end
+
+    def self.get_sf_client(params)
+      Salesforce::Client.new(params[:sf_login], params[:sf_password] + params[:sf_token], :server => params[:sf_server])
+    end
+
+    def self.download_metadata(downloaders_spec)
+      mods = []
+      c = Salesforce::Client.new(downloaders_spec["login"], downloaders_spec["password"] + downloaders_spec["token"])
+      downloaders_spec["input"].each do |mod|
+        sf_module = mod["module"]
+        output = mod["file"]
+        fields = mod["fields"]
+
+        u = c.describe(sf_module)
+        sf_fields = u[:describeSObjectResponse][:result][:fields]
+        mods << mod.merge({
+          "fields" => fields.map do |f|
+            found_field = sf_fields.find {|sf| sf[:name] == f["name"]}
+            # begin
+            if found_field.nil?
+              fail "Field \"#{f["name"]}\" was not found in the \"#{sf_module}\" module" 
+            end
+            f.merge(found_field)
+          end
+        })
+      end
+      downloaders_spec.merge({"input" => mods})
+    end
+
+    def self.save_metadata(filename, data)
+      File.open(filename, "w") do |file|
+        builder = Builder::XmlMarkup.new(:target => file, :indent=>2)
+        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"  
+        csv_metadata(builder, data)
+      end
+    end
+
+    def self.translate_field(what)
+      dict = {
+        nil             => "string",
+        "id"            => "string",
+        "string"        => "string",
+        "textarea"      => "string",
+        "email"         => "string",
+        "phone"         => "string",
+        "boolean"       => "string",
+        "picklist"      => "string",
+        "reference"     => "string",
+        "datetime"      => "date",
+        "date"          => "date",
+        "currency"      => "string",
+        "percent"       => "string",
+        "int"           => "string",
+        "multipicklist" => "string",
+        "combobox"      => "string",
+        "attribute"     => "string",
+        "fact"          => "string"
+      }
+      fail "#{what} not included in translation dictionary from SF to Clover Types" unless dict.include?(what)
+      dict[what]
+    end
+
+    def self.translate(fields)
+      fields
+      # fields.map do |f|
+      #   f.merge({
+      #     :type => translate_field(f[:type])
+      #   })
+      # end
+    end
+
+
+
+    def self.find_by_name(fields, name)
+      fields.find do |m|
+        xname = m["module"]
+        output = m["file"] || xname
+        output == name
+      end
+    end
+
+    def self.find_by_dataset(fields, name)
+      fields.find do |m|
+        md = m["module"]
+        file = m["file"]
+        dataset = m["dataset"]
+        (dataset || file || md) == name
+      end
+    end
+
+
+    def self.merge_modules(spec, base_spec)
+      modules = spec["input"]
+      resolved_modules = modules.map do |mod|
+        mod_name = mod["file"] || mod["module"]
+        fields = mod["fields"]
+        default = find_by_name(base_spec["input"], mod_name)
+        if default.nil?
+          mod
+        else
+          clashing_fields = ((default["fields"] || []) & (mod["fields"]))
+          unless clashing_fields.empty?
+            fail "There are fields \"#{clashing_fields.join(', ')}\" that are clashing with the default definition" 
+          end
+          master_fields = default["fields"]
+          redefinitions = fields.find_all {|f| f.has_key?(:acts_as)}
+          exclusions = redefinitions.reduce([]) {|memo, e| memo.concat([*e[:acts_as]])}
+          mod.merge({
+            "fields" => (master_fields.reject {|mf| exclusions.include? mf["name"]}) + fields
+          })
+        end
+      end
+      spec.merge({
+        "input" => resolved_modules
+      })
+    end
+
+    def self.generate_incremental_select(spec)
+      if spec[:condition].nil? || spec[:condition].empty?
+        spec[:condition] = "SystemModstamp > ${#{spec[:id]}_START} AND SystemModstamp <= ${#{spec[:id]}_END}"
+      else
+        spec[:condition] += "AND SystemModstamp > ${#{spec[:id]}_START} AND SystemModstamp <= ${#{spec[:id]}_END}"
+      end
+      generate_select(spec)
+    end
+
+    def self.generate_select(spec)
+      fields = spec[:fields].map do |f| 
+        f.has_key?(:multi_currency) ? "convertCurrency(#{f[:name]})" : f[:name]
+      end
+      condition = spec[:condition].nil?() ? "" : "WHERE #{spec[:condition]}"
+      limit = spec[:limit].nil?() ? "" : "LIMIT #{spec[:limit]}"
+      "SELECT #{fields.join(', ')} FROM #{spec[:object]} #{condition} #{limit}"
+    end
+
+    def self.normalize_module(spec)
+      fields = spec.has_key?("fields") ? spec["fields"].map {|f| f.kind_of?(Hash) ? f : {"name" => f} } : []
+      spec.merge({
+        "fields" => fields
+      })
+    end
+
+    def self.normalize_description(spec)
+      spec.merge({
+        "input" => spec["input"].map {|f| normalize_module(f) }
+      })
+    end
+
+    def self.sf_metadata_for_module(spec)
+      sf_metadata_for_fields(spec[:fields])
+    end
+
+    def self.sf_metadata_for_fields(spec)
+      spec.reduce([]) {|acc, f| acc.concat([{
+        :name => f[:name],
+        :type => f[:type] || "string"
+      }])}
+    end
+
+    def self.clover_metadata_for_module(spec)
+      clover_metadata_for_fields(spec[:fields])
+    end
+
+
+    def self.clover_metadata_for_fields(spec)
+      spec.reduce([]) do |acc, f|
+        transformation = f[:acts_as] ? f[:acts_as].map {|i| {:type => f[:type] || "string", :name => i}} : [{:type => f[:type] || "string", :name => f[:name]}]
+        acc.concat(transformation)
+      end
+    end
+
+    def self.transformation_acts_as(spec)
+      spec[:fields].reduce([]) do |acc, f|
+        transformation = f[:acts_as] ? f[:acts_as].map {|i| [f[:name], i]} : [[f[:name], f[:name]]]  
+        acc.concat(transformation)
+      end
+    end
+
+    # def self.to_port(node, port)
+    #   "#{to_id(node)}:#{port.to_s}"
+    # end
+
+    def self.to_id(node)
+      node[:id]
+    end
+    
+    $id = 0
+    def self.get_id()
+      $id += 1
+    end
+    
+    def self.create_trash_meta(builder)
+      builder.Metadata({:id   => "trash_metadata"}) do |builder|
+        csv_metadata(builder, {
+          :name   => "trash_metadata",
+          :fields => [{:type => "string", :name => "all"}]
+        })
+      end
+    end
+
+    def self.create_lookup_meta(builder)
+      builder.Metadata({:id   => "lookup_metadata"}) do |builder|
+        csv_metadata(builder, {
+          :name   => "lookup_metadata",
+          :fields => [{:type => "string", :name => "key"}, {:type => "string", :name => "value"}]
+        })
+      end
+    end
+    
+        
+    def self.create_run_graph(file, options={})
+      subgraphs = options[:subgraphs]
+
+      File.open(file, "w") do |file|
+        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+        builder.Graph({
+          :name => "Run graph"
+        }) do
+          builder.Global do
+            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+            property_file(builder, {:id => "params_params", :fileURL => "params.txt"})
+            create_trash_meta(builder)
+            create_lookup_meta(builder)
+                      
+          end
+          phase = 0
+          subgraphs.each do |subgraph|
+            builder.Phase(:number => phase+1) do
+              id1 = get_id
+              id2 = get_id
+              ctl = "function integer generate() {$out.0.all = \"FLOW=#{subgraph[:flow]}\";return OK;}"
+              build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata", :id => get_id()}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.txt", :outputFieldNames => "false", :quotedStrings => "false"}))
+            end
+            builder.Phase(:number => phase+2) do
+              
+              id1 = get_id
+              id2 = get_id
+              ctl = "function integer generate() {$out.0.all = \"NAME=#{subgraph[:name]}\";return OK;}"
+              build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata", :id => get_id()}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.txt", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
+            end  
+            
+            builder.Phase(:number => phase+3) do
+              build_node2(builder, GoodData::CloverGenerator::Nodes.run_graph2({:guiName => subgraph[:name], :name => subgraph[:name], :id => subgraph[:flow], :graphName => subgraph[:file]}))
+            end
+            phase += 4
+          end
+        end
+      end
+    end
+    
+    def self.create_incremental_downloader_run_graph(file, sources, options={})
+      # subgraphs = options[:subgraphs]
+
+      merged_sources = sources.reduce([]) do |memo, source|
+        merged_source = memo.find {|s| s[:object] == source[:object]}
+        if merged_source
+          merged_source[:fields] = (merged_source[:fields] + source[:fields]).uniq_by {|f| f[:name]}
+        else
+          memo.push(source)
+        end
+        memo
+      end
+
+      File.open(file, "w") do |file|
+        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+        builder.Graph({
+          :name => "Run graph"
+        }) do
+          builder.Global do
+            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+            property_file(builder, {:id => "params_params", :fileURL => "params.txt"})
+            create_trash_meta(builder)
+            create_lookup_meta(builder)
+            build_node2(builder, GoodData::CloverGenerator::Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => GoodData::CloverGenerator::Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
+          
+          end
+          phase = 0
+
+
+          merged_sources.each do |source|
+            module_name = source[:object]
+            file    = source[:id] || module_name
+            dataset = file || module_name
+
+
+            normalize_code = <<HEREDOC
+boolean done = false;
+function integer count() {
+  
+  if (indexOf($in.0.key, "#{dataset}_LAST_RUN") != -1) {
+    return 4;
+  }
+  else {
+    return 0;
+  }
+}
+
+string last_run = null;
+string end_date = null;
+
+function integer transform(integer idx) {
+  if (last_run == null) {
+    last_run = $in.0.value;
+  }
+  if (end_date == null) {
+    end_date = jodaDate2str(today(), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
+  }
+
+
+  if (idx == 1) {
+    $out.0.all = "#{dataset}_TRUNCATE_DATE=" + jodaDate2str(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'), "yyyy-MM-dd HH:mm:ss", 'en_US', 'UTC');
+  } else if (idx == 2) {
+    $out.0.all = "#{dataset}_START=" + last_run;
+  } else if (idx == 3) {
+    $out.0.all = "#{dataset}_END=" + end_date;
+  } else {
+    $out.0.all = "#{dataset}_LAST_RUN=" + end_date;
+  }
+
+  return OK;
+}
+
+HEREDOC
+
+              
+            
+
+            builder.Phase(:number => phase += 1) do
+            generate_func = <<HEREDOC
+function integer generate() {
+$out.0.key = "#{dataset}_LAST_RUN";
+$out.0.value = "1970-01-01T00:00:00.000+00:00";
+return OK;
+}
+HEREDOC
+
+              join_func = <<HEREDOC
+function integer transform() {
+  $out.0.key = nvl2($in.1.value, $in.1.key, $in.0.key);
+  $out.0.value = nvl2($in.1.value, $in.1.value, $in.0.value);
+return OK;
+}
+HEREDOC
+
+              build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => "generator_#{dataset}", :id => "generator_#{dataset}", :generate => generate_func}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.lookup_reader_writer2({:lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{dataset}" }))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.hash_join2({:id => "join_#{dataset}", :joinType => "leftOuter", :joinKey => "$key=$key", :transformation => join_func}))
+
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "join_#{dataset}:0", :fromNode => "generator_#{dataset}:0", :metadata => "lookup_metadata", :id => get_id()}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "join_#{dataset}:1", :fromNode => "gd_lookup_reader_#{dataset}:0", :metadata => "lookup_metadata", :id => get_id()}))
+
+              build_node2(builder, GoodData::CloverGenerator::Nodes.normalizer2({:name => "normalizer_#{dataset}", :id => "normalizer_#{dataset}", :normalize => normalize_code }))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "normalizer_#{dataset}:0", :fromNode => "join_#{dataset}:0", :metadata => "lookup_metadata", :id => get_id()}))
+
+              build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:quotedStrings => "false", :name => "params_writer_#{dataset}", :id => "params_writer_#{dataset}", :fileURL => "params.txt", :outputFieldNames => "false", :append => "true"}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "params_writer_#{dataset}:0", :fromNode => "normalizer_#{dataset}:0", :metadata => "trash_metadata", :id => get_id()}))
+            end
+          end
+          builder.Phase(:number => phase += 1) do
+            build_node2(builder, GoodData::CloverGenerator::Nodes.run_graph2({:guiName => "incremental", :name => "incremental_downloaders", :id => "downlaoders", :graphName => "graphs/incremental.grf"}))
+          end
+
+        end
+      end
+    end
+
+
+
+    def self.build_attribute_df(dataset, attribute)
+      {
+        :gd_dataset_attribute_display_form => {
+  			  :id => "label.#{dataset[:gd_name] || dataset[:id]}.#{attribute[:name]}",
+  			  :title => "label",
+  			  :prettyId => "label_#{dataset[:gd_name] || dataset[:id]}_#{attribute[:name]}",
+  			  :referenceKey => true,
+  			  :assignedMetadataField => attribute[:meta] || attribute[:name]
+			  }
+		  }
+    end
+    
+    def self.build_label_df(dataset, attribute, label)
+      {
+        :gd_dataset_attribute_display_form => {
+				  :id => "label.#{dataset[:gd_name] || dataset[:id]}.#{attribute[:name]}.#{label[:name]}",
+				  :title => "label",
+				  :prettyId => "label_#{dataset[:gd_name] || dataset[:id]}_#{attribute[:name]}_#{label[:name]}",
+				  :referenceKey => false,
+				  :assignedMetadataField => label[:meta] || label[:name]
+			  }
+      }
+    end
+    
+    def self.build_gd_dataset_loader_json(dataset)
+      # binding.pry
+      {
+        :gd_dataset => {
+          :attributes => dataset[:fields].find_all {|d| d[:type] == "attribute" || d[:type] == "connection_point"}.map do |attribute|
+            {
+              :gd_dataset_attribute => {
+                :id => "attr.#{dataset[:gd_name] || dataset[:id]}.#{attribute[:name]}",
+                :title => "XXX",
+                :prettyId => "attr_#{dataset[:gd_name] || dataset[:id]}_#{attribute[:name]}",
+                :selectedDisplayForm => build_attribute_df(dataset, attribute),
+                :assignedMetadataField => attribute[:meta] || attribute[:name],
+                :displayForms => dataset[:fields].find_all {|f| f[:type] == "label" && f[:for] == attribute[:name]}.map do |label|
+                  build_label_df(dataset, attribute, label)
+                end.concat([build_attribute_df(dataset, attribute)])
+              }
+            }
+          end,
+          :simpleFacts => dataset[:fields].find_all {|d| d[:type] == "fact"}.map do |fact|
+            {
+              :gd_dataset_fact => {
+                :id   => "fact.#{dataset[:gd_name] || dataset[:id]}.#{fact[:name]}",
+                :title => "FFF",
+        				:prettyId => "fact_#{dataset[:gd_name] || dataset[:id]}_#{fact[:name]}",
+                :type => "DECIMAL",
+                :assignedMetadataField => fact[:meta]
+              }
+            }
+          end,
+          :dateFacts => dataset[:fields].find_all {|d| d[:type] == "date"}.map do |date_attribute|
+            {
+        			:gd_dataset_fact => {
+        				:id => "dt.#{dataset[:gd_name] || dataset[:id]}.#{date_attribute[:fact] || date_attribute[:name]}",
+        				:title => "TTT",
+        				:prettyId => "dt_#{dataset[:gd_name] || dataset[:id]}_#{date_attribute[:fact] || date_attribute[:name]}",
+        				:type => "DATE",
+        				:assignedMetadataField => nil
+        			}
+        		}
+          end,
+          :dateAttributes =>  dataset[:fields].find_all {|d| d[:type] == "date"}.map do |date_attribute|
+            {
+        			:gd_dataset_date_attribute => {
+        				:id => "#{date_attribute[:dd]}.date",
+        				:title => "DDD",
+        				:prettyId => "#{date_attribute[:name]}",
+        				:assignedMetadataField => date_attribute[:meta],
+        				:associatedFact => {
+            			:gd_dataset_fact => {
+            				:id => "dt.#{dataset[:gd_name] || dataset[:id]}.#{date_attribute[:name]}",
+            				:title => "TTT",
+            				:prettyId => "dt_#{dataset[:gd_name] || dataset[:id]}_#{date_attribute[:name]}",
+            				:type => "DATE",
+            				:assignedMetadataField => nil
+            			}
+            		},
+            		:displayForms => [
+            		  {
+          					:gd_dataset_attribute_display_form => {
+          						:id => "#{date_attribute[:dd]}.date.yyyymmdd",
+          						:title => "yyyy-mm-dd (#{date_attribute[:dd]})",
+          						:prettyId => "#{date_attribute[:dd]}_date_yyyymmdd",
+          						:referenceKey => true,
+          						:assignedMetadataField => nil
+          					}
+          				}
+            		],
+            		:selectedDisplayForm => {
+        					:gd_dataset_attribute_display_form => {
+        						:id => "#{date_attribute[:dd]}.date.yyyymmdd",
+        						:title => "yyyy-mm-dd (#{date_attribute[:dd]})",
+        						:prettyId => "#{date_attribute[:dd]}_date_yyyymmdd",
+        						:referenceKey => true,
+        						:assignedMetadataField => nil
+        					}
+        				}
+        			}
+        		}
+          end,
+          :referencedAttributes => dataset[:fields].find_all {|d| d[:type] == "reference"}.map do |ref_attribute|
+            {
+              :gd_dataset_attribute => {
+        				:id => "attr.#{ref_attribute[:schema]}.#{ref_attribute[:ref]}",
+        				:title => "attr.#{ref_attribute[:schema]}.#{ref_attribute[:ref]}",
+        				:prettyId => "attr_#{ref_attribute[:schema]}_#{ref_attribute[:ref]}",
+        				:displayForms => [],
+        				:selectedDisplayForm => {
+        					:gd_dataset_attribute_display_form => {
+        						:id => "label.#{ref_attribute[:schema]}.#{ref_attribute[:ref]}",
+        						:title => "attr.#{ref_attribute[:schema]}.#{ref_attribute[:ref]}",
+        						:prettyId => "attr_#{ref_attribute[:schema]}_#{ref_attribute[:ref]}",
+        						:referenceKey => false,
+        						:assignedMetadataField => nil
+        					}
+        				},
+        				:assignedMetadataField => ref_attribute[:meta]
+        			}
+      			}
+          end,
+          :fieldsWithMetadataConflict => [],
+      		:entitiesWithoutMetadata => [],
+      		:entitiesWithoutValue => [],
+      		:datesWithFactConflict => [],
+      		:unassigneddateFacts => [],
+      		:datasetTitle => dataset[:gd_name] || dataset[:id],
+          :datasetId => "dataset.#{dataset[:gd_name] || dataset[:id]}"
+        }
+      }
+    end
+
+    def self.create_es_write_json(spec)
+      {
+        :entityName => spec[:id] || spec[:object],
+        :fieldsMapping => spec[:fields].reduce({}) do |memo, f|
+
+          if f.has_key?(:acts_as)
+          
+            f[:acts_as].each do |a|
+              type = case a
+              when "Id"
+                "recordid"
+              when "timestamp"
+                "timestamp"
+              else
+                f[:type] || "attribute"
+              end
+              memo[a] = {
+                :name => a,
+                :type => type
+              }
+            end
+          else
+            type = case f[:name]
+            when "Id"
+              "recordid"
+            when "timestamp"
+              "timestamp"
+            else
+              f[:type] || "attribute"
+            end
+
+            memo[f[:name]] = {
+              :name => f[:name],
+              :type => (f[:name] == "Id" ? "recordid" : f[:type] || "attribute")
+            }
+          end
+          memo
+        end
+
+
+      }
+    end
+
+
+    def self.create_moving_graph(file, options={})
+      source = options[:source]
+      target = options[:target]
+      operation = options[:operation]
+      force = options[:force] || false
+      
+      File.open(file, "w") do |file|
+        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+        builder.Graph({
+          :name => "File Copy"
+        }) do
+          builder.Global do
+            builder.Metadata({:id   => "list_metadata"}) do |builder|
+              csv_metadata(builder, {
+                :name => "list_metadata",
+                :fields => [{:name=>"filePath", :type=>"string"}]
+              })
+            end
+            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+          end
+          builder.Phase(:number => 0) do
+
+            transformation_source = "function integer transform() {\n" + ([["filePath", "filePath"]].map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+            build_node2(builder, GoodData::CloverGenerator::Nodes.file_list2(:baseURL => target, :id => "file_list", :transformation => transformation_source))
+
+            build_node2(builder, GoodData::CloverGenerator::Nodes.file_delete2(:baseURL => "${filePath}", :id => "file_delete"))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "file_delete:0", :fromNode => "file_list:0", :metadata => "list_metadata", :id => get_id()}))
+          end  
+          builder.Phase(:number => 1) do
+            build_node2(builder, GoodData::CloverGenerator::Nodes.file_copy2({:sourcePath => source, :targetPath => target, :operation => operation, :id => "file_copy"}))
+          end
+        end
+      end
+    end
+
+    def self.create_uploading_graph(file, options={})
+      metadata = options[:metadata]
+      dataset_infos = [options[:datasets]]
+      input_file = options[:input_file]
+      
+      File.open(file, "w") do |file|
+        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+        builder.Graph({
+          :name => "Goodsales Downloader"
+        }) do
+          builder.Global do
+            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+            dataset_infos.each do |dataset_info|
+              dataset = dataset_info[:id]
+              builder.Metadata({:id => "#{dataset}_load"}) do |builder| 
+                csv_metadata(builder, metadata)
+              end
+            end
+          end
+          builder.Phase(:number => 0) do
+            dataset_infos.each do |dataset_info|
+              dataset = dataset_info[:id]
+              gd_dataset = dataset_info[:gd_name] || dataset_info[:id]
+              to_svinstvo = build_gd_dataset_loader_json(dataset_info)
+              build_node2(builder, GoodData::CloverGenerator::Nodes.reader2({:name => "#{dataset} CSV Loader", :id => "#{dataset}_loader", :fileURL => "${PROJECT}/data/#{dataset}.csv"}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{dataset}_load:0", :fromNode => "#{dataset}_loader:0", :metadata => "#{dataset}_load", :id => get_id()}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.gd_loader2({:name => "#{dataset} Loader", :id => "#{dataset}_load", :dataset => "dataset.#{gd_dataset}", :datasetFieldMappings => to_svinstvo.to_json}))
+            end
+          end  
+        end
+      end
+    end
+    
+    def self.create_es_uploading_graph(file, options={})
+      metadata = options[:metadata]
+      dataset_infos = [options[:datasets]]
+      input_file = options[:input_file]
+      
+      File.open(file, "w") do |file|
+        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+        builder.Graph({
+          :name => "Goodsales Downloader"
+        }) do
+          builder.Global do
+            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+            dataset_infos.each do |dataset_info|
+              dataset = dataset_info[:id]
+              builder.Metadata({:id => "#{dataset}_load"}) do |builder| 
+                csv_metadata(builder, metadata)
+              end
+            end
+          end
+          builder.Phase(:number => 0) do
+            dataset_infos.each do |dataset_info|
+              dataset = dataset_info[:id]
+              gd_dataset = dataset_info[:gd_name] || dataset_info[:id]
+              to_svinstvo = build_gd_dataset_loader_json(dataset_info)
+              build_node2(builder, GoodData::CloverGenerator::Nodes.reader2({:name => "#{dataset} CSV Loader", :id => "#{dataset}_loader", :fileURL => "${PROJECT}/data/#{dataset}.csv"}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{dataset}_load:0", :fromNode => "#{dataset}_loader:0", :metadata => "#{dataset}_load", :id => get_id()}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.gd_loader2({:name => "#{dataset} Loader", :id => "#{dataset}_load", :dataset => "dataset.#{gd_dataset}", :datasetFieldMappings => to_svinstvo.to_json}))
+            end
+          end  
+        end
+      end
+    end
+    
+    def self.create_metadata(mod)
+      module_name = mod[:object]
+      file    = mod["file"] || module_name
+      # dataset = mod["dataset"] || file || module_name        
+      {
+        :name => file,
+        :fields => translate(clover_metadata_for_module(mod))
+      }
+    end
+
+
+    def self.create_sf_downloading_graph(file, sources, options={})
+      metadata = options[:metadata]
+      s3_backup = true && options[:s3_backup]
+
+      File.open(file, "w") do |file|
+      builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+      builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      builder.Graph({
+        :name => "Goodsales Downloader"
+      }) do
+        builder.Global do
+          sources.each do |mod|
+            module_name = mod[:object]
+            file    = mod[:id] || module_name
+            dataset = file || module_name        
+
+            builder.Metadata({:id   => "#{file}_sf_metadata"}) do |builder|
+              csv_metadata(builder, {
+                :name => "#{file}_sf_metadata",
+                :fields => translate(sf_metadata_for_module(mod))
+              })
+            end
+            builder.Metadata({:id   => "#{file}_clover_metadata"}) do |builder|
+              csv_metadata(builder, {
+                :name   => "#{file}_clover_metadata",
+                :fields => translate(clover_metadata_for_module(mod))
+              })
+            end
+          end
+
+          sf_connection(builder, {})
+          property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
+          property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
+          property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
+          property(builder, {:id => "SFDC_PASSWORD", :value => options[:password]})
+          property(builder, {:id => "SFDC_TOKEN", :value => options[:token]})
+          property(builder, {:id => "SFDC_USERNAME", :value => options[:login]})
+          property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+        end
+
+        builder.Phase(:number => 0) do
+          sources.each do |mod|
+            module_name = mod[:object]
+            file    = mod[:id] || module_name
+            dataset = file || module_name
+            
+            fields = mod[:fields]
+            mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
+            add = fields.map do |f|
+              "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
+            end
+        
+            stuff = mapping + add.join(",") + "]}}"
+                        
+            build_node2(builder, GoodData::CloverGenerator::Nodes.sfdc_reader2({:name => "#{file} SF Writer", :id => "#{file}_sf", :soql => generate_select(mod), :sfdcConnection => "SFDC", :fieldsMapping => stuff}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_reformat:0", :fromNode => "#{file}_sf:0", :metadata => "#{file}_sf_metadata", :id => get_id()}))
+            
+            transformation_source = "function integer transform() {\n" + (transformation_acts_as(mod).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+            build_node2(builder, GoodData::CloverGenerator::Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_reformat", :transformation => transformation_source}))
+            
+            
+            build_node2(builder, GoodData::CloverGenerator::Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reformat:0", :metadata => "#{file}_clover_metadata", :id => get_id()}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))
+            if s3_backup then build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "bucket", :outputFieldNames => true, :quotedStrings => false})) end
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_clover_metadata", :id => get_id()}))
+            if s3_backup then build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_clover_metadata", :id => get_id()})) end
+          end
+        end  
+      end
+      end
+    end
+
+    def self.create_es_downloading_graph(file, sources, options={})
+      metadata = options[:metadata]
+      s3_backup = true && options[:s3_backup]
+
+      File.open(file, "w") do |file|
+      builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+      builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      builder.Graph({
+        :name => "Goodsales Downloader"
+      }) do
+        builder.Global do
+          sources.each do |mod|
+            module_name = mod[:object]
+            file    = mod[:id] || module_name
+            dataset = file || module_name        
+
+            es_metadata = GoodData::CloverGenerator::DSL::Metadata.new({
+                :name   => "#{file}_es_metadata",
+                :fields => clover_metadata_for_module(mod)
+              })
+
+            builder.Metadata({:id   => "#{file}_es_metadata"}) do |builder|
+                csv_metadata(builder, es_metadata.change do |m|
+                  m.remove("timestamp")
+                end.to_hash)
+              end
+          end
+
+          property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+        end
+
+        builder.Phase(:number => 0) do
+          sources.each do |mod|
+            module_name = mod[:object]
+            file    = mod[:id] || module_name
+            dataset = file || module_name
+            
+            es_metadata = GoodData::CloverGenerator::DSL::Metadata.new({
+              :name   => "#{file}_es_metadata",
+              :fields => clover_metadata_for_module(mod)
+            })
+            es_metadata = es_metadata.change do |m|
+              m.remove("timestamp")
+            end.to_hash
+
+            fields = es_metadata[:fields]
+
+            e = Es::Entity.new("x", {
+              :file => "none",
+              :fields => fields.map do |f|
+                name = f[:name]
+                if name == "Id"
+                  Es::Field.new('Id', 'recordid')
+                else
+                  Es::Field.new(name, 'attribute')
+                end
+              end,
+              :timeframe => Es::Timeframe::parse("latest")
+            })
+            
+
+            stuff = {
+              :entityName => dataset,
+              :fieldsMapping => fields.inject({}) do |memo, field|
+                name = field[:name]
+                memo[name] = name
+                memo
+              end,
+              :eventStoreFieldToTypeMapping => fields.inject({}) do |memo, field|
+                name = field[:name]
+                if name == "Id"
+                  memo[name] = "recordid"
+                else
+                  memo[name] = "attribute"
+                end
+
+                memo
+              end,
+              :outputMetadataName => "#{file}_es_metadata"
+            }
+
+            readmap = {
+              :columns => e.to_extract_fragment('pid')["readTask"]["readMap"].first[:columns],
+              :populates => e.to_extract_fragment('pid')["readTask"]["readMap"].first[:populates]
+            }
+
+            # binding.pry      
+            build_node2(builder, GoodData::CloverGenerator::Nodes.es_reader2({:name => "#{file} ES Reader", :id => "#{file}_es", :entityFieldsMapping => stuff.to_json , :readMap => readmap.to_json}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_es:0", :metadata => "#{file}_es_metadata", :id => get_id()}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "${PROJECT}/data/#{dataset.downcase}.csv", :outputFieldNames => "true", :makeDirs => "true"}))
+            if s3_backup then build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "bucket", :outputFieldNames => true, :quotedStrings => false})) end
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_es_metadata", :id => get_id()}))
+            if s3_backup then build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_es_metadata", :id => get_id()})) end
+          end
+        end  
+      end
+      end
+    end
+
+    def self.create_incremental_downloading_graph(file, sources, options={})
+      metadata = options[:metadata]
+      store = options[:store] || "${GDC_EVENTSTORE}"
+      
+
+      merged_sources = sources.reduce([]) do |memo, source|
+        merged_source = memo.find {|s| s[:object] == source[:object]}
+        if merged_source
+          merged_source[:fields] = (merged_source[:fields] + source[:fields]).uniq_by {|f| f[:name]}
+        else
+          memo.push(source)
+        end
+        memo
+      end
+
+      File.open(file, "w") do |file|
+        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+        builder.Graph({:name => "Goodsales incremental Downloader"}) do
+          builder.Global do
+            property_file(builder, {:id => "params_params", :fileURL => "params.txt"})
+            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+
+            create_lookup_meta(builder)
+            merged_sources.each do |mod|
+              module_name = mod[:object]
+              file    = mod[:id] || module_name
+
+              sf_metadata = GoodData::CloverGenerator::DSL::Metadata.new({
+                :name => "#{file}_sf_metadata",
+                :fields => sf_metadata_for_module(mod)
+              })
+
+              clover_metadata = GoodData::CloverGenerator::DSL::Metadata.new({
+                :name   => "#{file}_clover_metadata",
+                :fields => clover_metadata_for_module(mod)
+              })
+
+
+              builder.Metadata({:id   => "#{file}_sf_metadata"}) do |builder|
+                csv_metadata(builder, sf_metadata.to_hash)
+              end
+
+              builder.Metadata({:id   => "#{file}_clover_metadata"}) do |builder|
+                csv_metadata(builder, clover_metadata.to_hash)
+              end
+
+              begin
+                builder.Metadata({:id   => "#{file}_es_metadata"}) do |builder|
+                  csv_metadata(builder, clover_metadata.change do |m|
+                    m.remove("timestamp")
+                    m.add(:name => "timestamp", :type => "date")
+                  end.to_hash)
+                end
+              rescue GoodData::CloverGenerator::DSL::RemoveMetadataFieldError => e 
+                exit_now!("Removing field \"#{e.field}\" failed from metadata \"#{e.metadata.name}\" for source \"#{file}\".")
+              end
+              # build_node2(builder, GoodData::CloverGenerator::Nodes.lookup2({:lookupTable => "params_lookup", :fileURL => "${PROJECT}/params.txt", :id => "params_lookup_id", :key => "key", :metadata => "lookup_metadata", :name => "params_lookup"}))
+              build_node2(builder, GoodData::CloverGenerator::Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => GoodData::CloverGenerator::Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
+
+            end
+
+            sf_connection(builder, {})
+            property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
+            property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
+            property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
+            property(builder, {:id => "SFDC_PASSWORD", :value => options[:password]})
+            property(builder, {:id => "SFDC_TOKEN", :value => options[:token]})
+            property(builder, {:id => "SFDC_USERNAME", :value => options[:login]})
+            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+          end
+
+          
+
+
+          phase = 1
+
+          merged_sources.each do |mod|
+
+            module_name = mod[:object]
+            file    = mod[:id] || module_name
+            dataset = file || module_name
+            s3_backup = true
+
+
+            builder.Phase(:number => phase += 1) do
+              build_node2(builder, GoodData::CloverGenerator::Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
+            end
+
+            builder.Phase(:number => phase += 1) do
+
+
+            fields = mod[:fields]
+            mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
+            add = fields.map do |f|
+              "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
+            end
+        
+            stuff = mapping + add.join(",") + "]}}"
+
+            build_node2(builder, GoodData::CloverGenerator::Nodes.sfdc_reader2({:name => "#{file} SF Writer", :id => "#{file}_sf", :soql => generate_incremental_select(mod), :sfdcConnection => "SFDC", :fieldsMapping => stuff, :mandatoryFields => fields.map {|f| f[:name] + ";"}.join}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_reformat:0", :fromNode => "#{file}_sf:0", :metadata => "#{file}_sf_metadata", :id => get_id()}))
+            
+            transformation_source = "function integer transform() {\n" + (transformation_acts_as(mod).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+            es_transformation_source = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.timestamp = str2date($in.0.timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");;\nreturn OK;\n}"
+
+            build_node2(builder, GoodData::CloverGenerator::Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_reformat", :transformation => transformation_source}))
+            
+            
+            build_node2(builder, GoodData::CloverGenerator::Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reformat:0", :metadata => "#{file}_clover_metadata", :id => get_id()}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:enabled => "disabled", :name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))
+
+            build_node2(builder, GoodData::CloverGenerator::Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_es_reformat", :transformation => es_transformation_source}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_es_reformat:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_clover_metadata", :id => get_id()}))
+
+            if s3_backup then
+              build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:enabled => "disabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}", :outputFieldNames => true, :quotedStrings => false}))
+            end
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_clover_metadata", :id => get_id()}))
+            if s3_backup then
+              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_clover_metadata", :id => get_id()}))
+            end
+            
+
+            build_node2(builder, GoodData::CloverGenerator::Nodes.sort2({:sortKey => "timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_es_sort"}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_es_sort:0", :fromNode => "#{file}_es_reformat:0", :metadata => "#{file}_es_metadata", :id => get_id()}))
+            
+            build_node2(builder, GoodData::CloverGenerator::Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_es", :store => store, :entityFieldsMapping => create_es_write_json(mod).to_json}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_es:0", :fromNode => "#{file}_es_sort:0", :metadata => "#{file}_es_metadata", :id => get_id()}))
+    
+
+            end
+
+          builder.Phase(:number => phase += 1) do
+                        generate_func = <<HEREDOC
+function integer generate() {
+date time_end = jodaStr2date("${#{dataset}_END}",["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC');
+$out.0.key = "#{dataset}_LAST_RUN";
+$out.0.value = jodaDate2str(time_end,"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", 'en_US', 'UTC');
+return OK;
+}
+HEREDOC
+
+            build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:guiName => dataset, :name => "generator_#{phase}", :id => "generator_#{phase}", :generate => generate_func}))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.lookup_reader_writer2({:guiName => dataset, :lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{phase}" }))
+            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "gd_lookup_reader_#{phase}:0", :fromNode => "generator_#{phase}:0", :metadata => "lookup_metadata", :id => get_id()}))  
+
+
+          end
+
+          end
+
+          
+
+          # builder.Phase(:number => phase += 1) do
+          #   build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:guiName => "generator1", :name => "generator1", :id => "generator1", :generate => generate_func}))
+          #   build_node2(builder, GoodData::CloverGenerator::Nodes.lookup_reader_writer2({:lookupTable => "gdLookup0", :id => "gd_lookup_reader", :graphName => "incremental.grf"}))
+          #   build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "gd_lookup_reader:0", :fromNode => "generator1:0", :metadata => "lookup_metadata", :id => get_id()}))  
+
+          # end
+
+
+        end
+      end
+    end
+
+  end
+end