gd_bam 0.0.15 → 0.1.0

data/lib/nodes/clover_gen.rb

@@ -1,27 +1,8 @@
-module Enumerable
-  def uniq_by
-    seen = Hash.new { |h,k| h[k] = true; false }
-    reject { |v| seen[yield(v)] }
-  end
-end
-
-class Hash
-  #pass single or array of keys, which will be removed, returning the remaining hash
-  def remove!(*keys)
-    keys.each{|key| self.delete(key) }
-    self
-  end
-
-  #non-destructive version
-  def remove(*keys)
-    self.dup.remove!(*keys)
-  end
-end
-
-
 module GoodData 
   module CloverGenerator
 
+    include GoodData::Bam
+
     ADD_NA_TRANSFORMATION = <<-trasnf
     function integer generate() {
     	integer number_of_fields = length($out.0);
@@ -40,53 +21,6 @@ module GoodData
     }
     trasnf
 
-    def self.parse_json(file)
-      begin
-        JSON.parse(File.read(file), :symbolize_names => true)
-      rescue RuntimeError => e
-        puts "Error parsing \"#{file}\": #{e.inspect}"
-      end
-    end
-
-
-    def self.metadata(builder, description)
-      builder.Record({
-          :fieldDelimiter => description[:fieldDelimiter],
-          :name => description[:name],
-          :recordDelimiter => description[:recordDelimiter],
-          :type => description[:type]
-        }) do |record|
-        description[:fields].each do |field|
-          builder.Field :name => field[:name], :type => field[:type], :nullable => "true"
-        end
-      end
-    end
-
-    def self.csv_metadata(builder, description)
-
-      sf_description = description.merge({
-        :fieldDelimiter => ",",
-        :recordDelimiter => "\\n",
-        :type => "delimited",
-      })
-      metadata(builder, sf_description)
-    end
-
-
-    def self.sf_connection(builder, data)
-      builder.Connection({
-        :clientId => "${SFDC_CLIENT_ID}",
-        :id => "SFDC",
-        :loginHostname => "${SFDC_LOGIN_HOSTNAME}",
-        :name => "${SFDC_NAME}",
-        :password => "${SFDC_PASSWORD}",
-        :passwordEncrypted => "true",
-        :token => "${SFDC_TOKEN}",
-        :type => "SFDC",
-        :username => "${SFDC_USERNAME}",
-        :passwordEncrypted => "false"
-      })
-    end
 
     def self.external_metadata_link(builder, data)
       builder.Metadata({
@@ -95,83 +29,6 @@ module GoodData
       })
     end
 
-    def self.property(builder, data)
-      builder.Property({
-        :id => data[:id],
-        :name => data[:name] || data[:id],
-        :value => data[:value]
-      })
-    end
-
-
-    def self.property_file(builder, data)
-      builder.Property({
-        :id => data[:id],
-        :fileURL => data[:fileURL]
-      })
-    end
-
-    def graph2(file, stuff)
-      metadata = stuff[:metadata]
-      nodes = stuff[:nodes]
-      connections = stuff[:connections]
-      File.open(file, "w") do |file|
-        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-        builder.Graph({
-          :name => "Goodsales Downloader"
-        }) do
-          builder.Global do
-            metadata.each do |m|
-              build_metadata2(builder, m)
-            end
-            connections.each do |conn|
-              build_node2(builder, conn)
-            end
-        
-          end
-          builder.Phase(:number => 0) do
-            nodes.each do |node|
-              build_node2(builder, node)
-            end
-          end  
-        end
-      end
-    end
-
-    def self.build_node2(builder, node)
-      if node[:type] == GoodData::CloverGenerator::Nodes::EDGE
-        builder.tag!("Edge", node)
-      elsif node[:type] == GoodData::CloverGenerator::Nodes::SF_CONNECTION
-        builder.tag!("Connection", node)
-      elsif node[:type] == GoodData::CloverGenerator::Nodes::FILE_LIST
-        builder.tag!("Node", node.remove(:transformation)) do |xml|
-          xml.attr({:name => "outputMapping"}) do |attr|
-            transformation = node[:transformation]
-            attr.cdata! transformation
-          end
-        end
-      elsif node[:type] == GoodData::CloverGenerator::Nodes::REFORMAT || node[:type] == GoodData::CloverGenerator::Nodes::EXT_HASH_JOIN
-        builder.tag!("Node", node.remove(:transformation)) do |xml|
-          xml.attr({:name => "transform"}) do |attr|
-            transformation = node[:transformation]
-            attr.cdata! transformation
-          end
-        end
-      elsif node[:type] == GoodData::CloverGenerator::Nodes::DATA_GENERATOR
-        builder.tag!("Node", node.remove(:transformation)) do |xml|
-          xml.attr({:name => "generate"}) do |attr|
-            transformation = node[:generate]
-            attr.cdata! transformation
-          end
-        end
-      elsif node[:type] == GoodData::CloverGenerator::Nodes::PERSISTENT_LOOKUP || node[:type] == GoodData::CloverGenerator::Nodes::GD_LOOKUP
-        builder.tag!("LookupTable", node)
-      else
-        builder.tag!("Node", node)
-      end
-    end
-
     def self.run_ctl(builder, ctl)
       id1 = rand(10000000000)
       id2 = rand(10000000000)
@@ -182,15 +39,6 @@ module GoodData
       build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata", :id => "#{id3}"}))
     end
 
-    def build_metadata2(builder, node)
-      builder.Metadata({:id => node[:id]}) do    
-        csv_metadata(builder, {
-          :name   => "account_sf_reformat",
-          :fields => translate(node[:fields])
-        })
-      end
-    end
-
     def sf_field_mapping(stuff)
       fields = stuff["fields"]
       mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
@@ -210,70 +58,57 @@ module GoodData
       }
     end
 
-    def self.validate_gd_datasets_metadata(datasets, gd_datasets)
-
-      init = {
-        :datasets => {}
-      }
-
-      datasets.reduce(init) do |memo, dataset|
-        gd_name = dataset[:gd_name] || dataset[:id]
-        gd_dataset = gd_datasets.find {|d| d["dataSet"]["meta"]["identifier"].gsub("dataset.", "") == gd_name}
-        gd_dataset_names = gd_datasets.map {|d| d["dataSet"]["meta"]["identifier"].gsub("dataset.", "")}.find_all {|d| d.index(".dt").nil?}
-
-        next(memo) if gd_dataset.nil?
-
-        refs = dataset[:fields].find_all {|f| f[:type] == "reference"}
-        refs.each do |ref|
-          (memo[:datasets][gd_name] ||= []) << ref unless gd_dataset_names.include?(ref[:schema])
-          ref_attr_indentifier = "attr.#{ref[:schema]}.#{ref[:ref]}"
-          (memo[:datasets][gd_name] ||= []) << ref if GoodData::MdObject[ref_attr_indentifier].nil?  
-        end
-
-        labels = dataset[:fields].find_all {|f| f[:type] == "label"}
-        gd_labels = gd_dataset['dataSet']['content']['dataLoadingColumns'].map {|label| GoodData.get(label)}
-        labels.each do |label|
-          label_name = label[:name]
-          label_identifier = "label.#{gd_name}.#{label[:for]}.#{label_name}"
-          (memo[:datasets][gd_name] ||= []) << label if GoodData::MdObject[label_identifier].nil?
-        end
-
-        facts = dataset[:fields].find_all {|f| f[:type] == "fact"}
-        gd_facts = gd_dataset['dataSet']['content']['facts'].map {|fact| GoodData.get(fact)}
-        facts.each do |fact|
-          fact_name = fact[:name]
-          unless gd_facts.any? {|a| a['fact']['meta']['identifier'] == "fact.#{gd_name}.#{fact_name}"}
-            (memo[:datasets][gd_name] ||= []) << fact
-          end
-        end
-
-        attributes = dataset[:fields].find_all {|f| f[:type] == "attribute" || f[:type] == "connection_point"}
-        gd_attributes = gd_dataset['dataSet']['content']['attributes'].map {|attr| GoodData.get(attr)}
-        attributes.each do |attr|
-          attr_name = attr[:name]
-          unless gd_attributes.any? {|a| a['attribute']['meta']['identifier'] == "attr.#{gd_name}.#{attr_name}"}
-            (memo[:datasets][gd_name] ||= []) << attr
-          end
-        end
-        memo
-      end
-    end
-
+    # def self.validate_gd_datasets_metadata(datasets, gd_datasets)
+    # 
+    #   init = {
+    #     :datasets => {}
+    #   }
+    # 
+    #   datasets.reduce(init) do |memo, dataset|
+    #     gd_name = dataset[:gd_name] || dataset[:id]
+    #     gd_dataset = gd_datasets.find {|d| d["dataSet"]["meta"]["identifier"].gsub("dataset.", "") == gd_name}
+    #     gd_dataset_names = gd_datasets.map {|d| d["dataSet"]["meta"]["identifier"].gsub("dataset.", "")}.find_all {|d| d.index(".dt").nil?}
+    # 
+    #     next(memo) if gd_dataset.nil?
+    # 
+    #     refs = dataset[:fields].find_all {|f| f[:type] == "reference"}
+    #     refs.each do |ref|
+    #       (memo[:datasets][gd_name] ||= []) << ref unless gd_dataset_names.include?(ref[:schema])
+    #       ref_attr_indentifier = "attr.#{ref[:schema]}.#{ref[:ref]}"
+    #       (memo[:datasets][gd_name] ||= []) << ref if GoodData::MdObject[ref_attr_indentifier].nil?  
+    #     end
+    # 
+    #     labels = dataset[:fields].find_all {|f| f[:type] == "label"}
+    #     gd_labels = gd_dataset['dataSet']['content']['dataLoadingColumns'].map {|label| GoodData.get(label)}
+    #     labels.each do |label|
+    #       label_name = label[:name]
+    #       label_identifier = "label.#{gd_name}.#{label[:for]}.#{label_name}"
+    #       (memo[:datasets][gd_name] ||= []) << label if GoodData::MdObject[label_identifier].nil?
+    #     end
+    # 
+    #     facts = dataset[:fields].find_all {|f| f[:type] == "fact"}
+    #     gd_facts = gd_dataset['dataSet']['content']['facts'].map {|fact| GoodData.get(fact)}
+    #     facts.each do |fact|
+    #       fact_name = fact[:name]
+    #       unless gd_facts.any? {|a| a['fact']['meta']['identifier'] == "fact.#{gd_name}.#{fact_name}"}
+    #         (memo[:datasets][gd_name] ||= []) << fact
+    #       end
+    #     end
+    # 
+    #     attributes = dataset[:fields].find_all {|f| f[:type] == "attribute" || f[:type] == "connection_point"}
+    #     gd_attributes = gd_dataset['dataSet']['content']['attributes'].map {|attr| GoodData.get(attr)}
+    #     attributes.each do |attr|
+    #       attr_name = attr[:name]
+    #       unless gd_attributes.any? {|a| a['attribute']['meta']['identifier'] == "attr.#{gd_name}.#{attr_name}"}
+    #         (memo[:datasets][gd_name] ||= []) << attr
+    #       end
+    #     end
+    #     memo
+    #   end
+    # end
 
-    def self.validate_sf_metadata(sf_client, sources)
-      sources.reduce({}) do |memo, source|
-        sf_object = source[:object]
-        u = sf_client.describe(sf_object)
-        sf_fields = u[:describeSObjectResponse][:result][:fields].map {|field| field[:name]}
-        fields_to_validate = source[:fields].map {|field| field[:name]}
-        memo[sf_object] = (fields_to_validate - sf_fields)
-        memo
-      end
-    end
 
-    def self.get_sf_client(params)
-      Salesforce::Client.new(params[:sf_login], params[:sf_password] + params[:sf_token], :server => params[:sf_server])
-    end
+    
 
     def self.download_metadata(downloaders_spec)
       mods = []
@@ -299,14 +134,6 @@ module GoodData
       downloaders_spec.merge({"input" => mods})
     end
 
-    def self.save_metadata(filename, data)
-      File.open(filename, "w") do |file|
-        builder = Builder::XmlMarkup.new(:target => file, :indent=>2)
-        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"  
-        csv_metadata(builder, data)
-      end
-    end
-
     def self.translate_field(what)
       dict = {
         nil             => "string",
@@ -332,79 +159,6 @@ module GoodData
       dict[what]
     end
 
-    def self.translate(fields)
-      fields
-      # fields.map do |f|
-      #   f.merge({
-      #     :type => translate_field(f[:type])
-      #   })
-      # end
-    end
-
-
-
-    def self.find_by_name(fields, name)
-      fields.find do |m|
-        xname = m["module"]
-        output = m["file"] || xname
-        output == name
-      end
-    end
-
-    def self.find_by_dataset(fields, name)
-      fields.find do |m|
-        md = m["module"]
-        file = m["file"]
-        dataset = m["dataset"]
-        (dataset || file || md) == name
-      end
-    end
-
-
-    def self.merge_modules(spec, base_spec)
-      modules = spec["input"]
-      resolved_modules = modules.map do |mod|
-        mod_name = mod["file"] || mod["module"]
-        fields = mod["fields"]
-        default = find_by_name(base_spec["input"], mod_name)
-        if default.nil?
-          mod
-        else
-          clashing_fields = ((default["fields"] || []) & (mod["fields"]))
-          unless clashing_fields.empty?
-            fail "There are fields \"#{clashing_fields.join(', ')}\" that are clashing with the default definition" 
-          end
-          master_fields = default["fields"]
-          redefinitions = fields.find_all {|f| f.has_key?(:acts_as)}
-          exclusions = redefinitions.reduce([]) {|memo, e| memo.concat([*e[:acts_as]])}
-          mod.merge({
-            "fields" => (master_fields.reject {|mf| exclusions.include? mf["name"]}) + fields
-          })
-        end
-      end
-      spec.merge({
-        "input" => resolved_modules
-      })
-    end
-
-    def self.generate_incremental_select(spec)
-      if spec[:condition].nil? || spec[:condition].empty?
-        spec[:condition] = "SystemModstamp > ${#{spec[:id]}_START} AND SystemModstamp <= ${#{spec[:id]}_END}"
-      else
-        spec[:condition] += " AND SystemModstamp > ${#{spec[:id]}_START} AND SystemModstamp <= ${#{spec[:id]}_END}"
-      end
-      generate_select(spec)
-    end
-
-    def self.generate_select(spec)
-      fields = spec[:fields].map do |f| 
-        f.has_key?(:multi_currency) ? "convertCurrency(#{f[:name]})" : f[:name]
-      end
-      condition = spec[:condition].nil?() ? "" : "WHERE #{spec[:condition]}"
-      limit = spec[:limit].nil?() ? "" : "LIMIT #{spec[:limit]}"
-      "SELECT #{fields.join(', ')} FROM #{spec[:object]} #{condition} #{limit}"
-    end
-
     def self.normalize_module(spec)
       fields = spec.has_key?("fields") ? spec["fields"].map {|f| f.kind_of?(Hash) ? f : {"name" => f} } : []
       spec.merge({
@@ -434,20 +188,13 @@ module GoodData
     end
 
 
-    def self.clover_metadata_for_fields(spec)
-      spec.reduce([]) do |acc, f|
+    def self.clover_metadata_for_fields(fields)
+      fields.reduce([]) do |acc, f|
         transformation = f[:acts_as] ? f[:acts_as].map {|i| {:type => f[:type] || "string", :name => i}} : [{:type => f[:type] || "string", :name => f[:name]}]
         acc.concat(transformation)
       end
     end
 
-    def self.transformation_acts_as(spec)
-      spec[:fields].reduce([]) do |acc, f|
-        transformation = f[:acts_as] ? f[:acts_as].map {|i| [f[:name], i]} : [[f[:name], f[:name]]]  
-        acc.concat(transformation)
-      end
-    end
-
     # def self.to_port(node, port)
     #   "#{to_id(node)}:#{port.to_s}"
     # end
@@ -456,192 +203,11 @@ module GoodData
       node[:id]
     end
     
-    $id = 0
-    def self.get_id()
-      $id += 1
-    end
-    
-    def self.create_trash_meta(builder)
-      builder.Metadata({:id   => "trash_metadata"}) do |builder|
-        csv_metadata(builder, {
-          :name   => "trash_metadata",
-          :fields => [{:type => "string", :name => "all"}]
-        })
-      end
-    end
 
-    def self.create_lookup_meta(builder)
-      builder.Metadata({:id   => "lookup_metadata"}) do |builder|
-        csv_metadata(builder, {
-          :name   => "lookup_metadata",
-          :fields => [{:type => "string", :name => "key"}, {:type => "string", :name => "value"}]
-        })
-      end
-    end
-    
-        
-    def self.create_run_graph(file, options={})
-      subgraphs = options[:subgraphs]
 
-      File.open(file, "w") do |file|
-        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-        builder.Graph({
-          :name => "Run graph"
-        }) do
-          builder.Global do
-            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-            property_file(builder, {:id => "params_params", :fileURL => "params.txt"})
-            create_trash_meta(builder)
-            create_lookup_meta(builder)
-                      
-          end
-          phase = 0
-          subgraphs.each do |subgraph|
-            builder.Phase(:number => phase+1) do
-              id1 = get_id
-              id2 = get_id
-              ctl = "function integer generate() {$out.0.all = \"FLOW=#{subgraph[:flow]}\";return OK;}"
-              build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata", :id => get_id()}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.txt", :outputFieldNames => "false", :quotedStrings => "false"}))
-            end
-            builder.Phase(:number => phase+2) do
-              
-              id1 = get_id
-              id2 = get_id
-              ctl = "function integer generate() {$out.0.all = \"NAME=#{subgraph[:name]}\";return OK;}"
-              build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata", :id => get_id()}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.txt", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
-            end  
-            
-            builder.Phase(:number => phase+3) do
-              build_node2(builder, GoodData::CloverGenerator::Nodes.run_graph2({:guiName => subgraph[:name], :name => subgraph[:name], :id => subgraph[:flow], :graphName => subgraph[:file]}))
-            end
-            phase += 4
-          end
-        end
-      end
-    end
     
-    def self.create_incremental_downloader_run_graph(file, sources, options={})
-      # subgraphs = options[:subgraphs]
-
-      merged_sources = sources.reduce([]) do |memo, source|
-        merged_source = memo.find {|s| s[:object] == source[:object]}
-        if merged_source
-          merged_source[:fields] = (merged_source[:fields] + source[:fields]).uniq_by {|f| f[:name]}
-        else
-          memo.push(source)
-        end
-        memo
-      end
-
-      File.open(file, "w") do |file|
-        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-        builder.Graph({
-          :name => "Run graph"
-        }) do
-          builder.Global do
-            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-            property_file(builder, {:id => "params_params", :fileURL => "params.txt"})
-            create_trash_meta(builder)
-            create_lookup_meta(builder)
-            build_node2(builder, GoodData::CloverGenerator::Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => GoodData::CloverGenerator::Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
-          
-          end
-          phase = 0
-
-
-          merged_sources.each do |source|
-            module_name = source[:object]
-            file    = source[:id] || module_name
-            dataset = file || module_name
-
-
-            normalize_code = <<HEREDOC
-boolean done = false;
-function integer count() {
-  
-  if (indexOf($in.0.key, "#{dataset}_LAST_RUN") != -1) {
-    return 4;
-  }
-  else {
-    return 0;
-  }
-}
-
-string last_run = null;
-string end_date = null;
-
-function integer transform(integer idx) {
-  if (last_run == null) {
-    last_run = $in.0.value;
-  }
-  if (end_date == null) {
-    end_date = jodaDate2str(today(), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
-  }
-
-
-  if (idx == 1) {
-    $out.0.all = "#{dataset}_TRUNCATE_DATE=" + jodaDate2str(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'), "yyyy-MM-dd HH:mm:ss", 'en_US', 'UTC');
-  } else if (idx == 2) {
-    $out.0.all = "#{dataset}_START=" + last_run;
-  } else if (idx == 3) {
-    $out.0.all = "#{dataset}_END=" + end_date;
-  } else {
-    $out.0.all = "#{dataset}_LAST_RUN=" + end_date;
-  }
-
-  return OK;
-}
-
-HEREDOC
-
-              
-            
-
-            builder.Phase(:number => phase += 1) do
-            generate_func = <<HEREDOC
-function integer generate() {
-$out.0.key = "#{dataset}_LAST_RUN";
-$out.0.value = "1970-01-01T00:00:00.000+00:00";
-return OK;
-}
-HEREDOC
-
-              join_func = <<HEREDOC
-function integer transform() {
-  $out.0.key = nvl2($in.1.value, $in.1.key, $in.0.key);
-  $out.0.value = nvl2($in.1.value, $in.1.value, $in.0.value);
-return OK;
-}
-HEREDOC
-
-              build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:name => "generator_#{dataset}", :id => "generator_#{dataset}", :generate => generate_func}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.lookup_reader_writer2({:lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{dataset}" }))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.hash_join2({:id => "join_#{dataset}", :joinType => "leftOuter", :joinKey => "$key=$key", :transformation => join_func}))
-
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "join_#{dataset}:0", :fromNode => "generator_#{dataset}:0", :metadata => "lookup_metadata", :id => get_id()}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "join_#{dataset}:1", :fromNode => "gd_lookup_reader_#{dataset}:0", :metadata => "lookup_metadata", :id => get_id()}))
-
-              build_node2(builder, GoodData::CloverGenerator::Nodes.normalizer2({:name => "normalizer_#{dataset}", :id => "normalizer_#{dataset}", :normalize => normalize_code }))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "normalizer_#{dataset}:0", :fromNode => "join_#{dataset}:0", :metadata => "lookup_metadata", :id => get_id()}))
-
-              build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:quotedStrings => "false", :name => "params_writer_#{dataset}", :id => "params_writer_#{dataset}", :fileURL => "params.txt", :outputFieldNames => "false", :append => "true"}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "params_writer_#{dataset}:0", :fromNode => "normalizer_#{dataset}:0", :metadata => "trash_metadata", :id => get_id()}))
-            end
-          end
-          builder.Phase(:number => phase += 1) do
-            build_node2(builder, GoodData::CloverGenerator::Nodes.run_graph2({:guiName => "incremental", :name => "incremental_downloaders", :id => "downlaoders", :graphName => "graphs/incremental.grf"}))
-          end
-
-        end
-      end
-    end
-
+        
+   
 
 
     def self.build_attribute_df(dataset, attribute)
@@ -777,120 +343,8 @@ HEREDOC
       }
     end
 
-    def self.create_es_write_json(spec)
-      {
-        :entityName => spec[:id] || spec[:object],
-        :fieldsMapping => spec[:fields].reduce({}) do |memo, f|
-
-          if f.has_key?(:acts_as)
-          
-            f[:acts_as].each do |a|
-              type = case a
-              when "Id"
-                "recordid"
-              when "timestamp"
-                "timestamp"
-              else
-                f[:type] || "attribute"
-              end
-              memo[a] = {
-                :name => a,
-                :type => type
-              }
-            end
-          else
-            type = case f[:name]
-            when "Id"
-              "recordid"
-            when "timestamp"
-              "timestamp"
-            else
-              f[:type] || "attribute"
-            end
-
-            memo[f[:name]] = {
-              :name => f[:name],
-              :type => (f[:name] == "Id" ? "recordid" : f[:type] || "attribute")
-            }
-          end
-          memo
-        end
-
-
-      }
-    end
-
-
-    def self.create_moving_graph(file, options={})
-      source = options[:source]
-      target = options[:target]
-      operation = options[:operation]
-      force = options[:force] || false
-      
-      File.open(file, "w") do |file|
-        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-        builder.Graph({
-          :name => "File Copy"
-        }) do
-          builder.Global do
-            builder.Metadata({:id   => "list_metadata"}) do |builder|
-              csv_metadata(builder, {
-                :name => "list_metadata",
-                :fields => [{:name=>"filePath", :type=>"string"}]
-              })
-            end
-            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-          end
-          builder.Phase(:number => 0) do
-
-            transformation_source = "function integer transform() {\n" + ([["filePath", "filePath"]].map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
-            build_node2(builder, GoodData::CloverGenerator::Nodes.file_list2(:baseURL => target, :id => "file_list", :transformation => transformation_source))
 
-            build_node2(builder, GoodData::CloverGenerator::Nodes.file_delete2(:baseURL => "${filePath}", :id => "file_delete"))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "file_delete:0", :fromNode => "file_list:0", :metadata => "list_metadata", :id => get_id()}))
-          end  
-          builder.Phase(:number => 1) do
-            build_node2(builder, GoodData::CloverGenerator::Nodes.file_copy2({:sourcePath => source, :targetPath => target, :operation => operation, :id => "file_copy"}))
-          end
-        end
-      end
-    end
 
-    def self.create_uploading_graph(file, options={})
-      metadata = options[:metadata]
-      dataset_infos = [options[:datasets]]
-      input_file = options[:input_file]
-      
-      File.open(file, "w") do |file|
-        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-        builder.Graph({
-          :name => "Goodsales Downloader"
-        }) do
-          builder.Global do
-            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-            dataset_infos.each do |dataset_info|
-              dataset = dataset_info[:id]
-              builder.Metadata({:id => "#{dataset}_load"}) do |builder| 
-                csv_metadata(builder, metadata)
-              end
-            end
-          end
-          builder.Phase(:number => 0) do
-            dataset_infos.each do |dataset_info|
-              dataset = dataset_info[:id]
-              gd_dataset = dataset_info[:gd_name] || dataset_info[:id]
-              to_svinstvo = build_gd_dataset_loader_json(dataset_info)
-              build_node2(builder, GoodData::CloverGenerator::Nodes.reader2({:name => "#{dataset} CSV Loader", :id => "#{dataset}_loader", :fileURL => "${PROJECT}/data/#{dataset}.csv"}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{dataset}_load:0", :fromNode => "#{dataset}_loader:0", :metadata => "#{dataset}_load", :id => get_id()}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.gd_loader2({:name => "#{dataset} Loader", :id => "#{dataset}_load", :dataset => "dataset.#{gd_dataset}", :datasetFieldMappings => to_svinstvo.to_json}))
-            end
-          end  
-        end
-      end
-    end
-    
     def self.create_es_uploading_graph(file, options={})
       metadata = options[:metadata]
       dataset_infos = [options[:datasets]]
@@ -920,361 +374,20 @@ HEREDOC
               build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{dataset}_load:0", :fromNode => "#{dataset}_loader:0", :metadata => "#{dataset}_load", :id => get_id()}))
               build_node2(builder, GoodData::CloverGenerator::Nodes.gd_loader2({:name => "#{dataset} Loader", :id => "#{dataset}_load", :dataset => "dataset.#{gd_dataset}", :datasetFieldMappings => to_svinstvo.to_json}))
             end
-          end  
+          end
         end
       end
     end
     
     def self.create_metadata(mod)
-      module_name = mod[:object]
-      file    = mod["file"] || module_name
+      module_name = mod.object
+      file    = mod.file || module_name
       # dataset = mod["dataset"] || file || module_name        
       {
         :name => file,
-        :fields => translate(clover_metadata_for_module(mod))
+        :fields => clover_metadata_for_module(mod)
       }
     end
 
-
-    def self.create_sf_downloading_graph(file, sources, options={})
-      metadata = options[:metadata]
-      s3_backup = true && options[:s3_backup]
-
-      File.open(file, "w") do |file|
-      builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-      builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-      builder.Graph({
-        :name => "Goodsales Downloader"
-      }) do
-        builder.Global do
-          sources.each do |mod|
-            module_name = mod[:object]
-            file    = mod[:id] || module_name
-            dataset = file || module_name        
-
-            builder.Metadata({:id   => "#{file}_sf_metadata"}) do |builder|
-              csv_metadata(builder, {
-                :name => "#{file}_sf_metadata",
-                :fields => translate(sf_metadata_for_module(mod))
-              })
-            end
-            builder.Metadata({:id   => "#{file}_clover_metadata"}) do |builder|
-              csv_metadata(builder, {
-                :name   => "#{file}_clover_metadata",
-                :fields => translate(clover_metadata_for_module(mod))
-              })
-            end
-          end
-
-          sf_connection(builder, {})
-          property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
-          property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
-          property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
-          property(builder, {:id => "SFDC_PASSWORD", :value => options[:password]})
-          property(builder, {:id => "SFDC_TOKEN", :value => options[:token]})
-          property(builder, {:id => "SFDC_USERNAME", :value => options[:login]})
-          property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-        end
-
-        builder.Phase(:number => 0) do
-          sources.each do |mod|
-            module_name = mod[:object]
-            file    = mod[:id] || module_name
-            dataset = file || module_name
-            
-            fields = mod[:fields]
-            mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
-            add = fields.map do |f|
-              "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
-            end
-        
-            stuff = mapping + add.join(",") + "]}}"
-            mandatory_fields = fields.reject {|f| f[:is_mandatory] == false }.map {|f| f[:name] + ";"}.join
-
-            build_node2(builder, GoodData::CloverGenerator::Nodes.sfdc_reader2({:name => "#{file} SF Writer", :id => "#{file}_sf", :soql => generate_select(mod), :sfdcConnection => "SFDC", :fieldsMapping => stuff, :mandatoryFields => mandatory_fields}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_reformat:0", :fromNode => "#{file}_sf:0", :metadata => "#{file}_sf_metadata", :id => get_id()}))
-            
-            transformation_source = "function integer transform() {\n" + (transformation_acts_as(mod).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
-            build_node2(builder, GoodData::CloverGenerator::Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_reformat", :transformation => transformation_source}))
-            
-            
-            build_node2(builder, GoodData::CloverGenerator::Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reformat:0", :metadata => "#{file}_clover_metadata", :id => get_id()}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))
-            if s3_backup then build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "bucket", :outputFieldNames => true, :quotedStrings => false})) end
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_clover_metadata", :id => get_id()}))
-            if s3_backup then build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_clover_metadata", :id => get_id()})) end
-          end
-        end  
-      end
-      end
-    end
-
-    def self.create_es_downloading_graph(file, sources, options={})
-      metadata = options[:metadata]
-      s3_backup = true && options[:s3_backup]
-
-      File.open(file, "w") do |file|
-      builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-      builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-      builder.Graph({
-        :name => "Goodsales Downloader"
-      }) do
-        builder.Global do
-          sources.each do |mod|
-            module_name = mod[:object]
-            file    = mod[:id] || module_name
-            dataset = file || module_name        
-
-            es_metadata = GoodData::CloverGenerator::DSL::Metadata.new({
-                :name   => "#{file}_es_metadata",
-                :fields => clover_metadata_for_module(mod)
-              })
-
-            builder.Metadata({:id   => "#{file}_es_metadata"}) do |builder|
-                csv_metadata(builder, es_metadata.change do |m|
-                  m.remove("timestamp")
-                end.to_hash)
-              end
-          end
-
-          property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-        end
-
-        builder.Phase(:number => 0) do
-          sources.each do |mod|
-            module_name = mod[:object]
-            file    = mod[:id] || module_name
-            dataset = file || module_name
-            
-            es_metadata = GoodData::CloverGenerator::DSL::Metadata.new({
-              :name   => "#{file}_es_metadata",
-              :fields => clover_metadata_for_module(mod)
-            })
-            es_metadata = es_metadata.change do |m|
-              m.remove("timestamp")
-            end.to_hash
-
-            fields = es_metadata[:fields]
-
-            e = Es::Entity.new("x", {
-              :file => "none",
-              :fields => fields.map do |f|
-                name = f[:name]
-                if name == "Id"
-                  Es::Field.new('Id', 'recordid')
-                else
-                  Es::Field.new(name, 'attribute')
-                end
-              end,
-              :timeframe => Es::Timeframe::parse("latest")
-            })
-            
-
-            stuff = {
-              :entityName => dataset,
-              :fieldsMapping => fields.inject({}) do |memo, field|
-                name = field[:name]
-                memo[name] = name
-                memo
-              end,
-              :eventStoreFieldToTypeMapping => fields.inject({}) do |memo, field|
-                name = field[:name]
-                if name == "Id"
-                  memo[name] = "recordid"
-                else
-                  memo[name] = "attribute"
-                end
-
-                memo
-              end,
-              :outputMetadataName => "#{file}_es_metadata"
-            }
-
-            readmap = {
-              :columns => e.to_extract_fragment('pid')["readTask"]["readMap"].first[:columns],
-              :populates => e.to_extract_fragment('pid')["readTask"]["readMap"].first[:populates]
-            }
-
-            build_node2(builder, GoodData::CloverGenerator::Nodes.es_reader2({:name => "#{file} ES Reader", :id => "#{file}_es", :entityFieldsMapping => stuff.to_json , :readMap => readmap.to_json}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_es:0", :metadata => "#{file}_es_metadata", :id => get_id()}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "${PROJECT}/data/#{dataset.downcase}.csv", :outputFieldNames => "true", :makeDirs => "true"}))
-            if s3_backup then build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "bucket", :outputFieldNames => true, :quotedStrings => false})) end
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_es_metadata", :id => get_id()}))
-            if s3_backup then build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_es_metadata", :id => get_id()})) end
-          end
-        end  
-      end
-      end
-    end
-
-    def self.create_incremental_downloading_graph(file, sources, options={})
-      metadata = options[:metadata]
-      store = options[:store] || "${GDC_EVENTSTORE}"
-      s3_backup = true && options[:s3_backup]
-
-      merged_sources = sources.reduce([]) do |memo, source|
-        merged_source = memo.find {|s| s[:object] == source[:object]}
-        if merged_source
-          merged_source[:fields] = (merged_source[:fields] + source[:fields]).uniq_by {|f| f[:name]}
-        else
-          memo.push(source)
-        end
-        memo
-      end
-
-      File.open(file, "w") do |file|
-        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-        builder.Graph({:name => "Goodsales incremental Downloader"}) do
-          builder.Global do
-            property_file(builder, {:id => "params_params", :fileURL => "params.txt"})
-            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-
-            create_lookup_meta(builder)
-            merged_sources.each do |mod|
-              module_name = mod[:object]
-              file    = mod[:id] || module_name
-
-              sf_metadata = GoodData::CloverGenerator::DSL::Metadata.new({
-                :name => "#{file}_sf_metadata",
-                :fields => sf_metadata_for_module(mod)
-              })
-
-              clover_metadata = GoodData::CloverGenerator::DSL::Metadata.new({
-                :name   => "#{file}_clover_metadata",
-                :fields => clover_metadata_for_module(mod)
-              })
-
-
-              builder.Metadata({:id   => "#{file}_sf_metadata"}) do |builder|
-                csv_metadata(builder, sf_metadata.to_hash)
-              end
-
-              builder.Metadata({:id   => "#{file}_clover_metadata"}) do |builder|
-                csv_metadata(builder, clover_metadata.to_hash)
-              end
-
-              begin
-                builder.Metadata({:id   => "#{file}_es_metadata"}) do |builder|
-                  csv_metadata(builder, clover_metadata.change do |m|
-                    m.remove("timestamp")
-                    m.add(:name => "timestamp", :type => "date")
-                  end.to_hash)
-                end
-              rescue GoodData::CloverGenerator::DSL::RemoveMetadataFieldError => e 
-                exit_now!("Removing field \"#{e.field}\" failed from metadata \"#{e.metadata.name}\" for source \"#{file}\".")
-              end
-              # build_node2(builder, GoodData::CloverGenerator::Nodes.lookup2({:lookupTable => "params_lookup", :fileURL => "${PROJECT}/params.txt", :id => "params_lookup_id", :key => "key", :metadata => "lookup_metadata", :name => "params_lookup"}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => GoodData::CloverGenerator::Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
-
-            end
-
-            sf_connection(builder, {})
-            property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
-            property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
-            property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
-            property(builder, {:id => "SFDC_PASSWORD", :value => options[:password]})
-            property(builder, {:id => "SFDC_TOKEN", :value => options[:token]})
-            property(builder, {:id => "SFDC_USERNAME", :value => options[:login]})
-            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-          end
-
-          
-
-
-          phase = 1
-
-          merged_sources.each do |mod|
-
-            module_name = mod[:object]
-            file    = mod[:id] || module_name
-            dataset = file || module_name
-            
-
-            builder.Phase(:number => phase += 1) do
-              build_node2(builder, GoodData::CloverGenerator::Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
-            end
-
-            builder.Phase(:number => phase += 1) do
-
-
-            fields = mod[:fields]
-            mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
-            add = fields.map do |f|
-              "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
-            end
-        
-            stuff = mapping + add.join(",") + "]}}"
-            mandatory_fields = fields.reject {|f| f[:is_mandatory] == false }.map {|f| f[:name] + ";"}.join
-
-            build_node2(builder, GoodData::CloverGenerator::Nodes.sfdc_reader2({:name => "#{file} SF Writer", :id => "#{file}_sf", :soql => generate_incremental_select(mod), :sfdcConnection => "SFDC", :fieldsMapping => stuff, :mandatoryFields => mandatory_fields}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_reformat:0", :fromNode => "#{file}_sf:0", :metadata => "#{file}_sf_metadata", :id => get_id()}))
-            
-            transformation_source = "function integer transform() {\n" + (transformation_acts_as(mod).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
-            es_transformation_source = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.timestamp = str2date($in.0.timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");;\nreturn OK;\n}"
-
-            build_node2(builder, GoodData::CloverGenerator::Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_reformat", :transformation => transformation_source}))
-            
-            
-            build_node2(builder, GoodData::CloverGenerator::Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reformat:0", :metadata => "#{file}_clover_metadata", :id => get_id()}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:enabled => "disabled", :name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))
-
-            build_node2(builder, GoodData::CloverGenerator::Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_es_reformat", :transformation => es_transformation_source}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_es_reformat:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_clover_metadata", :id => get_id()}))
-
-            if s3_backup then
-              build_node2(builder, GoodData::CloverGenerator::Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true, :quotedStrings => false}))
-            end
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_clover_metadata", :id => get_id()}))
-            if s3_backup then
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_clover_metadata", :id => get_id()}))
-            end
-            
-
-            build_node2(builder, GoodData::CloverGenerator::Nodes.sort2({:sortKey => "timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_es_sort"}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_es_sort:0", :fromNode => "#{file}_es_reformat:0", :metadata => "#{file}_es_metadata", :id => get_id()}))
-            
-            build_node2(builder, GoodData::CloverGenerator::Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_es", :store => store, :entityFieldsMapping => create_es_write_json(mod).to_json}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{file}_es:0", :fromNode => "#{file}_es_sort:0", :metadata => "#{file}_es_metadata", :id => get_id()}))
-    
-
-            end
-
-          builder.Phase(:number => phase += 1) do
-                        generate_func = <<HEREDOC
-function integer generate() {
-date time_end = jodaStr2date("${#{dataset}_END}",["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC');
-$out.0.key = "#{dataset}_LAST_RUN";
-$out.0.value = jodaDate2str(time_end,"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", 'en_US', 'UTC');
-return OK;
-}
-HEREDOC
-
-            build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:guiName => dataset, :name => "generator_#{phase}", :id => "generator_#{phase}", :generate => generate_func}))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.lookup_reader_writer2({:guiName => dataset, :lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{phase}" }))
-            build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "gd_lookup_reader_#{phase}:0", :fromNode => "generator_#{phase}:0", :metadata => "lookup_metadata", :id => get_id()}))  
-
-
-          end
-
-          end
-
-          
-
-          # builder.Phase(:number => phase += 1) do
-          #   build_node2(builder, GoodData::CloverGenerator::Nodes.data_generator2({:guiName => "generator1", :name => "generator1", :id => "generator1", :generate => generate_func}))
-          #   build_node2(builder, GoodData::CloverGenerator::Nodes.lookup_reader_writer2({:lookupTable => "gdLookup0", :id => "gd_lookup_reader", :graphName => "incremental.grf"}))
-          #   build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "gd_lookup_reader:0", :fromNode => "generator1:0", :metadata => "lookup_metadata", :id => get_id()}))  
-
-          # end
-
-
-        end
-      end
-    end
-
   end
 end