gd_bam 0.0.15 → 0.1.0

data/lib/generators/downloaders.rb

@@ -0,0 +1,449 @@
+module GoodData
+  module Bam
+    module Generators
+      module Downloaders
+
+        include GoodData::Bam
+        include GoodData::CloudConnect
+
+        def self.create_incremental_downloader_run_graph(file, taps, options={})
+
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({
+              :name => "Run graph"
+            }) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+                Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
+                Helpers::create_trash_meta(builder)
+                Helpers::create_lookup_meta(builder)
+                Helpers::create_file_list_meta(builder)
+                Helpers::create_run_graph_failure_metadata(builder)
+
+                Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
+
+              end
+              phase = 0
+              sf_taps = Taps.get_salesforce(taps)
+
+              sf_taps.each do |tap|
+                module_name = tap[:object]
+                file    = tap[:id] || module_name
+                dataset = file || module_name
+
+
+            normalize_code = <<HEREDOC
+boolean done = false;
+function integer count() {
+
+  if (indexOf($in.0.key, "#{dataset}_LAST_RUN") != -1) {
+    return 4;
+  }
+  else {
+    return 0;
+  }
+}
+
+string last_run = null;
+string end_date = null;
+
+function integer transform(integer idx) {
+  if (last_run == null) {
+    last_run = $in.0.value;
+  }
+  if (end_date == null) {
+    end_date = jodaDate2str(today(), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
+  }
+
+
+  if (idx == 1) {
+    $out.0.all = "#{dataset}_TRUNCATE_DATE=" + jodaDate2str(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'), "yyyy-MM-dd HH:mm:ss", 'en_US', 'UTC');
+  } else if (idx == 2) {
+    $out.0.all = "#{dataset}_START=" + last_run;
+  } else if (idx == 3) {
+    $out.0.all = "#{dataset}_END=" + end_date;
+  } else {
+    $out.0.all = "#{dataset}_LAST_RUN=" + end_date;
+  }
+
+  return OK;
+}
+
+HEREDOC
+
+
+
+
+                builder.Phase(:number => phase += 1) do
+            generate_func = <<HEREDOC
+function integer generate() {
+$out.0.key = "#{dataset}_LAST_RUN";
+$out.0.value = "1970-01-01T00:00:00.000+00:00";
+return OK;
+}
+HEREDOC
+
+              join_func = <<HEREDOC
+function integer transform() {
+  $out.0.key = nvl2($in.1.value, $in.1.key, $in.0.key);
+  $out.0.value = nvl2($in.1.value, $in.1.value, $in.0.value);
+return OK;
+}
+HEREDOC
+
+                  Core::build_node2(builder, Nodes.data_generator2({:name => "generator_#{dataset}", :id => "generator_#{dataset}", :generate => generate_func}))
+                  Core::build_node2(builder, Nodes.lookup_reader_writer2({:lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{dataset}" }))
+                  Core::build_node2(builder, Nodes.hash_join2({:id => "join_#{dataset}", :joinType => "leftOuter", :joinKey => "$key=$key", :transformation => join_func}))
+
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "join_#{dataset}:0", :fromNode => "generator_#{dataset}:0", :metadata => "lookup_metadata"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "join_#{dataset}:1", :fromNode => "gd_lookup_reader_#{dataset}:0", :metadata => "lookup_metadata"}))
+
+                  Core::build_node2(builder, Nodes.normalizer2({:name => "normalizer_#{dataset}", :id => "normalizer_#{dataset}", :normalize => normalize_code }))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "normalizer_#{dataset}:0", :fromNode => "join_#{dataset}:0", :metadata => "lookup_metadata"}))
+
+                  Core::build_node2(builder, Nodes.writer2({:quotedStrings => "false", :name => "params_writer_#{dataset}", :id => "params_writer_#{dataset}", :fileURL => "params.prm", :outputFieldNames => "false", :append => "true"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "params_writer_#{dataset}:0", :fromNode => "normalizer_#{dataset}:0", :metadata => "trash_metadata"}))
+                end
+              end
+              builder.Phase(:number => phase += 1) do
+                if !sf_taps.empty?
+                  Core::build_node2(builder, Nodes.run_graph2({:guiName => "incremental", :name => "incremental_downloaders", :id => "downlaoders", :graphName => "graphs/incremental.grf"}))
+                end
+              end
+
+              file_taps = Taps.get_file(taps)
+
+              file_taps.each do |tap|
+                source = tap[:source]
+                id =  tap[:id]
+
+
+            reformat_func = <<HEREDOC
+
+function integer transform() {
+    $out.0.filePath = replace($in.0.filePath, "${GDC_WEBDAV_HOST}", replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + ":${GD_PASSWORD}@${GDC_WEBDAV_HOST}");
+    $out.0.fileName = $in.0.fileName;
+    return ALL;
+}
+HEREDOC
+
+                builder.Phase(:number => phase += 1) do
+                  Core::build_node2(builder, Nodes.file_list2({:id => "#{id}_file_list", :name => "#{id}_file_list", :dataPolicy => "Strict", :baseURL => "#{tap[:source]}", :output_mapping => Nodes::MAP_ALL}))
+                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat", :id => "#{id}_reformat", :transformation => reformat_func}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat:0", :fromNode => "#{id}_file_list:0", :metadata => "file_list"}))
+                  Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_writer", :fileURL => "data/#{id}_files_to_read.csv"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_writer:0", :fromNode => "#{id}_reformat:0", :metadata => "file_list"}))
+                end
+
+                builder.Phase(:number => phase += 1) do
+                  ctl = "function integer generate() {$out.0.all = \"#{id}_SKIP_LINES=0\";return OK;}"
+                  Core::build_node2(builder, Nodes.data_generator2({:name => "#{id}_generator", :id => "#{id}_generator", :generate => ctl}))
+                  Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id}_csv_writer", :fileURL => "#{id}_counter.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_csv_writer:0", :fromNode => "#{id}_generator:0", :metadata => "trash_metadata"}))
+                end
+
+               subgraph_reformat_func = <<HEREDOC
+
+function integer transform() {
+    $out.0.all = "graphs/#{id}_loop.grf";  
+    return ALL;
+}
+HEREDOC
+
+            fail_reformat = <<HEREDOC
+function integer transform() {
+  raiseError("Loop failed");
+}
+HEREDOC
+
+
+                builder.Phase(:number => phase += 1) do
+                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat graph", :id => "#{id}_reformat_graph", :transformation => subgraph_reformat_func}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_graph:0", :fromNode => "#{id}_reformat:1", :metadata => "file_list"}))
+                  Core::build_node2(builder, Nodes.run_graph2({
+                    :guiName => id,
+                    :name => id,
+                    :id => "#{id}_run_graph"
+                  }))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_run_graph:0", :fromNode => "#{id}_reformat_graph:0", :metadata => "trash_metadata"}))
+                  Core::build_node2(builder, Nodes.reformat2({:name => "#{id} Reformat fail", :id => "#{id}_reformat_fail", :transformation => fail_reformat}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_reformat_fail:0", :fromNode => "#{id}_run_graph:1", :metadata => "run_graph_failure_metadata"}))
+                  Core::build_node2(builder, Nodes.trash2({:name => "#{id}_trash", :id => "#{id}_trash", :debugPrint => true}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{id}_trash:0", :fromNode => "#{id}_reformat_fail:0", :metadata => "run_graph_failure_metadata"}))
+                end
+              end
+
+
+            end
+          end
+        end
+
+        def self.create_incremental_sf_downloading_graph(file, taps, options={})
+          metadata = options[:metadata]
+          store = options[:store] || "${GDC_EVENTSTORE}"
+          s3_backup = options[:s3_backup]
+
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({:name => "Goodsales incremental Downloader"}) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+
+                Helpers::create_lookup_meta(builder)
+                taps.each do |tap|
+                  module_name = tap[:object]
+                  file = tap[:id] || module_name
+
+                  builder.Metadata({:id   => "#{file}_s3_metadata"}) do |builder|
+                    Helpers::csv_metadata(builder, GoodData::Bam::Metadata.add_field(Metadata.get_target_metadata(Tap.prepare_for_s3_backup(tap)), {:name => "Timestamp"}))
+                  end
+
+                  tap = Tap.prepare_for_sf_downloader(tap)
+                  builder.Metadata({:id   => "#{file}_source_metadata"}) do |builder|
+                    Helpers::csv_metadata(builder, Metadata.get_source_metadata(tap))
+                  end
+                  
+                  builder.Metadata({:id   => "#{file}_es_metadata"}) do |builder|
+                    Helpers::csv_metadata(builder, Metadata.add_timestamp_as_date(Metadata.get_target_metadata(tap)))
+                  end
+
+                  Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
+
+                end
+
+                Helpers::sf_connection(builder, {})
+                Helpers::property(builder, {:id => "SFDC_CLIENT_ID", :value => "gooddata/gooddata/"})
+                Helpers::property(builder, {:id => "SFDC_LOGIN_HOSTNAME", :value => options[:sf_server] || "login.salesforce.com"})
+                Helpers::property(builder, {:id => "SFDC_NAME", :value => "Salesforce connection"})
+                Helpers::property(builder, {:id => "SFDC_PASSWORD", :value => options[:sf_password]})
+                Helpers::property(builder, {:id => "SFDC_TOKEN", :value => options[:sf_token]})
+                Helpers::property(builder, {:id => "SFDC_USERNAME", :value => options[:sf_login]})
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+              end
+
+              phase = 1
+
+              taps.each do |tap|
+                module_name = tap[:object]
+                file    = tap[:id]
+                dataset = file || module_name
+                source_file  = tap[:source]
+                
+                has_timestamp = Tap.has_output_field?(tap, "Timestamp")
+                timestamp_field = Tap.find_output_field(tap, "Timestamp")
+                id_field = Tap.find_output_field(tap, "Id")
+                tap = Tap.prepare_for_sf_downloader(tap)
+                builder.Phase(:number => phase += 1) do
+                  Core::build_node2(builder, Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
+                end
+
+                builder.Phase(:number => phase += 1) do
+                  fields = tap[:fields]
+                  mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
+                  add = fields.map do |f|
+                    "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
+                  end
+                  stuff = mapping + add.join(",") + "]}}"
+
+                  mandatory_fields = fields.reject {|f| f[:is_mandatory] == false }.map {|f| f[:name] + ";"}.join
+
+                  Core::build_node2(builder, Nodes.sfdc_reader2({:name => "#{file} SF Reader", :id => "#{file}_reader", :soql => Helpers::generate_incremental_select(tap), :sfdcConnection => "SFDC", :fieldsMapping => stuff, :mandatoryFields => mandatory_fields}))
+
+
+                  es_transformation_source = if has_timestamp
+                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = str2date($in.0.#{timestamp_field[:name]},\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");;\nreturn OK;\n}"
+                  else
+                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = long2date(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"
+                  end
+
+                  Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reader:0", :metadata => "#{file}_source_metadata"}))
+                  Core::build_node2(builder, Nodes.writer2({:enabled => "disabled", :name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))
+
+                  Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_es_reformat", :transformation => es_transformation_source}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_reformat:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_source_metadata"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_source_metadata"}))
+                  
+                  if s3_backup then
+                    if has_timestamp
+
+                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
+
+                    else
+                      Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_s3_reformat", :transformation => "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = toString(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3_reformat:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
+
+                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_s3_reformat:0", :metadata => "#{file}_s3_metadata"}))
+                      
+                    end
+                  end
+
+
+                  Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_es_sort"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_sort:0", :fromNode => "#{file}_es_reformat:0", :metadata => "#{file}_es_metadata"}))
+
+                  Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json(Tap.add_timestamp_field(tap)).to_json}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es:0", :fromNode => "#{file}_es_sort:0", :metadata => "#{file}_es_metadata"}))
+
+                end
+
+                builder.Phase(:number => phase += 1) do
+                              generate_func = <<HEREDOC
+    function integer generate() {
+  date time_end = jodaStr2date("${#{dataset}_END}",["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC');
+  $out.0.key = "#{dataset}_LAST_RUN";
+  $out.0.value = jodaDate2str(time_end,"yyyy-MM-dd'T'HH:mm:ss.SSSZZ", 'en_US', 'UTC');
+  return OK;
+}
+HEREDOC
+
+                  Core::build_node2(builder, Nodes.data_generator2({:guiName => dataset, :name => "generator_#{phase}", :id => "generator_#{phase}", :generate => generate_func}))
+                  Core::build_node2(builder, Nodes.lookup_reader_writer2({:guiName => dataset, :lookupTable => "gdLookup0", :id => "gd_lookup_reader_#{phase}" }))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "gd_lookup_reader_#{phase}:0", :fromNode => "generator_#{phase}:0", :metadata => "lookup_metadata"}))
+
+                end
+              end
+            end
+          end
+        end
+
+        def self.generate_downloaders(home, project, params)
+          home = Pathname(home)
+
+          incremental_taps = Taps.get_incremental(project[:taps])
+
+          sf_taps = Taps.get_salesforce(incremental_taps)
+          file_taps = Taps.get_file(incremental_taps)
+
+          create_incremental_downloader_run_graph(home + "main.grf", incremental_taps)
+          create_incremental_sf_downloading_graph(home + "incremental.grf", sf_taps, params)
+
+          file_taps.each do |tap|
+            id = tap[:id]
+            Helpers::loop_over_file(home + "#{tap[:id]}_loop.grf", {
+              :token => id,
+              :file_to_loop => "data/#{id}_files_to_read.csv",
+              :graph_to_run => "graphs/#{id}_download.grf"
+            })
+            create_incremental_file_downloading_graph(home + "#{tap[:id]}_download.grf", [tap], params)
+          
+          end
+        end
+
+        def self.create_incremental_file_downloading_graph(file, taps, options={})
+          metadata = options[:metadata]
+          store = options[:store] || "${GDC_EVENTSTORE}"
+          s3_backup = options[:s3_backup]
+          File.open(file, "w") do |file|
+            builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+            builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+            builder.Graph({:name => "Goodsales incremental Downloader"}) do
+              builder.Global do
+                Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+                Helpers::create_lookup_meta(builder)
+                taps.each do |tap|
+                  module_name = tap[:object]
+                  file    = tap[:id] || module_name
+
+                  Helpers::property_file(builder, {:id => "params_params", :fileURL => "#{file}_item.prm"})
+
+                  builder.Metadata({:id   => "#{file}_s3_metadata"}) do |builder|
+                    Helpers::csv_metadata(builder, GoodData::Bam::Metadata.add_field(Metadata.get_target_metadata(Tap.prepare_for_s3_backup(tap)), {:name => "Timestamp"}))
+                  end
+
+                  tap = Tap.prepare_for_sf_downloader(tap)
+                  builder.Metadata({:id   => "#{file}_source_metadata"}) do |builder|
+                    Helpers::csv_metadata(builder, Metadata.get_source_metadata(tap))
+                  end
+
+                  builder.Metadata({:id   => "#{file}_es_metadata"}) do |builder|
+                    Helpers::csv_metadata(builder, GoodData::Bam::Metadata.add_timestamp_as_date(Metadata.get_target_metadata(tap)))
+                  end
+                  Core::build_node2(builder, Nodes.lookup2({:name => "gdLookup0", :id => "gdLookup0", :type => Nodes::GD_LOOKUP, :metadata => "lookup_metadata"}))
+                end
+              end
+
+              phase = 1
+
+              taps.each do |tap|
+                module_name = tap[:object]
+                file    = tap[:id]
+                dataset = file || module_name
+                source_file  = tap[:source]
+
+                
+                has_timestamp = GoodData::Bam::Tap.has_output_field?(tap, "Timestamp")
+                timestamp_field = GoodData::Bam::Tap.find_output_field(tap, "Timestamp")
+                id_field = GoodData::Bam::Tap.find_output_field(tap, "Id")
+
+                tap = Tap.prepare_for_sf_downloader(tap)
+
+                builder.Phase(:number => phase += 1) do
+                  fields = tap[:fields]
+                  mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
+                  add = fields.map do |f|
+                    "{\"xmlFieldMapping\":{\"name\":\"#{f[:name]}\",\"label\":\"#{f[:label]}\",\"xmlPath\":\"#{f[:name]}\",\"metadataField\":\"#{f[:name]}\"}}"
+                  end
+                  stuff = mapping + add.join(",") + "]}}"
+
+                  mandatory_fields = fields.reject {|f| f[:is_mandatory] == false }.map {|f| f[:name] + ";"}.join
+
+                  Core::build_node2(builder, Nodes.reader2({:name => "#{file} File Reader", :id => "#{file}_reader", :fileURL =>  "${FILE}"}))
+                  es_transformation_source = if has_timestamp
+                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = str2date($in.0.#{timestamp_field[:name]},\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");;\nreturn OK;\n}"
+                  else
+                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = long2date(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"
+                  end
+
+                  # build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_reformat", :transformation => transformation_source}))
+
+                  Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_copy:0", :fromNode => "#{file}_reader:0", :metadata => "#{file}_source_metadata"}))
+                  Core::build_node2(builder, Nodes.writer2({:enabled => "disabled", :name => "#{file} CSV Writer", :id => "#{file}_csv", :fileURL => "data/#{dataset.downcase}.csv", :outputFieldNames => "true"}))
+
+                  Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_es_reformat", :transformation => es_transformation_source}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_reformat:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_source_metadata"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_source_metadata"}))
+                  
+                  if s3_backup 
+                    if has_timestamp
+                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
+                    else
+                      Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_s3_reformat", :transformation => "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = toString(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3_reformat:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
+                
+                      Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
+                      Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_s3_reformat:0", :metadata => "#{file}_s3_metadata"}))
+                    end
+                    # Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
+                    # Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
+                  end
+                  
+
+                  Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_es_sort"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_sort:0", :fromNode => "#{file}_es_reformat:0", :metadata => "#{file}_es_metadata"}))
+                  Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json(Tap.add_timestamp_field(tap)).to_json}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es:0", :fromNode => "#{file}_es_sort:0", :metadata => "#{file}_es_metadata"}))
+                end
+
+                builder.Phase(:number => phase += 1) do
+                  Core::build_node2(builder, Nodes.file_delete2({:guiName => "#{file}_file_delete", :name => "#{file}_file_delete", :id => "#{file}_file_delete", :baseURL => "${FILE}"}))
+                end
+
+              end
+
+            end
+          end
+        end
+
+      end
+    end
+  end
+end