gd_bam 0.1.34 → 0.1.35

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.34'
+  VERSION = '0.1.35'
 end

data/lib/cloud_connect/dsl/helpers.rb

@@ -98,7 +98,7 @@ module GoodData
       def self.sink_out_gd_metadata(metadata, sink)
         metadata_fields = metadata[:fields]
         sink_facts = GoodData::Bam::Sink.get_fact_fields(sink).map {|f| f[:meta]}
-        x = metadata_fields.map {|f| sink_facts.include?(f[:name]) ? f.merge({:type => "decimal"}) : f}
+        x = metadata_fields.map {|f| sink_facts.include?(f[:name]) ? f.merge({:type => "decimal", :length => 20, :scale => 3}) : f}
         GoodData::Bam::Metadata.create(metadata.merge({:fields => x}))
       end
 
@@ -110,7 +110,12 @@ module GoodData
             :type => description[:type]
           }) do |record|
           description[:fields].each do |field|
-            builder.Field :name => field[:name], :type => field[:type] || "string", :nullable => "true"
+            # binding.pry if field.has_key?(:length)
+            cloned_field = field.clone
+            cloned_field[:type] = "string" if cloned_field[:type].blank?
+            cloned_field[:nullable] = "true" if cloned_field[:nullable].blank?
+
+            builder.Field cloned_field
           end
         end
       end

data/lib/generators/downloaders.rb

@@ -38,19 +38,29 @@ module GoodData
 function integer count() {
 
   // This would run for the first time since we do not have anything stored in the custom properties hash
-  if (indexOf($in.0.key, "#{dataset}_LAST_RUN") != -1) {
+  //if (indexOf($in.0.key, "#{dataset}_LAST_RUN") != -1) {
     return 4;
-  }
-  else {
-    return 0;
-  }
+  //}
+  //else {
+  //  return 0;
+  //}
 }
 
 string last_run = null;
 long last_run_ts = null;
 string end_date = null;
-long now = date2long(today());
-long ten_mins_ago = now - (60 * 10) ;
+date now = today();
+
+string p_year = date2str(now, "yyyy");
+string p_month = date2str(now, "MM");
+string p_day = date2str(now, "dd");
+string p_hour = date2str(now, "HH");
+string p_minute = date2str(now, "mm");
+string p_second = date2str(now, "ss");
+
+string now_date_concatenated = concat(p_year, '-', p_month, '-', p_day, 'T', p_hour, ':', p_minute, ':', p_second, '.000');
+long now_without_milis = date2long(str2date(now_date_concatenated, "yyyy-MM-dd'T'HH:mm:ss.SSS"));
+long ten_mins_ago = now_without_milis - (60L * 10L * 1000L);
 
 
 function integer transform(integer idx) {
@@ -59,6 +69,11 @@ function integer transform(integer idx) {
     last_run_ts = date2long(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'));
   }
 
+  printErr(ten_mins_ago);
+  printErr(date2long(now));
+  printErr(last_run_ts);
+  
+
   if (end_date == null) {
     if (ten_mins_ago < last_run_ts) {
       end_date = jodaDate2str(long2date(last_run_ts), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
@@ -66,7 +81,7 @@ function integer transform(integer idx) {
       end_date = jodaDate2str(long2date(ten_mins_ago), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
     }
   }
-
+  
   if (idx == 1) {
     $out.0.all = "#{dataset}_TRUNCATE_DATE=" + jodaDate2str(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'), "yyyy-MM-dd HH:mm:ss", 'en_US', 'UTC');
   } else if (idx == 2) {
@@ -74,7 +89,8 @@ function integer transform(integer idx) {
   } else if (idx == 3) {
     $out.0.all = "#{dataset}_END=" + end_date;
   } else {
-    $out.0.all = "#{dataset}_LAST_RUN=" + end_date;
+    $out.0.all = "#{dataset}_LAST_RUN=" + end_date; 
+    //+ last_run_ts + " " + ten_mins_ago;
   }
 
   return OK;
@@ -320,7 +336,9 @@ HEREDOC
                   es_transformation_source = if has_timestamp
                     "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = str2date($in.0.#{timestamp_field[:name]},\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");;\nreturn OK;\n}"
                   else
-                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = long2date(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"
+                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = jodaStr2date(\"${#{dataset}_END}\",[\"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"], 'en_US', 'UTC', 'UTC');\nreturn OK;\n}"
+                    
+                    
                   end
 
                   Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
@@ -338,7 +356,7 @@ HEREDOC
                       Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
 
                     else
-                      Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_s3_reformat", :transformation => "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = toString(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"}))
+                      Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_s3_reformat", :transformation => "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = \"${#{dataset}_END}\";\nreturn OK;\n}"}))
                       Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3_reformat:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
 
                       Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
@@ -450,7 +468,6 @@ HEREDOC
 
                 (tap[:fields] - [id_field, timestamp_field]).each_with_index do |field, i|
                   f = field[:name]
-
                   selects = if (f == "CreatedDate") then
                     [{
                       :object => sf_object,
@@ -465,6 +482,7 @@ HEREDOC
                   else
                     objects_to_get.reduce([]) do |memo, o|
                       fields = client.fields(o)
+                      fields_with_details = client.fields(sf_object, :with_details => true)
                       # how the history is stored does not rely on table naming convention. It is a couple of conventionsmangled together.This piece of code is trying to recognize which it is and produce the right downloader
                       generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|fl| fields.include?(fl)}
                       specific_field_history = ["NewValue", "OldValue", "#{sf_object}Id"].all? {|fl| fields.include?(fl)}
@@ -483,8 +501,10 @@ HEREDOC
                         ]
                       elsif (specific_field_history && !field[:multi_currency])
                         # This is like the previous with the difference that it specifically names the id of the record. Like OpportunityId, AccountId, etc.
+                        sf_field = fields_with_details.find {|sf_f| sf_f[:name] == f}
+                        sf_field_type = sf_field[:type]
                         [
-                          "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
+                          sf_field_type == "reference" ? "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f.chomp('Id')}'" : "SELECT NewValue, CreatedDate, #{sf_object}Id FROM #{o} WHERE Field = '#{f}'",
                           "{\"xmlFieldsMapping\":{\"xmlFields\":[
                           {\"xmlFieldMapping\":{\"name\":\"NewValue\",\"label\":\"NewValue\",\"xmlPath\":\"NewValue\",\"metadataField\":\"Value\"}},
                           {\"xmlFieldMapping\":{\"name\":\"CreatedDate\",\"label\":\"CreatedDate\",\"xmlPath\":\"CreatedDate\",\"metadataField\":\"Timestamp\"}},
@@ -547,7 +567,7 @@ HEREDOC
 
               phase = 1
 
-              taps_memo.values.each_slice(6).to_a.each do |slice|
+              taps_memo.values.each_slice(4).to_a.each do |slice|
                 builder.Phase(:number => phase +=1) do
                   slice.each do |field|
                 # taps_memo.each do |key, field|
@@ -747,7 +767,7 @@ HEREDOC
                   es_transformation_source = if has_timestamp
                     "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = str2date($in.0.#{timestamp_field[:name]},\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\nreturn OK;\n}"
                   else
-                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = long2date(${GRAPH_STARTED_TIMESTAMP});\nreturn OK;\n}"
+                    "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Id = $in.0.#{id_field[:name]};\n$out.0.Timestamp = jodaStr2date(\"${#{dataset}_END}\",[\"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"], 'en_US', 'UTC', 'UTC');\nreturn OK;\n}"
                   end
 
                   Core::build_node2(builder, Nodes.copy2({:name => "#{file} copy", :id => "#{file}_copy"}))
@@ -757,7 +777,7 @@ HEREDOC
                   Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_es_reformat", :transformation => es_transformation_source}))
                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_reformat:0", :fromNode => "#{file}_copy:1", :metadata => "#{file}_source_metadata"}))
                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_csv:0", :fromNode => "#{file}_copy:0", :metadata => "#{file}_source_metadata"}))
-                  
+
                   if s3_backup 
                     if has_timestamp
                       Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
@@ -772,7 +792,6 @@ HEREDOC
                     # Core::build_node2(builder, Nodes.writer2({:enabled => "enabled", :name => "#{file} s3 Writer", :id => "#{file}_s3", :fileURL => "https://${S3_ACCESS_KEY_ID}:\`replace(\"${S3_SECRET_ACCESS_KEY}\",\"/\",\"%2F\")\`@${S3_BUCKETNAME}.s3.amazonaws.com/${GDC_PROJECT_ID}/#{file}/#{file}_\`date2long(today())\`", :outputFieldNames => true}))
                     # Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_s3:0", :fromNode => "#{file}_copy:2", :metadata => "#{file}_source_metadata"}))
                   end
-                  
 
                   Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_es_sort"}))
                   Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_es_sort:0", :fromNode => "#{file}_es_reformat:0", :metadata => "#{file}_es_metadata"}))

data/lib/generators/etl.rb

@@ -144,7 +144,7 @@ function decimal deal_with_scientific_notation(string a_number) {
   if (isnull(a_number) || a_number == "") {
     return null;
   } else {
-    return str2decimal(a_number);
+    return floor(str2decimal(a_number)*1000D)/1000D;
   }
 }
 
@@ -304,6 +304,7 @@ HEREDOC
                 end
                 
                 
+                # Skipping the one row that is added by ES
                 transformation_source = if Metadata::has_field?(metadata_from_es, "Snapshot")
                   "function integer transform() {\n" + (Tap.source_target_mappging(tap).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\n$out.0.Snapshot = date2str(dateAdd(str2date(\"1900-01-01\", \"yyyy-MM-dd\"),  str2long($in.0.Snapshot), day), \"yyyy-MM-dd\");\nreturn OK;\n}"
                 else
@@ -311,17 +312,53 @@ HEREDOC
                 end
                 Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "reformat", :transformation => transformation_source}))
                 Core::build_node2(builder, Nodes.edge2({:toNode => "reformat:0", :fromNode => "es:0", :metadata => "es_metadata"}))
+
+                # Filter out deleted Records
                 if tap[:has_deleted_records] == true
                   filter_deleted_function = "function integer transform() {\nif ($in.0.IsDeleted == \"true\") {return SKIP;} else {$out.0.* = $in.0.*; return OK;};\n}"
                   Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "filter_deleted", :transformation => filter_deleted_function}))
                   Core::build_node2(builder, Nodes.edge2({:toNode => "filter_deleted:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "filter_deleted:0", :metadata => "output_metadata"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "dummy_reformat_1:0", :fromNode => "filter_deleted:0", :metadata => "output_metadata"}))
                 else
-                  Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "dummy_reformat_1:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
                 end
 
+                Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "dummy_reformat_1", :transformation => Nodes::MAP_ALL}))
+                
+                # Propagating a value from latest snapshot to all snapshots if necessary
+                if tap[:propagate_latest_value_for] && !tap[:propagate_latest_value_for].empty?
+
+                  retain_transform = <<HEREDOC
+#{tap[:propagate_latest_value_for].map {|f| "string " + f + " = \"\";"}.join("\n")}
+string id = null;
+string previousId = null;
+
+  function integer transform() {
+  	$out.0.* = $in.0.*;
+  	id = $in.0.Id;
+  	if (id != previousId) {
+  		#{tap[:propagate_latest_value_for].map {|f| f + " = $in.0." + f + ";"}.join("\n")}
+  	}
+  	#{tap[:propagate_latest_value_for].map {|f| "$out.0." + f + " = " + f + ";"}.join("\n")}
+  	previousId = id;
+  	return ALL;
+  }
+HEREDOC
+                  fields_to_propagate = tap[:propagate_latest_value_for]
+                  propagate_values_function = "string lastId;\nfunction integer transform() {\nif (lastId !=$in.0.Id) {$in.0.*=$out.0.*;} else {$in.0.*=$out.0.*; #{fields_to_propagate.map{|f| "$out.0.#{f} = \"XXXXX\""}.join("\n")}};lastId=$in.0.Id \nreturn OK;\n}"
+                  Core::build_node2(builder, Nodes.sort2({:sortKey => "Id(a);Snapshot(d)", :id => "sort_for_history_retain"}));
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "sort_for_history_retain:0", :fromNode => "dummy_reformat_1:0", :metadata => "output_metadata"}))
+                  Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "retain_values_reformat", :transformation => retain_transform}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "retain_values_reformat:0", :fromNode => "sort_for_history_retain:0", :metadata => "output_metadata"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "dummy_reformat_2:0", :fromNode => "retain_values_reformat:0", :metadata => "output_metadata"}))
+                else
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "dummy_reformat_2:0", :fromNode => "dummy_reformat_1:0", :metadata => "output_metadata"}))
+                end
+                
+                Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "dummy_reformat_2", :transformation => Nodes::MAP_ALL}))
+                
                 Core::build_node2(builder, Nodes.copy2({:name => "Copy", :id => "copy"}))
-                # Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "dummy_reformat_2:0", :metadata => "output_metadata"}))
                 Core::build_node2(builder, Nodes.writer2({:name => "CSV Writer", :id => "csv", :fileURL => "${PROJECT}/data/#{tap[:id]}.csv", :outputFieldNames => "true", :makeDirs => "true"}))
                 Core::build_node2(builder, Nodes.edge2({:toNode => "csv:0", :fromNode => "copy:0", :metadata => "output_metadata"}))
               end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.1.34
+  version: 0.1.35
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-01 00:00:00.000000000 Z
+date: 2013-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -258,7 +258,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.0.2
+        version: 0.0.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -266,7 +266,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.0.2
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
   name: gd_es
   requirement: !ruby/object:Gem::Requirement
@@ -320,11 +320,9 @@ files:
 - lib/base/step.rb
 - lib/base/tap.rb
 - lib/base/taps.rb
-- lib/base/taps.rb.orig
 - lib/cloud_connect/dsl/cc.rb
 - lib/cloud_connect/dsl/es_helpers.rb
 - lib/cloud_connect/dsl/helpers.rb
-- lib/cloud_connect/dsl/helpers.rb.orig
 - lib/cloud_connect/dsl/nodes.rb
 - lib/cloud_connect/dsl/sf_helpers.rb
 - lib/cloud_connect/dsl/structure_helpers.rb

data/lib/base/taps.rb.orig

@@ -1,35 +0,0 @@
-module GoodData 
-  module Bam
-    module Taps
-
-      def self.get_incremental(taps)
-        taps.find_all {|t| t[:direct] == false || t[:direct].nil? }
-      end
-
-      def self.get_salesforce(taps)
-        taps.find_all {|t| t[:source_type] == :salesforce}
-      end
-
-      def self.get_file(taps)
-        taps.find_all {|t| [:web, :file].include?(t[:source_type]) }
-      end
-
-<<<<<<< HEAD
-      def self.get_deleted_records_info(taps, client)
-        deleted = client.modules.map {|i| {:name => i[:name], :flag => i[:replicateable]}}.reduce({}) {|memo, x| memo.merge({x[:name] => x[:flag]})}
-        taps.map do |t|
-          if t[:source_type] == :salesforce
-            t.merge({:has_deleted_records => deleted[t[:object]]})
-          else
-            t.merge({:has_deleted_records => false})
-          end
-        end
-=======
-      def self.get_box(taps)
-        taps.find_all {|t| [:box].include?(t[:source_type]) }
->>>>>>> Box init
-      end
-
-    end
-  end
-end

data/lib/cloud_connect/dsl/helpers.rb.orig

@@ -1,229 +0,0 @@
-module GoodData
-  module CloudConnect
-    module Helpers
-
-      def self.property(builder, data)
-        builder.Property({
-          :id => data[:id],
-          :name => data[:name] || data[:id],
-          :value => data[:value]
-        })
-      end
-
-      def self.property_file(builder, data)
-        builder.Property({
-          :id => data[:id],
-          :fileURL => data[:fileURL]
-        })
-      end
-
-<<<<<<< HEAD
-      def self.create_deleted_records_meta(builder)
-        builder.Metadata({:id => "deleted_records_metadata"}) do |builder|
-          csv_metadata(builder, {
-            :name   => "deleted_records_metadata",
-            :fields => [
-              {:type => "string", :name => "Id"},
-              {:type => "string", :name => "Timestamp"}
-=======
-      def self.create_http_request_meta(builder)
-        builder.Metadata({:id => "http_metadata"}) do |builder|
-          csv_metadata(builder, {
-            :name   => "http_metadata",
-            :fields => [
-              {:type => "string", :name => "response"},
-              {:type => "string", :name => "all"}
->>>>>>> Box init
-            ]
-          })
-        end
-      end
-
-      def self.create_trash_meta(builder)
-        builder.Metadata({:id => "trash_metadata"}) do |builder|
-          csv_metadata(builder, {
-            :name   => "trash_metadata",
-            :fields => [{:type => "string", :name => "all"}]
-          })
-        end
-      end
-
-      def self.save_metadata(filename, data)
-        dirname = File.dirname(filename)
-        FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
- 
-        File.open(filename, "w") do |file|
-          builder = Builder::XmlMarkup.new(:target => file, :indent=>2)
-          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"  
-          csv_metadata(builder, data)
-        end
-      end
-
-      def self.create_moving_graph(file, options={})
-        source = options[:source]
-        target = options[:target]
-        operation = options[:operation]
-        force = options[:force] || false
-
-        File.open(file, "w") do |file|
-          builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-          builder.Graph({
-            :name => "File Copy"
-          }) do
-            builder.Global do
-              builder.Metadata({:id   => "list_metadata"}) do |builder|
-                Helpers::csv_metadata(builder, {
-                  :name => "list_metadata",
-                  :fields => [{:name=>"filePath", :type=>"string"}]
-                })
-              end
-              Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-            end
-            builder.Phase(:number => 0) do
-
-              transformation_source = "function integer transform() {\n" + ([["filePath", "filePath"]].map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
-              Core::build_node2(builder, Nodes.file_list2(:baseURL => target, :id => "file_list", :output_mapping => transformation_source))
-
-              Core::build_node2(builder, Nodes.file_delete2(:baseURL => "${filePath}", :id => "file_delete"))
-              Core::build_node2(builder, Nodes.edge2({:toNode => "file_delete:0", :fromNode => "file_list:0", :metadata => "list_metadata"}))
-            end  
-            builder.Phase(:number => 1) do
-              Core::build_node2(builder, Nodes.file_copy2({:sourcePath => source, :targetPath => target, :operation => operation, :id => "file_copy"}))
-            end
-          end
-        end
-      end
-
-      def self.csv_metadata(builder, description)
-        sf_description = description.merge({
-          :fieldDelimiter => ",",
-          :recordDelimiter => "\\n",
-          :type => "delimited",
-        })
-        metadata(builder, sf_description)
-      end
-
-      # prepares a sink for sinking into GD. This means dealing with stuff like scientific notation which is handled bu uploading as a decimal and not string
-      def self.sink_out_gd_metadata(metadata, sink)
-        metadata_fields = metadata[:fields]
-        sink_facts = GoodData::Bam::Sink.get_fact_fields(sink).map {|f| f[:meta]}
-        x = metadata_fields.map {|f| sink_facts.include?(f[:name]) ? f.merge({:type => "decimal"}) : f}
-        GoodData::Bam::Metadata.create(metadata.merge({:fields => x}))
-      end
-
-      def self.metadata(builder, description)
-        builder.Record({
-            :fieldDelimiter => description[:fieldDelimiter],
-            :name => description[:name],
-            :recordDelimiter => description[:recordDelimiter],
-            :type => description[:type]
-          }) do |record|
-          description[:fields].each do |field|
-            builder.Field :name => field[:name], :type => field[:type] || "string", :nullable => "true"
-          end
-        end
-      end
-
-      def self.create_lookup_meta(builder)
-        builder.Metadata({:id   => "lookup_metadata"}) do |builder|
-          csv_metadata(builder, {
-            :name   => "lookup_metadata",
-            :fields => [{:type => "string", :name => "key"}, {:type => "string", :name => "value"}]
-          })
-        end
-      end
-
-      def self.create_file_list_meta(builder)
-        builder.Metadata({:id   => "file_list"}) do |builder|
-          csv_metadata(builder, {
-            :name   => "file_list",
-            :fields => [
-              {:name => "filePath", :type => "string"},
-              {:name => "fileName", :type => "string"}
-            ]
-          })
-        end
-      end
-
-      def self.create_read_error_meta(builder)
-         builder.Metadata({:id   => "reader_error_metadata"}) do |builder|
-           csv_metadata(builder, {
-             :name   => "reader_error_metadata",
-             :fields => [
-               {:name => "line_number", :type => "integer"},
-               {:name => "field_number", :type => "integer"},
-               {:name => "record", :type => "string"},
-               {:name => "message", :type => "string"},
-               {:name => "file", :type => "string"}
-             ]
-           })
-         end
-      end
-
-      def self.create_run_graph_failure_metadata(builder)
-        builder.Metadata({:id   => "run_graph_failure_metadata"}) do |builder|
-          csv_metadata(builder, {
-            :name   => "run_graph_failure_metadata",
-            :fields => [
-              {:type => "string", :name => "graph"},
-              {:type => "string", :name => "result"},
-              {:type => "string", :name => "description"},
-              {:type => "string", :name => "message"},
-              {:type => "decimal", :name => "duration"}
-            ]
-          })
-        end
-      end
-
-      def self.create_run_graph(file, options={})
-        subgraphs = options[:subgraphs]
-        flow = options[:flow]
-        File.open(file, "w") do |file|
-          builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-          builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-          builder.Graph({
-            :name => "Run graph"
-          }) do
-            builder.Global do
-              property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-              property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
-              create_trash_meta(builder)
-              create_lookup_meta(builder)
-
-            end
-            phase = 0
-            subgraphs.each do |subgraph|
-              name = GoodData::Bam::Step.step_name(GoodData::Bam::Graph.create(:path => subgraph))
-              # Pathname(subgraph).basename.to_s.chomp(Pathname(subgraph).extname)
-              builder.Phase(:number => phase+1) do
-                id1 = GoodData::CloudConnect::Nodes.get_id
-                id2 = GoodData::CloudConnect::Nodes.get_id
-                ctl = "function integer generate() {$out.0.all = \"FLOW=#{flow}\";return OK;}"
-                Core::build_node2(builder, Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
-                Core::build_node2(builder, Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
-                Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :quotedStrings => "false"}))
-              end
-              builder.Phase(:number => phase+2) do
-
-                id1 = GoodData::CloudConnect::Nodes.get_id
-                id2 = GoodData::CloudConnect::Nodes.get_id
-                ctl = "function integer generate() {$out.0.all = \"NAME=#{name}\";return OK;}"
-                Core::build_node2(builder, Nodes.data_generator2({:name => id1, :id => id1, :generate => ctl}))
-                Core::build_node2(builder, Nodes.edge2({:toNode => "#{id2}:0", :fromNode => "#{id1}:0", :metadata => "trash_metadata"}))
-                Core::build_node2(builder, Nodes.writer2({:name => "PARAMS CSV Writer", :id => "#{id2}", :fileURL => "params.prm", :outputFieldNames => "false", :append => "true", :quotedStrings => "false"}))
-              end  
-
-              builder.Phase(:number => phase+3) do
-                Core::build_node2(builder, Nodes.run_graph2({:guiName => name, :name => name, :id => flow, :graphName => subgraph}))
-              end
-              phase += 4
-            end
-          end
-        end
-      end
-    
-
-    end
-  end
-end