gd_bam 0.1.28 → 0.1.29

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.28'
+  VERSION = '0.1.29'
 end

data/lib/base/graph.rb

@@ -16,7 +16,6 @@ module GoodData
           })
         else
           # it is a URI
-          # binding.pry
           {
             :inputs => [],
             :outputs => []

data/lib/base/project.rb

@@ -81,6 +81,12 @@ module GoodData
         Project.create(project.merge({:sinks => sinks}))
       end
 
+      def self.mark_deleted_taps(project, options)
+        taps = project[:taps]
+        enriched_taps = Taps.get_deleted_records_info(taps, GoodData::Bam::Commands.get_sf_client(options))
+        Project.create(project.merge({:taps => enriched_taps}))
+      end
+
       def self.load_flows_definitions(project, path)
         p = Pathname(path)
         flows = Pathname.glob(p + "*.rb").map do |p|
@@ -96,6 +102,7 @@ module GoodData
         p = Project.load_taps(project, home + 'taps')
         p = Project.load_sinks(p, home + 'sinks')
         p = Project.set_graph_repos(p, paths)
+        p = Project.mark_deleted_taps(p, options)
         p = Project.load_flows_definitions(p, home + "flows")
         p = Compiler.compile_dsl(p)
       end

data/lib/base/tap.rb

@@ -161,7 +161,9 @@ module GoodData
       def self.prepare_for_es_downloader(tap)
         tap = Tap.pull_id_from_output_to_input(tap)
         tap = Tap.remove_timestamp_field(Tap.pull_timestamp_from_output_to_input(tap))
-        tap[:snapshots] == true ? Tap.add_field(tap, {:name => "Snapshot"}) : tap
+        tap = tap[:snapshots] == true ? Tap.add_field(tap, {:name => "Snapshot"}) : tap
+        tap = tap[:has_deleted_records] == true ? Tap.add_field(tap, {:name => "IsDeleted"}) : tap
+        tap = tap[:has_deleted_records] == true ? Tap.add_field(tap, {:name => "DeletedAt"}) : tap
       end
 
       def self.prepare_for_s3_backup(tap)

data/lib/base/taps.rb

@@ -14,6 +14,17 @@ module GoodData
         taps.find_all {|t| [:web, :file].include?(t[:source_type]) }
       end
 
+      def self.get_deleted_records_info(taps, client)
+        deleted = client.modules.map {|i| {:name => i[:name], :flag => i[:replicateable]}}.reduce({}) {|memo, x| memo.merge({x[:name] => x[:flag]})}
+        taps.map do |t|
+          if t[:source_type] == :salesforce
+            t.merge({:has_deleted_records => deleted[t[:object]]})
+          else
+            t.merge({:has_deleted_records => false})
+          end
+        end
+      end
+
     end
   end
 end

data/lib/cloud_connect/dsl/helpers.rb

@@ -17,6 +17,18 @@ module GoodData
         })
       end
 
+      def self.create_deleted_records_meta(builder)
+        builder.Metadata({:id => "deleted_records_metadata"}) do |builder|
+          csv_metadata(builder, {
+            :name   => "deleted_records_metadata",
+            :fields => [
+              {:type => "string", :name => "Id"},
+              {:type => "string", :name => "Timestamp"}
+            ]
+          })
+        end
+      end
+
       def self.create_trash_meta(builder)
         builder.Metadata({:id => "trash_metadata"}) do |builder|
           csv_metadata(builder, {

data/lib/cloud_connect/dsl/nodes.rb

@@ -8,6 +8,7 @@ module GoodData
       DEDUP = "DEDUP"
       READER = "DATA_READER"
       SF_READER = "SF_READER"
+      SF_DELETED_READER = "SF_DELETED_READER"
       REFORMAT = "REFORMAT"
       FILE_BACKUP = "FILE_BACKUP"
       PERSISTENT_LOOKUP = "persistentLookup"
@@ -189,7 +190,23 @@ HEREDOC
           :required => []
         })
       end
-      
+
+
+      def self.sfdc_deleted_records(data, options={})
+        local_defaults = {
+          :type => SF_DELETED_READER,
+          :retryDelay => 100,
+          :startDateTime => "`date2str(dateadd(today(), -20, day), \"yyyy-MM-dd'T'00:00:00.000'Z'\")`"
+        }
+        defaults = local_defaults.merge(options[:defaults] || {})
+
+        base_node(defaults.merge(data), {
+          :allowed => [:entity, :sfdcConnection],
+          :defaults => defaults,
+          :required => [:entity, :sfdcConnection, :startDateTime]
+        })
+      end
+
 
       def self.sfdc_reader2(data, options={})
         local_defaults = {

data/lib/generators/downloaders.rb

@@ -55,7 +55,7 @@ long ten_mins_ago = now - (60 * 10) ;
 
 function integer transform(integer idx) {
   if (last_run == null) {
-    last_run = $in.0.value;
+    last_run = jodaDate2str(jodaStr2date($in.0.value, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
     last_run_ts = date2long(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'));
   }
 
@@ -203,6 +203,18 @@ HEREDOC
                 Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
 
                 Helpers::create_lookup_meta(builder)
+                Helpers::create_deleted_records_meta(builder)
+                builder.Metadata({:id   => "reformat_deleted_records"}) do |builder|
+                  Helpers::csv_metadata(builder, {
+                    :name   => "reformat_deleted_records",
+                    :fields => [
+                      {:type => "string", :name => "Id"},
+                      {:type => "date", :name => "Timestamp"},
+                      {:type => "string", :name => "DeletedAt"},
+                      {:type => "string", :name => "IsDeleted"}]
+                  })
+                end
+
                 taps.each do |tap|
                   module_name = tap[:object]
                   file = tap[:id] || module_name
@@ -246,10 +258,52 @@ HEREDOC
                 timestamp_field = Tap.find_output_field(tap, "Timestamp")
                 id_field = Tap.find_output_field(tap, "Id")
                 tap = Tap.prepare_for_sf_downloader(tap)
+                deleted_records_tap = Tap.create({
+                  :source => "salesforce",
+                  :object => module_name,
+                  :id => dataset,
+                  :fields => [
+                    {:name => "Id"},
+                    {:name => "Timestamp"},
+                    {:name => "DeletedAt"},
+                    {:name => "IsDeleted"}]
+                })
+
+                if tap[:has_deleted_records]
+                  # binding.pry
+                  builder.Phase(:number => phase += 1) do
+                    ctl = "function integer generate() {$out.0.Id = \"GD_FAKE_DELETED_RECORD\";$out.0.Timestamp = $out.0.Timestamp = str2date(\"2038-01-01T00:00:00.000+0000\", \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");$out.0.IsDeleted = \"true\";$out.0.DeletedAt = \"2038-01-01T00:00:00.000+0000\";return OK;}"
+                    Core::build_node2(builder, Nodes.data_generator2({:name => "#{file}_fake_deleted_records_generator", :id => "#{file}_fake_deleted_records_generator", :generate => ctl}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_fake_deleted_records_es:0", :fromNode => "#{file}_fake_deleted_records_generator:0", :metadata => "reformat_deleted_records"}))
+                    Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_fake_deleted_records_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json(deleted_records_tap).to_json}))
+                  end
+                end
+
                 builder.Phase(:number => phase += 1) do
                   Core::build_node2(builder, Nodes.es_truncate2({:guiName => dataset, :store => store, :entity => dataset, :timestamp => "${#{dataset}_TRUNCATE_DATE}", :name => "#{module_name} es truncate", :id => "#{module_name}_es_truncate"}))
                 end
 
+                if tap[:has_deleted_records]
+                  builder.Phase(:number => phase += 1) do
+                    Core::build_node2(builder, Nodes.sfdc_deleted_records({
+                      :sfdcConnection => "SFDC",
+                      :entity => module_name,
+                      :id => "#{file}_deleted_records_downlader",
+                      :startDateTime => "`iif((${GRAPH_STARTED_TIMESTAMP} - 3600 * 20 * 24 * 1000) > date2long(str2date(replace(\"${#{dataset}_START}\", \"\\\\+00:00\", \"\+0000\"), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\")), date2str(long2date(${GRAPH_STARTED_TIMESTAMP} - 20 * 3600 * 24 * 1000), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"), date2str(str2date(replace(\"${#{dataset}_START}\", \"\\\\+00:00\", \"\+0000\"), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"))`",
+                      :endDateTime => "${#{dataset}_END}"
+                      # "`str2date(replace(\"${#{dataset}_END}\", \"\\\\+00:00\", \"\+0900\"), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"))`"
+                    }))
+
+                    es_transformation_source = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = str2date($in.0.Timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\n$out.0.DeletedAt = jodaDate2str(jodaStr2date($in.0.Timestamp, [\"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"], 'en_US', 'UTC', 'UTC'), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\", \"en_US\", 'UTC');\n$out.0.IsDeleted = \"true\";\nreturn OK;\n}"
+                    Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_deleted_records_reformat", :transformation => es_transformation_source}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_deleted_records_reformat:0", :fromNode => "#{file}_deleted_records_downlader:0", :metadata => "deleted_records_metadata"}))
+                    Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_deleted_records_es_sort"}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_deleted_records_es_sort:0", :fromNode => "#{file}_deleted_records_reformat:0", :metadata => "reformat_deleted_records"}))
+                    Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_deleted_records_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json(deleted_records_tap).to_json}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_deleted_records_es:0", :fromNode => "#{file}_deleted_records_es_sort:0", :metadata => "reformat_deleted_records"}))
+                  end
+                end
+
                 builder.Phase(:number => phase += 1) do
                   fields = tap[:fields]
                   mapping = "{\"xmlFieldsMapping\":{\"xmlFields\":["
@@ -340,6 +394,18 @@ HEREDOC
                 Helpers::property_file(builder, {:id => "params_params", :fileURL => "params.prm"})
                 Helpers::property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
 
+                Helpers::create_deleted_records_meta(builder)
+                builder.Metadata({:id   => "reformat_deleted_records"}) do |builder|
+                  Helpers::csv_metadata(builder, {
+                    :name   => "reformat_deleted_records",
+                    :fields => [
+                      {:type => "string", :name => "Id"},
+                      {:type => "date", :name => "Timestamp"},
+                      {:type => "string", :name => "DeletedAt"},
+                      {:type => "string", :name => "IsDeleted"}]
+                  })
+                end
+
                 Helpers::create_lookup_meta(builder)
                 builder.Metadata({:id   => "data_metadata"}) do |builder|
                   Helpers::csv_metadata(builder, {:name => "data_metadata",:fields => [{:name => "Id"}, {:name => "Timestamp"}, {:name => "Value"}]})
@@ -509,7 +575,43 @@ HEREDOC
                   end
                 end
               end
-              taps_with_history.each do |tap|
+
+              taps_with_history.uniq_by {|x| x[:id]}.each do |tap|
+
+                # Download the Deleted records for these taps. since we will not have a chance to download them later
+
+                if tap[:has_deleted_records]
+                  module_name = tap[:object]
+                  dataset = file = tap[:id]
+                  builder.Phase(:number => phase += 1) do
+                    Core::build_node2(builder, Nodes.sfdc_deleted_records({
+                      :sfdcConnection => "SFDC",
+                      :entity => module_name,
+                      :id => "#{file}_deleted_records_downlader",
+                      :startDateTime => "`iif((${GRAPH_STARTED_TIMESTAMP} - 3600 * 20 * 24 * 1000) > date2long(str2date(replace(\"${#{dataset}_START}\", \"\\\\+00:00\", \"\+0000\"), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\")), date2str(long2date(${GRAPH_STARTED_TIMESTAMP} - 20 * 3600 * 24 * 1000), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"), date2str(str2date(replace(\"${#{dataset}_START}\", \"\\\\+00:00\", \"\+0000\"), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"))`",
+                      :endDateTime => "${#{dataset}_END}"
+                      # "`str2date(replace(\"${#{dataset}_END}\", \"\\\\+00:00\", \"\+0900\"), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"))`"
+                    }))
+                    deleted_records_tap = Tap.create({
+                      :source => "salesforce",
+                      :object => module_name,
+                      :id => dataset,
+                      :fields => [
+                        {:name => "Id"},
+                        {:name => "Timestamp"},
+                        {:name => "DeletedAt"},
+                        {:name => "IsDeleted"}]
+                    })
+                    es_transformation_source = "function integer transform() {\n$out.0.* = $in.0.*;\n$out.0.Timestamp = str2date($in.0.Timestamp,\"joda:yyyy-MM-dd'T'HH:mm:ss.SSSZZ\");\n$out.0.DeletedAt = jodaDate2str(jodaStr2date($in.0.Timestamp, [\"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\"], 'en_US', 'UTC', 'UTC'), \"yyyy-MM-dd'T'HH:mm:ss.SSSZZ\", \"en_US\", 'UTC');\n$out.0.IsDeleted = \"true\";\nreturn OK;\n}"
+                    Core::build_node2(builder, Nodes.reformat2({:name => "#{file} Reformat", :id => "#{file}_deleted_records_reformat", :transformation => es_transformation_source}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_deleted_records_reformat:0", :fromNode => "#{file}_deleted_records_downlader:0", :metadata => "deleted_records_metadata"}))
+                    Core::build_node2(builder, Nodes.sort2({:sortKey => "Timestamp(a)",:name => "#{file} es Sort", :id => "#{file}_deleted_records_es_sort"}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_deleted_records_es_sort:0", :fromNode => "#{file}_deleted_records_reformat:0", :metadata => "reformat_deleted_records"}))
+                    Core::build_node2(builder, Nodes.es_writer2({:name => "#{file} es Writer", :id => "#{file}_deleted_records_es", :store => store, :entityFieldsMapping => Helpers::create_es_write_json(deleted_records_tap).to_json}))
+                    Core::build_node2(builder, Nodes.edge2({:toNode => "#{file}_deleted_records_es:0", :fromNode => "#{file}_deleted_records_es_sort:0", :metadata => "reformat_deleted_records"}))
+                  end
+                end
+
                 # puts "#{tap[:id]}_LAST_RUN"
                 builder.Phase(:number => phase += 1) do
                 generate_func = <<HEREDOC

data/lib/generators/etl.rb

@@ -221,7 +221,7 @@ HEREDOC
             }) do
               builder.Global do
 
-                builder.Metadata({:id   => "clover_metadata"}) do |builder|
+                builder.Metadata({:id  => "clover_metadata"}) do |builder|
                   Helpers::csv_metadata(builder, Metadata.get_target_metadata(tap))
                 end
 
@@ -311,9 +311,17 @@ HEREDOC
                 end
                 Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "reformat", :transformation => transformation_source}))
                 Core::build_node2(builder, Nodes.edge2({:toNode => "reformat:0", :fromNode => "es:0", :metadata => "es_metadata"}))
+                if tap[:has_deleted_records] == true
+                  filter_deleted_function = "function integer transform() {\nif ($in.0.IsDeleted == \"true\") {return SKIP;} else {$out.0.* = $in.0.*; return OK;};\n}"
+                  Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "filter_deleted", :transformation => filter_deleted_function}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "filter_deleted:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "filter_deleted:0", :metadata => "output_metadata"}))
+                else
+                  Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
+                end
 
                 Core::build_node2(builder, Nodes.copy2({:name => "Copy", :id => "copy"}))
-                Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
+                # Core::build_node2(builder, Nodes.edge2({:toNode => "copy:0", :fromNode => "reformat:0", :metadata => "output_metadata"}))
                 Core::build_node2(builder, Nodes.writer2({:name => "CSV Writer", :id => "csv", :fileURL => "${PROJECT}/data/#{tap[:id]}.csv", :outputFieldNames => "true", :makeDirs => "true"}))
                 Core::build_node2(builder, Nodes.edge2({:toNode => "csv:0", :fromNode => "copy:0", :metadata => "output_metadata"}))
               end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.1.28
+  version: 0.1.29
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-05 00:00:00.000000000 Z
+date: 2013-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake