gd_bam 0.1.21 → 0.1.22

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.21'
+  VERSION = '0.1.22'
 end

data/lib/cloud_connect/dsl/sf_helpers.rb

@@ -25,31 +25,33 @@ module GoodData
 
       def self.generate_incremental_select(tap)
         if tap[:condition].blank?
-          tap[:condition] = "SystemModstamp > ${#{tap[:id]}_START} AND SystemModstamp <= ${#{tap[:id]}_END}"
+          tap[:condition] = "SystemModstamp > ${#{tap[:id]}_START}"
+          # AND SystemModstamp <= ${#{tap[:id]}_END}
         else
-          tap[:condition] += " AND SystemModstamp > ${#{tap[:id]}_START} AND SystemModstamp <= ${#{tap[:id]}_END}"
+          tap[:condition] += " AND SystemModstamp > ${#{tap[:id]}_START}"
+           # AND SystemModstamp <= ${#{tap[:id]}_END}
         end
         generate_select(tap)
       end
 
-      def self.generate_history_query(client, field, sf_object, for_sf_object)
-        fields = client.fields(sf_object)
-        generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|f| fields.include?(f)}
-        specific_field_history = ["NewValue", "OldValue", "#{for_sf_object}Id"].all? {|f| fields.include?(f)}
-        specific_history = ["SystemModstamp", "#{for_sf_object}Id"].all? {|f| fields.include?(f)}
-        
-
-        if generic_field_history
-          "SELECT NewValue, CreatedDate, ParentId FROM #{sf_object} WHERE Field = '#{field}'"
-        elsif specific_field_history
-          "SELECT NewValue, CreatedDate, ParentId FROM #{sf_object} WHERE Field = '#{field}'"
-        elsif specific_history
-          return nil unless fields.include?(field)
-          "SELECT #{field}, SystemModstamp, #{for_sf_object}Id FROM #{sf_object}"
-        else
-          fail "Unrecognized fields configuration for historization in SF."
-        end
-      end
+      # def self.generate_history_query(client, field, sf_object, for_sf_object)
+      #   fields = client.fields(sf_object)
+      #   generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|f| fields.include?(f)}
+      #   specific_field_history = ["NewValue", "OldValue", "#{for_sf_object}Id"].all? {|f| fields.include?(f)}
+      #   specific_history = ["SystemModstamp", "#{for_sf_object}Id"].all? {|f| fields.include?(f)}
+      #   
+      # 
+      #   if generic_field_history
+      #     "SELECT NewValue, CreatedDate, ParentId FROM #{sf_object} WHERE Field = '#{field}'"
+      #   elsif specific_field_history
+      #     "SELECT NewValue, CreatedDate, ParentId FROM #{sf_object} WHERE Field = '#{field}'"
+      #   elsif specific_history
+      #     return nil unless fields.include?(field)
+      #     "SELECT #{field}, SystemModstamp, #{for_sf_object}Id FROM #{sf_object}"
+      #   else
+      #     fail "Unrecognized fields configuration for historization in SF."
+      #   end
+      # end
 
       def self.sf_connection(builder, data)
         builder.Connection({

data/lib/generators/downloaders.rb

@@ -35,9 +35,9 @@ module GoodData
 
 
             normalize_code = <<HEREDOC
-boolean done = false;
 function integer count() {
 
+  // This would run for the first time since we do not have anything stored in the custom properties hash
   if (indexOf($in.0.key, "#{dataset}_LAST_RUN") != -1) {
     return 4;
   }
@@ -47,17 +47,26 @@ function integer count() {
 }
 
 string last_run = null;
+long last_run_ts = null;
 string end_date = null;
+long now = date2long(today());
+long ten_mins_ago = now - (60 * 10) ;
+
 
 function integer transform(integer idx) {
   if (last_run == null) {
     last_run = $in.0.value;
+    last_run_ts = date2long(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'));
   }
+
   if (end_date == null) {
-    end_date = jodaDate2str(today(), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
+    if (ten_mins_ago < last_run_ts) {
+      end_date = jodaDate2str(long2date(last_run_ts), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
+    } else {
+      end_date = jodaDate2str(long2date(ten_mins_ago), "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "en_US", 'UTC');
+    }
   }
 
-
   if (idx == 1) {
     $out.0.all = "#{dataset}_TRUNCATE_DATE=" + jodaDate2str(jodaStr2date(last_run, ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"], 'en_US', 'UTC', 'UTC'), "yyyy-MM-dd HH:mm:ss", 'en_US', 'UTC');
   } else if (idx == 2) {

data/lib/generators/etl.rb

@@ -278,10 +278,11 @@ HEREDOC
                   Core::build_node2(builder, Nodes.es_reader2({:name => "ES Reader", :id => "es", :entityFieldsMapping => stuff.to_json , :readMap => read_map.to_json, :timeIntervals => time_frames.to_json}))
                 end
                 
+                
                 transformation_source = if Metadata::has_field?(metadata_from_es, "Snapshot")
                   "function integer transform() {\n" + (Tap.source_target_mappging(tap).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\n$out.0.Snapshot = date2str(dateAdd(str2date(\"1900-01-01\", \"yyyy-MM-dd\"),  str2long($in.0.Snapshot), day), \"yyyy-MM-dd\");\nreturn OK;\n}"
                 else
-                  "function integer transform() {\n" + (Tap.source_target_mappging(tap).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
+                  "long count = 0;\nfunction integer transform() {\ncount = count + 1; if (count == 1) {return SKIP;}\n" + (Tap.source_target_mappging(tap).map {|t| "$out.0.#{t.last} = $in.0.#{t.first};"}.join("\n")) + "\nreturn OK;\n}"
                 end
                 Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "reformat", :transformation => transformation_source}))
                 Core::build_node2(builder, Nodes.edge2({:toNode => "reformat:0", :fromNode => "es:0", :metadata => "es_metadata"}))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.1.21
+  version: 0.1.22
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-15 00:00:00.000000000 Z
+date: 2013-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake