gd_bam 0.1.19 → 0.1.20

data/bin/bam

@@ -191,6 +191,11 @@ command :debug do |c|
     flows = project[:flows].find_all do |flow|
       GoodData::Bam::Flow.find_graphs(flow).map {|s| GoodData::Bam::Step.get_graph_path(s).basename}.any? {|name| name == debug_path}
     end
+    
+    if flows.empty?
+      fail "Graph \"#{debug_path}\" is not used in any flow."
+    end
+    
     index = if flows.count > 1
       m = HighLine::Menu.new
       m.choose do |menu|
@@ -248,8 +253,9 @@ command :project do |c|
 
   c.action do |global_options,options,args|
     fail "You need to specify token to create a project" if options[:token].nil?
-    
+binding.pry
     GoodData::Bam::Commands::connect_to_gd()
+    binding.pry
     pid = case options[:blueprint]
       when "goodsales"
         "nt935rwzls50zfqwy6dh62tabu8h0ocy"

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.19'
+  VERSION = '0.1.20'
 end

data/lib/base/sink.rb

@@ -17,6 +17,10 @@ module GoodData
         !find_field(sink, name).nil?
       end
 
+      def self.get_date_fields(sink)
+        sink[:fields].find_all {|f| f[:type] == "date"}
+      end
+
       def self.add_target_type(sink)
         target = sink[:target].to_s
         path = Pathname(target)

data/lib/commands/sf_commands.rb

@@ -14,7 +14,11 @@ module GoodData
           u = sf_client.describe(sf_object)
           sf_fields = u[:describeSObjectResponse][:result][:fields].map {|field| field[:name]}
           fields_to_validate = tap[:fields].map {|field| field[:name]}
-          memo[sf_object] = (fields_to_validate - sf_fields)
+          if memo.has_key?(sf_object) 
+            memo[sf_object] = memo[sf_object].concat(fields_to_validate - sf_fields)
+          else
+            memo[sf_object] = (fields_to_validate - sf_fields)
+          end
           memo
         end
       end

data/lib/generators/etl.rb

@@ -109,11 +109,36 @@ module GoodData
                   Helpers::csv_metadata(builder, state[:metadata][input_name])
                 end
               end
+
               builder.Phase(:number => 0) do
                 gd_dataset = sink[:gd_name] || sink[:id]
                 to_svinstvo = GoodData::CloverGenerator::build_gd_dataset_loader_json(sink)
                 Core::build_node2(builder, Nodes.reader2({:name => "CSV Loader", :id => "loader", :fileURL => "${PROJECT}/data/#{input_name}.csv"}))
-                Core::build_node2(builder, Nodes.edge2({:toNode => "gd_load:0", :fromNode => "loader:0", :metadata => "load_metadata"}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "gd_load:0", :fromNode => "reformat:0", :metadata => "load_metadata"}))
+
+                transform = <<HEREDOC
+function string gd_date_compliance(string a_date) {
+  if (a_date != null && a_date != "" && left(a_date, 4) < "1950") {
+    return '1950-01-01';
+  }
+  if (a_date != null && a_date != "" && left(a_date, 4) > "2049") {
+    return '2049-01-01';
+  }
+  return a_date;
+}
+
+function integer transform() {
+    $out.0.* = $in.0.*;
+    #{Sink.get_date_fields(sink).map do |f|
+      "$out.0.#{f[:meta]} = gd_date_compliance($in.0.#{f[:meta]});"
+    end.join("\n")}
+
+    return ALL;
+}
+HEREDOC
+
+                Core::build_node2(builder, Nodes.reformat2({:name => "Reformat", :id => "reformat", :transformation => transform}))
+                Core::build_node2(builder, Nodes.edge2({:toNode => "reformat:0", :fromNode => "loader:0", :metadata => "load_metadata"}))
                 Core::build_node2(builder, Nodes.gd_loader2({:name => "GD Loader", :id => "gd_load", :dataset => "dataset.#{gd_dataset}", :datasetFieldMappings => to_svinstvo.to_json}))
               end
             end

data/lib/nodes/clover_gen.rb

@@ -345,39 +345,39 @@ module GoodData
 
 
 
-    def self.create_es_uploading_graph(file, options={})
-      metadata = options[:metadata]
-      dataset_infos = [options[:datasets]]
-      input_file = options[:input_file]
-      
-      File.open(file, "w") do |file|
-        builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
-        builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
-        builder.Graph({
-          :name => "Goodsales Downloader"
-        }) do
-          builder.Global do
-            property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
-            dataset_infos.each do |dataset_info|
-              dataset = dataset_info[:id]
-              builder.Metadata({:id => "#{dataset}_load"}) do |builder| 
-                csv_metadata(builder, metadata)
-              end
-            end
-          end
-          builder.Phase(:number => 0) do
-            dataset_infos.each do |dataset_info|
-              dataset = dataset_info[:id]
-              gd_dataset = dataset_info[:gd_name] || dataset_info[:id]
-              to_svinstvo = build_gd_dataset_loader_json(dataset_info)
-              build_node2(builder, GoodData::CloverGenerator::Nodes.reader2({:name => "#{dataset} CSV Loader", :id => "#{dataset}_loader", :fileURL => "${PROJECT}/data/#{dataset}.csv"}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{dataset}_load:0", :fromNode => "#{dataset}_loader:0", :metadata => "#{dataset}_load", :id => get_id()}))
-              build_node2(builder, GoodData::CloverGenerator::Nodes.gd_loader2({:name => "#{dataset} Loader", :id => "#{dataset}_load", :dataset => "dataset.#{gd_dataset}", :datasetFieldMappings => to_svinstvo.to_json}))
-            end
-          end
-        end
-      end
-    end
+    # def self.create_es_uploading_graph(file, options={})
+    #   metadata = options[:metadata]
+    #   dataset_infos = [options[:datasets]]
+    #   input_file = options[:input_file]
+    #   
+    #   File.open(file, "w") do |file|
+    #     builder = Builder::XmlMarkup.new(:target=>file, :indent=>2)
+    #     builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+    #     builder.Graph({
+    #       :name => "GoodData Uploader"
+    #     }) do
+    #       builder.Global do
+    #         property_file(builder, {:id => "workspace_params", :fileURL => "workspace.prm"})
+    #         dataset_infos.each do |dataset_info|
+    #           dataset = dataset_info[:id]
+    #           builder.Metadata({:id => "#{dataset}_load"}) do |builder| 
+    #             csv_metadata(builder, metadata)
+    #           end
+    #         end
+    #       end
+    #       builder.Phase(:number => 0) do
+    #         dataset_infos.each do |dataset_info|
+    #           dataset = dataset_info[:id]
+    #           gd_dataset = dataset_info[:gd_name] || dataset_info[:id]
+    #           to_svinstvo = build_gd_dataset_loader_json(dataset_info)
+    #           build_node2(builder, GoodData::CloverGenerator::Nodes.reader2({:name => "#{dataset} CSV Loader", :id => "#{dataset}_loader", :fileURL => "${PROJECT}/data/#{dataset}.csv"}))
+    #           build_node2(builder, GoodData::CloverGenerator::Nodes.edge2({:toNode => "#{dataset}_load:0", :fromNode => "#{dataset}_loader:0", :metadata => "#{dataset}_load", :id => get_id()}))
+    #           build_node2(builder, GoodData::CloverGenerator::Nodes.gd_loader2({:name => "#{dataset} Loader", :id => "#{dataset}_load", :dataset => "dataset.#{gd_dataset}", :datasetFieldMappings => to_svinstvo.to_json}))
+    #         end
+    #       end
+    #     end
+    #   end
+    # end
     
     def self.create_metadata(mod)
       module_name = mod.object

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.1.19
+  version: 0.1.20
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-01 00:00:00.000000000 Z
+date: 2013-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake