gd_bam 0.1.26 → 0.1.27

data/bin/bam

@@ -326,7 +326,16 @@ command :model_sync do |c|
   c.switch :dry
 
   c.action do |global_options,options,args|
-    GoodData::Bam::Commands::model_sync('.', options)
+    
+    params = PARAMS.merge({
+      :project_name => "downloaders-#{PARAMS[:project_name]}",
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)]
+    })
+    home = '.'
+    home = Pathname(home)
+    GoodData::Bam::Commands::model_sync('.', params)
   end
 end
 

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.26'
+  VERSION = '0.1.27'
 end

data/lib/base/errors.rb

@@ -15,7 +15,7 @@ module GoodData
     class IdInTapNotPresentError < RuntimeError
 
       def initialize(tap)
-        super("Every tap needs to have an output field Id defined. Tap \"#{tap[:id]}\" does not have one.")
+        super("Tap \"#{tap[:id]}\" does not have Id field. Every non direct tap has to have field name Id on the output. Consider adding an appropriate field and/or using acts_as.")
       end
     end
 

data/lib/base/graph.rb

@@ -4,14 +4,28 @@ module GoodData
 
       def self.create(graph)
         fail "Graph should have graph to execute. You defined \"#{graph}\"" if graph[:path].blank?
-        {
-          :inputs => [],
-          :outputs => []
-        }.merge(graph).merge({
-          :path => Pathname(graph[:path]),
-          :name => Pathname(graph[:path]).basename.to_s,
-          :type => :graph
-        })
+        if(!URI.parse(graph[:path].to_s).absolute?)
+          # it is a path
+          {
+            :inputs => [],
+            :outputs => []
+          }.merge(graph).merge({
+            :path => Pathname(graph[:path]),
+            :name => Pathname(graph[:path]).basename.to_s,
+            :type => :graph
+          })
+        else
+          # it is a URI
+          # binding.pry
+          {
+            :inputs => [],
+            :outputs => []
+          }.merge(graph).merge({
+            :path => URI(graph[:path]),
+            :name => URI(graph[:path]),
+            :type => :graph
+          })
+        end
       end
 
       def self.get_path(graph)

data/lib/base/repo.rb

@@ -11,6 +11,9 @@ module GoodData
         when :hash
           fail "Repo has to have content defined" if repository[:content].nil? || !repository[:content].is_a?(Hash)
           repository
+        when :http
+          repository.merge({
+            :base => URI(repository[:base])})
         end
       end
       
@@ -26,6 +29,18 @@ module GoodData
           end
         when :hash
           Graph.create(repository[:content][path])
+        when :http
+          path = repository[:base] + URI(path)
+          puts path
+          begin
+            RestClient.get(path.to_s)
+            Graph.create({
+              :path => path,
+              :repository => repository
+            })
+          rescue RestClient::ResourceNotFound => e  
+            nil
+          end
         else
           fail "not implemented"
         end

data/lib/base/tap.rb

@@ -139,19 +139,23 @@ module GoodData
         }))
       end
 
+
+      # 
+      # This method prepares the tap for data after downloading from SF but BEFORE it goes to ES
+      # We want to be able to load to ES as unchanged data as posible. Only Id and Timestamp (if provided is needed)
+      # other acts as should be used on the output of ES. The reason is that even if we change acts_as we should still pick up the same data
       def self.prepare_for_sf_downloader(tap)
-        # begin
-          id_field = Tap.find_output_field(tap, "Id")
-          fail IdInTapNotPresentError.new(tap) if id_field.nil?
-          time_field = Tap.find_output_field(tap, "Timestamp")
-          tap = Tap.remove_acts_as(tap)
-          fields = tap[:fields].map {|f| f[:name] == id_field[:name] ? f.merge(:acts_as => ["Id"]) : f}
-          if time_field
-            fields = fields.map {|f| f[:name] == time_field[:name] ? f.merge(:acts_as => ["Timestamp"]) : f}
-          end
-          Tap.create(tap.merge({
-            :fields => fields
-          }))
+        id_field = Tap.find_output_field(tap, "Id")
+        Tap.validate(tap)
+        time_field = Tap.find_output_field(tap, "Timestamp")
+        tap = Tap.remove_acts_as(tap)
+        fields = tap[:fields].map {|f| f[:name] == id_field[:name] ? f.merge(:acts_as => ["Id"]) : f}
+        if time_field
+          fields = fields.map {|f| f[:name] == time_field[:name] ? f.merge(:acts_as => ["Timestamp"]) : f}
+        end
+        Tap.create(tap.merge({
+          :fields => fields
+        }))
       end
 
       def self.prepare_for_es_downloader(tap)
@@ -168,6 +172,18 @@ module GoodData
         tap[:fields].reject {|f| f[:is_mandatory] == false }
       end
 
+      def self.validate(tap)
+        assert_presence_of_id_field(tap)
+      end
+
+      # Every incremental tap has to have 
+      def self.assert_presence_of_id_field(tap)
+        id_present = Tap.has_output_field?(tap, "Id")
+
+        direct = tap[:direct]
+        fail IdInTapNotPresentError.new(tap) if (!id_present && !direct)
+      end
+
     end
   end
 end

data/lib/commands/gd_commands.rb

@@ -20,8 +20,8 @@ module GoodData
 
       def self.model_sync(path, options)
         dry_run = options[:dry]
-        project = build_project(path, options)
-        datasets = project.sinks
+        project = GoodData::Bam::Project.build_project(path, options)
+        datasets = project[:sinks]
         model_update_dir = Pathname('model_update')
         cl_home = ENV['CL_HOME'] || PARAMS['CL_HOME'] || fail("Home of cl tool cannot be found. Either set up CL_HOME in your env with 'export CL_HOME=path/to/cl or set it up in your params.json. Point to the directory of CL not to the bin dir.'")
         cl_home = Pathname(cl_home) + 'bin/gdi.sh'
@@ -56,7 +56,7 @@ module GoodData
               end
             end
             template_name = dry_run ? "update_dataset_dry.script.erb" : "update_dataset.script.erb"
-            render_template(template_name, PARAMS.merge({"config_file" => dataset_path.expand_path}), :to_file => 'update_dataset.script')
+            Utils::render_template(template_name, PARAMS.merge({"config_file" => dataset_path.expand_path}), :to_file => 'update_dataset.script')
             puts "Generate #{ds[:id]}"
 
             system("#{cl_home} update_dataset.script --username #{PARAMS[:gd_login]} --password #{PARAMS[:gd_pass]}")

data/lib/compiler/compiler.rb

@@ -51,6 +51,7 @@ module GoodData
           when :tap
             tap = Project.find_tap_by_id(project, id)
             fail "Tap \"#{id}\" which was used in flow \"#{current_step[:flow_id]}\" is not defined" if tap.nil?
+            Tap.validate(tap)
             Flow.add_step(flow_memo, Tap.create(tap.merge(current_step)))
           when :sink
             sink = Project.find_sink_by_id(project, id)

data/lib/compiler/etl_visitor.rb

@@ -140,7 +140,6 @@ module GoodData
 
         def visit_direct_file_tap(node, state)
           graph_path = Step.get_graph_path(node)
-          tap = Tap.prepare_for_sf_downloader(node)
           GoodData::Bam::Generators::Etl::create_file_downloading_graph(graph_path, node, state)
           state.merge({
             :graphs => state[:graphs].concat([graph_path]),

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gd_bam
 version: !ruby/object:Gem::Version
-  version: 0.1.26
+  version: 0.1.27
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-04 00:00:00.000000000 Z
+date: 2013-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake