gd_bam 0.1.3 → 0.1.5

data/bin/bam

@@ -49,14 +49,14 @@ command :sf_jack_in do |c|
   end
 end
 
-desc 'Jacks into project.'
-# arg_name 'Describe arguments to new here'
-command :project_jack_in do |c|
-
-  c.action do |global_options,options,args|
-    GoodData::Bam::Commands::project_jack_in(PARAMS)
-  end
-end
+# desc 'Jacks into project.'
+# # arg_name 'Describe arguments to new here'
+# command :project_jack_in do |c|
+# 
+#   c.action do |global_options,options,args|
+#     GoodData::Bam::Commands::project_jack_in(PARAMS)
+#   end
+# end
 
 
 desc 'Validatates connection to SalesForce.'
@@ -125,6 +125,26 @@ command :generate_downloaders do |c|
   end
 end
 
+command :generate_history_downloaders do |c|
+
+  c.desc 's3 backup'
+  c.arg_name 'backup'
+  c.flag :backup
+
+  c.action do |global_options,options,args|
+    params = global_options[:user_params]
+    params = PARAMS.merge({
+      :project_name => "downloaders-#{PARAMS[:project_name]}",
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)]
+    })
+    GoodData::Bam::Commands::generate_history_downloaders(".", params)
+  end
+end
+
+
+
 desc 'Validates that the tap has the fields it is claimed it should have. This is supposed to make the mitigate errors during deploy.'
 # arg_name 'Describe arguments to new here'
 command :sf_taps_validate do |c|
@@ -269,7 +289,7 @@ command :model_sync do |c|
   c.switch :dry
 
   c.action do |global_options,options,args|
-    GoodData::Bam::Commands::model_sync(options)
+    GoodData::Bam::Commands::model_sync('.', options)
   end
 end
 

data/bin/bam.orig

@@ -0,0 +1,375 @@
+#!/usr/bin/env ruby
+require 'gli'
+require 'bam'
+
+include GLI::App
+
+program_desc 'Describe your application here'
+
+version Bam::VERSION
+
+# desc 'Describe some switch here'
+# switch [:s,:switch]
+# 
+desc 'Verbose'
+default_value false
+arg_name 'verbose'
+switch [:v,:verbose]
+
+desc 'Http logger'
+default_value false
+arg_name 'logger'
+switch [:l,:logger]
+
+desc 'Generates clover project based on information in current directory. The default ouptut is the directory ./clover-project'
+# arg_name 'Describe arguments to new here'
+command :generate do |c|
+  
+  c.desc 'generate only specified flow'
+  c.arg_name 'only'
+  c.flag :only
+  
+  c.action do |global_options,options,args|
+    params = PARAMS.merge(options).merge({
+      :project_name => "etl-#{PARAMS[:project_name]}",
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => Pathname('./local_graphs').expand_path),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)
+      ]})
+    GoodData::Bam::Commands::generate('.', params)
+  end
+end
+
+desc 'Jacks into SF.'
+# arg_name 'Describe arguments to new here'
+command :sf_jack_in do |c|
+
+  c.action do |global_options,options,args|
+    GoodData::Bam::Commands::sf_jack_in(PARAMS)
+  end
+end
+
+# desc 'Jacks into project.'
+# # arg_name 'Describe arguments to new here'
+# command :project_jack_in do |c|
+# 
+#   c.action do |global_options,options,args|
+#     GoodData::Bam::Commands::project_jack_in(PARAMS)
+#   end
+# end
+
+
+desc 'Validatates connection to SalesForce.'
+# arg_name 'Describe arguments to new here'
+command :sf_validate_connection do |c|
+
+  c.action do |global_options,options,args|
+    GoodData::Bam::Commands::sf_validate_connection(PARAMS)
+  end
+end
+
+
+desc 'Generate data validator.'
+# arg_name 'Describe arguments to new here'
+command :generate_validator do |c|
+  c.action do |global_options,options,args|
+    GoodData::Bam::Commands::generate_validators('.', PARAMS.merge(options).merge({
+      :project_name => "validator-#{PARAMS[:project_name]}",
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)
+      ]
+      }))
+  end
+end
+
+desc 'Run data validator.'
+# arg_name 'Describe arguments to new here'
+command :run_validator do |c|
+
+  c.desc 'Checker process ID'
+  c.arg_name 'process'
+  c.flag :process
+
+  c.action do |global_options,options,args|
+    process = options[:process]
+    files = args.map {|f| Pathname(f)}
+    files.each do |f|
+      fail "Provded file \"#{f}\" does not exist." unless File.exist?(f)
+    end
+    fail "You need to specify process" if process.blank?
+    
+    GoodData::Bam::Commands::connect_to_gd()
+    GoodData::Bam::Commands::run_validator(process, files, PARAMS)
+  end
+end
+
+
+desc 'Generates clover project for downloaders.'
+# arg_name 'Describe arguments to new here'
+command :generate_downloaders do |c|
+
+  c.desc 's3 backup'
+  c.arg_name 'backup'
+  c.flag :backup
+
+  c.action do |global_options,options,args|
+    params = global_options[:user_params]
+    params = PARAMS.merge({
+      :project_name => "downloaders-#{PARAMS[:project_name]}",
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)]
+    })
+    GoodData::Bam::Commands::generate_downloaders(".", params)
+  end
+end
+
+command :generate_history_downloaders do |c|
+
+  c.desc 's3 backup'
+  c.arg_name 'backup'
+  c.flag :backup
+
+  c.action do |global_options,options,args|
+    params = global_options[:user_params]
+    params = PARAMS.merge({
+      :project_name => "downloaders-#{PARAMS[:project_name]}",
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)]
+    })
+<<<<<<< HEAD
+    GoodData::Bam::Commands::generate_downloaders(".", params)
+=======
+    GoodData::Bam::Commands::generate_history_downloaders(".", params)
+>>>>>>> Added historical SF downloader
+  end
+end
+
+
+
+desc 'Validates that the tap has the fields it is claimed it should have. This is supposed to make the mitigate errors during deploy.'
+# arg_name 'Describe arguments to new here'
+command :sf_taps_validate do |c|
+  c.action do |global_options,options,args|
+    verbose = global_options[:v]
+    params = PARAMS.merge({
+      :graph_repos => [
+        GoodData::Bam::Repository.create(:type => :file, :base => './local_graphs'),
+        GoodData::Bam::Repository.create(:type => :file, :base => GoodData::CloverGenerator::BAM_DEFINED_GRAPHS_ROOT)]})
+    result = GoodData::Bam::Commands::validate_sf_taps(params)
+
+    error = false
+    result.each_pair do |obj, fields|
+      if fields.empty?
+        puts HighLine::color("GOOD", :green) + " #{obj}"
+      else
+        error = true
+        puts HighLine::color("BAD", :red) + " #{obj} [" + fields.join(', ') + "]"
+      end
+    end
+    exit_now!("Errors found",exit_code=1) if error
+  end
+end
+
+desc "Prepares params.prm file for debugging that particular graph"
+command :debug do |c|
+  c.action do |global_options,options,args|
+    fail "Arguments should be passed as \"project\" \"flow\" \"graph\"" if args.length < 3
+    GoodData::Bam::Commands::set_up_debug(args.first, args[1], args[2])
+  end
+end
+
+desc 'Validates that the tap has the fields it is claimed it should have. This is supposed to make the mitigate errors during deploy.'
+# arg_name 'Describe arguments to new here'
+command :docs do |c|
+  c.action do |global_options,options,args|
+    GoodData::Bam::Commands::generate_docs
+  end
+end
+
+desc 'Lists processes for the project.'
+# arg_name 'Describe arguments to new here'
+command :procs do |c|
+
+  c.desc 'procs for all projects'
+  c.arg_name 'all'
+  c.switch :all
+
+  c.action do |global_options,options,args|
+    out = GoodData::Bam::Commands::procs_list(options)
+    out.each do |proc|
+      puts proc.join(',')
+    end
+  end
+end
+
+
+desc 'Creates project'
+command :project do |c|
+
+  c.desc 'blueprint name. Currently support goodsales'
+  c.arg_name 'blueprint'
+  c.flag :blueprint
+
+  c.desc 'token'
+  c.arg_name 'token'
+  c.flag :token
+
+  c.action do |global_options,options,args|
+    fail "You need to specify token to create a project" if options[:token].nil?
+    
+    GoodData::Bam::Commands::connect_to_gd()
+    pid = case options[:blueprint]
+      when "goodsales"
+        "nt935rwzls50zfqwy6dh62tabu8h0ocy"
+    end
+
+    params = PARAMS.merge({:token => options[:token]})
+
+    new_project = if pid
+      GoodData::Bam::Commands.clone_project(pid, params)
+    else
+      GoodData::Bam::Commands.create_project(params)
+    end
+
+    puts "Your project pid is #{new_project}"
+  end
+
+end
+
+
+desc 'Generates structures'
+arg_name 'what you want to generate project, tap, flow, sink'
+command :scaffold do |c|
+
+  c.desc 'blueprint name. Currently support goodsales'
+  c.arg_name 'blueprint'
+  c.flag :blueprint
+
+  c.action do |global_options,options,args|
+    command = args.first
+    fail "You did not provide what I should scaffold. I can generate project, tap, flow, sink, graph_template nothing else" unless ["project", "tap", "flow", "sink", "graph_template"].include?(command) 
+    case command
+    when "project"
+      directory = args[1]
+      fail "Directory has to be provided as an argument. See help" if directory.nil?
+      if options[:blueprint].nil?
+        GoodData::Bam::Commands::setup_bash_structure(directory)  
+      else
+        case options[:blueprint]
+        when "goodsales"
+          system "git clone git://github.com/gooddata/goodsales_base.git #{directory}"
+        end
+      end
+    when "flow"
+      name = args[1]
+      fail "Name of the flow has to be provided as an argument. See help" if name.nil?
+      GoodData::Bam::Commands::setup_flow(name)
+    when "tap"
+      name = args[1]
+      fail "Name of the tap has to be provided as an argument. See help" if name.nil?
+      GoodData::Bam::Commands::setup_tap(name)
+    when "sink"
+      name = args[1]
+      fail "Name of the sink has to be provided as an argument. See help" if name.nil?
+      GoodData::Bam::Commands::setup_sink(name)
+    when "graph_template"
+      name = args[1]
+      target = args[2]
+      fail "Name of the template has to be provided as an argument. See help" if name.nil?
+      fail "Name of the target has to be provided as an argument. See help" if target.nil?
+      GoodData::Bam::Commands::generate_graph_template(name, target)
+    end
+  end
+end
+
+desc 'Runs the project on server'
+command :model_sync do |c|
+
+  c.desc 'do not execute'
+  c.arg_name 'dry'
+  c.switch :dry
+
+  c.action do |global_options,options,args|
+    GoodData::Bam::Commands::model_sync('.', options)
+  end
+end
+
+
+desc 'Deploys the project on server and schedules it'
+command :deploy do |c|
+  
+  c.desc 'existing process id under which it is going to be redeployed'
+  c.arg_name 'process'
+  c.flag :process
+
+  c.desc 'name of the process'
+  c.arg_name 'name'
+  c.flag :name
+
+  c.action do |global_options,options,args|
+    dir = args.first
+    fail "You have to specify directory to deploy as an argument" if dir.nil?
+    fail "Specified directory does not exist" unless File.exist?(dir)
+    
+    GoodData::Bam::Commands::connect_to_gd()
+    params = PARAMS.merge(global_options.merge(options))
+    response = GoodData::Bam::Commands::deploy(dir, params)
+  end
+end
+
+desc 'Runs the project on server'
+command :run do |c|
+
+  c.desc 'email to addresses when the run is finished'
+  c.arg_name 'email'
+  c.flag :email
+
+  c.action do |global_options,options,args|
+
+    dir = args.first
+    fail "You have to specify directory to deploy as an argument" if dir.nil?
+    fail "Specified directory does not exist" unless File.exist?(dir)
+
+    params = PARAMS.merge(global_options.merge(options))
+    GoodData::Bam::Commands::connect_to_gd()
+    GoodData::Bam::Commands::run(dir, params)
+  end
+end
+
+
+pre do |global_options,command,options,args|
+
+  logger = Logger.new(STDOUT) if global_options[:l]
+  GoodData.logger = logger
+  params = GoodData::Bam::Utils::get_user_params('.')
+  s3_backup = GoodData::Bam::Utils::should_backup_to_s3?(params)
+  PARAMS = params.merge({:s3_backup => s3_backup})
+
+  # Pre logic here
+  # Return true to proceed; false to abort and not call the
+  # chosen command
+  # Use skips_pre before a command to skip this block
+  # on that command only
+
+  true
+end
+
+post do |global_options,command,options,args|
+  # Post logic here
+  # Use skips_post before a command to skip this
+  # block on that command only
+  verbose = global_options[:v]
+  puts HighLine::color("DONE", :green) if verbose
+end
+
+on_error do |exception|
+  pp exception.backtrace
+  # Error logic here
+  # return false to skip default error handling
+  true
+end
+
+exit run(ARGV)

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.1.3'
+  VERSION = '0.1.5'
 end

data/lib/base/project.rb

@@ -42,7 +42,7 @@ module GoodData
         found.first
       end
 
-      def self.load_taps(project, path="./taps/")
+      def self.load_taps(project, path)
         p = Pathname(path)
         taps = Pathname.glob(p + "*.json").map do |p|
           Tap.create(JSON.parse(File.read(p), :symbolize_names => true))
@@ -64,7 +64,7 @@ module GoodData
         Project.create(project.merge({:graph_repos => repos}))
       end
 
-      def self.load_sinks(project, path="./sinks/")
+      def self.load_sinks(project, path)
         p = Pathname(path)
         sinks = Pathname.glob(p + "*.json").map do |p|
           Sink.create(JSON.parse(File.read(p), :symbolize_names => true))
@@ -72,7 +72,7 @@ module GoodData
         Project.create(project.merge({:sinks => sinks}))
       end
 
-      def self.load_flows_definitions(project, path="./flows/")
+      def self.load_flows_definitions(project, path)
         p = Pathname(path)
         flows = Pathname.glob(p + "*.rb").map do |p|
           GoodData::Bam::DSL.class_eval(File.read p).to_hash
@@ -80,13 +80,14 @@ module GoodData
         Project.create(project.merge({:flows_definitions => flows}))
       end
 
-      def self.build_project(options={})
+      def self.build_project(home='.', options={})
+        home = Pathname(home)
         project = Project.create({})
         paths = options[:graph_repos] || []
-        p = Project.load_taps(project)
-        p = Project.load_sinks(p)
+        p = Project.load_taps(project, home + 'taps')
+        p = Project.load_sinks(p, home + 'sinks')
         p = Project.set_graph_repos(p, paths)
-        p = Project.load_flows_definitions(p)
+        p = Project.load_flows_definitions(p, home + "flows")
         p = Compiler.compile_dsl(p)
       end
 

data/lib/base/tap.rb

@@ -162,6 +162,10 @@ module GoodData
         Tap.remove_acts_as(tap)
       end
 
+      def self.get_mandatory_fields(tap)
+        tap[:fields].reject {|f| f[:is_mandatory] == false }
+      end
+
     end
   end
 end

data/lib/cloud_connect/dsl/es_helpers.rb

@@ -34,7 +34,7 @@ module GoodData
                 f[:type] || "attribute"
               end
 
-              memo[f[:name]] = {
+              memo[f[:meta] || f[:name]] = {
                 :name => f[:name],
                 :type => type
               }

data/lib/cloud_connect/dsl/sf_helpers.rb

@@ -2,8 +2,20 @@ module GoodData
   module CloudConnect
     module Helpers
 
-      def self.generate_select(tap)
-        fields = tap[:fields].map do |f| 
+      def self.objects_for_history(client, tap) 
+        sf_module = tap[:object]
+        modules = client.modules.map {|o| o[:name]}
+        modules.find_all {|m| ["#{sf_module}History", "#{sf_module}FieldHistory"].include?(m) }
+      end
+
+      def self.generate_select(tap, options={})
+        field = options[:only]
+        fields = if field.nil?
+          tap[:fields]
+        else
+          tap[:fields].find_all {|f| f[:name] == field}
+        end
+        fields = fields.map do |f| 
           f.has_key?(:multi_currency) ? "convertCurrency(#{f[:name]})" : f[:name]
         end
         condition = tap[:condition].nil?() ? "" : "WHERE #{tap[:condition]}"
@@ -20,6 +32,25 @@ module GoodData
         generate_select(tap)
       end
 
+      def self.generate_history_query(client, field, sf_object, for_sf_object)
+        fields = client.fields(sf_object)
+        generic_field_history = ["NewValue", "OldValue", "ParentId"].all? {|f| fields.include?(f)}
+        specific_field_history = ["NewValue", "OldValue", "#{for_sf_object}Id"].all? {|f| fields.include?(f)}
+        specific_history = ["SystemModstamp", "#{for_sf_object}Id"].all? {|f| fields.include?(f)}
+        
+
+        if generic_field_history
+          "SELECT NewValue, CreatedDate, ParentId FROM #{sf_object} WHERE Field = '#{field}'"
+        elsif specific_field_history
+          "SELECT NewValue, CreatedDate, ParentId FROM #{sf_object} WHERE Field = '#{field}'"
+        elsif specific_history
+          return nil unless fields.include?(field)
+          "SELECT #{field}, SystemModstamp, #{for_sf_object}Id FROM #{sf_object}"
+        else
+          fail "Unrecognized fields configuration for historization in SF."
+        end
+      end
+
       def self.sf_connection(builder, data)
         builder.Connection({
           :clientId => "${SFDC_CLIENT_ID}",

data/lib/commands/commands.rb

@@ -13,16 +13,16 @@ module GoodData
         render_template(template_name, PARAMS, :to_file => USER_DEFINED_GRAPHS_ROOT + target)
       end
 
-      def self.project_jack_in(params)
-        project = GoodData::Bam::Project.build_project
-      end
+      # def self.project_jack_in(params)
+      #   project = GoodData::Bam::Project.build_project
+      # end
 
       def self.generate_validators(home, params)
         home = Pathname(home)
         clobber_contract_project(home, params)
         setup_contract_project(home, params)
 
-        project = GoodData::Bam::Project.build_project(params)
+        project = GoodData::Bam::Project.build_project(home, params)
         file_taps = Taps.get_file(project[:taps])
         file_taps = file_taps.find_all {|t| t[:validation_source]}
         file_taps.each do |t|
@@ -90,16 +90,24 @@ module GoodData
       def self.generate_downloaders(home, params)
         home = Pathname(home)
         clobber_downloaders_project(home)
-        project = GoodData::Bam::Project.build_project(params)
+        project = GoodData::Bam::Project.build_project(home, params)
         setup_downloaders_project(home, params)
         GoodData::Bam::Generators::Downloaders.generate_downloaders(home + DOWNLOADERS_HOME + "graphs", project, params)
       end
 
+      def self.generate_history_downloaders(home, params)
+        home = Pathname(home)
+        clobber_downloaders_project(home)
+        project = GoodData::Bam::Project.build_project(params)
+        setup_downloaders_project(home, params)
+        GoodData::Bam::Generators::Downloaders.generate_history_downloaders(home + DOWNLOADERS_HOME + "graphs", project, params)
+      end
+
       def self.generate(home, params)
         # fail "The flow you specified was not found" if flows.empty? && !only_flow.nil?
-        GoodData::Bam::Commands::clobber_etl_project('.')
-        GoodData::Bam::Commands::setup_etl_project('.', params)
-        GoodData::Bam::Generators::Etl.generate('.', params)
+        clobber_etl_project(home)
+        setup_etl_project(home, params)
+        GoodData::Bam::Generators::Etl.generate(home, params)
       end
 
     end

data/lib/commands/docs_commands.rb

@@ -3,7 +3,7 @@ module GoodData
     module Commands
 
       def self.generate_docs(path=".")
-        project = GoodData::Bam::Project.build_project
+        project = GoodData::Bam::Project.build_project(path)
 
         taps = ""
         project[:taps].each do |tap|

data/lib/commands/gd_commands.rb

@@ -18,9 +18,9 @@ module GoodData
         end
       end
 
-      def self.model_sync(options)
+      def self.model_sync(path, options)
         dry_run = options[:dry]
-        project = build_project
+        project = build_project(path, options)
         datasets = project.sinks
         model_update_dir = Pathname('model_update')
         cl_home = ENV['CL_HOME'] || PARAMS['CL_HOME'] || fail("Home of cl tool cannot be found. Either set up CL_HOME in your env with 'export CL_HOME=path/to/cl or set it up in your params.json. Point to the directory of CL not to the bin dir.'")

data/lib/commands/sf_commands.rb

@@ -3,7 +3,7 @@ module GoodData
     module Commands
 
       def self.validate_sf_taps(params)
-        project = GoodData::Bam::Project.build_project(params)
+        project = GoodData::Bam::Project.build_project(params[:home], params)
         client = get_sf_client(params)
         validate_sf_metadata(client, Taps.get_salesforce(project[:taps]))
       end

data/lib/compiler/etl_visitor.rb

@@ -6,7 +6,7 @@ module GoodData
         def visit(node, state)
           case node[:type]
           when :project
-            FileUtils::cd(GoodData::Bam::Commands::ETL_HOME) do
+            FileUtils::cd(Pathname(state[:home]) + GoodData::Bam::Commands::ETL_HOME) do
               flows = if state[:only]
                 node[:flows].find_all {|f| f[:name] == state[:only]}
               else