gd_bam 0.0.15 → 0.1.0

data/lib/commands/commands.rb

@@ -0,0 +1,110 @@
+module GoodData
+  module Bam
+    module Commands
+
+      include GoodData::CloudConnect
+
+      VALIDATORS_HOME = "validators-project"
+      DOWNLOADERS_HOME = "downloaders-project"
+      ETL_HOME = "etl-project"
+
+      def self.generate_graph_template(name, target)
+        template_name = "#{name}_template.grf.erb"
+        render_template(template_name, PARAMS, :to_file => USER_DEFINED_GRAPHS_ROOT + target)
+      end
+
+      def self.project_jack_in(params)
+        project = GoodData::Bam::Project.build_project
+      end
+
+      def self.generate_validators(home, params)
+        home = Pathname(home)
+        clobber_contract_project(home, params)
+        setup_contract_project(home, params)
+
+        project = GoodData::Bam::Project.build_project(params)
+        file_taps = Taps.get_file(project[:taps])
+        file_taps = file_taps.find_all {|t| t[:validation_source]}
+        file_taps.each do |t|
+          puts "Will generate validator for tap \"#{t[:id]}\""
+        end
+        GoodData::Bam::Validators.generate_checker_file_list(home + VALIDATORS_HOME + "graphs/main.grf", file_taps)
+
+        file_taps.each do |tap|
+          id = tap[:id]
+          GoodData::Bam::Validators.generate_contract(home + VALIDATORS_HOME + "graphs/#{id}_checker.grf", tap)
+          Helpers::loop_over_file(home + VALIDATORS_HOME + "graphs/#{id}_loop.grf", {
+            :token => id,
+            :file_to_loop => "data/#{id}_files_to_read.csv",
+            :graph_to_run => "graphs/#{id}_checker.grf"
+          })
+          
+          GoodData::Bam::Validators.generate_checker_success(home + VALIDATORS_HOME + "graphs/#{id}_success.grf", tap)
+          GoodData::Bam::Validators.backup_files_from_glob(home + VALIDATORS_HOME + "graphs/#{id}_failure.grf", {
+            :files_to_backup => ["${DATA}/validation_output_*.csv", "${DATA}/*_files_to_read.csv"],
+            :remote_url => "`\"https://\" + replace(replace(\"${GD_LOGIN}\",\"@\",\"%40\"),\"\\\\+\",\"%2B\") + \":${GD_PASSWORD}@${GDC_WEBDAV_HOST}/project-uploads/${GDC_PROJECT_ID}/validation_reports/report_${NOW}/\"`"
+          })
+        end
+      end
+
+      def self.setup_contract_project(home, params={})
+        home = Pathname(home)
+        paths = Utils::get_paths(home + VALIDATORS_HOME)
+        Utils::setup_basic_project(paths, params)
+      end
+
+      def self.clobber_contract_project(home, params={})
+        home = Pathname(home)
+        paths = Utils::get_paths(home + VALIDATORS_HOME)
+        clobber_project(paths)
+      end
+
+      def self.setup_etl_project(home, params={})
+        home = Pathname(home)
+        paths = Utils::get_paths(home + ETL_HOME)
+        Utils::setup_basic_project(paths, params)
+      end
+
+      def self.clobber_etl_project(home, params={})
+        home = Pathname(home)
+        paths = Utils::get_paths(home + ETL_HOME)
+        clobber_project(paths)
+      end
+
+      def self.clobber_downloaders_project(home, params={})
+        home = Pathname(home)
+        paths = Utils::get_paths(home + DOWNLOADERS_HOME)
+        clobber_project(paths)
+      end
+
+      def self.setup_downloaders_project(home, params={})
+        home = Pathname(home)
+        paths = Utils::get_paths(home + DOWNLOADERS_HOME)
+        Utils::setup_basic_project(paths, params)
+      end
+
+      def self.clobber_project(paths)
+        FileUtils::rm_rf(paths[:home_path])
+      end
+
+      def self.generate_downloaders(home, params)
+        home = Pathname(home)
+        clobber_downloaders_project(home)
+        project = GoodData::Bam::Project.build_project(params)
+        setup_downloaders_project(home, params)
+        GoodData::Bam::Generators::Downloaders.generate_downloaders(home + DOWNLOADERS_HOME + "graphs", project, params)
+      end
+
+      def self.generate(home, params)
+        project = GoodData::Bam::Project.build_project(params)
+
+        # fail "The flow you specified was not found" if flows.empty? && !only_flow.nil?
+        fail "There are no flows to generate from" if project[:flows].empty?
+
+        v = Compiler::EtlVisitor.new
+        v.visit(project, params)
+      end
+
+    end
+  end
+end

data/lib/commands/deployment.rb

@@ -0,0 +1,217 @@
+module GoodData
+  module Bam
+    module Commands
+
+      def self.upload_files(files, now)
+        upload_paths = files.map do |file|
+          file = Pathname(file)
+          ext = file.extname
+          "#{file.to_s.chomp(ext)}_#{now}#{ext}"
+        end
+
+        Dir.mktmpdir do |dir|
+          dir = Pathname(dir)
+          files.zip(upload_paths).each do |f|
+            FileUtils::cp(f[0], dir + f[1])
+            system "curl -u #{PARAMS[:gd_login]}:#{PARAMS[:gd_pass]} -k -T #{dir+f[1]} https://secure-di.gooddata.com/project-uploads/#{PARAMS[:project_pid]}/"
+            puts "Uploaded #{f[1]}"
+          end
+        end
+      end
+
+      def self.clean_files(files, now)
+        upload_paths = files.map do |file|
+          file = Pathname(file)
+          ext = file.extname
+          "#{file.to_s.chomp(ext)}_#{now}#{ext}"
+        end
+
+        upload_paths.each do |f|
+          system "curl -u #{PARAMS[:gd_login]}:#{PARAMS[:gd_pass]} -k  -X DELETE https://secure-di.gooddata.com/project-uploads/#{PARAMS[:project_pid]}/#{f}"
+          puts "Uploaded #{f[1]}"
+        end
+      end
+
+      def self.execute_process(link, dir)
+        dir = Pathname(dir)
+        result = GoodData.post(link, {
+          :execution => {
+           :graph => dir + "graphs/main.grf",
+           :params => {}  
+          }
+        })
+        begin
+          GoodData.poll(result, "executionTask")
+        rescue RestClient::RequestFailed => e
+
+        ensure
+          result = GoodData.get(result["executionTask"]["links"]["detail"])
+          if result["executionDetail"]["status"] == "ERROR"
+            fail "Runing process failed. You can look at a log here #{result["executionDetail"]["logFileName"]}"
+          end
+        end
+        result
+      end
+
+      def self.run(dir, options={})
+        email = options[:email]
+        verbose = options[:v]
+
+        dir = Pathname(dir)
+
+        deploy(dir, options.merge(:name => "Temporary deploy[#{dir}][#{PARAMS[:project_name]}]")) do |deploy_response|
+          puts HighLine::color("Executing", HighLine::BOLD) if verbose
+          if email.nil?
+            result = execute_process(deploy_response["process"]["links"]["executions"], dir)
+          else
+            create_email_channel(options) do |channel_response|
+              subscribe_on_finish(:success, channel_response, deploy_response, options)
+              result = execute_process(deploy_response["process"]["links"]["executions"], dir)
+            end
+          end
+        end
+      end
+
+      def self.subscribe_on_finish(event_type, channel, deploy, options={})
+        channel_uri = channel["channelConfiguration"]["meta"]["uri"]
+        process_id = deploy["process"]["links"]["self"].split('/').last
+        event_id = case event_type
+          when :success
+            "dataload.process.finish.ok"
+          when :failure
+            "dataload.process.finish.error"
+          else
+            fail "You specified unknown event \"#{event_type}\""
+          end
+
+        templates = {
+          :success => {
+            :message => "Just wanted to let you know that ETL for \"#{deploy["process"]["name"]}\" Succeeded",
+            :subject => "GoodData ETL SUCCESS: process with name \"#{deploy["process"]["name"]}\""
+          },
+          :failure => {
+            :message => "Just wanted to let you know that ETL for \"#{deploy["process"]["name"]}\" Failed",
+            :subject => "GoodData ETL FAILURE: process with name \"#{deploy["process"]["name"]}\""
+          }
+        }
+
+        templates_to_be_used = templates[event_type]
+
+        data = {
+          :subscription => {
+            :triggers => [
+              {
+                :projectEventTrigger => {
+                  :types => [event_id]
+                }
+              }
+            ],
+            :condition => {
+              :condition => {
+                :expression => "params.PROCESS_ID==\"#{process_id}\""
+              }
+            },
+            :subject => {
+              :template => {
+                :expression => templates_to_be_used[:subject]
+              }
+            },
+            :message => {
+              :template => {
+                :expression => templates_to_be_used[:message]
+              }
+            },
+            :channels => [channel_uri],
+            :meta => {
+              :title => "Notification for process #{process_id}"
+            }
+          }
+        }
+        profile_id = GoodData.connection.user["profile"].split("/").last
+        GoodData.post("/gdc/projects/#{PARAMS[:project_pid]}/users/#{profile_id}/subscriptions", data)
+      end
+
+      def self.deploy(dir, options={}, &block) 
+        verbose = options[:verbose] || false
+        if block
+          begin
+            res = deploy_graph(dir, options)
+            block.call(res)
+          ensure
+            self_link = res["process"]["links"]["self"]
+            GoodData.delete(self_link)
+          end
+        else
+          deploy_graph(dir, options)
+        end
+      end
+
+      def self.deploy_graph(dir, options={}) 
+        dir = Pathname(dir)
+        deploy_name = options[:name] || "#{PARAMS[:project_name]}"
+        verbose = options[:verbose] || false
+        puts HighLine::color("Deploying #{dir}", HighLine::BOLD) if verbose
+        res = nil
+
+        Tempfile.open("deploy-graph-archive") do |temp|
+          Zip::ZipOutputStream.open(temp.path) do |zio|
+            Dir.glob(dir + "**/*") do |item|
+              puts "including #{item}" if verbose
+              unless File.directory?(item)
+                zio.put_next_entry(item)
+                zio.print IO.read(item)
+              end
+            end
+          end
+
+          GoodData.connection.upload(temp.path)
+          process_id = options[:process]
+
+          data = {
+              :process => {
+                :name => deploy_name,
+                :path => "/uploads/#{File.basename(temp.path)}"
+              }
+            }
+          res = if process_id.nil?
+            GoodData.post("/gdc/projects/#{PARAMS[:project_pid]}/dataload/processes", data)
+          else
+            GoodData.put("/gdc/projects/#{PARAMS[:project_pid]}/dataload/processes/#{process_id}", data)
+          end
+        end
+        puts HighLine::color("Deploy DONE #{dir}", HighLine::BOLD) if verbose
+        res
+      end
+
+      def self.create_email_channel(options={}, &block)
+        email = options[:email]
+
+        data = {
+          :channelConfiguration => {
+            :configuration => {
+              :emailConfiguration => {
+                :to => email
+              }
+            },
+            :meta => {
+              :title => "temporary email channel"
+            }
+          }
+        }
+        profile_id = GoodData.connection.user["profile"].split("/").last
+        res = GoodData.post("/gdc/account/profile/#{profile_id}/channelConfigurations", data)
+        self_link = res["channelConfiguration"]["meta"]["uri"]
+        if block
+          begin
+            block.call(res)
+          ensure
+            GoodData.delete(self_link)
+          end
+        else
+          res
+        end
+      end
+
+    end
+  end
+end

data/lib/commands/docs_commands.rb

@@ -0,0 +1,41 @@
+module GoodData
+  module Bam
+    module Commands
+
+      def self.generate_docs(path=".")
+        project = GoodData::Bam::Project.build_project
+
+        taps = ""
+        project[:taps].each do |tap|
+          taps += "####{tap[:object]}"
+          taps += "\n"
+          tap[:fields].each do |f|
+            if f[:acts_as]
+              taps += "    #{f[:name]} -> #{f[:acts_as].join(", ")}"
+            else
+              taps += "    #{f[:name]}"
+            end
+            taps += "\n"
+          end
+
+          taps += "\n"
+        end
+
+        sinks = ""
+        project[:sinks].each do |sink|
+          name = sink[:gd_name] || sink[:id]
+          sinks += "####{name}\n"
+          sink[:fields].each do |field|
+            name = field[:name] || "#{field[:schema]}:#{field[:ref]}"
+            type = field[:type]
+            sinks += "    #{type.upcase} #{field[:meta]} => #{name}\n"
+          end
+          sinks += "\n"
+
+        end
+
+        GoodData::Bam::Utils.render_template("README.md.erb", PARAMS.merge(:taps => taps, :sinks => sinks), :to_file => 'README.md', :root => Pathname(path))
+      end
+    end
+  end
+end

data/lib/commands/gd_commands.rb

@@ -0,0 +1,95 @@
+module GoodData
+  module Bam
+    module Commands
+
+      def self.connect_to_gd(options={})
+          GoodData.connect(PARAMS[:gd_login], PARAMS[:gd_pass])
+          GoodData.project = PARAMS[:project_pid] if !PARAMS[:project_pid].nil? && !PARAMS[:project_pid].empty?
+      end
+
+      def self.set_up_debug(project, flow, graph)
+        fail "Project \"#{project}\" was not found" unless File.exist?(project)
+        fail "Project \"#{project}\" is not a directory" unless File.directory?(project)
+        # :TODO make the proram really check there is a flow and a graph etc before writing to the file
+        project = Pathname(project)
+        File.open(project + 'params.prm', 'w') do |f|
+          f << "FLOW=#{flow}\n"
+          f << "NAME=#{graph}\n"
+        end
+      end
+
+      def self.model_sync(options)
+        dry_run = options[:dry]
+        project = build_project
+        datasets = project.sinks
+        model_update_dir = Pathname('model_update')
+        cl_home = ENV['CL_HOME'] || PARAMS['CL_HOME'] || fail("Home of cl tool cannot be found. Either set up CL_HOME in your env with 'export CL_HOME=path/to/cl or set it up in your params.json. Point to the directory of CL not to the bin dir.'")
+        cl_home = Pathname(cl_home) + 'bin/gdi.sh'
+
+        FileUtils::mkdir_p(model_update_dir)
+        File.open(model_update_dir + 'dummy', 'w')
+        FileUtils::cd(model_update_dir) do
+          datasets.each do |ds|
+            dataset_path = Pathname("cl_file_#{ds[:id]}")
+            File.open(dataset_path, "w") do |temp|
+              builder = Builder::XmlMarkup.new(:target=>temp, :indent=>2)
+              builder.schema do |builder|
+                builder.name(ds[:gd_name])
+                builder.title(ds[:gd_name])
+                builder.columns do |b|
+                  ds[:fields].each do |f|
+                    builder.column do |builder|
+                      builder.title(f[:name])
+                      builder.ldmType(f[:type].upcase)
+                      builder.reference(f[:for]) if f.has_key?(:for)
+                      builder.reference(f[:ref]) if f.has_key?(:ref)
+                      builder.schemaReference(f[:schema]) if f.has_key?(:schema)
+                      if f[:type] == "date"
+                        builder.schemaReference("#{f[:dd]}")
+                        builder.name("#{f[:name]}")
+                      else
+                        builder.name(f[:name] || f[:ref])
+                      end
+                    end
+                  end
+                end
+              end
+            end
+            template_name = dry_run ? "update_dataset_dry.script.erb" : "update_dataset.script.erb"
+            render_template(template_name, PARAMS.merge({"config_file" => dataset_path.expand_path}), :to_file => 'update_dataset.script')
+            puts "Generate #{ds[:id]}"
+
+            system("#{cl_home} update_dataset.script --username #{PARAMS[:gd_login]} --password #{PARAMS[:gd_pass]}")
+            File.delete(dataset_path)        
+          end
+        end
+        FileUtils::rm_rf(model_update_dir)
+      end
+
+      def self.create_project(options={})
+        project_name = options[:project_name]
+
+        pr = {
+          :project => {
+            :content => {
+              :guidedNavigation => 1,
+              :driver => "Pg",
+              :authorizationToken => options[:token]
+            },
+            :meta => {
+              :title => project_name,
+              :summary => "Testing Project"
+            }
+          }
+        }
+        result = GoodData.post("/gdc/projects/", pr)
+        uri = result["uri"]
+        while(GoodData.get(uri)["project"]["content"]["state"] == "LOADING")
+          sleep(5)
+        end
+        p = GoodData::Project[uri]
+        p.obj_id
+      end
+    end
+  end
+end

data/lib/commands/scaffold_commands.rb

@@ -0,0 +1,103 @@
+module GoodData
+  module Bam
+    module Commands
+
+      def self.generate_graph_template(name, target)
+        template_name = "#{name}_template.grf.erb"
+        Utils::render_template(template_name, PARAMS, :to_file => GoodData::CloverGenerator::USER_DEFINED_GRAPHS_ROOT + target)
+      end
+
+      def self.setup_bash_structure(name)
+        fail "Directory \"#{name}\" already exists. Please remove it if you want to move forward." if File.exist?(name)
+        FileUtils::mkdir_p name
+        FileUtils::cd(name) do
+          Utils::render_template("params.json.erb", PARAMS, :to_file => 'params.json')
+          ['flows', 'sinks', 'taps'].each do |dir|
+            FileUtils::mkdir_p dir
+          end
+
+          setup_flow('user')
+          Utils::render_template("tap.json.erb", PARAMS, :to_file => 'taps/source_example.json')
+          Utils::render_template("sink.json.erb", PARAMS, :to_file => 'sinks/dataset_example.json')
+        end
+      end
+
+      def self.setup_flow(name)
+        Utils::render_template("flow.rb.erb", {:flow_name => name}, :to_file => "flows/#{name}.rb")
+      end
+
+      def self.setup_tap(name)
+        Utils::render_template("tap.json.erb", {:tap_name => name}, :to_file => "taps/#{name}.json")
+      end
+
+      def self.setup_sink(name)
+        Utils::render_template("sink.json.erb", {:sink_name => name}, :to_file => "sinks/#{name}.json")
+      end
+
+      def self.clone_project(pid, options={})
+        project_name = options[:project_name]
+        fail "project name has to be filled in" if project_name.blank?
+        with_users  = options[:with_users]
+
+        export = {
+          :exportProject => {
+            :exportUsers => with_users ? 1 : 0,
+            :exportData => 1
+          }
+        }
+
+        result = GoodData.post("/gdc/md/#{pid}/maintenance/export", export)
+        token = result["exportArtifact"]["token"]
+        status_url = result["exportArtifact"]["status"]["uri"]
+
+        state = GoodData.get(status_url)["taskState"]["status"]
+        while state == "RUNNING"
+          sleep 5
+          result = GoodData.get(status_url) 
+          state = result["taskState"]["status"]
+        end
+
+        old_project = GoodData::Project[pid]
+
+        pr = {
+          :project => {
+            :content => {
+              :guidedNavigation => 1,
+              :driver => "Pg",
+              :authorizationToken => options[:token]
+            },
+            :meta => {
+              :title => project_name,
+              :summary => "Testing Project"
+            }
+          }
+        }
+        result = GoodData.post("/gdc/projects/", pr)
+        uri = result["uri"]
+        while(GoodData.get(uri)["project"]["content"]["state"] == "LOADING")
+          sleep(5)
+        end
+
+        new_project = GoodData::Project[uri]
+
+        import = {
+          :importProject => {
+            :token => token
+          }
+        }
+
+        result = GoodData.post("/gdc/md/#{new_project.obj_id}/maintenance/import", import)
+        status_url = result["uri"]
+        state = GoodData.get(status_url)["taskState"]["status"]
+        while state == "RUNNING"
+          sleep 5
+          result = GoodData.get(status_url) 
+          state = result["taskState"]["status"]
+        end
+        GoodData.post "/gdc/projects/#{new_project.obj_id}/eventStore/stores", {:store => {:storeId => "es_0"}}
+        new_project.obj_id
+      end
+
+    end
+  end
+end

data/lib/commands/sf_commands.rb

@@ -0,0 +1,37 @@
+module GoodData
+  module Bam
+    module Commands
+
+      def self.validate_sf_taps(params)
+        project = GoodData::Bam::Project.build_project(params)
+        client = get_sf_client(params)
+        validate_sf_metadata(client, Taps.get_salesforce(project[:taps]))
+      end
+
+      def self.validate_sf_metadata(sf_client, taps)
+        taps.reduce({}) do |memo, tap|
+          sf_object = tap[:object]
+          u = sf_client.describe(sf_object)
+          sf_fields = u[:describeSObjectResponse][:result][:fields].map {|field| field[:name]}
+          fields_to_validate = tap[:fields].map {|field| field[:name]}
+          memo[sf_object] = (fields_to_validate - sf_fields)
+          memo
+        end
+      end
+
+      def self.sf_jack_in(params)
+        client = get_sf_client(params)
+        client.pry
+      end
+
+      def self.sf_validate_connection(params)
+        get_sf_client(params)
+      end
+
+      def self.get_sf_client(params)
+        Salesforce::Client.new(params[:sf_login], params[:sf_password] + params[:sf_token], :server => params[:sf_server])
+      end
+
+    end
+  end
+end

data/lib/commands/validators.rb

@@ -0,0 +1,19 @@
+module GoodData
+  module Bam
+    module Commands
+
+      include GoodData::CloudConnect
+
+      def self.run_validator(process, files, params)
+        now = Time.now.to_i.to_s
+        upload_files(files, now)
+        begin
+          execute_process("/gdc/projects/#{params[:project_pid]}/dataload/processes/#{process}/executions", VALIDATORS_HOME)
+        rescue
+          clean_files(files, now)
+          exit_now!("Data validation failed failed. Please look here for details. \"https://secure-di.gooddata.com/project-uploads/#{params[:project_pid]}/validation_reports/\"")
+        end
+      end
+    end
+  end
+end

data/lib/compatibility.rb

@@ -0,0 +1,19 @@
+module Enumerable
+  def uniq_by
+    seen = Hash.new { |h,k| h[k] = true; false }
+    reject { |v| seen[yield(v)] }
+  end
+end
+
+class Hash
+  #pass single or array of keys, which will be removed, returning the remaining hash
+  def remove!(*keys)
+    keys.each{|key| self.delete(key) }
+    self
+  end
+
+  #non-destructive version
+  def remove(*keys)
+    self.dup.remove!(*keys)
+  end
+end