gooddata 0.6.0.pre6 → 0.6.0.pre7

data/lib/gooddata/bricks/middleware/restforce_middleware.rb

@@ -27,19 +27,18 @@ module GoodData::Bricks
           :oauth_token    => oauth_token,
           :refresh_token  => refresh_token
         }
-      else
-        fail "Salesforce middleware failed while trying to log in. Either salesforce_username, salesforce_password, salesforce_token or salesforce_oauth_token, salesforce_refresh_token are needed. Additionally you have to specify salesforce_client_id and salesforce_client_secret parameters in both cases"
       end
 
+      client = if credentials
+        credentials.merge!({
+          :client_id      => client_id,
+          :client_secret  => client_secret,
+        })
+        credentials[:host] = host unless host.nil?
 
-      credentials.merge!({
-        :client_id      => client_id,
-        :client_secret  => client_secret,
-      })
-      credentials[:host] = host unless host.nil?
-
-      Restforce.log = true if params[:salesforce_client_logger]
-      client = Restforce.new(credentials)
+        Restforce.log = true if params[:salesforce_client_logger]
+        Restforce.new(credentials)
+      end
       @app.call(params.merge(:salesforce_client => client))
     end
 

data/lib/gooddata/client.rb

@@ -1,5 +1,6 @@
 require 'gooddata/version'
 require 'gooddata/connection'
+require 'gooddata/helpers'
 
 # fastercsv is built in Ruby 1.9
 if RUBY_VERSION < "1.9"
@@ -94,6 +95,17 @@ module GoodData
       
     end
 
+    def logging_on
+      if logger.is_a? NilLogger
+        GoodData::logger = Logger.new(STDOUT)
+      end
+    end
+
+    def logging_off
+      GoodData::logger = NilLogger.new
+    end
+
+
     # Hepler for starting with SST easier
     # === Parameters
     #
@@ -123,7 +135,7 @@ module GoodData
       old_project = GoodData.project
       begin
         GoodData.use(project)
-        bl.call(project)
+        bl.call(GoodData.project)
       rescue Exception => e
         fail e
       ensure
@@ -251,6 +263,14 @@ module GoodData
       }))
     end
 
+    def download_form_user_webdav(file, where, options={})
+      u = URI(connection.options[:webdav_server] || GoodData.project.links["uploads"])
+      url = URI.join(u.to_s.chomp(u.path.to_s), "/uploads/")
+      connection.download(file, where, options.merge({
+        :staging_url  => url
+      }))
+    end
+
     def poll(result, key, options={})
       sleep_interval = options[:sleep_interval] || 10
       link = result[key]["links"]["poll"]

data/lib/gooddata/commands/auth.rb

@@ -1,80 +1,88 @@
 module GoodData::Command
-  class Auth < Base
-    def connect
-      unless defined? @connected
-        GoodData.connect user, password, url, :auth_token => auth_token
-        @connected = true
+  class Auth
+    
+    class << self
+      def connect
+        unless defined? @connected
+          GoodData.connect({
+            :login    => user,
+            :password => password,
+            :server   => url,
+            :auth_token => auth_token
+          })
+          @connected = true
+        end
+        @connected
       end
-      @connected
-    end
-
-    def user
-      ensure_credentials
-      @credentials[:username]
-    end
-
-    def password
-      ensure_credentials
-      @credentials[:password]
-    end
 
-    def url
-      ensure_credentials
-      @credentials[:url]
-    end
+      def user
+        ensure_credentials
+        @credentials[:username]
+      end
 
-    def auth_token
-      ensure_credentials
-      @credentials[:auth_token]
-    end
+      def password
+        ensure_credentials
+        @credentials[:password]
+      end
 
-    def credentials_file
-      "#{home_directory}/.gooddata"
-    end
+      def url
+        ensure_credentials
+        @credentials[:url]
+      end
 
-    def ensure_credentials
-      return if defined? @credentials
-      unless @credentials = read_credentials
-        @credentials = ask_for_credentials
+      def auth_token
+        ensure_credentials
+        @credentials[:auth_token]
       end
-      @credentials
-    end
 
-    def read_credentials
-      if File.exists?(credentials_file) then
-        config = File.read(credentials_file)
-        JSON.parser.new(config, :symbolize_names => true).parse
+      def credentials_file
+        "#{GoodData::Helpers.home_directory}/.gooddata"
       end
-    end
 
-    def ask_for_credentials
-      puts "Enter your GoodData credentials."
-      user = ask("Email")
-      password = ask("Password", :secret => true)
-      auth_token = ask("Authorization Token")
-      { :username => user, :password => password, :auth_token => auth_token }
-    end
+      def ensure_credentials
+        return if defined? @credentials
+        unless @credentials = read_credentials
+          @credentials = ask_for_credentials
+        end
+        @credentials
+      end
 
-    def store
-      credentials = ask_for_credentials
+      def read_credentials
+        if File.exists?(credentials_file) then
+          config = File.read(credentials_file)
+          JSON.parser.new(config, :symbolize_names => true).parse
+        end
+      end
 
-      ovewrite = if File.exist?(credentials_file)
-        ask "Overwrite existing stored credentials", :answers => %w(y n)
-      else
-        'y'
+      def ask_for_credentials
+        puts "Enter your GoodData credentials."
+        user = HighLine::ask("Email")
+        password = HighLine::ask("Password") { |q| q.echo = "x" }
+        auth_token = HighLine::ask("Authorization Token")
+        { :username => user, :password => password, :auth_token => auth_token }
       end
 
-      if ovewrite == 'y'
-        File.open(credentials_file, 'w', 0600) do |f|
-          f.puts JSON.pretty_generate(credentials)
+      def store
+        credentials = ask_for_credentials
+
+        ovewrite = if File.exist?(credentials_file)
+          HighLine::ask("Overwrite existing stored credentials (y/n)")
+           # { |q| q.validate = /[y,n]/ }
+        else
+          'y'
+        end
+        if ovewrite == 'y'
+          File.open(credentials_file, 'w', 0600) do |f|
+            f.puts JSON.pretty_generate(credentials)
+          end
+        else
+          puts 'Aborting...'
         end
-      else
-        puts 'Aborting...'
       end
-    end
 
-    def unstore
-      FileUtils.rm_f(credentials_file)
+      def unstore
+        FileUtils.rm_f(credentials_file)
+      end
     end
   end
 end

data/lib/gooddata/commands/process.rb

@@ -2,74 +2,42 @@ module GoodData::Command
   class Process
 
     def self.list(options={})
-      # with project usage
-      processes = GoodData::Process[:all]
+      GoodData.with_project(options[:project_id]) do
+        processes = GoodData::Process[:all]
+      end
     end
 
     def self.get(options={})
       id = options[:process_id]
       fail "Unspecified process id" if id.nil?
-      GoodData::Process[id]
+      
+      GoodData.with_project(options[:project_id]) do
+        GoodData::Process[id]
+      end
     end
 
-    def self.with_deploy(dir, options={}, &block) 
+    def self.deploy(dir, options={})
       verbose = options[:verbose] || false
-      if block
-        begin
-          res = deploy_graph(dir, options)
-          block.call(res)
-        ensure
-          # self_link = res["process"]["links"]["self"]
-          # GoodData.delete(self_link)
-        end
-      else
-        deploy_graph(dir, options)
+      GoodData.with_project(options[:project_id]) do
+        deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
       end
     end
 
-    def self.deploy_graph(dir, options={}) 
-      dir = Pathname(dir) || fail("Directory is not specified")
-      fail "\"#{dir}\" is not a directory" unless dir.directory?
-      project_id = options[:project_id] || fail("Project Id has to be specified")
-      
-
-      type = options[:type] || fail("Type of deployment is not specified")
-      deploy_name = options[:name]
+    def self.with_deploy(dir, options={}, &block) 
       verbose = options[:verbose] || false
-      project_pid = options[:project_pid]
-      puts HighLine::color("Deploying #{dir}", HighLine::BOLD) if verbose
-      res = nil
-
-      Tempfile.open("deploy-graph-archive") do |temp|
-        
-        Zip::OutputStream.open(temp.path) do |zio|
-          Dir.glob(dir + "**/*") do |item|
-            puts "including #{item}" if verbose
-            unless File.directory?(item)
-              zio.put_next_entry(item)
-              zio.print IO.read(item)
-            end
+      GoodData.with_project(options[:project_id]) do
+        if block
+          begin
+            res = deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
+            block.call(res)
+          ensure
+            # self_link = res["process"]["links"]["self"]
+            # GoodData.delete(self_link)
           end
-        end
-
-        GoodData.upload_to_user_webdav(temp.path)
-        process_id = options[:process]
-        
-        data = {
-          :process => {
-            :name => deploy_name,
-            :path => "/uploads/#{File.basename(temp.path)}",
-            :type => type
-          }
-        }
-        res = if process_id.nil?
-          GoodData.post("/gdc/projects/#{project_pid}/dataload/processes", data)
         else
-          GoodData.put("/gdc/projects/#{project_pid}/dataload/processes/#{process_id}", data)
+          deploy_graph(dir, options.merge({:files_to_exclude => [options[:params]]}))
         end
       end
-      puts HighLine::color("Deploy DONE #{dir}", HighLine::BOLD) if verbose
-      res
     end
 
     def self.execute_process(link, dir, options={})
@@ -78,8 +46,8 @@ module GoodData::Command
       if type == :ruby 
         result = GoodData.post(link, {
           :execution => {
-           :graph => (dir + "main.rb").to_s,
-           :params => options[:params]
+           :graph => ("./main.rb").to_s,
+           :params => options[:expanded_params]
           }
         })
         begin
@@ -122,16 +90,61 @@ module GoodData::Command
 
       with_deploy(dir, options.merge(:name => name)) do |deploy_response|
         puts HighLine::color("Executing", HighLine::BOLD) if verbose
-        # if email.nil?
-        #   result = execute_process(deploy_response["process"]["links"]["executions"], dir, options)
-        # else
-          # create_email_channel(options) do |channel_response|
-            # subscribe_on_finish(:success, channel_response, deploy_response, options)
-            result = execute_process(deploy_response["process"]["links"]["executions"], dir, options)
-          # end
-        # end
+        result = execute_process(deploy_response["process"]["links"]["executions"], dir, options)
       end
     end
 
+    private
+    def self.deploy_graph(dir, options={})
+      dir = Pathname(dir) || fail("Directory is not specified")
+      fail "\"#{dir}\" is not a directory" unless dir.directory?
+      files_to_exclude = options[:files_to_exclude].map {|p| Pathname(p)}
+
+      # project_id = options[:project_id] || fail("Project Id has to be specified")
+      
+
+      type = options[:type] || "GRAPH"
+      deploy_name = options[:name]
+      verbose = options[:verbose] || false
+      
+      puts HighLine::color("Deploying #{dir}", HighLine::BOLD) if verbose
+      res = nil
+
+      Tempfile.open("deploy-graph-archive") do |temp|
+        Zip::OutputStream.open(temp.path) do |zio|
+          FileUtils::cd(dir) do
+
+            files_to_pack = Dir.glob("./**/*").reject {|f| files_to_exclude.include?(Pathname(dir) + f)}
+            files_to_pack.each do |item|
+              puts "including #{item}" if verbose
+              unless File.directory?(item)
+                zio.put_next_entry(item)
+                zio.print IO.read(item)
+              end
+            end
+          end
+        end
+
+        GoodData.upload_to_user_webdav(temp.path)
+        process_id = options[:process]
+        
+        data = {
+          :process => {
+            :name => deploy_name,
+            :path => "/uploads/#{File.basename(temp.path)}",
+            :type => type
+          }
+        }
+        res = if process_id.nil?
+          GoodData.post("/gdc/projects/#{GoodData.project.pid}/dataload/processes", data)
+        else
+          GoodData.put("/gdc/projects/#{GoodData.project.pid}/dataload/processes/#{process_id}", data)
+        end
+      end
+      puts HighLine::color("Deploy DONE #{dir}", HighLine::BOLD) if verbose
+      res
+    end
+
+
   end
 end

data/lib/gooddata/commands/runners.rb

@@ -4,18 +4,18 @@ module GoodData::Command
   class Runners
 
     def self.run_ruby_locally(brick_dir, options={})
-      pid = options[:project]
+      pid = options[:project_id]
       fail "You have to specify a project ID" if pid.nil?
       fail "You have to specify directory of the brick run" if brick_dir.nil?
       fail "You specified file as a birck run directory. You have to specify directory." if File.exist?(brick_dir) && !File.directory?(brick_dir)
 
-      params = options[:params] || {}
+      params = options[:expanded_params] || {}
 
       GoodData.connection.connect!
       sst = GoodData.connection.cookies[:cookies]["GDCAuthSST"]
       pwd = Pathname.new(Dir.pwd)
       logger_stream = STDOUT
-      
+
       server_uri = URI(options[:server]) unless options[:server].nil?
       scheme = server_uri.nil? ? "" : server_uri.scheme
       hostname = server_uri.nil? ? "" : server_uri.host
@@ -24,12 +24,15 @@ script_body = <<-script_body
       require 'fileutils'
       FileUtils::cd(\"#{pwd+brick_dir}\") do\
         require 'bundler/setup'
-        eval(File.read(\"main.rb\")).call({
+
+        $SCRIPT_PARAMS = {
           :GDC_SST => \"#{sst}\",
           :GDC_PROJECT_ID => \"#{pid}\",
           :GDC_PROTOCOL => \"#{scheme}\",
-          :GDC_SERVER => \"#{hostname}\"
-        }.merge(#{params}))
+          :GDC_SERVER => \"#{hostname}\",
+          :GDC_LOGGER_FILE => STDOUT
+        }.merge(#{params})
+        eval(File.read(\"./main.rb\"))
       end
 script_body
 

data/lib/gooddata/connection.rb

@@ -74,7 +74,7 @@ module GoodData
       @username   = username
       @password   = password
       @url        = options[:server] || DEFAULT_URL
-      @auth_token = options.delete(:auth_token)
+      @auth_token = options.delete(:token)
       @options    = options
 
       @server = create_server_connection(@url, @options)
@@ -118,9 +118,10 @@ module GoodData
     #
     #   Connection.new(username, password).post '/gdc/projects', { ... }
     def post(path, data, options = {})
-      payload = data.is_a?(Hash) ? data.to_json : data
-      GoodData.logger.debug "POST #{@server}#{path}, payload: #{payload}"
+      
+      GoodData.logger.debug("POST #{@server}#{path}, payload: #{scrub_params(data, [:password, :login, :authorizationToken])}")
       ensure_connection
+      payload = data.is_a?(Hash) ? data.to_json : data
       b = Proc.new { @server[path].post payload, cookies }
       process_response(options, &b)
     end
@@ -196,6 +197,7 @@ module GoodData
     # /uploads/ resources are special in that they use a different
     # host and a basic authentication.
     def upload(file, options={})
+
       ensure_connection
 
       dir = options[:directory] || ''
@@ -236,8 +238,7 @@ module GoodData
       filename = options[:filename] || options[:stream] ? "randome-filename.txt" : File.basename(file)
 
       # Upload the file
-      puts "uploading the file #{URI.join(url, filename).to_s}"
-      
+      # puts "uploading the file #{URI.join(url, filename).to_s}"
       req = RestClient::Request.new({
         :method => :put,
         :url => URI.join(url, filename).to_s,
@@ -246,24 +247,35 @@ module GoodData
           :user_agent => GoodData.gem_version_string,
         },
         :payload => payload,
-        :raw_response => true
-      }.merge(cookies))
+        :raw_response => true,
+        :user => @username,
+        :password => @password
+      })
+      # .merge(cookies))
       resp = req.execute
       true
     end
 
-    def download(what, where)
-      stage_url = @options[:webdav_server] || @url.sub(/\./, '-di.')
-      url = stage_url + STAGE_PATH + what
-      File.open(where, 'w') do |f|
-        resp = RestClient::Request.execute({
-          :method => 'GET',
-          :url => url,
-          :user => @username,
-          :password => @password,
-          :timeout => 0
-        }) do |chunk, x, y|
-          f.write chunk
+    def download(what, where, options={})
+      staging_uri = options[:staging_url].to_s
+      url = staging_uri + what
+      req = RestClient::Request.new({
+        :method => 'GET',
+        :url => url,
+        :user => @username,
+        :password => @password
+      })
+
+      if where.is_a?(String)
+        File.open(where, 'w') do |f|
+          req.execute do |chunk, x, y|
+            f.write chunk
+          end
+        end
+      else
+        # Assume it is a IO stream
+        req.execute do |chunk, x, y|
+          where.write chunk
         end
       end
     end
@@ -285,7 +297,7 @@ module GoodData
     end
 
     def connect
-      # GoodData.logger.info "Connecting to GoodData..."
+      GoodData.logger.info "Connecting to GoodData..."
       @status = :connecting
       authenticate
     end
@@ -350,5 +362,18 @@ module GoodData
         authenticate
       end
     end
+
+    def scrub_params(params, keys)
+      keys = keys.reduce([]) {|memo, k| memo.concat([k.to_s, k.to_sym])}
+
+      new_params = Marshal.load(Marshal.dump(params))
+      GoodData::Helpers.hash_dfs(new_params) do |k, key|
+        keys.each do |key_to_scrub|
+          k[key_to_scrub] = ("*" * k[key_to_scrub].length) if k && k.has_key?(key_to_scrub)
+        end
+      end      
+      new_params
+    end
+
   end
 end