gooddata 0.6.11 → 0.6.12

data/lib/gooddata/{rest/connections/connections.rb → models/user_filters/user_filters.rb}

@@ -3,6 +3,7 @@
 require 'pathname'
 
 base = Pathname(__FILE__).dirname.expand_path
+
 Dir.glob(base + '*.rb').each do |file|
   require_relative file
 end

data/lib/gooddata/models/user_filters/variable_user_filter.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+
+module GoodData
+  class VariableUserFilter < UserFilter
+    # Creates or updates the variable user filter on the server
+    #
+    # @return [String]
+    def save
+      res = client.post(uri, :variable => @json)
+      @json['uri'] = res['uri']
+      self
+    end
+  end
+end

data/lib/gooddata/rest/client.rb

@@ -4,7 +4,7 @@ require 'rest-client'
 
 require_relative '../helpers/auth_helpers'
 
-require_relative 'connections/connections'
+require_relative 'connection'
 require_relative 'object_factory'
 
 require_relative '../mixins/inspector'
@@ -19,7 +19,7 @@ module GoodData
       #################################
       # Constants
       #################################
-      DEFAULT_CONNECTION_IMPLEMENTATION = Connections::RestClientConnection
+      DEFAULT_CONNECTION_IMPLEMENTATION = GoodData::Rest::Connection
 
       #################################
       # Class variables
@@ -60,6 +60,11 @@ module GoodData
         # @param password [String] Password to be used for authentication
         # @return [GoodData::Rest::Client] Client
         def connect(username, password, opts = { :verify_ssl => true })
+          if username.nil? && password.nil?
+            username = ENV['GD_GEM_USER']
+            password = ENV['GD_GEM_PASSWORD']
+          end
+
           new_opts = opts.dup
           if username.is_a?(Hash) && username.key?(:sst_token)
             new_opts[:sst_token] = username[:sst_token]
@@ -83,7 +88,7 @@ module GoodData
 
           if client
             at_exit do
-              # puts client.connection.stats_table if client && client.connection
+              puts client.connection.stats_table if client && client.connection && (GoodData.stats_on? || client.stats_on?)
             end
           end
 
@@ -104,18 +109,8 @@ module GoodData
         end
 
         # Retry block if exception thrown
-        def retryable(options = {}, &_block)
-          opts = { :tries => 1, :on => Exception }.merge(options)
-
-          retry_exception, retries = opts[:on], opts[:tries]
-
-          begin
-            return yield
-          rescue retry_exception
-            retry if (retries -= 1) > 0
-          end
-
-          yield
+        def retryable(options = {}, &block)
+          GoodData::Rest::Connection.retryable(options, &block)
         end
 
         alias_method :client, :connection
@@ -145,7 +140,7 @@ module GoodData
       end
 
       def create_project(options = { title: 'Project', auth_token: ENV['GD_PROJECT_TOKEN'] })
-        GoodData::Project.create(options.merge(client: self))
+        GoodData::Project.create({ client: self }.merge(options))
       end
 
       def create_project_from_blueprint(blueprint, options = {})
@@ -192,8 +187,24 @@ module GoodData
         nil
       end
 
-      def user
-        create(GoodData::Profile, @connection.user)
+      def user(id = nil)
+        if id
+          create(GoodData::Profile, get(id))
+        else
+          create(GoodData::Profile, @connection.user)
+        end
+      end
+
+      def stats_off
+        @stats = false
+      end
+
+      def stats_on
+        @stats = true
+      end
+
+      def stats_on? # rubocop:disable Style/TrivialAccessors
+        @stats
       end
 
       #######################
@@ -253,7 +264,7 @@ module GoodData
 
         while response.code == code
           sleep sleep_interval
-          GoodData::Rest::Client.retryable(:tries => 3, :on => RestClient::InternalServerError) do
+          GoodData::Rest::Client.retryable(:tries => 3, :refresh_token => proc { connection.refresh_token }) do
             sleep sleep_interval
             response = get(link, :process => false)
           end
@@ -279,7 +290,7 @@ module GoodData
         response = get(link)
         while bl.call(response)
           sleep sleep_interval
-          GoodData::Rest::Client.retryable(:tries => 3, :on => RestClient::InternalServerError) do
+          GoodData::Rest::Client.retryable(:tries => 3, :refresh_token => proc { connection.refresh_token }) do
             sleep sleep_interval
             response = get(link)
           end
@@ -318,7 +329,7 @@ module GoodData
         @connection.download source_relative_path, target_file_path, options
       end
 
-      def download_from_user_webdav(source_relative_path, target_file_path, options = {})
+      def download_from_user_webdav(source_relative_path, target_file_path, options = { :client => GoodData.client, :project => project })
         download(source_relative_path, target_file_path, options.merge(
             :directory => options[:directory],
             :staging_url => get_user_webdav_url(options)

data/lib/gooddata/rest/connection.rb

@@ -23,6 +23,16 @@ module GoodData
         :headers => DEFAULT_HEADERS
       }
 
+      RETRYABLE_ERRORS = [
+        RestClient::InternalServerError,
+        RestClient::RequestTimeout,
+        RestClient::MethodNotAllowed,
+        SystemCallError,
+        Timeout::Error
+      ]
+
+      RETRYABLE_ERRORS << Net::ReadTimeout if Net.const_defined?(:ReadTimeout)
+
       class << self
         def construct_login_payload(username, password)
           res = {
@@ -34,6 +44,36 @@ module GoodData
           }
           res
         end
+
+        # Retry block if exception thrown
+        def retryable(options = {}, &_block)
+          opts = { :tries => 1, :on => RETRYABLE_ERRORS }.merge(options)
+
+          retry_exception, retries = opts[:on], opts[:tries]
+
+          unless retry_exception.is_a?(Array)
+            retry_exception = [retry_exception]
+          end
+
+          retry_time = 1
+          begin
+            return yield
+          rescue RestClient::Forbidden => e # , RestClient::Unauthorized => e
+            raise e unless options[:refresh_token]
+            options[:refresh_token].call
+            retry if (retries -= 1) > 0
+          rescue RestClient::TooManyRequests
+            GoodData.logger.warn "Too many requests, retrying in #{retry_time} seconds"
+            sleep retry_time
+            retry_time *= 1.5
+            retry
+          rescue *retry_exception => e
+            GoodData.logger.warn e.inspect
+            retry if (retries -= 1) > 0
+          end
+
+          yield
+        end
       end
 
       attr_reader :cookies
@@ -41,17 +81,39 @@ module GoodData
       attr_reader :user
 
       def initialize(opts)
+        @stats = {}
+        @opts = opts
+
+        @headers = DEFAULT_HEADERS.dup
         @user = nil
+        @server = nil
 
-        @stats = {}
         @opts = opts
+        headers = opts[:headers] || {}
+        @headers.merge! headers
 
         # Initialize cookies
         reset_cookies!
+
+        @at_exit_handler_installed = nil
       end
 
       # Connect using username and password
       def connect(username, password, options = {})
+        server = options[:server] || DEFAULT_URL
+        @server = RestClient::Resource.new server, DEFAULT_LOGIN_PAYLOAD
+
+        # Install at_exit handler first
+        unless @at_exit_handler_installed
+          begin
+            at_exit { disconnect if @user }
+          rescue RestClient::Unauthorized
+            GoodData.logger.info 'Already logged out'
+          ensure
+            @at_exit_handler_installed = true
+          end
+        end
+
         # Reset old cookies first
         if options[:sst_token]
           merge_cookies!('GDCAuthSST' => options[:sst_token])
@@ -62,8 +124,8 @@ module GoodData
           credentials = Connection.construct_login_payload(username, password)
           @auth = post(LOGIN_PATH, credentials, :dont_reauth => true)['userLogin']
 
-          @user = get(@auth['profile'])
           refresh_token :dont_reauth => true
+          @user = get(@auth['profile'])
         end
       end
 
@@ -71,7 +133,12 @@ module GoodData
       def disconnect
         # TODO: Wrap somehow
         url = @auth['state']
-        delete url if url
+
+        begin
+          delete url if url
+        rescue RestClient::Unauthorized
+          GoodData.logger.info 'Already disconnected'
+        end
 
         @auth = nil
         @server = nil
@@ -80,6 +147,72 @@ module GoodData
         reset_cookies!
       end
 
+      def download(what, where, options = {})
+        # handle the path (directory) given in what
+        ilast_slash = what.rindex('/')
+        if ilast_slash.nil?
+          what_dir = ''
+        else
+          # take the directory from the path
+          what_dir = what[0..ilast_slash - 1]
+          # take the filename from the path
+          what = what[ilast_slash + 1..-1]
+        end
+
+        option_dir = options[:directory] || ''
+        option_dir = option_dir[0..-2] if option_dir[-1] == '/'
+
+        # join the otion dir with the what_dir
+        # [option dir empty, what dir empty] => the joined dir
+        dir_hash = {
+          [true, true] => '',
+          [true, false] => what_dir,
+          [false, true] => option_dir,
+          [false, false] => "#{what_dir}/#{option_dir}"
+        }
+        dir = dir_hash[[option_dir.empty?, what_dir.empty?]]
+
+        staging_uri = options[:staging_url].to_s
+
+        base_url = dir.empty? ? staging_uri : URI.join(staging_uri, "#{dir}/").to_s
+        url = URI.join(base_url, CGI.escape(what)).to_s
+
+        b = proc do
+          raw = {
+            :headers => {
+              :user_agent => GoodData.gem_version_string
+            },
+            :method => :get,
+            :url => url
+          }.merge(cookies)
+
+          if where.is_a?(IO)
+            RestClient::Request.execute(raw) do |chunk, _x, response|
+              if response.code.to_s != '200'
+                fail ArgumentError, "Error downloading #{url}. Got response: #{response.code} #{response} #{response.body}"
+              end
+              where.write chunk
+            end
+          else
+            # Assume it is a string or file
+            File.open(where, 'w') do |f|
+              RestClient::Request.execute(raw) do |chunk, _x, response|
+                if response.code.to_s != '200'
+                  fail ArgumentError, "Error downloading #{url}. Got response: #{response.code} #{response} #{response.body}"
+                end
+                f.write chunk
+              end
+            end
+          end
+        end
+
+        res = nil
+        GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
+          res = b.call
+        end
+        res
+      end
+
       def refresh_token(_options = {})
         begin # rubocop:disable RedundantBegin
           get TOKEN_PATH, :dont_reauth => true # avoid infinite loop GET fails with 401
@@ -99,29 +232,47 @@ module GoodData
       # HTTP DELETE
       #
       # @param uri [String] Target URI
-      def delete(uri, _options = {})
-        fail NotImplementedError "DELETE #{uri}"
+      def delete(uri, options = {})
+        GoodData.logger.debug "DELETE: #{@server.url}#{uri}"
+        profile "DELETE #{uri}" do
+          b = proc { @server[uri].delete cookies }
+          process_response(options, &b)
+        end
       end
 
       # HTTP GET
       #
       # @param uri [String] Target URI
-      def get(uri, _options = {})
-        fail NotImplementedError "GET #{uri}"
+      def get(uri, options = {}, &user_block)
+        GoodData.logger.debug "GET: #{@server.url}#{uri}"
+        profile "GET #{uri}" do
+          b = proc { @server[uri].get(cookies, &user_block) }
+          process_response(options, &b)
+        end
       end
 
       # HTTP PUT
       #
       # @param uri [String] Target URI
-      def put(uri, _data, _options = {})
-        fail NotImplementedError "PUT #{uri}"
+      def put(uri, data, options = {})
+        payload = data.is_a?(Hash) ? data.to_json : data
+        GoodData.logger.debug "PUT: #{@server.url}#{uri}, #{scrub_params(data, [:password, :login, :authorizationToken])}"
+        profile "PUT #{uri}" do
+          b = proc { @server[uri].put payload, cookies }
+          process_response(options, &b)
+        end
       end
 
       # HTTP POST
       #
       # @param uri [String] Target URI
-      def post(uri, _data, _options = {})
-        fail NotImplementedError "POST #{uri}"
+      def post(uri, data, options = {})
+        GoodData.logger.debug "POST: #{@server.url}#{uri}, #{scrub_params(data, [:password, :login, :authorizationToken])}"
+        profile "POST #{uri}" do
+          payload = data.is_a?(Hash) ? data.to_json : data
+          b = proc { @server[uri].post payload, cookies }
+          process_response(options, &b)
+        end
       end
 
       # Reader method for SST token
@@ -155,12 +306,133 @@ module GoodData
         cookies[:cookies]['GDCAuthTT']
       end
 
+      # Uploads a file to GoodData server
+      def upload(file, options = {})
+        def do_stream_file(uri, filename, _options = {})
+          puts "uploading the file #{uri}"
+
+          to_upload = File.new(filename)
+          cookies_str = cookies[:cookies].map { |cookie| "#{cookie[0]}=#{cookie[1]}" }.join(';')
+          req = Net::HTTP::Put.new(uri.path, 'User-Agent' => GoodData.gem_version_string, 'Cookie' => cookies_str)
+          req.content_length = to_upload.size
+          req.body_stream = to_upload
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+          response = nil
+          GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
+            response = http.start { |client| client.request(req) }
+          end
+          response
+        end
+
+        def webdav_dir_exists?(url)
+          method = :get
+          GoodData.logger.debug "#{method}: #{url}"
+
+          b = proc do
+            raw = {
+              :method => method,
+              :url => url,
+              :headers => @headers
+            }
+            begin
+              RestClient::Request.execute(raw.merge(cookies))
+            rescue RestClient::Exception => e
+              false if e.http_code == 404
+            end
+          end
+
+          res = nil
+          GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
+            res = b.call
+          end
+          res
+        end
+
+        def create_webdav_dir_if_needed(url)
+          return if webdav_dir_exists?(url)
+
+          method = :mkcol
+          GoodData.logger.debug "#{method}: #{url}"
+          b = proc do
+            raw = {
+              :method => method,
+              :url => url,
+              :headers => @headers
+            }.merge(cookies)
+            RestClient::Request.execute(raw)
+          end
+
+          GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
+            b.call
+          end
+        end
+
+        dir = options[:directory] || ''
+        staging_uri = options[:staging_url].to_s
+        url = dir.empty? ? staging_uri : URI.join(staging_uri, "#{dir}/").to_s
+
+        # Make a directory, if needed
+        create_webdav_dir_if_needed url unless dir.empty?
+
+        webdav_filename = options[:filename] || File.basename(file)
+        do_stream_file URI.join(url, CGI.escape(webdav_filename)), file
+      end
+
       private
 
       def merge_cookies!(cookies)
         @cookies[:cookies].merge! cookies
       end
 
+      def process_response(options = {}, &block)
+        # begin
+        #   # Simply try again when ConnectionReset, ConnectionRefused etc.. (see e.g. MSF-7591)
+        #   response = GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => Proc.new { refresh_token }) do
+        #     block.call
+        #   end
+        # rescue RestClient::Unauthorized
+        #   raise $ERROR_INFO if options[:dont_reauth]
+        #   GoodData::Rest::Connection.retryable(:tries => 2) do
+        #     refresh_token
+        #     response = block.call
+        #   end
+        # end
+
+        response = GoodData::Rest::Connection.retryable(:tries => 2, :refresh_token => proc { refresh_token }) do
+          block.call
+        end
+
+        merge_cookies! response.cookies
+        content_type = response.headers[:content_type]
+        return response if options[:process] == false
+
+        if content_type == 'application/json' || content_type == 'application/json;charset=UTF-8'
+          result = response.to_str == '""' ? {} : MultiJson.load(response.to_str)
+          GoodData.logger.debug "Request ID: #{response.headers[:x_gdc_request]} - Response: #{result.inspect}"
+        elsif ['text/plain;charset=UTF-8', 'text/plain; charset=UTF-8', 'text/plain'].include?(content_type)
+          result = response
+          GoodData.logger.debug 'Response: plain text'
+        elsif content_type == 'application/zip'
+          result = response
+          GoodData.logger.debug 'Response: a zipped stream'
+        elsif response.headers[:content_length].to_s == '0'
+          result = nil
+          GoodData.logger.debug 'Response: Empty response possibly 204'
+        elsif response.code == 204
+          result = nil
+          GoodData.logger.debug 'Response: 204 no content'
+        else
+          fail "Unsupported response content type '%s':\n%s" % [content_type, response.to_str[0..127]]
+        end
+        result
+      rescue RestClient::Exception => e
+        GoodData.logger.debug "Response: #{e.response}"
+        raise $ERROR_INFO
+      end
+
       def profile(title, &block)
         t1 = Time.now
         res = block.call